def standard_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['end'])
moments = market_client.get_moments(symbols=['AAPL'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
self.assertEqual(len(STOCKS_DAILY['AAPL']['records']), len(moments))
for record in STOCKS_DAILY['AAPL']['records']:
record['symbol'] = 'AAPL'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def test_add_later_moment(self):
moment_later = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_LATER])
self.day_series.add_moment(moment_later)
moment_1 = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1])
self.day_series.add_moment(moment_1)
moment_earlier = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER])
self.day_series.add_moment(moment_earlier)
expected_timestamps = [
IntuitiveDateConverter.to_epoch_s(x) for x in [
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][
self.I_RECORD_LATER]['epoch_ms']
]
]
expected_moments = [moment_earlier, moment_1, moment_later]
self.assertEqual(expected_timestamps, self.day_series.timestamps)
self.assertEqual(
expected_moments,
[self.day_series.moments[x] for x in self.day_series.timestamps])
def __init__(self, data_ingest_manager: DataIngestManager,
traders: List[Trader]) -> List:
self.data_ingest_manager = data_ingest_manager
self.traders = traders
self.today_day_str = IntuitiveDateConverter.to_day_str(datetime.now())
self.today_epoch_s = IntuitiveDateConverter.to_epoch_s(
self.today_day_str)
def test_date_str_representation(self):
moment_data = {
'symbol': 'TEST',
'epoch_s': IntuitiveDateConverter.to_epoch_s('2021-01-01')
}
moment = Moment.from_dict(moment_data)
self.assertEqual('2021-01-01', moment.date_str)
def _populate_recent_register(self, stocks: List[Stock]):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
IntuitiveDateConverter.to_datetime(self.today_epoch_s) -
timedelta(days=20))
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
IntuitiveDateConverter.to_datetime(self.today_epoch_s) -
timedelta(days=1))
self.data_ingest_manager.populate_warehouse(stocks,
start_epoch_s,
end_epoch_s,
batch_size=1)
register = self.data_ingest_manager.create_register_from_warehouse(
stocks, start_epoch_s, end_epoch_s)
return register
def run(self, candidates: List[Stock], current_holdings: List[Stock]):
self.today_day_str = IntuitiveDateConverter.to_day_str(datetime.now())
self.today_epoch_s = IntuitiveDateConverter.to_epoch_s(
self.today_day_str)
all_stocks = candidates + current_holdings
register = self._populate_register(all_stocks)
signals = self.apply_traders(register, candidates, current_holdings)
return signals
def _run_and_evaluate_to_epoch_s(self,
conversion_input: object,
expected_output_type: type,
expected_output_value: object) -> None:
idc = IntuitiveDateConverter()
actual_output = idc.to_epoch_s(conversion_input)
actual_output_type = type(actual_output)
self.assertEqual(expected_output_type, actual_output_type)
self.assertEqual(expected_output_value, actual_output)
def test_moment_from_dict_3(self):
moment_data = {
'Symbol': 'TEST',
'EPOCH_S': IntuitiveDateConverter.to_epoch_s('2017-04-30'),
'oPen': 10.5,
'cloSE': 11.7,
'high': 12,
'loW': 9.2,
'VOLume': 5598
}
moment = Moment.from_dict(moment_data)
self.assertEqual('TEST', moment.symbol)
self.assertEqual(IntuitiveDateConverter.to_epoch_s('2017-04-30'),
moment.epoch_s)
self.assertEqual(10.5, moment.open)
self.assertEqual(11.7, moment.close)
self.assertEqual(12, moment.high)
self.assertEqual(9.2, moment.low)
self.assertEqual(5598, moment.volume)
self.assertEqual('2017-04-30', moment.date_str)
def multiple_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['end'])
moments = market_client.get_moments(symbols=['BYND', 'GOOG'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
n_expected = len(STOCKS_DAILY['BYND']['records']) + len(
STOCKS_DAILY['GOOG']['records'])
self.assertEqual(n_expected, len(moments))
for record in STOCKS_DAILY['BYND']['records']:
record['symbol'] = 'BYND'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
for record in STOCKS_DAILY['GOOG']['records']:
record['symbol'] = 'GOOG'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def get(self, stock: Stock, day: str, offset=0) -> Moment:
"""
Retrieve a moment by stock and date.
"""
epoch_s = IntuitiveDateConverter.to_epoch_s(day)
if offset == 0:
return self.series[stock].get(epoch_s)
_series = self.series.get(stock, None)
if _series is None:
return None
if epoch_s not in _series.timestamps:
return None
_ix = _series.timestamps.index(epoch_s)
_ix += offset
if _ix < 0:
return None
if _ix >= len(_series.timestamps):
return None
return _series.get(_series.timestamps[_ix])
def get(self, day: str) -> Moment:
"""
Retrieve a moment by date.
"""
epoch_s = IntuitiveDateConverter.to_epoch_s(day)
return self.moments.get(epoch_s, None)
def get_moments(self, stocks: List[Stock], start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
"""
Retrieve all daily market data for symbols and date range and convert into list of moments.
Retries after a pause if API rate limit is reached.
"""
moments = list()
for i_0 in range(0, len(stocks), BATCH_SIZE):
i_f = i_0 + BATCH_SIZE
i_f = len(stocks) if i_f > len(stocks) else i_f
stock_batch = stocks[i_0:i_f]
for _ in range(MAX_RETRIES):
try:
stock_batch_str = ','.join(
[s.to_str() for s in stock_batch])
print(f'api request for symbols: {stock_batch_str}')
content = self.make_api_call(stock_batch, start_epoch_s,
end_epoch_s)
except requests.exceptions.ConnectionError as conerr:
print(conerr)
print(
f'Schedule to retry after {RETRY_PAUSE_CONNECTION_ERROR} seconds'
)
time.sleep(RETRY_PAUSE_CONNECTION_ERROR)
except ChunkedEncodingError as _:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
else:
if 'code' in content:
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
break
if len(stocks) == 1:
content = {stocks[0].symbol: content}
for symbol, data in content.items():
if 'values' not in data:
if data['code'] == ERROR_CODE_NO_DATA_AVAILABLE:
print(f'No data available for {symbol}. Skipping.')
continue
print('unexpected response:')
print(content)
exchange = data['meta']['exchange']
for record in data['values']:
day_data = dict()
day_data['exchange'] = exchange
day_data['symbol'] = symbol
day_data['epoch_s'] = IntuitiveDateConverter.to_epoch_s(
record['datetime'])
day_data['open'] = float(record['open'])
day_data['close'] = float(record['close'])
day_data['high'] = float(record['high'])
day_data['low'] = float(record['low'])
day_data['volume'] = int(record['volume'])
moment = Moment(**day_data)
moments.append(moment)
return moments
def get_technical_indicator(self, stock: Stock, indicator_name: str,
start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
start_date = IntuitiveDateConverter.to_day_str(start_epoch_s)
end_datetime = IntuitiveDateConverter.to_datetime(end_epoch_s)
end_datetime += timedelta(days=1)
end_date = IntuitiveDateConverter.to_day_str(end_datetime)
request_url = f'https://api.twelvedata.com/{indicator_name.lower()}?symbol=' \
f'{stock.symbol}&exchange={stock.exchange}&interval=1day&start_date=' \
f'{start_date}&end_date={end_date}&apikey={self.api_key}'
for _ in range(MAX_RETRIES):
try:
print(
f'API call for technical indicator {indicator_name.lower()} for {stock.to_str()}'
)
response = requests.get(request_url)
except ChunkedEncodingError:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
except Exception as e:
print("Stopping retry loop because of un-handled error")
raise e
else:
content = response.content
content = json.loads(content)
if 'code' in content.keys():
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
if 'values' not in content:
print('unexpected response:')
print(content)
break
moments = list()
if 'values' not in content:
return moments
for item in content['values']:
epoch_s = IntuitiveDateConverter.to_epoch_s(item['datetime'])
for k, v in item.items():
if k == 'datetime':
continue
if k == indicator_name:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
indicator_name: item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
else:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
f'{indicator_name}_{k}': item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
return moments