def test_add_later_moment(self):
moment_later = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_LATER])
self.day_series.add_moment(moment_later)
moment_1 = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1])
self.day_series.add_moment(moment_1)
moment_earlier = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER])
self.day_series.add_moment(moment_earlier)
expected_timestamps = [
IntuitiveDateConverter.to_epoch_s(x) for x in [
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][
self.I_RECORD_LATER]['epoch_ms']
]
]
expected_moments = [moment_earlier, moment_1, moment_later]
self.assertEqual(expected_timestamps, self.day_series.timestamps)
self.assertEqual(
expected_moments,
[self.day_series.moments[x] for x in self.day_series.timestamps])
def test_get(self):
moments = list()
for symbol in ('AAPL', 'BYND'):
_chunk = [Moment.from_dict(x) for x in STOCKS_DAILY[symbol]['records']]
for moment in _chunk:
moment.symbol = symbol
moments += _chunk
self.symbol_day_register.add_moments(moments)
for symbol in ('AAPL', 'BYND'):
for _ in range(3):
i = random.randint(0, len(STOCKS_DAILY[symbol]['records']) - 1)
moment = Moment.from_dict(STOCKS_DAILY[symbol]['records'][i])
moment.symbol = symbol
self.assertEqual(moment, self.symbol_day_register.get(symbol, moment.epoch_s))
def test_store_moments(self):
moments = get_test_moments()
wc = SqliteDayWarehouseClient(SQLITE_TEST_DB_PATH)
wc.store_moments(moments)
con = sqlite3.connect(SQLITE_TEST_DB_PATH)
cur = con.cursor()
cur.execute("""SELECT COUNT(*) FROM day;""")
result = cur.fetchall()
self.assertEqual(len(moments), result[0][0])
cur.execute(""" SELECT
symbol, date, open, close, high, low, volume, epoch_s
FROM day;""")
result = cur.fetchall()
for r in result:
moment = Moment.from_dict({
'symbol': r[0],
'epoch_s': r[7],
'open': r[2],
'close': r[3],
'high': r[4],
'low': r[5],
'volume': r[6]
})
self.assertIn(moment, moments)
def test_add_moment(self):
moment_1 = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1])
self.day_series.add_moment(moment_1)
self.assertEqual(moment_1.epoch_s, self.day_series.timestamps[0])
self.assertEqual(
moment_1, self.day_series.moments[self.day_series.timestamps[0]])
def test_date_str_representation(self):
moment_data = {
'symbol': 'TEST',
'epoch_s': IntuitiveDateConverter.to_epoch_s('2021-01-01')
}
moment = Moment.from_dict(moment_data)
self.assertEqual('2021-01-01', moment.date_str)
def add_moment(self, moment: Moment):
"""
Add a single moment. Re-establish chronological date order.
"""
stock = moment.get_stock()
if stock not in self.series:
self.series[stock] = DaySeries()
self.series[stock].add_moment(moment)
def test_get(self):
moments = [
Moment.from_dict(x) for x in STOCKS_DAILY['AAPL']['records']
]
moments_shuffled = moments.copy()
random.shuffle(moments_shuffled)
self.assertNotEqual(moments, moments_shuffled)
self.day_series.add_moments(moments_shuffled)
self.assertEqual(moments, self.day_series.get_moments())
def get_test_moments() -> List[Moment]:
"""
Return list of moments that can be used for tests.
"""
moments = list()
for symbol in STOCKS_DAILY.keys():
_chunk = [Moment.from_dict(x) for x in STOCKS_DAILY[symbol]['records']]
for moment in _chunk:
moment.symbol = symbol
moments += _chunk
return moments
def test_moment_from_dict_1(self):
moment_data = {'symbol': 'TEST'}
moment = Moment.from_dict(moment_data)
self.assertEqual('TEST', moment.symbol)
self.assertIsNone(moment.epoch_s)
self.assertIsNone(moment.open)
self.assertIsNone(moment.close)
self.assertIsNone(moment.high)
self.assertIsNone(moment.low)
self.assertIsNone(moment.volume)
def get_moments(self, symbols: List[str], start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
"""
Retrieve all daily market data for symbols and date range and convert into list of moments.
Retries after a pause if API rate limit is reached.
"""
moments = list()
for symbol in symbols:
print(f'api request for symbol: {symbol}')
for _ in range(MAX_RETRIES):
try:
content = self.make_api_call(symbol, start_epoch_s,
end_epoch_s)
except requests.exceptions.HTTPError as httpe:
if '429 Client Error: Too Many Requests for url' in str(
httpe):
print(httpe)
print(
f'Schedule to retry after {RETRY_PAUSE_S} seconds')
time.sleep(RETRY_PAUSE_S)
except requests.exceptions.ConnectionError as conerr:
print(conerr)
print(
f'Schedule to retry after {RETRY_PAUSE_CONNECTION_ERROR} seconds'
)
time.sleep(RETRY_PAUSE_CONNECTION_ERROR)
else:
break
if 'results' not in content:
print('Warning: nothing returned for symbol: ', symbol)
continue
for record in content['results']:
day_data = dict()
day_data['symbol'] = symbol
day_data['epoch_s'] = record['t']
day_data['open'] = record['o']
day_data['close'] = record['c']
day_data['high'] = record['h']
day_data['low'] = record['l']
day_data['volume'] = record['v']
moment = Moment.from_dict(day_data)
moments.append(moment)
time.sleep(REGULAR_PAUSE)
return moments
def test_add_moments(self):
moments = [
Moment.from_dict(x) for x in STOCKS_DAILY['AAPL']['records']
]
moments_shuffled = moments.copy()
random.shuffle(moments_shuffled)
self.assertNotEqual(moments, moments_shuffled)
self.day_series.add_moments(moments_shuffled)
expected_timestamps = [m.epoch_s for m in moments]
self.assertEqual(expected_timestamps, self.day_series.timestamps)
self.assertEqual(
moments,
[self.day_series.moments[x] for x in self.day_series.timestamps])
def multiple_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['end'])
moments = market_client.get_moments(symbols=['BYND', 'GOOG'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
n_expected = len(STOCKS_DAILY['BYND']['records']) + len(
STOCKS_DAILY['GOOG']['records'])
self.assertEqual(n_expected, len(moments))
for record in STOCKS_DAILY['BYND']['records']:
record['symbol'] = 'BYND'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
for record in STOCKS_DAILY['GOOG']['records']:
record['symbol'] = 'GOOG'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def standard_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['end'])
moments = market_client.get_moments(symbols=['AAPL'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
self.assertEqual(len(STOCKS_DAILY['AAPL']['records']), len(moments))
for record in STOCKS_DAILY['AAPL']['records']:
record['symbol'] = 'AAPL'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def test_add_multiple_moments(self):
moments = list()
for symbol in ('AAPL', 'BYND'):
_chunk = [Moment.from_dict(x) for x in STOCKS_DAILY[symbol]['records']]
for moment in _chunk:
moment.symbol = symbol
moments += _chunk
moments_shuffled = moments.copy()
random.shuffle(moments_shuffled)
self.assertNotEqual(moments, moments_shuffled)
self.symbol_day_register.add_moments(moments_shuffled)
actual_moments = list()
for symbol in ('AAPL', 'BYND'):
actual_moments += self.symbol_day_register.series[symbol].get_moments()
self.assertEqual(moments, actual_moments)
def get_moments(self, stocks: List[Stock], start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
"""
Get moments based on exchange, symbol, start, end filters.
"""
moments = list()
exchanges = list(set([s.exchange for s in stocks]))
for exchange in exchanges:
symbols = [s.symbol for s in stocks]
symbols_str = '\',\''.join(symbols)
symbols_str = '\'' + symbols_str + '\''
sql = f"""
SELECT
*
FROM day
WHERE TRUE
AND symbol IN ({symbols_str})
AND exchange = %s
AND epoch_s >= %s
AND epoch_s <= %s;
"""
params = [exchange, start_epoch_s, end_epoch_s]
self.cur.execute(sql, params)
result = self.cur.fetchall()
field_names = [
description[0] for description in self.cur.description
]
for r in result:
data = dict()
for i in range(len(field_names)):
data[field_names[i]] = r[i]
del data['date']
moments.append(Moment(**data))
return moments
def get_moments(self, symbols: List[str], start: datetime,
end: datetime) -> List[Moment]:
"""
Retrieve all daily market data for symbols and date range and convert into list of moments.
Retries after a pause if API rate limit is reached.
"""
moments = list()
days = [start + timedelta(x) for x in range((end - start).days)]
for symbol in symbols:
for day in days:
print(day)
for _ in range(MAX_RETRIES):
try:
content = self.make_api_call(symbol, day)
except requests.exceptions.HTTPError as httpe:
if '429 Client Error: Too Many Requests for url' in str(
httpe):
print(httpe)
print(
f'Schedule to retry after {RETRY_PAUSE_S} seconds'
)
time.sleep(RETRY_PAUSE_S)
else:
break
print(content)
if content['closingTrades'] is None:
break
day_data = dict()
day_data['symbol'] = symbol
day_data['date'] = datetime.strptime(content['day'][0:10],
DAY_FORMAT)
day_data['open'] = content['open']
day_data['close'] = content['close']
moment = Moment.from_dict(day_data)
moments.append(moment)
return moments
def _populate_today_register(self, stocks: List[Stock]):
register = SymbolDayRegister()
moments = list()
for stock in stocks:
prices = self.data_ingest_manager.market_client.get_real_time_price(
[stock])
for price in prices:
moment_data = {
'epoch_s': self.today_epoch_s,
'exchange': price['exchange'],
'symbol': price['symbol'],
'close': float(price['price'])
}
moment = Moment(**moment_data)
moments.append(moment)
register.add_moments(moments)
return register
def test_moment_from_dict_3(self):
moment_data = {
'Symbol': 'TEST',
'EPOCH_S': IntuitiveDateConverter.to_epoch_s('2017-04-30'),
'oPen': 10.5,
'cloSE': 11.7,
'high': 12,
'loW': 9.2,
'VOLume': 5598
}
moment = Moment.from_dict(moment_data)
self.assertEqual('TEST', moment.symbol)
self.assertEqual(IntuitiveDateConverter.to_epoch_s('2017-04-30'),
moment.epoch_s)
self.assertEqual(10.5, moment.open)
self.assertEqual(11.7, moment.close)
self.assertEqual(12, moment.high)
self.assertEqual(9.2, moment.low)
self.assertEqual(5598, moment.volume)
self.assertEqual('2017-04-30', moment.date_str)
def get_technical_indicator(self, stock: Stock, indicator_name: str,
start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
start_date = IntuitiveDateConverter.to_day_str(start_epoch_s)
end_datetime = IntuitiveDateConverter.to_datetime(end_epoch_s)
end_datetime += timedelta(days=1)
end_date = IntuitiveDateConverter.to_day_str(end_datetime)
request_url = f'https://api.twelvedata.com/{indicator_name.lower()}?symbol=' \
f'{stock.symbol}&exchange={stock.exchange}&interval=1day&start_date=' \
f'{start_date}&end_date={end_date}&apikey={self.api_key}'
for _ in range(MAX_RETRIES):
try:
print(
f'API call for technical indicator {indicator_name.lower()} for {stock.to_str()}'
)
response = requests.get(request_url)
except ChunkedEncodingError:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
except Exception as e:
print("Stopping retry loop because of un-handled error")
raise e
else:
content = response.content
content = json.loads(content)
if 'code' in content.keys():
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
if 'values' not in content:
print('unexpected response:')
print(content)
break
moments = list()
if 'values' not in content:
return moments
for item in content['values']:
epoch_s = IntuitiveDateConverter.to_epoch_s(item['datetime'])
for k, v in item.items():
if k == 'datetime':
continue
if k == indicator_name:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
indicator_name: item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
else:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
f'{indicator_name}_{k}': item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
return moments
def get_moments(self, stocks: List[Stock], start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
"""
Retrieve all daily market data for symbols and date range and convert into list of moments.
Retries after a pause if API rate limit is reached.
"""
moments = list()
for i_0 in range(0, len(stocks), BATCH_SIZE):
i_f = i_0 + BATCH_SIZE
i_f = len(stocks) if i_f > len(stocks) else i_f
stock_batch = stocks[i_0:i_f]
for _ in range(MAX_RETRIES):
try:
stock_batch_str = ','.join(
[s.to_str() for s in stock_batch])
print(f'api request for symbols: {stock_batch_str}')
content = self.make_api_call(stock_batch, start_epoch_s,
end_epoch_s)
except requests.exceptions.ConnectionError as conerr:
print(conerr)
print(
f'Schedule to retry after {RETRY_PAUSE_CONNECTION_ERROR} seconds'
)
time.sleep(RETRY_PAUSE_CONNECTION_ERROR)
except ChunkedEncodingError as _:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
else:
if 'code' in content:
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
break
if len(stocks) == 1:
content = {stocks[0].symbol: content}
for symbol, data in content.items():
if 'values' not in data:
if data['code'] == ERROR_CODE_NO_DATA_AVAILABLE:
print(f'No data available for {symbol}. Skipping.')
continue
print('unexpected response:')
print(content)
exchange = data['meta']['exchange']
for record in data['values']:
day_data = dict()
day_data['exchange'] = exchange
day_data['symbol'] = symbol
day_data['epoch_s'] = IntuitiveDateConverter.to_epoch_s(
record['datetime'])
day_data['open'] = float(record['open'])
day_data['close'] = float(record['close'])
day_data['high'] = float(record['high'])
day_data['low'] = float(record['low'])
day_data['volume'] = int(record['volume'])
moment = Moment(**day_data)
moments.append(moment)
return moments
def test_add_moment(self):
moment = Moment.from_dict(STOCKS_DAILY['AAPL']['records'][0])
moment.symbol = 'AAPL'
self.symbol_day_register.add_moment(moment)
self.assertIn('AAPL', self.symbol_day_register.series.keys())
self.assertEqual(moment, self.symbol_day_register.series['AAPL'].get_moments()[0])