def __init__(self, data_ingest_manager: DataIngestManager,
traders: List[Trader]) -> List:
self.data_ingest_manager = data_ingest_manager
self.traders = traders
self.today_day_str = IntuitiveDateConverter.to_day_str(datetime.now())
self.today_epoch_s = IntuitiveDateConverter.to_epoch_s(
self.today_day_str)
def run(self, candidates: List[Stock], current_holdings: List[Stock]):
self.today_day_str = IntuitiveDateConverter.to_day_str(datetime.now())
self.today_epoch_s = IntuitiveDateConverter.to_epoch_s(
self.today_day_str)
all_stocks = candidates + current_holdings
register = self._populate_register(all_stocks)
signals = self.apply_traders(register, candidates, current_holdings)
return signals
def _run_and_evaluate_to_epoch_s(self,
conversion_input: object,
expected_output_type: type,
expected_output_value: object) -> None:
idc = IntuitiveDateConverter()
actual_output = idc.to_epoch_s(conversion_input)
actual_output_type = type(actual_output)
self.assertEqual(expected_output_type, actual_output_type)
self.assertEqual(expected_output_value, actual_output)
def _build_url(self, exchange: str, symbols: List[str],
start_epoch_s: float, end_epoch_s: float) -> str:
"""
Build url from parameters and api key
"""
start_str = IntuitiveDateConverter.to_day_str(start_epoch_s)
end_datetime = IntuitiveDateConverter.to_datetime(end_epoch_s)
end_datetime += timedelta(days=1)
end_str = IntuitiveDateConverter.to_day_str(end_datetime)
symbols_str = ','.join(symbols)
url = f'https://api.twelvedata.com/time_series?exchange={exchange}&symbol={symbols_str}&interval=1day&' \
f'start_date={start_str}&end_date={end_str}&apikey={self.api_key}'
return url
def _build_url(self, symbol: str, start_epoch_s: float,
end_epoch_s: float) -> str:
"""
Build url from parameters and api key
"""
unit_str = 'day'
range_str = '1'
start_str = IntuitiveDateConverter.to_day_str(start_epoch_s)
end_str = IntuitiveDateConverter.to_day_str(end_epoch_s)
url = f'https://api.polygon.io/v2/aggs/ticker/{symbol}/range/{range_str}/{unit_str}/{start_str}/{end_str}?' \
f'apiKey={self.api_key}'
return url
def run(self):
all_timestamps = self.register.get_all_timestamps()
for epoch_s in all_timestamps:
for trader_id in range(len(self.trader_ensembles)):
trader_ensemble = self.trader_ensembles[trader_id]
self.trader_ensembles[trader_id].execute(
epoch_s, self.register)
net_worth_mean, net_worth_median, net_worth_sd = \
trader_ensemble.performance.get_net_worth_estimate(epoch_s)
cash_mean, cash_median, cash_sd = trader_ensemble.performance.get_cash_estimate(
epoch_s)
date = IntuitiveDateConverter.to_day_str(epoch_s)
self.campaign_recorder.record(
date, trader_id,
self.campaign_recorder.METRIC_NET_WORTH_MEAN,
net_worth_mean)
self.campaign_recorder.record(
date, trader_id,
self.campaign_recorder.METRIC_NET_WORTH_MEDIAN,
net_worth_median)
self.campaign_recorder.record(
date, trader_id,
self.campaign_recorder.METRIC_NET_WORTH_SD, net_worth_sd)
self.campaign_recorder.record(
date, trader_id, self.campaign_recorder.METRIC_CASH_MEAN,
cash_mean)
self.campaign_recorder.record(
date, trader_id, self.campaign_recorder.METRIC_CASH_MEDIAN,
cash_median)
self.campaign_recorder.record(
date, trader_id, self.campaign_recorder.METRIC_CASH_SD,
cash_sd)
def test_add_later_moment(self):
moment_later = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_LATER])
self.day_series.add_moment(moment_later)
moment_1 = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1])
self.day_series.add_moment(moment_1)
moment_earlier = Moment.from_dict(
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER])
self.day_series.add_moment(moment_earlier)
expected_timestamps = [
IntuitiveDateConverter.to_epoch_s(x) for x in [
STOCKS_DAILY['AAPL']['records'][self.I_RECORD_EARLIER]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][self.I_RECORD_1]
['epoch_ms'], STOCKS_DAILY['AAPL']['records'][
self.I_RECORD_LATER]['epoch_ms']
]
]
expected_moments = [moment_earlier, moment_1, moment_later]
self.assertEqual(expected_timestamps, self.day_series.timestamps)
self.assertEqual(
expected_moments,
[self.day_series.moments[x] for x in self.day_series.timestamps])
def standard_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['AAPL']['end'])
moments = market_client.get_moments(symbols=['AAPL'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
self.assertEqual(len(STOCKS_DAILY['AAPL']['records']), len(moments))
for record in STOCKS_DAILY['AAPL']['records']:
record['symbol'] = 'AAPL'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def test_date_str_representation(self):
moment_data = {
'symbol': 'TEST',
'epoch_s': IntuitiveDateConverter.to_epoch_s('2021-01-01')
}
moment = Moment.from_dict(moment_data)
self.assertEqual('2021-01-01', moment.date_str)
def _populate_recent_register(self, stocks: List[Stock]):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
IntuitiveDateConverter.to_datetime(self.today_epoch_s) -
timedelta(days=20))
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
IntuitiveDateConverter.to_datetime(self.today_epoch_s) -
timedelta(days=1))
self.data_ingest_manager.populate_warehouse(stocks,
start_epoch_s,
end_epoch_s,
batch_size=1)
register = self.data_ingest_manager.create_register_from_warehouse(
stocks, start_epoch_s, end_epoch_s)
return register
def test_moment_from_dict_3(self):
moment_data = {
'Symbol': 'TEST',
'EPOCH_S': IntuitiveDateConverter.to_epoch_s('2017-04-30'),
'oPen': 10.5,
'cloSE': 11.7,
'high': 12,
'loW': 9.2,
'VOLume': 5598
}
moment = Moment.from_dict(moment_data)
self.assertEqual('TEST', moment.symbol)
self.assertEqual(IntuitiveDateConverter.to_epoch_s('2017-04-30'),
moment.epoch_s)
self.assertEqual(10.5, moment.open)
self.assertEqual(11.7, moment.close)
self.assertEqual(12, moment.high)
self.assertEqual(9.2, moment.low)
self.assertEqual(5598, moment.volume)
self.assertEqual('2017-04-30', moment.date_str)
def run(self):
all_timestamps = self.register.get_all_timestamps()
for epoch_s in all_timestamps:
self.trader_ensemble.execute(epoch_s, self.register)
net_worth_mean, net_worth_median, net_worth_sd = \
self.trader_ensemble.performance.get_net_worth_estimate(epoch_s)
cash_mean, cash_median, cash_sd = \
self.trader_ensemble.performance.get_cash_estimate(epoch_s)
date = IntuitiveDateConverter.to_day_str(epoch_s)
print(
f'{date}, net worth: {net_worth_mean} +- {2 * net_worth_sd} [median: {net_worth_median}], '
f'cash: {cash_mean} +- {2 * cash_sd} [median: {cash_median}]')
def multiple_symbol_query(self, market_client):
start_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['start'])
end_epoch_s = IntuitiveDateConverter.to_epoch_s(
STOCKS_DAILY['BYND']['end'])
moments = market_client.get_moments(symbols=['BYND', 'GOOG'],
start_epoch_s=start_epoch_s,
end_epoch_s=end_epoch_s)
n_expected = len(STOCKS_DAILY['BYND']['records']) + len(
STOCKS_DAILY['GOOG']['records'])
self.assertEqual(n_expected, len(moments))
for record in STOCKS_DAILY['BYND']['records']:
record['symbol'] = 'BYND'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
for record in STOCKS_DAILY['GOOG']['records']:
record['symbol'] = 'GOOG'
moment = Moment.from_dict(record)
self.assertIn(moment, moments)
def determine_buys(self, epoch_s: float,
register: SymbolDayRegister) -> List[str]:
"""
Apply rules that trigger buy order
"""
stocks = list(register.keys())
random.shuffle(stocks)
to_buy = list()
for stock in stocks:
buy_flag = True
prices = [None] * (self.n_crash + 1)
for i in range(self.n_crash):
close_today = register.get_close(stock, epoch_s, -i)
if close_today is None:
buy_flag = False
break
close_yesterday = register.get_close(stock, epoch_s, -i - 1)
if close_yesterday is None:
buy_flag = False
break
gain = 100 * (close_today / close_yesterday - 1)
if gain > self.gain_crash:
buy_flag = False
break
prices[i] = close_today
prices[i + 1] = close_yesterday
if buy_flag:
to_buy.append(stock)
date = IntuitiveDateConverter.to_day_str(epoch_s)
print(
f'Identified BUY flag: {date} : {stock.to_str()} : {prices}'
)
if self.logger is not None:
message = f'Identified BUY flag: {date} : {stock.to_str()} : {prices}'
self.logger.log_message(message)
if len(to_buy) >= self.n_buy_per_day:
break
return to_buy
def determine_sells(self, epoch_s: float,
register: SymbolDayRegister) -> List[str]:
"""
Apply rules that trigger sell order
"""
stocks = self.holdings.keys()
to_sell = list()
for stock in stocks:
sell_flag = True
prices = [None] * (self.n_down + 1)
for i in range(self.n_down):
close_today = register.get_close(stock, epoch_s, -i)
if close_today is None:
sell_flag = False
break
close_yesterday = register.get_close(stock, epoch_s, -i - 1)
if close_yesterday is None:
sell_flag = False
break
gain = 100 * (close_today / close_yesterday - 1)
if gain > self.gain_down:
sell_flag = False
break
prices[i] = close_today
prices[i + 1] = close_yesterday
if sell_flag:
to_sell.append(stock)
date = IntuitiveDateConverter.to_day_str(epoch_s)
if self.logger is not None:
message = f'Identified SELL flag: {date} : {stock.to_str()} : {prices}'
self.logger.log_message(message)
return to_sell
def get(self, stock: Stock, day: str, offset=0) -> Moment:
"""
Retrieve a moment by stock and date.
"""
epoch_s = IntuitiveDateConverter.to_epoch_s(day)
if offset == 0:
return self.series[stock].get(epoch_s)
_series = self.series.get(stock, None)
if _series is None:
return None
if epoch_s not in _series.timestamps:
return None
_ix = _series.timestamps.index(epoch_s)
_ix += offset
if _ix < 0:
return None
if _ix >= len(_series.timestamps):
return None
return _series.get(_series.timestamps[_ix])
def get_moments(self, stocks: List[Stock], start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
"""
Retrieve all daily market data for symbols and date range and convert into list of moments.
Retries after a pause if API rate limit is reached.
"""
moments = list()
for i_0 in range(0, len(stocks), BATCH_SIZE):
i_f = i_0 + BATCH_SIZE
i_f = len(stocks) if i_f > len(stocks) else i_f
stock_batch = stocks[i_0:i_f]
for _ in range(MAX_RETRIES):
try:
stock_batch_str = ','.join(
[s.to_str() for s in stock_batch])
print(f'api request for symbols: {stock_batch_str}')
content = self.make_api_call(stock_batch, start_epoch_s,
end_epoch_s)
except requests.exceptions.ConnectionError as conerr:
print(conerr)
print(
f'Schedule to retry after {RETRY_PAUSE_CONNECTION_ERROR} seconds'
)
time.sleep(RETRY_PAUSE_CONNECTION_ERROR)
except ChunkedEncodingError as _:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
else:
if 'code' in content:
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
break
if len(stocks) == 1:
content = {stocks[0].symbol: content}
for symbol, data in content.items():
if 'values' not in data:
if data['code'] == ERROR_CODE_NO_DATA_AVAILABLE:
print(f'No data available for {symbol}. Skipping.')
continue
print('unexpected response:')
print(content)
exchange = data['meta']['exchange']
for record in data['values']:
day_data = dict()
day_data['exchange'] = exchange
day_data['symbol'] = symbol
day_data['epoch_s'] = IntuitiveDateConverter.to_epoch_s(
record['datetime'])
day_data['open'] = float(record['open'])
day_data['close'] = float(record['close'])
day_data['high'] = float(record['high'])
day_data['low'] = float(record['low'])
day_data['volume'] = int(record['volume'])
moment = Moment(**day_data)
moments.append(moment)
return moments
def get(self, day: str) -> Moment:
"""
Retrieve a moment by date.
"""
epoch_s = IntuitiveDateConverter.to_epoch_s(day)
return self.moments.get(epoch_s, None)
def get_technical_indicator(self, stock: Stock, indicator_name: str,
start_epoch_s: float,
end_epoch_s: float) -> List[Moment]:
start_date = IntuitiveDateConverter.to_day_str(start_epoch_s)
end_datetime = IntuitiveDateConverter.to_datetime(end_epoch_s)
end_datetime += timedelta(days=1)
end_date = IntuitiveDateConverter.to_day_str(end_datetime)
request_url = f'https://api.twelvedata.com/{indicator_name.lower()}?symbol=' \
f'{stock.symbol}&exchange={stock.exchange}&interval=1day&start_date=' \
f'{start_date}&end_date={end_date}&apikey={self.api_key}'
for _ in range(MAX_RETRIES):
try:
print(
f'API call for technical indicator {indicator_name.lower()} for {stock.to_str()}'
)
response = requests.get(request_url)
except ChunkedEncodingError:
print(
f'intermittent ChunkedEncodingError. Retrying in {RETRY_PAUSE_CHUNK_ERROR}s'
)
time.sleep(RETRY_PAUSE_CHUNK_ERROR)
print('retrying')
continue
except Exception as e:
print("Stopping retry loop because of un-handled error")
raise e
else:
content = response.content
content = json.loads(content)
if 'code' in content.keys():
if content['code'] == ERROR_CODE_TOO_MANY_REQUESTS:
print(
f'Too many requests. Pausing for {RETRY_PAUSE_S} seconds before retrying.'
)
time.sleep(RETRY_PAUSE_S)
continue
if 'values' not in content:
print('unexpected response:')
print(content)
break
moments = list()
if 'values' not in content:
return moments
for item in content['values']:
epoch_s = IntuitiveDateConverter.to_epoch_s(item['datetime'])
for k, v in item.items():
if k == 'datetime':
continue
if k == indicator_name:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
indicator_name: item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
else:
moment_data = {
'epoch_s': epoch_s,
'exchange': stock.exchange,
'symbol': stock.symbol,
f'{indicator_name}_{k}': item[k]
}
moment = Moment(**moment_data)
moments.append(moment)
return moments