def forming_estimation(self, exchange, symbol, timeframe): long_key = jh.key(exchange, symbol, timeframe) short_key = jh.key(exchange, symbol, '1m') required_1m_to_complete_count = jh.timeframe_to_one_minutes(timeframe) current_1m_count = len(self.get_storage(exchange, symbol, '1m')) dif = current_1m_count % required_1m_to_complete_count return dif, long_key, short_key
def init_storage(self, bucket_size: int = 1000) -> None: for c in config['app']['considering_candles']: exchange, symbol = c[0], c[1] # initiate the '1m' timeframes key = jh.key(exchange, symbol, timeframes.MINUTE_1) self.storage[key] = DynamicNumpyArray((bucket_size, 6)) for timeframe in config['app']['considering_timeframes']: key = jh.key(exchange, symbol, timeframe) # ex: 1440 / 60 + 1 (reserve one for forming candle) total_bigger_timeframe = int((bucket_size / jh.timeframe_to_one_minutes(timeframe)) + 1) self.storage[key] = DynamicNumpyArray((total_bigger_timeframe, 6))
def get_btc_and_eth_candles(): candles = {} candles[jh.key(exchanges.SANDBOX, 'BTCUSDT')] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTCUSDT', 'candles': fake_range_candle_from_range_prices(range(101, 200)) } candles[jh.key(exchanges.SANDBOX, 'ETHUSDT')] = { 'exchange': exchanges.SANDBOX, 'symbol': 'ETHUSDT', 'candles': fake_range_candle_from_range_prices(range(1, 100)) } return candles
def get_btc_and_eth_candles(): candles = { jh.key(exchanges.SANDBOX, 'BTC-USDT'): { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': candles_from_close_prices(range(101, 200)), } } candles[jh.key(exchanges.SANDBOX, 'ETH-USDT')] = { 'exchange': exchanges.SANDBOX, 'symbol': 'ETH-USDT', 'candles': candles_from_close_prices(range(1, 100)) } return candles
def __init__(self, training_candles: ndarray, testing_candles: ndarray, optimal_total: int, cpu_cores: int, csv: bool, json: bool, start_date: str, finish_date: str) -> None: if len(router.routes) != 1: raise NotImplementedError('optimize_mode mode only supports one route at the moment') self.strategy_name = router.routes[0].strategy_name self.optimal_total = optimal_total self.exchange = router.routes[0].exchange self.symbol = router.routes[0].symbol self.timeframe = router.routes[0].timeframe StrategyClass = jh.get_strategy_class(self.strategy_name) self.strategy_hp = StrategyClass.hyperparameters(None) solution_len = len(self.strategy_hp) if solution_len == 0: raise exceptions.InvalidStrategy('Targeted strategy does not implement a valid hyperparameters() method.') super().__init__( iterations=2000 * solution_len, population_size=solution_len * 100, solution_len=solution_len, options={ 'strategy_name': self.strategy_name, 'exchange': self.exchange, 'symbol': self.symbol, 'timeframe': self.timeframe, 'strategy_hp': self.strategy_hp, 'csv': csv, 'json': json, 'start_date': start_date, 'finish_date': finish_date, } ) if cpu_cores > cpu_count(): raise ValueError(f'Entered cpu cores number is more than available on this machine which is {cpu_count()}') elif cpu_cores == 0: self.cpu_cores = cpu_count() else: self.cpu_cores = cpu_cores self.training_candles = training_candles self.testing_candles = testing_candles key = jh.key(self.exchange, self.symbol) training_candles_start_date = jh.timestamp_to_time(self.training_candles[key]['candles'][0][0]).split('T')[0] training_candles_finish_date = jh.timestamp_to_time(self.training_candles[key]['candles'][-1][0]).split('T')[0] testing_candles_start_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][0][0]).split('T')[0] testing_candles_finish_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][-1][0]).split('T')[0] self.training_initial_candles = [] self.testing_initial_candles = [] for c in config['app']['considering_candles']: self.training_initial_candles.append( required_candles.load_required_candles(c[0], c[1], training_candles_start_date, training_candles_finish_date)) self.testing_initial_candles.append( required_candles.load_required_candles(c[0], c[1], testing_candles_start_date, testing_candles_finish_date))
def add_trade(self, trade: np.ndarray, exchange: str, symbol: str) -> None: key = jh.key(exchange, symbol) if not len(self.temp_storage[key] ) or trade[0] - self.temp_storage[key][0][0] < 1000: self.temp_storage[key].append(trade) else: arr = self.temp_storage[key] buy_arr = np.array(list(filter(lambda x: x[3] == 1, arr))) sell_arr = np.array(list(filter(lambda x: x[3] == 0, arr))) generated = np.array([ # timestamp arr[0][0], # price (weighted average) (arr[:][:, 1] * arr[:][:, 2]).sum() / arr[:][:, 2].sum(), # buy_qty 0 if not len(buy_arr) else buy_arr[:, 2].sum(), # sell_qty 0 if not len(sell_arr) else sell_arr[:, 2].sum(), # buy_count len(buy_arr), # sell_count len(sell_arr) ]) if jh.is_collecting_data(): store_trade_into_db(exchange, symbol, generated) else: self.storage[key].append(generated) self.temp_storage[key].flush() self.temp_storage[key].append(trade)
def get_past_ticker(self, exchange: str, symbol: str, number_of_tickers_ago: int): if number_of_tickers_ago > 120: raise ValueError('Max accepted value for number_of_tickers_ago is 120') number_of_tickers_ago = abs(number_of_tickers_ago) key = jh.key(exchange, symbol) return self.storage[key][-1 - number_of_tickers_ago]
def test_modifying_stop_loss_after_part_of_position_is_already_reduced_with_stop_loss( ): set_up([ (exchanges.SANDBOX, 'BTCUSD', timeframes.MINUTE_1, 'Test14'), ]) generated_candles = fake_range_candle_from_range_prices( list(range(1, 10)) + list(range(10, 1, -1))) candles = {} key = jh.key(exchanges.SANDBOX, 'BTCUSD') candles[key] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTCUSD', 'candles': generated_candles } backtest_mode.run('2019-04-01', '2019-04-02', candles) assert len(store.completed_trades.trades) == 1 t1: CompletedTrade = store.completed_trades.trades[0] assert t1.type == 'long' assert t1.entry_price == 7 assert t1.exit_price == (4 * 2 + 6) / 3 assert t1.take_profit_at == 13 assert t1.stop_loss_at == (4 * 2 + 6) / 3 assert t1.qty == 1.5 assert t1.fee == 0
def test_forming_candles(): reset_config() router.set_routes([(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_5, 'Test19')]) router.set_extra_candles([(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_15)]) store.reset(True) candles = {} key = jh.key(exchanges.SANDBOX, 'BTC-USDT') candles[key] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': test_candles_0 } backtest_mode.run('2019-04-01', '2019-04-02', candles) # use math.ceil because it must include forming candle too assert len( store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_5)) == math.ceil(1382 / 5) assert len( store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_15)) == math.ceil(1382 / 15)
def add_orderbook(self, exchange: str, symbol: str, asks: list, bids: list): """ :param exchange: :param symbol: :param asks: :param bids: """ key = jh.key(exchange, symbol) self.temp_storage[key]['asks'] = asks self.temp_storage[key]['bids'] = bids # generate new numpy formatted orderbook if it is # either the first time, or that it has passed # 1000 milliseconds since the last time if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now( ) - self.temp_storage[key]['last_updated_timestamp'] >= 1000: self.temp_storage[key]['last_updated_timestamp'] = jh.now() formatted_orderbook = self.format_orderbook(exchange, symbol) if jh.is_collecting_data(): store_orderbook_into_db(exchange, symbol, formatted_orderbook) else: self.storage[key].append(formatted_orderbook)
def test_backtesting_three_routes(): reset_config() router.set_routes([ (exchanges.SANDBOX, 'BTCUSD', timeframes.MINUTE_5, 'Test19'), (exchanges.SANDBOX, 'ETHUSD', timeframes.MINUTE_5, 'Test19'), (exchanges.SANDBOX, 'XRPUSD', timeframes.MINUTE_15, 'Test19'), ]) store.reset(True) candles = {} routes = router.routes for r in routes: key = jh.key(r.exchange, r.symbol) candles[key] = { 'exchange': r.exchange, 'symbol': r.symbol, 'candles': fake_range_candle(5 * 3 * 20) } # assert that strategy hasn't been initiated before running backtest_mode() assert r.strategy is None # run backtest (dates are fake just to pass) backtest_mode.run('2019-04-01', '2019-04-02', candles) # there must be three positions present with the updated current_price assert len(store.positions.storage) == 3 for r in routes: # r3's '15m' timeframe makes r1 and r2 to support # '15' timeframe as well. r1 and r2 also make r3 # to support '5m' timeframe also. r_one_min = store.candles.get_candles(r.exchange, r.symbol, '1m') r_five_min = store.candles.get_candles(r.exchange, r.symbol, '5m') r_fifteen_min = store.candles.get_candles(r.exchange, r.symbol, '15m') # assert the count of present candles assert len(r_one_min) == (5 * 3) * 20 assert len(r_five_min) == 20 * 3 assert len(r_fifteen_min) == 20 r_first_1 = r_one_min[0] r_last_1 = r_one_min[-1] r_first_5 = r_five_min[0] r_last_5 = r_five_min[-1] r_first_15 = r_fifteen_min[0] r_last_15 = r_fifteen_min[-1] # assert timestamps assert r_first_1[0] == r_first_5[0] assert r_last_1[0] == (r_last_5[0] + 60000 * 4) assert r_last_5[0] == (r_last_15[0] + 60000 * 10) # assert positions p = selectors.get_position(r.exchange, r.symbol) assert p.is_close is True last_candle = store.candles.get_candles(r.exchange, r.symbol, '1m')[-1] assert p.current_price == last_candle[2] # assert that the strategy has been initiated assert r.strategy is not None
def test_forming_candles(): reset_config() routes = [{ 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': timeframes.MINUTE_5, 'strategy': 'Test19' }] extra_routes = [{ 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': timeframes.MINUTE_15 }] candles = {} key = jh.key(exchanges.SANDBOX, 'BTC-USDT') candles[key] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': test_candles_0 } backtest_mode.run(False, {}, routes, extra_routes, '2019-04-01', '2019-04-02', candles) # use math.ceil because it must include forming candle too assert len( store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_5)) == math.ceil(1382 / 5) assert len( store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_15)) == math.ceil(1382 / 15)
def init_storage(self): """ """ for c in config['app']['considering_candles']: key = jh.key(c[0], c[1]) self.storage[key] = DynamicNumpyArray((60, 5), drop_at=120)
def test_strategy_properties(): set_up([ (exchanges.SANDBOX, 'ETHUSDT', timeframes.MINUTE_5, 'Test19'), (exchanges.SANDBOX, 'BTCUSDT', timeframes.MINUTE_5, 'Test19'), ]) candles = {} routes = router.routes for r in routes: key = jh.key(r.exchange, r.symbol) candles[key] = { 'exchange': r.exchange, 'symbol': r.symbol, 'candles': fake_range_candle((5 * 3) * 20) } # run backtest (dates are fake just to pass) backtest_mode.run('2019-04-01', '2019-04-02', candles) for r in routes: s: Strategy = r.strategy assert s.name == r.strategy_name assert s.symbol == r.symbol assert s.exchange == r.exchange assert s.timeframe == r.timeframe assert s.trade is None assert s._is_executing is False assert s._is_initiated is True np.testing.assert_equal(s.current_candle, store.candles.get_current_candle(r.exchange, r.symbol, r.timeframe)) np.testing.assert_equal(s.candles, store.candles.get_candles(r.exchange, r.symbol, r.timeframe)) assert s.position == selectors.get_position(r.exchange, r.symbol) assert s.orders == store.orders.get_orders(r.exchange, r.symbol)
def test_modifying_stop_loss_after_part_of_position_is_already_reduced_with_stop_loss( ): set_up() routes = [{ 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': timeframes.MINUTE_1, 'strategy': 'Test14' }] generated_candles = candles_from_close_prices( list(range(1, 10)) + list(range(10, 1, -1))) candles = {} key = jh.key(exchanges.SANDBOX, 'BTC-USDT') candles[key] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': generated_candles } backtest_mode.run(False, {}, routes, [], '2019-04-01', '2019-04-02', candles) assert len(store.completed_trades.trades) == 1 t1: CompletedTrade = store.completed_trades.trades[0] assert t1.type == 'long' assert t1.entry_price == 7 assert t1.exit_price == (4 * 2 + 6) / 3 assert t1.qty == 1.5 assert t1.fee == 0
def test_portfolio_value(): set_up() routes = [ { 'exchange': exchanges.SANDBOX, 'symbol': 'ETH-USDT', 'timeframe': '5m', 'strategy': 'TestPortfolioValue' }, { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': '5m', 'strategy': 'TestPortfolioValue' }, ] candles = {} for r in routes: key = jh.key(r['exchange'], r['symbol']) candles[key] = { 'exchange': r['exchange'], 'symbol': r['symbol'], 'candles': range_candles((5 * 3) * 20) } # run backtest (dates are fake just to pass) backtest_mode.run(False, {}, routes, [], '2019-04-01', '2019-04-02', candles)
def get_btc_candles(): return { jh.key(exchanges.SANDBOX, 'BTC-USDT'): { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': fake_range_candle_from_range_prices(range(1, 100)), } }
def get_downtrend_candles(): candles = {} candles[jh.key(exchanges.SANDBOX, 'BTC-USDT')] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': fake_range_candle_from_range_prices(range(100, 10, -1)) } return candles
def get_downtrend_candles(): return { jh.key(exchanges.SANDBOX, 'BTC-USDT'): { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': candles_from_close_prices(range(100, 10, -1)), } }
def test_can_pass_strategy_as_class(): class TestStrategy(Strategy): def should_long(self): return False def should_short(self): return False def should_cancel(self): return False def go_long(self): pass def go_short(self): pass fake_candles = candles_from_close_prices( [101, 102, 103, 104, 105, 106, 107, 108, 109, 110]) exchange_name = 'Fake Exchange' symbol = 'FAKE-USDT' timeframe = '1m' config = { 'starting_balance': 10_000, 'fee': 0, 'futures_leverage': 2, 'futures_leverage_mode': 'cross', 'exchange': exchange_name, 'settlement_currency': 'USDT', 'warm_up_candles': 0 } routes = [ { 'exchange': exchange_name, 'strategy': TestStrategy, 'symbol': symbol, 'timeframe': timeframe }, ] extra_routes = [] candles = { jh.key(exchange_name, symbol): { 'exchange': exchange_name, 'symbol': symbol, 'candles': fake_candles, }, } result = research.backtest(config, routes, extra_routes, candles) # result must have None values because the strategy makes no decisions assert result['metrics'] == { 'net_profit_percentage': 0, 'total': 0, 'win_rate': 0 } assert result['charts'] is None assert result['logs'] is None
def get_storage(self, exchange, symbol, timeframe): key = jh.key(exchange, symbol, timeframe) try: return self.storage[key] except KeyError: raise RouteNotFound( "Bellow route is required but missing in your routes:\n('{}', '{}', '{}')" .format(exchange, symbol, timeframe))
def get_tickers(self, exchange: str, symbol: str): """ :param exchange: :param symbol: :return: """ key = jh.key(exchange, symbol) return self.storage[key][:]
def test_backtesting_one_route(): reset_config() router.set_routes([ (exchanges.SANDBOX, 'BTC-USDT', timeframes.MINUTE_5, 'Test19') ]) config['env']['exchanges'][exchanges.SANDBOX]['type'] = 'margin' store.reset(True) candles = {} key = jh.key(exchanges.SANDBOX, 'BTC-USDT') candles[key] = { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'candles': fake_range_candle(5 * 20) } routes = router.routes # assert that strategy hasn't been initiated before running backtest_mode() assert routes[0].strategy is None # run backtest (dates are fake just to pass) backtest_mode.run('2019-04-01', '2019-04-02', candles) one_min = store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', '1m') five_min = store.candles.get_candles(exchanges.SANDBOX, 'BTC-USDT', '5m') # assert the count of present candles assert len(five_min) == 20 assert len(one_min) == 20 * 5 first_1 = one_min[0] last_1 = one_min[-1] first_5 = five_min[0] last_5 = five_min[-1] # assert time in store assert store.app.time == last_1[0] + 60000 # assert timestamps assert first_1[0] == first_5[0] assert last_1[0] == (last_5[0] + 60000 * 4) # there must be only one positions present assert len(store.positions.storage) == 1 p = selectors.get_position(exchanges.SANDBOX, 'BTC-USDT') assert p.is_close assert p.current_price == last_1[2] assert p.current_price == last_5[2] # assert routes assert len(routes) == 1 assert routes[0].exchange == exchanges.SANDBOX assert routes[0].symbol == 'BTC-USDT' assert routes[0].timeframe == '5m' assert routes[0].strategy_name == 'Test19' # assert that the strategy has been initiated assert routes[0].strategy is not None
def get_storage(self, exchange: str, symbol: str, timeframe: str): key = jh.key(exchange, symbol, timeframe) try: return self.storage[key] except KeyError: raise RouteNotFound( f"Bellow route is required but missing in your routes:\n('{exchange}', '{symbol}', '{timeframe}')" )
def get_current_trade(self, exchange: str, symbol: str): """ :param exchange: :param symbol: :return: """ key = jh.key(exchange, symbol) return self.storage[key][-1]
def get_current_orderbook(self, exchange: str, symbol: str) -> np.ndarray: """ :param exchange: :param symbol: :return: """ key = jh.key(exchange, symbol) return self.storage[key][-1]
def get_best_ask(self, exchange: str, symbol: str) -> np.ndarray: """ :param exchange: :param symbol: :return: """ key = jh.key(exchange, symbol) return self.storage[key][-1][0][0]
def init_storage(self): for c in config['app']['considering_candles']: key = jh.key(c[0], c[1]) self.temp_storage[key] = { 'last_updated_timestamp': None, 'asks': [], 'bids': [] } self.storage[key] = DynamicNumpyArray((60, 2, 50, 2), drop_at=60)
def test_can_perform_backtest_with_multiple_routes(): set_up() routes = [ { 'exchange': exchanges.SANDBOX, 'symbol': 'ETH-USDT', 'timeframe': '5m', 'strategy': 'Test01' }, { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': '5m', 'strategy': 'Test02' }, ] candles = {} for r in routes: key = jh.key(r['exchange'], r['symbol']) candles[key] = { 'exchange': r['exchange'], 'symbol': r['symbol'], 'candles': range_candles((5 * 3) * 20) } # run backtest (dates are fake just to pass) backtest_mode.run(False, {}, routes, [], '2019-04-01', '2019-04-02', candles) for r in router.routes: s: Strategy = r.strategy p = s.position assert p.is_close is True assert len(s.orders) == 3 o: Order = s.orders[0] short_candles = store.candles.get_candles(r.exchange, r.symbol, '1m') assert o.price == short_candles[4][2] assert o.price == s.candles[0][2] assert o.created_at == short_candles[4][0] + 60_000 assert o.is_executed is True assert s.orders[0].role == order_roles.OPEN_POSITION assert s.orders[0].type == order_types.MARKET assert s.orders[2].role == order_roles.CLOSE_POSITION assert s.orders[2].type == order_types.STOP assert s.orders[1].role == order_roles.CLOSE_POSITION assert s.orders[1].type == order_types.LIMIT assert s.trade is None assert len(store.completed_trades.trades) == 2 # assert one is long and the other is a short trade assert (store.completed_trades.trades[0].type == 'long' and store.completed_trades.trades[1].type == 'short') or ( store.completed_trades.trades[0].type == 'short' and store.completed_trades.trades[1].type == 'long')
def test_multiple_routes_can_communicate_with_each_other(): set_up() routes = [ { 'exchange': exchanges.SANDBOX, 'symbol': 'ETH-USDT', 'timeframe': '5m', 'strategy': 'Test03' }, { 'exchange': exchanges.SANDBOX, 'symbol': 'BTC-USDT', 'timeframe': '5m', 'strategy': 'Test03' }, ] candles = {} for r in routes: key = jh.key(r['exchange'], r['symbol']) candles[key] = { 'exchange': r['exchange'], 'symbol': r['symbol'], 'candles': range_candles((5 * 3) * 20) } # run backtest (dates are fake just to pass) backtest_mode.run(False, {}, routes, [], '2019-04-01', '2019-04-02', candles) assert len(store.completed_trades.trades) == 1 for r in router.routes: s: Strategy = r.strategy p = s.position assert p.is_close is True o: Order = s.orders[0] short_candles = store.candles.get_candles(r.exchange, r.symbol, '1m') assert o.created_at == short_candles[4][0] + 60_000 if r.strategy.trades_count == 0: assert len(s.orders) == 1 # assert that the order got canceled assert o.is_canceled is True assert s.orders[0].role == order_roles.OPEN_POSITION assert s.orders[0].type == order_types.LIMIT elif r.strategy.trades_count == 1: assert len(s.orders) == 3 assert o.is_executed is True assert s.orders[0].role == order_roles.OPEN_POSITION assert s.orders[0].type == order_types.LIMIT assert s.orders[2].role == order_roles.CLOSE_POSITION assert s.orders[2].type == order_types.STOP assert s.orders[1].role == order_roles.CLOSE_POSITION assert s.orders[1].type == order_types.LIMIT