def test_get_candles_including_forming(): set_up() candles_to_add = fake_range_candle(14) store.candles.batch_add_candle(candles_to_add, 'Sandbox', 'BTCUSD', '1m') store.candles.add_candle( generate_candle_from_one_minutes('5m', candles_to_add[0:5], False), 'Sandbox', 'BTCUSD', '5m') store.candles.add_candle( generate_candle_from_one_minutes('5m', candles_to_add[5:10], False), 'Sandbox', 'BTCUSD', '5m') assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '5m')) == 3 assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '1m')) == 14 candles = store.candles.get_candles('Sandbox', 'BTCUSD', '5m') assert candles[0][0] == candles_to_add[0][0] assert candles[-1][2] == candles_to_add[13][2] assert candles[-1][0] == candles_to_add[10][0] # add third one while still a forming candle. Now since # we already have forming, get_candles() must not # append another forming candle to the end. store.candles.add_candle( generate_candle_from_one_minutes('5m', candles_to_add[10:14], True), 'Sandbox', 'BTCUSD', '5m') assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '5m')) == 3 assert candles[-1][2] == candles_to_add[13][2] assert candles[-1][0] == candles_to_add[10][0]
def get_current_candle(self, exchange, symbol, timeframe) -> np.ndarray: # no need to worry for forming candles when timeframe == 1m if timeframe == '1m': arr: DynamicNumpyArray = self.get_storage(exchange, symbol, '1m') if len(arr) == 0: return np.zeros((0, 6)) else: return arr[-1] # other timeframes dif, long_key, short_key = self.forming_estimation( exchange, symbol, timeframe) long_count = len(self.get_storage(exchange, symbol, timeframe)) short_count = len(self.get_storage(exchange, symbol, '1m')) # complete candle if dif == 0: if long_count == 0: return np.zeros((0, 6)) else: return self.storage[long_key][-1] # generate forming else: return generate_candle_from_one_minutes( timeframe, self.storage[short_key][short_count - dif:short_count], True)
def generate_bigger_timeframes(self, candle: np.ndarray, exchange: str, symbol: str, with_execution: bool, is_forming_candle: bool): if not jh.is_live(): return for timeframe in config['app']['considering_timeframes']: # skip '1m' if timeframe == '1m': continue last_candle = self.get_current_candle(exchange, symbol, timeframe) generate_from_count = int((candle[0] - last_candle[0]) / 60_000) required_for_complete_candle = jh.timeframe_to_one_minutes( timeframe) short_candles = self.get_candles(exchange, symbol, '1m')[-1 - generate_from_count:] if generate_from_count == (required_for_complete_candle - 1) and not is_forming_candle: is_forming_candle = False else: is_forming_candle = True # update latest candle generated_candle = generate_candle_from_one_minutes( timeframe, short_candles, True) self.add_candle(generated_candle, exchange, symbol, timeframe, with_execution, with_generation=False, is_forming_candle=is_forming_candle)
def inject_required_candles_to_store(candles: np.ndarray, exchange: str, symbol: str): """ generate and add required candles to the candle store """ # batch add 1m candles: store.candles.batch_add_candle(candles, exchange, symbol, '1m', with_generation=False) # loop to generate, and add candles (without execution) for i in range(len(candles)): for timeframe in config['app']['considering_timeframes']: # skip 1m. already added if timeframe == '1m': continue num = jh.timeframe_to_one_minutes(timeframe) if (i + 1) % num == 0: generated_candle = generate_candle_from_one_minutes( timeframe, candles[(i - (num - 1)):(i + 1)], True) store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False, with_generation=False)
def get_candles(self, exchange: str, symbol: str, timeframe: str) -> np.ndarray: # no need to worry for forming candles when timeframe == 1m if timeframe == '1m': arr: DynamicNumpyArray = self.get_storage(exchange, symbol, '1m') if len(arr) == 0: return np.zeros((0, 6)) else: return arr[:] # other timeframes dif, long_key, short_key = self.forming_estimation( exchange, symbol, timeframe) long_count = len(self.get_storage(exchange, symbol, timeframe)) short_count = len(self.get_storage(exchange, symbol, '1m')) if dif == 0 and long_count == 0: return np.zeros((0, 6)) # complete candle if dif == 0 or self.storage[long_key][:long_count][-1][ 0] == self.storage[short_key][short_count - dif][0]: return self.storage[long_key][:long_count] # generate forming else: return np.concatenate( (self.storage[long_key][:long_count], np.array((generate_candle_from_one_minutes( timeframe, self.storage[short_key][short_count - dif:short_count], True), ))), axis=0)
def generate_bigger_timeframes(self, candle: np.ndarray, exchange: str, symbol: str, with_execution: bool) -> None: if not jh.is_live(): return for timeframe in config['app']['considering_timeframes']: # skip '1m' if timeframe == '1m': continue last_candle = self.get_current_candle(exchange, symbol, timeframe) generate_from_count = int((candle[0] - last_candle[0]) / 60_000) short_candles = self.get_candles(exchange, symbol, '1m')[-1 - generate_from_count:] # update latest candle generated_candle = generate_candle_from_one_minutes( timeframe, short_candles, accept_forming_candles=True) self.add_candle(generated_candle, exchange, symbol, timeframe, with_execution, with_generation=False)
def test_get_past_candle(): set_up() candles_to_add = fake_range_candle(14) store.candles.batch_add_candle(candles_to_add, 'Sandbox', 'BTCUSD', '1m') store.candles.add_candle( generate_candle_from_one_minutes('5m', candles_to_add[0:5]), 'Sandbox', 'BTCUSD', '5m') store.candles.add_candle( generate_candle_from_one_minutes('5m', candles_to_add[5:10]), 'Sandbox', 'BTCUSD', '5m') assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '1m')) == 14 # get_past_candle() should act as if there are 3 candles (last one being the forming) np.testing.assert_equal( store.candles.get_past_candle('Sandbox', 'BTCUSD', '5m', 1), store.candles.storage[jh.key('Sandbox', 'BTCUSD', '5m')][1]) np.testing.assert_equal( store.candles.get_past_candle('Sandbox', 'BTCUSD', '5m', 0), generate_candle_from_one_minutes('5m', candles_to_add[10:], True))
def get_candles(exchange: str, symbol: str, timeframe: str): from jesse.services.db import database database.open_connection() from jesse.services.candle import generate_candle_from_one_minutes from jesse.models.utils import fetch_candles_from_db symbol = symbol.upper() num_candles = 210 one_min_count = jh.timeframe_to_one_minutes(timeframe) finish_date = jh.now(force_fresh=True) start_date = finish_date - (num_candles * one_min_count * 60_000) # fetch 1m candles from database candles = np.array( fetch_candles_from_db(exchange, symbol, start_date, finish_date)) # if there are no candles in the database, return [] if candles.size == 0: database.close_connection() return [] # leave out first candles until the timestamp of the first candle is the beginning of the timeframe timeframe_duration = one_min_count * 60_000 while candles[0][0] % timeframe_duration != 0: candles = candles[1:] # generate bigger candles from 1m candles if timeframe != '1m': generated_candles = [] for i in range(len(candles)): if (i + 1) % one_min_count == 0: bigger_candle = generate_candle_from_one_minutes( timeframe, candles[(i - (one_min_count - 1)):(i + 1)], True) generated_candles.append(bigger_candle) candles = generated_candles database.close_connection() return [{ 'time': int(c[0] / 1000), 'open': c[1], 'close': c[2], 'high': c[3], 'low': c[4], 'volume': c[5], } for c in candles]
def simulator(candles: Dict[str, Dict[str, Union[str, np.ndarray]]], hyperparameters=None) -> None: begin_time_track = time.time() key = '{}-{}'.format(config['app']['considering_candles'][0][0], config['app']['considering_candles'][0][1]) first_candles_set = candles[key]['candles'] length = len(first_candles_set) # to preset the array size for performance store.app.starting_time = first_candles_set[0][0] store.app.time = first_candles_set[0][0] # initiate strategies for r in router.routes: StrategyClass = jh.get_strategy_class(r.strategy_name) try: r.strategy = StrategyClass() except TypeError: raise exceptions.InvalidStrategy( "Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file." "\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies" ) except: raise r.strategy.name = r.strategy_name r.strategy.exchange = r.exchange r.strategy.symbol = r.symbol r.strategy.timeframe = r.timeframe # inject hyper parameters (used for optimize_mode) # convert DNS string into hyperparameters if r.dna and hyperparameters is None: hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.dna) # inject hyperparameters sent within the optimize mode if hyperparameters is not None: r.strategy.hp = hyperparameters # init few objects that couldn't be initiated in Strategy __init__ # it also injects hyperparameters into self.hp in case the route does not uses any DNAs r.strategy._init_objects() selectors.get_position(r.exchange, r.symbol).strategy = r.strategy # add initial balance save_daily_portfolio_balance() with click.progressbar(length=length, label='Executing simulation...') as progressbar: for i in range(length): # update time store.app.time = first_candles_set[i][0] + 60_000 # add candles for j in candles: short_candle = candles[j]['candles'][i] if i != 0: previous_short_candle = candles[j]['candles'][i - 1] short_candle = _get_fixed_jumped_candle( previous_short_candle, short_candle) exchange = candles[j]['exchange'] symbol = candles[j]['symbol'] store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False, with_generation=False) # print short candle if jh.is_debuggable('shorter_period_candles'): print_candle(short_candle, True, symbol) _simulate_price_change_effect(short_candle, exchange, symbol) # generate and add candles for bigger timeframes for timeframe in config['app']['considering_timeframes']: # for 1m, no work is needed if timeframe == '1m': continue count = jh.timeframe_to_one_minutes(timeframe) until = count - ((i + 1) % count) if (i + 1) % count == 0: generated_candle = generate_candle_from_one_minutes( timeframe, candles[j]['candles'][(i - (count - 1)):(i + 1)]) store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False, with_generation=False) # update progressbar if not jh.is_debugging() and not jh.should_execute_silently( ) and i % 60 == 0: progressbar.update(60) # now that all new generated candles are ready, execute for r in router.routes: count = jh.timeframe_to_one_minutes(r.timeframe) # 1m timeframe if r.timeframe == timeframes.MINUTE_1: r.strategy._execute() elif (i + 1) % count == 0: # print candle if jh.is_debuggable('trading_candles'): print_candle( store.candles.get_current_candle( r.exchange, r.symbol, r.timeframe), False, r.symbol) r.strategy._execute() # now check to see if there's any MARKET orders waiting to be executed store.orders.execute_pending_market_orders() if i != 0 and i % 1440 == 0: save_daily_portfolio_balance() if not jh.should_execute_silently(): if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print('\n') # print executed time for the backtest session finish_time_track = time.time() print( 'Executed backtest simulation in: ', '{} seconds'.format(round(finish_time_track - begin_time_track, 2))) for r in router.routes: r.strategy._terminate() store.orders.execute_pending_market_orders() # now that backtest is finished, add finishing balance save_daily_portfolio_balance()
def simulator( candles: dict, run_silently: bool, hyperparameters: dict = None ) -> None: begin_time_track = time.time() key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" first_candles_set = candles[key]['candles'] length = len(first_candles_set) # to preset the array size for performance try: store.app.starting_time = first_candles_set[0][0] except IndexError: raise IndexError('Check your "warm_up_candles" config value') store.app.time = first_candles_set[0][0] # initiate strategies for r in router.routes: # if the r.strategy is str read it from file if isinstance(r.strategy_name, str): StrategyClass = jh.get_strategy_class(r.strategy_name) # else it is a class object so just use it else: StrategyClass = r.strategy_name try: r.strategy = StrategyClass() except TypeError: raise exceptions.InvalidStrategy( "Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file." "\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies" ) except: raise r.strategy.name = r.strategy_name r.strategy.exchange = r.exchange r.strategy.symbol = r.symbol r.strategy.timeframe = r.timeframe # read the dna from strategy's dna() and use it for injecting inject hyperparameters # first convert DNS string into hyperparameters if len(r.strategy.dna()) > 0 and hyperparameters is None: hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.strategy.dna()) # inject hyperparameters sent within the optimize mode if hyperparameters is not None: r.strategy.hp = hyperparameters # init few objects that couldn't be initiated in Strategy __init__ # it also injects hyperparameters into self.hp in case the route does not uses any DNAs r.strategy._init_objects() selectors.get_position(r.exchange, r.symbol).strategy = r.strategy # add initial balance save_daily_portfolio_balance() progressbar = Progressbar(length, step=60) for i in range(length): # update time store.app.time = first_candles_set[i][0] + 60_000 # add candles for j in candles: short_candle = candles[j]['candles'][i] if i != 0: previous_short_candle = candles[j]['candles'][i - 1] short_candle = _get_fixed_jumped_candle(previous_short_candle, short_candle) exchange = candles[j]['exchange'] symbol = candles[j]['symbol'] store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False, with_generation=False) # print short candle if jh.is_debuggable('shorter_period_candles'): print_candle(short_candle, True, symbol) _simulate_price_change_effect(short_candle, exchange, symbol) # generate and add candles for bigger timeframes for timeframe in config['app']['considering_timeframes']: # for 1m, no work is needed if timeframe == '1m': continue count = jh.timeframe_to_one_minutes(timeframe) # until = count - ((i + 1) % count) if (i + 1) % count == 0: generated_candle = generate_candle_from_one_minutes( timeframe, candles[j]['candles'][(i - (count - 1)):(i + 1)]) store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False, with_generation=False) # update progressbar if not run_silently and i % 60 == 0: progressbar.update() sync_publish('progressbar', { 'current': progressbar.current, 'estimated_remaining_seconds': progressbar.estimated_remaining_seconds }) # now that all new generated candles are ready, execute for r in router.routes: count = jh.timeframe_to_one_minutes(r.timeframe) # 1m timeframe if r.timeframe == timeframes.MINUTE_1: r.strategy._execute() elif (i + 1) % count == 0: # print candle if jh.is_debuggable('trading_candles'): print_candle(store.candles.get_current_candle(r.exchange, r.symbol, r.timeframe), False, r.symbol) r.strategy._execute() # now check to see if there's any MARKET orders waiting to be executed store.orders.execute_pending_market_orders() if i != 0 and i % 1440 == 0: save_daily_portfolio_balance() if not run_silently: # print executed time for the backtest session finish_time_track = time.time() sync_publish('alert', { 'message': f'Successfully executed backtest simulation in: {round(finish_time_track - begin_time_track, 2)} seconds', 'type': 'success' }) for r in router.routes: r.strategy._terminate() store.orders.execute_pending_market_orders() # now that backtest is finished, add finishing balance save_daily_portfolio_balance()
def get_candles(exchange: str, symbol: str, timeframe: str, start_date: str, finish_date: str) -> np.ndarray: """ Returns candles from the database in numpy format :param exchange: str :param symbol: str :param timeframe: str :param start_date: str :param finish_date: str :return: np.ndarray """ exchange = exchange.title() symbol = symbol.upper() import arrow import jesse.helpers as jh from jesse.models import Candle from jesse.exceptions import CandleNotFoundInDatabase from jesse.services.candle import generate_candle_from_one_minutes start_date = jh.arrow_to_timestamp(arrow.get(start_date, 'YYYY-MM-DD')) finish_date = jh.arrow_to_timestamp(arrow.get(finish_date, 'YYYY-MM-DD')) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().int_timestamp * 1000: raise ValueError('Can\'t backtest the future!') # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_date, finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples() candles = np.array(tuple(candles_tuple)) # validate that there are enough candles for selected period if len(candles) == 0 or candles[-1][0] != finish_date or candles[0][ 0] != start_date: raise CandleNotFoundInDatabase( f'Not enough candles for {symbol}. Try running "jesse import-candles"' ) if timeframe == '1m': return candles generated_candles = [] for i in range(len(candles)): num = jh.timeframe_to_one_minutes(timeframe) if (i + 1) % num == 0: generated_candles.append( generate_candle_from_one_minutes( timeframe, candles[(i - (num - 1)):(i + 1)], True)) return np.array(generated_candles)
def simulator(candles, hyper_parameters=None): begin_time_track = time.time() key = '{}-{}'.format(config['app']['trading_exchanges'][0], config['app']['trading_symbols'][0]) first_candles_set = candles[key]['candles'] length = len(first_candles_set) # to preset the array size for performance store.app.starting_time = first_candles_set[0][0] # initiate strategies for r in router.routes: StrategyClass = jh.get_strategy_class(r.strategy_name) # convert DNS string into hyper_parameters if r.dna and hyper_parameters is None: hyper_parameters = jh.dna_to_hp(StrategyClass.hyper_parameters(), r.dna) r.strategy = StrategyClass() r.strategy.name = r.strategy_name r.strategy.exchange = r.exchange r.strategy.symbol = r.symbol r.strategy.timeframe = r.timeframe # init few objects that couldn't be initiated in Strategy __init__ r.strategy._init_objects() # inject hyper parameters (used for optimize_mode) if hyper_parameters is not None: r.strategy.hp = hyper_parameters selectors.get_position(r.exchange, r.symbol).strategy = r.strategy # add initial balance _save_daily_portfolio_balance() with click.progressbar(length=length, label='Executing simulation...') as progressbar: for i in range(length): # update time store.app.time = first_candles_set[i][0] + 60_000 # add candles for j in candles: short_candle = candles[j]['candles'][i] exchange = candles[j]['exchange'] symbol = candles[j]['symbol'] store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False, with_generation=False) # print short candle if jh.is_debuggable('shorter_period_candles'): print_candle(short_candle, True, symbol) _simulate_price_change_effect(short_candle, exchange, symbol) # generate and add candles for bigger timeframes for timeframe in config['app']['considering_timeframes']: # for 1m, no work is needed if timeframe == '1m': continue count = jh.timeframe_to_one_minutes(timeframe) until = count - ((i + 1) % count) if (i + 1) % count == 0: generated_candle = generate_candle_from_one_minutes( timeframe, candles[j]['candles'][(i - (count - 1)):(i + 1)]) store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False, with_generation=False) # update progressbar if not jh.is_debugging() and not jh.should_execute_silently( ) and i % 60 == 0: progressbar.update(60) # now that all new generated candles are ready, execute for r in router.routes: count = jh.timeframe_to_one_minutes(r.timeframe) # 1m timeframe if r.timeframe == timeframes.MINUTE_1: r.strategy._execute() elif (i + 1) % count == 0: # print candle if jh.is_debuggable('trading_candles'): print_candle( store.candles.get_current_candle( r.exchange, r.symbol, r.timeframe), False, r.symbol) r.strategy._execute() # now check to see if there's any MARKET orders waiting to be executed store.orders.execute_pending_market_orders() if i != 0 and i % 1440 == 0: _save_daily_portfolio_balance() if not jh.should_execute_silently(): if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print('\n') # print executed time for the backtest session finish_time_track = time.time() print( 'Executed backtest simulation in: ', '{} seconds'.format(round(finish_time_track - begin_time_track, 2))) for r in router.routes: r.strategy._terminate() # now that backtest is finished, add finishing balance _save_daily_portfolio_balance()