def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = '{}-{}'.format(config['app']['considering_candles'][0][0], config['app']['considering_candles'][0][1]) table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', str(round(np.average(change), 2)) + "%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs store_logs(json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns() # QuantStats' report if full_reports: quantstats.quantstats_tearsheet() else: print(jh.color('No trades were made.', 'yellow'))
def run( debug_mode, user_config: dict, routes: List[Dict[str, str]], extra_routes: List[Dict[str, str]], start_date: str, finish_date: str, candles: dict = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False ) -> None: if not jh.is_unit_testing(): # at every second, we check to see if it's time to execute stuff status_checker = Timeloop() @status_checker.job(interval=timedelta(seconds=1)) def handle_time(): if process_status() != 'started': raise exceptions.Termination status_checker.start() from jesse.config import config, set_config config['app']['trading_mode'] = 'backtest' # debug flag config['app']['debug_mode'] = debug_mode # inject config if not jh.is_unit_testing(): set_config(user_config) # set routes router.initiate(routes, extra_routes) store.app.set_session_id() register_custom_exception_handler() # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): sync_publish('general_info', { 'session_id': jh.get_session_id(), 'debug_mode': str(config['app']['debug_mode']), }) # candles info key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" sync_publish('candles_info', stats.candles_info(candles[key]['candles'])) # routes info sync_publish('routes_info', stats.routes(router.routes)) # run backtest simulation simulator(candles, run_silently=jh.should_execute_silently()) # hyperparameters (if any) if not jh.should_execute_silently(): sync_publish('hyperparameters', stats.hyperparameters(router.routes)) if not jh.should_execute_silently(): if store.completed_trades.count > 0: sync_publish('metrics', report.portfolio_metrics()) routes_count = len(router.routes) more = f"-and-{routes_count - 1}-more" if routes_count > 1 else "" study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}" store_logs(study_name, json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns(study_name) sync_publish('equity_curve', charts.equity_curve()) # QuantStats' report if full_reports: price_data = [] # load close candles for Buy and hold and calculate pct_change for index, c in enumerate(config['app']['considering_candles']): exchange, symbol = c[0], c[1] if exchange in config['app']['trading_exchanges'] and symbol in config['app']['trading_symbols']: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.close ).where( Candle.timestamp.between(jh.date_to_timestamp(start_date), jh.date_to_timestamp(finish_date) - 60000), Candle.exchange == exchange, Candle.symbol == symbol ).order_by(Candle.timestamp.asc()).tuples() candles = np.array(candles_tuple) timestamps = candles[:, 0] price_data.append(candles[:, 1]) price_data = np.transpose(price_data) price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample( 'D').mean() price_pct_change = price_df.pct_change(1).fillna(0) bh_daily_returns_all_routes = price_pct_change.mean(1) quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name) else: sync_publish('equity_curve', None) sync_publish('metrics', None) # close database connection from jesse.services.db import database database.close_connection()
def load_candles( start_date_str: str, finish_date_str: str) -> Dict[str, Dict[str, Union[str, np.ndarray]]]: start_date = jh.date_to_timestamp(start_date_str) finish_date = jh.date_to_timestamp(finish_date_str) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().int_timestamp * 1000: raise ValueError("Can't load candle data from the future!") # load and add required warm-up candles for backtest if jh.is_backtesting(): for c in config['app']['considering_candles']: required_candles.inject_required_candles_to_store( required_candles.load_required_candles(c[0], c[1], start_date_str, finish_date_str), c[0], c[1]) # download candles for the duration of the backtest candles = {} for c in config['app']['considering_candles']: exchange, symbol = c[0], c[1] key = jh.key(exchange, symbol) cache_key = '{}-{}-'.format(start_date_str, finish_date_str) + key cached_value = cache.get_value(cache_key) # if cache exists if cached_value: candles_tuple = cached_value # not cached, get and cache for later calls in the next 5 minutes else: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_date, finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() # validate that there are enough candles for selected period required_candles_count = (finish_date - start_date) / 60_000 if len(candles_tuple) == 0 or candles_tuple[-1][ 0] != finish_date or candles_tuple[0][0] != start_date: raise exceptions.CandleNotFoundInDatabase( 'Not enough candles for {}. Try running "jesse import-candles"' .format(symbol)) elif len(candles_tuple) != required_candles_count + 1: raise exceptions.CandleNotFoundInDatabase( 'There are missing candles between {} => {}'.format( start_date_str, finish_date_str)) # cache it for near future calls cache.set_value(cache_key, tuple(candles_tuple), expire_seconds=60 * 60 * 24 * 7) candles[key] = { 'exchange': exchange, 'symbol': symbol, 'candles': np.array(candles_tuple) } return candles
def run(exchange: str, symbol: str, start_date_str: str, skip_confirmation=False): try: start_timestamp = jh.arrow_to_timestamp( arrow.get(start_date_str, 'YYYY-MM-DD')) except: raise ValueError( 'start_date must be a string representing a date before today. ex: 2020-01-17' ) # more start_date validations today = arrow.utcnow().floor('day').int_timestamp * 1000 if start_timestamp == today: raise ValueError( "Today's date is not accepted. start_date must be a string a representing date BEFORE today." ) elif start_timestamp > today: raise ValueError( "Future's date is not accepted. start_date must be a string a representing date BEFORE today." ) click.clear() symbol = symbol.upper() until_date = arrow.utcnow().floor('day') start_date = arrow.get(start_timestamp / 1000) days_count = jh.date_diff_in_days(start_date, until_date) candles_count = days_count * 1440 exchange = exchange.title() try: driver: CandleExchange = drivers[exchange]() except KeyError: raise ValueError('{} is not a supported exchange'.format(exchange)) loop_length = int(candles_count / driver.count) + 1 # ask for confirmation if not skip_confirmation: click.confirm( 'Importing {} days candles from "{}" for "{}". Duplicates will be skipped. All good?' .format(days_count, exchange, symbol), abort=True, default=True) with click.progressbar(length=loop_length, label='Importing candles...') as progressbar: for _ in range(candles_count): temp_start_timestamp = start_date.int_timestamp * 1000 temp_end_timestamp = temp_start_timestamp + (driver.count - 1) * 60000 # to make sure it won't try to import candles from the future! LOL if temp_start_timestamp > jh.now_to_timestamp(): break # prevent duplicates calls to boost performance count = Candle.select().where( Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp), Candle.symbol == symbol, Candle.exchange == exchange).count() already_exists = count == driver.count if not already_exists: # it's today's candles if temp_end_timestamp < now if temp_end_timestamp > jh.now_to_timestamp(): temp_end_timestamp = arrow.utcnow().floor( 'minute').int_timestamp * 1000 - 60000 # fetch from market candles = driver.fetch(symbol, temp_start_timestamp) if not len(candles): click.clear() first_existing_timestamp = driver.get_starting_time(symbol) # if driver can't provide accurate get_starting_time() if first_existing_timestamp is None: raise CandleNotFoundInExchange( 'No candles exists in the market for this day: {} \n' 'Try another start_date'.format( jh.timestamp_to_time(temp_start_timestamp) [:10], )) # handle when there's missing candles during the period if temp_start_timestamp > first_existing_timestamp: # see if there are candles for the same date for the backup exchange, # if so, get those, if not, download from that exchange. driver.init_backup_exchange() if driver.backup_exchange is not None: candles = _get_candles_from_backup_exchange( exchange, driver.backup_exchange, symbol, temp_start_timestamp, temp_end_timestamp) else: if not skip_confirmation: print( jh.color( 'No candle exists in the market for {}\n'. format( jh.timestamp_to_time( temp_start_timestamp)[:10]), 'yellow')) click.confirm( 'First present candle is since {}. Would you like to continue?' .format( jh.timestamp_to_time( first_existing_timestamp)[:10]), abort=True, default=True) run( exchange, symbol, jh.timestamp_to_time(first_existing_timestamp) [:10], True) return # fill absent candles (if there's any) candles = _fill_absent_candles(candles, temp_start_timestamp, temp_end_timestamp) # store in the database if skip_confirmation: _insert_to_database(candles) else: threading.Thread(target=_insert_to_database, args=[candles]).start() # add as much as driver's count to the temp_start_time start_date = start_date.shift(minutes=driver.count) progressbar.update(1) # sleep so that the exchange won't get angry at us if not already_exists: time.sleep(driver.sleep_time)
def _get_candles_from_backup_exchange(exchange: str, backup_driver: CandleExchange, symbol: str, start_timestamp: int, end_timestamp: int): total_candles = [] # try fetching from database first backup_candles = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_timestamp, end_timestamp), Candle.exchange == backup_driver.name, Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples() already_exists = len( backup_candles) == (end_timestamp - start_timestamp) / 60_000 + 1 if already_exists: # loop through them and set new ID and exchange for c in backup_candles: total_candles.append({ 'id': jh.generate_unique_id(), 'symbol': symbol, 'exchange': exchange, 'timestamp': c[0], 'open': c[1], 'close': c[2], 'high': c[3], 'low': c[4], 'volume': c[5] }) return total_candles # try fetching from market now days_count = jh.date_diff_in_days(jh.timestamp_to_arrow(start_timestamp), jh.timestamp_to_arrow(end_timestamp)) # make sure it's rounded up so that we import maybe more candles, but not less if days_count < 1: days_count = 1 if type(days_count) is float and not days_count.is_integer(): days_count = math.ceil(days_count) candles_count = days_count * 1440 start_date = jh.timestamp_to_arrow(start_timestamp).floor('day') for _ in range(candles_count): temp_start_timestamp = start_date.int_timestamp * 1000 temp_end_timestamp = temp_start_timestamp + (backup_driver.count - 1) * 60000 # to make sure it won't try to import candles from the future! LOL if temp_start_timestamp > jh.now_to_timestamp(): break # prevent duplicates count = Candle.select().where( Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp), Candle.symbol == symbol, Candle.exchange == backup_driver.name).count() already_exists = count == backup_driver.count if not already_exists: # it's today's candles if temp_end_timestamp < now if temp_end_timestamp > jh.now_to_timestamp(): temp_end_timestamp = arrow.utcnow().floor( 'minute').int_timestamp * 1000 - 60000 # fetch from market candles = backup_driver.fetch(symbol, temp_start_timestamp) if not len(candles): raise CandleNotFoundInExchange( 'No candles exists in the market for this day: {} \n' 'Try another start_date'.format( jh.timestamp_to_time(temp_start_timestamp)[:10], )) # fill absent candles (if there's any) candles = _fill_absent_candles(candles, temp_start_timestamp, temp_end_timestamp) # store in the database _insert_to_database(candles) # add as much as driver's count to the temp_start_time start_date = start_date.shift(minutes=backup_driver.count) # sleep so that the exchange won't get angry at us if not already_exists: time.sleep(backup_driver.sleep_time) # now try fetching from database again. Why? because we might have fetched more # than what's needed, but we only want as much was requested. Don't worry, the next # request will probably fetch from database and there won't be any waste! backup_candles = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_timestamp, end_timestamp), Candle.exchange == backup_driver.name, Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples() already_exists = len( backup_candles) == (end_timestamp - start_timestamp) / 60_000 + 1 if already_exists: # loop through them and set new ID and exchange for c in backup_candles: total_candles.append({ 'id': jh.generate_unique_id(), 'symbol': symbol, 'exchange': exchange, 'timestamp': c[0], 'open': c[1], 'close': c[2], 'high': c[3], 'low': c[4], 'volume': c[5] }) return total_candles
def get_candles(exchange: str, symbol: str, timeframe: str, start_date: str, finish_date: str) -> np.ndarray: """ Returns candles from the database in numpy format :param exchange: str :param symbol: str :param timeframe: str :param start_date: str :param finish_date: str :return: np.ndarray """ exchange = exchange.title() symbol = symbol.upper() import arrow import jesse.helpers as jh from jesse.models import Candle from jesse.exceptions import CandleNotFoundInDatabase from jesse.services.candle import generate_candle_from_one_minutes start_date = jh.arrow_to_timestamp(arrow.get(start_date, 'YYYY-MM-DD')) finish_date = jh.arrow_to_timestamp(arrow.get(finish_date, 'YYYY-MM-DD')) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().int_timestamp * 1000: raise ValueError('Can\'t backtest the future!') # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_date, finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples() candles = np.array(tuple(candles_tuple)) # validate that there are enough candles for selected period if len(candles) == 0 or candles[-1][0] != finish_date or candles[0][ 0] != start_date: raise CandleNotFoundInDatabase( f'Not enough candles for {symbol}. Try running "jesse import-candles"' ) if timeframe == '1m': return candles generated_candles = [] for i in range(len(candles)): num = jh.timeframe_to_one_minutes(timeframe) if (i + 1) % num == 0: generated_candles.append( generate_candle_from_one_minutes( timeframe, candles[(i - (num - 1)):(i + 1)], True)) return np.array(generated_candles)
def get_training_and_testing_candles(start_date_str: str, finish_date_str: str): start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD')) finish_date = jh.arrow_to_timestamp( arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().timestamp * 1000: raise ValueError('Can\'t backtest the future!') candles = {} for exchange in config['app']['considering_exchanges']: for symbol in config['app']['considering_symbols']: key = jh.key(exchange, symbol) candles_tuple = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_date, finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() candles[key] = { 'exchange': exchange, 'symbol': symbol, 'candles': np.array(candles_tuple) } # validate that there are enough candles for selected period if len( candles[key]['candles'] ) == 0 or candles[key]['candles'][-1][0] != finish_date or candles[ key]['candles'][0][0] != start_date: raise Breaker( 'Not enough candles for {}. Try running "jesse import-candles"' .format(symbol)) # divide into training(85%) and testing(15%) sets training_candles = {} testing_candles = {} days_diff = jh.date_diff_in_days(jh.get_arrow(start_date), jh.get_arrow(finish_date)) divider_index = int(days_diff * 0.85) * 1440 for key in candles: training_candles[key] = { 'exchange': candles[key]['exchange'], 'symbol': candles[key]['symbol'], 'candles': candles[key]['candles'][0:divider_index], } testing_candles[key] = { 'exchange': candles[key]['exchange'], 'symbol': candles[key]['symbol'], 'candles': candles[key]['candles'][divider_index:], } return training_candles, testing_candles
def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', f"{str(round(np.average(change), 2))}%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs more = "" routes_count = len(router.routes) if routes_count > 1: more = f"-and-{routes_count-1}-more" study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}" store_logs(study_name, json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns(study_name) # QuantStats' report if full_reports: price_data = [] # load close candles for Buy and hold and calculate pct_change for index, c in enumerate( config['app']['considering_candles']): exchange, symbol = c[0], c[1] if exchange in config['app'][ 'trading_exchanges'] and symbol in config['app'][ 'trading_symbols']: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.close).where( Candle.timestamp.between( jh.date_to_timestamp(start_date), jh.date_to_timestamp(finish_date) - 60000), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() candles = np.array(candles_tuple) timestamps = candles[:, 0] price_data.append(candles[:, 1]) price_data = np.transpose(price_data) price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample('D').mean() price_pct_change = price_df.pct_change(1).fillna(0) bh_daily_returns_all_routes = price_pct_change.mean(1) quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name) else: print(jh.color('No trades were made.', 'yellow'))
def load_required_candles(exchange: str, symbol: str, start_date_str: str, finish_date_str: str): """ loads initial candles that required before executing strategies. 210 for the biggest timeframe and more for the rest """ start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD')) finish_date = jh.arrow_to_timestamp( arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().int_timestamp * 1000: raise ValueError('Can\'t backtest the future!') max_timeframe = jh.max_timeframe(config['app']['considering_timeframes']) short_candles_count = jh.get_config( 'env.data.warmup_candles_num', 210) * jh.timeframe_to_one_minutes(max_timeframe) pre_finish_date = start_date - 60_000 pre_start_date = pre_finish_date - short_candles_count * 60_000 # make sure starting from the beginning of the day instead pre_start_date = jh.get_arrow(pre_start_date).floor( 'day').int_timestamp * 1000 # update candles_count to count from the beginning of the day instead short_candles_count = int((pre_finish_date - pre_start_date) / 60_000) key = jh.key(exchange, symbol) cache_key = '{}-{}-{}'.format(jh.timestamp_to_date(pre_start_date), jh.timestamp_to_date(pre_finish_date), key) cached_value = cache.get_value(cache_key) # if cache exists if cached_value: candles_tuple = cached_value # not cached, get and cache for later calls in the next 5 minutes else: # fetch from database candles_tuple = tuple( Candle.select(Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(pre_start_date, pre_finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples()) # cache it for near future calls cache.set_value(cache_key, candles_tuple, expire_seconds=60 * 60 * 24 * 7) candles = np.array(candles_tuple) if len(candles) < short_candles_count + 1: first_existing_candle = tuple( Candle.select(Candle.timestamp).where( Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).limit(1).tuples()) if not len(first_existing_candle): raise CandleNotFoundInDatabase( 'No candle for {} {} is present in the database. Try importing candles.' .format(exchange, symbol)) first_existing_candle = first_existing_candle[0][0] last_existing_candle = tuple( Candle.select(Candle.timestamp).where( Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.desc()).limit(1).tuples())[0][0] first_backtestable_timestamp = first_existing_candle + ( pre_finish_date - pre_start_date) + (60_000 * 1440) # if first backtestable timestamp is in the future, that means we have some but not enough candles if first_backtestable_timestamp > jh.today(): raise CandleNotFoundInDatabase( 'Not enough candle for {} {} is present in the database. Jesse requires "210 * biggest_timeframe" warm-up candles. ' 'Try importing more candles from an earlier date.'.format( exchange, symbol)) raise CandleNotFoundInDatabase( 'Not enough candles for {} {} exists to run backtest from {} => {}. \n' 'First available date is {}\n' 'Last available date is {}'.format( exchange, symbol, start_date_str, finish_date_str, jh.timestamp_to_date(first_backtestable_timestamp), jh.timestamp_to_date(last_existing_candle), )) return candles
def run(exchange: str, symbol: str, start_date_str: str, skip_confirmation: bool = False, mode: str = 'candles') -> None: config['app']['trading_mode'] = mode # first, create and set session_id store.app.set_session_id() register_custom_exception_handler() # close database connection from jesse.services.db import database database.open_connection() # at every second, we check to see if it's time to execute stuff status_checker = Timeloop() @status_checker.job(interval=timedelta(seconds=1)) def handle_time(): if process_status() != 'started': raise exceptions.Termination status_checker.start() try: start_timestamp = jh.arrow_to_timestamp( arrow.get(start_date_str, 'YYYY-MM-DD')) except: raise ValueError( 'start_date must be a string representing a date before today. ex: 2020-01-17' ) # more start_date validations today = arrow.utcnow().floor('day').int_timestamp * 1000 if start_timestamp == today: raise ValueError( "Today's date is not accepted. start_date must be a string a representing date BEFORE today." ) elif start_timestamp > today: raise ValueError( "Future's date is not accepted. start_date must be a string a representing date BEFORE today." ) # We just call this to throw a exception in case of a symbol without dash jh.quote_asset(symbol) click.clear() symbol = symbol.upper() until_date = arrow.utcnow().floor('day') start_date = arrow.get(start_timestamp / 1000) days_count = jh.date_diff_in_days(start_date, until_date) candles_count = days_count * 1440 try: driver: CandleExchange = drivers[exchange]() except KeyError: raise ValueError(f'{exchange} is not a supported exchange') except TypeError: raise FileNotFoundError('You are missing the "plugins.py" file') loop_length = int(candles_count / driver.count) + 1 # ask for confirmation if not skip_confirmation: click.confirm( f'Importing {days_count} days candles from "{exchange}" for "{symbol}". Duplicates will be skipped. All good?', abort=True, default=True) progressbar = Progressbar(loop_length) for i in range(candles_count): temp_start_timestamp = start_date.int_timestamp * 1000 temp_end_timestamp = temp_start_timestamp + (driver.count - 1) * 60000 # to make sure it won't try to import candles from the future! LOL if temp_start_timestamp > jh.now_to_timestamp(): break # prevent duplicates calls to boost performance count = Candle.select().where( Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp), Candle.symbol == symbol, Candle.exchange == exchange).count() already_exists = count == driver.count if not already_exists: # it's today's candles if temp_end_timestamp < now if temp_end_timestamp > jh.now_to_timestamp(): temp_end_timestamp = arrow.utcnow().floor( 'minute').int_timestamp * 1000 - 60000 # fetch from market candles = driver.fetch(symbol, temp_start_timestamp) # check if candles have been returned and check those returned start with the right timestamp. # Sometimes exchanges just return the earliest possible candles if the start date doesn't exist. if not len(candles) or arrow.get( candles[0]['timestamp'] / 1000) > start_date: click.clear() first_existing_timestamp = driver.get_starting_time(symbol) # if driver can't provide accurate get_starting_time() if first_existing_timestamp is None: raise CandleNotFoundInExchange( f'No candles exists in the market for this day: {jh.timestamp_to_time(temp_start_timestamp)[:10]} \n' 'Try another start_date') # handle when there's missing candles during the period if temp_start_timestamp > first_existing_timestamp: # see if there are candles for the same date for the backup exchange, # if so, get those, if not, download from that exchange. if driver.backup_exchange is not None: candles = _get_candles_from_backup_exchange( exchange, driver.backup_exchange, symbol, temp_start_timestamp, temp_end_timestamp) else: temp_start_time = jh.timestamp_to_time( temp_start_timestamp)[:10] temp_existing_time = jh.timestamp_to_time( first_existing_timestamp)[:10] sync_publish( 'alert', { 'message': f'No candle exists in the market for {temp_start_time}. So ' f'Jesse started importing since the first existing date which is {temp_existing_time}', 'type': 'success' }) run(exchange, symbol, jh.timestamp_to_time(first_existing_timestamp)[:10], True) return # fill absent candles (if there's any) candles = _fill_absent_candles(candles, temp_start_timestamp, temp_end_timestamp) # store in the database if skip_confirmation: store_candles(candles) else: threading.Thread(target=store_candles, args=[candles]).start() # add as much as driver's count to the temp_start_time start_date = start_date.shift(minutes=driver.count) progressbar.update() sync_publish( 'progressbar', { 'current': progressbar.current, 'estimated_remaining_seconds': progressbar.estimated_remaining_seconds }) # sleep so that the exchange won't get angry at us if not already_exists: time.sleep(driver.sleep_time) # stop the status_checker time loop status_checker.stop() sync_publish( 'alert', { 'message': f'Successfully imported candles since {jh.timestamp_to_date(start_timestamp)} until today ({days_count} days). ', 'type': 'success' }) # if it is to skip, then it's being called from another process hence we should leave the database be if not skip_confirmation: # close database connection from jesse.services.db import database database.close_connection()