def get_starting_time(self, symbol: str): # hard-code few common symbols if symbol == 'BTCUSD': return jh.date_to_timestamp('2015-08-01') elif symbol == 'ETHUSD': return jh.date_to_timestamp('2016-01-01') payload = { 'sort': 1, 'limit': 5000, } response = requests.get(self.endpoint + '/trade:1D:t{}/hist'.format(symbol), params=payload) if response.status_code != 200: raise Exception(response.content) data = response.json() # wrong symbol entered if not len(data): raise exceptions.SymbolNotFound( "No candle exists for {} in Bitfinex. You're probably misspelling the symbol name." .format(symbol)) first_timestamp = int(data[0][0]) second_timestamp = first_timestamp + 60_000 * 1440 return second_timestamp
def get_starting_time(self, symbol: str): dashless_symbol = jh.dashless_symbol(symbol) # hard-code few common symbols if symbol == 'BTC-USD': return jh.date_to_timestamp('2015-08-01') elif symbol == 'ETH-USD': return jh.date_to_timestamp('2016-01-01') payload = { 'sort': 1, 'limit': 5000, } response = requests.get(f"{self.endpoint}/trade:1D:t{dashless_symbol}/hist", params=payload) if response.status_code != 200: raise Exception(response.content) data = response.json() # wrong symbol entered if not len(data): raise exceptions.SymbolNotFound( f"No candle exists for {symbol} in Bitfinex. You're probably misspelling the symbol name." ) # since the first timestamp doesn't include all the 1m # candles, let's start since the second day then first_timestamp = int(data[0][0]) second_timestamp = first_timestamp + 60_000 * 1440 return second_timestamp
def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = '{}-{}'.format(config['app']['considering_candles'][0][0], config['app']['considering_candles'][0][1]) table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', str(round(np.average(change), 2)) + "%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs store_logs(json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns() # QuantStats' report if full_reports: quantstats.quantstats_tearsheet() else: print(jh.color('No trades were made.', 'yellow'))
def load_candles( start_date_str: str, finish_date_str: str) -> Dict[str, Dict[str, Union[str, np.ndarray]]]: start_date = jh.date_to_timestamp(start_date_str) finish_date = jh.date_to_timestamp(finish_date_str) - 60000 # validate if start_date == finish_date: raise ValueError('start_date and finish_date cannot be the same.') if start_date > finish_date: raise ValueError('start_date cannot be bigger than finish_date.') if finish_date > arrow.utcnow().int_timestamp * 1000: raise ValueError("Can't load candle data from the future!") # load and add required warm-up candles for backtest if jh.is_backtesting(): for c in config['app']['considering_candles']: required_candles.inject_required_candles_to_store( required_candles.load_required_candles(c[0], c[1], start_date_str, finish_date_str), c[0], c[1]) # download candles for the duration of the backtest candles = {} for c in config['app']['considering_candles']: exchange, symbol = c[0], c[1] key = jh.key(exchange, symbol) cache_key = '{}-{}-'.format(start_date_str, finish_date_str) + key cached_value = cache.get_value(cache_key) # if cache exists if cached_value: candles_tuple = cached_value # not cached, get and cache for later calls in the next 5 minutes else: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low, Candle.volume).where( Candle.timestamp.between(start_date, finish_date), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() # validate that there are enough candles for selected period required_candles_count = (finish_date - start_date) / 60_000 if len(candles_tuple) == 0 or candles_tuple[-1][ 0] != finish_date or candles_tuple[0][0] != start_date: raise exceptions.CandleNotFoundInDatabase( 'Not enough candles for {}. Try running "jesse import-candles"' .format(symbol)) elif len(candles_tuple) != required_candles_count + 1: raise exceptions.CandleNotFoundInDatabase( 'There are missing candles between {} => {}'.format( start_date_str, finish_date_str)) # cache it for near future calls cache.set_value(cache_key, tuple(candles_tuple), expire_seconds=60 * 60 * 24 * 7) candles[key] = { 'exchange': exchange, 'symbol': symbol, 'candles': np.array(candles_tuple) } return candles
def run( debug_mode, user_config: dict, routes: List[Dict[str, str]], extra_routes: List[Dict[str, str]], start_date: str, finish_date: str, candles: dict = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False ) -> None: if not jh.is_unit_testing(): # at every second, we check to see if it's time to execute stuff status_checker = Timeloop() @status_checker.job(interval=timedelta(seconds=1)) def handle_time(): if process_status() != 'started': raise exceptions.Termination status_checker.start() from jesse.config import config, set_config config['app']['trading_mode'] = 'backtest' # debug flag config['app']['debug_mode'] = debug_mode # inject config if not jh.is_unit_testing(): set_config(user_config) # set routes router.initiate(routes, extra_routes) store.app.set_session_id() register_custom_exception_handler() # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): sync_publish('general_info', { 'session_id': jh.get_session_id(), 'debug_mode': str(config['app']['debug_mode']), }) # candles info key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" sync_publish('candles_info', stats.candles_info(candles[key]['candles'])) # routes info sync_publish('routes_info', stats.routes(router.routes)) # run backtest simulation simulator(candles, run_silently=jh.should_execute_silently()) # hyperparameters (if any) if not jh.should_execute_silently(): sync_publish('hyperparameters', stats.hyperparameters(router.routes)) if not jh.should_execute_silently(): if store.completed_trades.count > 0: sync_publish('metrics', report.portfolio_metrics()) routes_count = len(router.routes) more = f"-and-{routes_count - 1}-more" if routes_count > 1 else "" study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}" store_logs(study_name, json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns(study_name) sync_publish('equity_curve', charts.equity_curve()) # QuantStats' report if full_reports: price_data = [] # load close candles for Buy and hold and calculate pct_change for index, c in enumerate(config['app']['considering_candles']): exchange, symbol = c[0], c[1] if exchange in config['app']['trading_exchanges'] and symbol in config['app']['trading_symbols']: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.close ).where( Candle.timestamp.between(jh.date_to_timestamp(start_date), jh.date_to_timestamp(finish_date) - 60000), Candle.exchange == exchange, Candle.symbol == symbol ).order_by(Candle.timestamp.asc()).tuples() candles = np.array(candles_tuple) timestamps = candles[:, 0] price_data.append(candles[:, 1]) price_data = np.transpose(price_data) price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample( 'D').mean() price_pct_change = price_df.pct_change(1).fillna(0) bh_daily_returns_all_routes = price_pct_change.mean(1) quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name) else: sync_publish('equity_curve', None) sync_publish('metrics', None) # close database connection from jesse.services.db import database database.close_connection()
def test_date_to_timestamp(): assert jh.date_to_timestamp('2015-08-01') == 1438387200000
def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', f"{str(round(np.average(change), 2))}%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs more = "" routes_count = len(router.routes) if routes_count > 1: more = f"-and-{routes_count-1}-more" study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}" store_logs(study_name, json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns(study_name) # QuantStats' report if full_reports: price_data = [] # load close candles for Buy and hold and calculate pct_change for index, c in enumerate( config['app']['considering_candles']): exchange, symbol = c[0], c[1] if exchange in config['app'][ 'trading_exchanges'] and symbol in config['app'][ 'trading_symbols']: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.close).where( Candle.timestamp.between( jh.date_to_timestamp(start_date), jh.date_to_timestamp(finish_date) - 60000), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() candles = np.array(candles_tuple) timestamps = candles[:, 0] price_data.append(candles[:, 1]) price_data = np.transpose(price_data) price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample('D').mean() price_pct_change = price_df.pct_change(1).fillna(0) bh_daily_returns_all_routes = price_pct_change.mean(1) quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name) else: print(jh.color('No trades were made.', 'yellow'))