def handle_exception(exc_type, exc_value, exc_traceback) -> None: if issubclass(exc_type, KeyboardInterrupt): sys.excepthook(exc_type, exc_value, exc_traceback) return # handle Breaking exceptions if exc_type in [ exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(exc_type.__name__, exc_value), 'yellow')) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error('{}: {}'.format(exc_type.__name__, exc_value)) if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback)) else: print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(exc_type.__name__, exc_value), 'yellow')) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/paper-trade.txt'), 'red')) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/live-trade.txt'), 'red')) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/collect.txt'), 'red'))
def install_routes(): """ :return: """ considering_candles = set() # when importing market data, considering_candles is all we need if jh.is_collecting_data(): for r in router.market_data: considering_candles.add((r.exchange, r.symbol)) config['app']['considering_candles'] = tuple(considering_candles) return # validate routes for duplicates: # each exchange-symbol pair can be traded only once. for r in router.routes: considering_candles.add((r.exchange, r.symbol)) exchange = r.exchange symbol = r.symbol count = 0 for ro in router.routes: if ro.exchange == exchange and ro.symbol == symbol: count += 1 if count != 1: raise InvalidRoutes( 'each exchange-symbol pair can be traded only once. \nMore info: https://docs.jesse.trade/docs/routes.html#trading-multiple-routes') trading_exchanges = set() trading_timeframes = set() trading_symbols = set() for r in router.routes: trading_exchanges.add(r.exchange) trading_timeframes.add(r.timeframe) trading_symbols.add(r.symbol) considering_exchanges = trading_exchanges.copy() considering_timeframes = trading_timeframes.copy() considering_symbols = trading_symbols.copy() for e in router.extra_candles: considering_candles.add((e[0], e[1])) considering_exchanges.add(e[0]) considering_symbols.add(e[1]) considering_timeframes.add(e[2]) # 1m must be present at all times considering_timeframes.add('1m') config['app']['considering_candles'] = tuple(considering_candles) config['app']['considering_exchanges'] = tuple(considering_exchanges) config['app']['considering_symbols'] = tuple(considering_symbols) config['app']['considering_timeframes'] = tuple(considering_timeframes) config['app']['trading_exchanges'] = tuple(trading_exchanges) config['app']['trading_symbols'] = tuple(trading_symbols) config['app']['trading_timeframes'] = tuple(trading_timeframes)
def error(msg: str) -> None: if jh.app_mode() not in LOGGERS: _init_main_logger() # error logs should be logged as info logs as well info(msg) msg = str(msg) from jesse.store import store log_id = jh.generate_unique_id() log_dict = { 'id': log_id, 'timestamp': jh.now_to_timestamp(), 'message': msg } if jh.is_live() and jh.get_config('env.notifications.events.errors', True): # notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}") notify_urgently(f"ERROR:\n{msg}") notify(f'ERROR:\n{msg}') if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data() or jh.is_live(): sync_publish('error_log', log_dict) store.logs.errors.append(log_dict) if jh.is_live() or jh.is_optimizing(): msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}" logger = LOGGERS[jh.app_mode()] logger.error(msg) if jh.is_live(): from jesse.models.utils import store_log_into_db store_log_into_db(log_dict, 'error')
def info(msg: str, send_notification=False) -> None: if jh.app_mode() not in LOGGERS: _init_main_logger() msg = str(msg) from jesse.store import store log_id = jh.generate_unique_id() log_dict = { 'id': log_id, 'timestamp': jh.now_to_timestamp(), 'message': msg } store.logs.info.append(log_dict) if jh.is_collecting_data() or jh.is_live(): sync_publish('info_log', log_dict) if jh.is_live() or (jh.is_backtesting() and jh.is_debugging()): msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}" logger = LOGGERS[jh.app_mode()] logger.info(msg) if jh.is_live(): from jesse.models.utils import store_log_into_db store_log_into_db(log_dict, 'info') if send_notification: notify(msg)
def add_trade(self, trade: np.ndarray, exchange: str, symbol: str) -> None: key = jh.key(exchange, symbol) if not len(self.temp_storage[key] ) or trade[0] - self.temp_storage[key][0][0] < 1000: self.temp_storage[key].append(trade) else: arr = self.temp_storage[key] buy_arr = np.array(list(filter(lambda x: x[3] == 1, arr))) sell_arr = np.array(list(filter(lambda x: x[3] == 0, arr))) generated = np.array([ # timestamp arr[0][0], # price (weighted average) (arr[:][:, 1] * arr[:][:, 2]).sum() / arr[:][:, 2].sum(), # buy_qty 0 if not len(buy_arr) else buy_arr[:, 2].sum(), # sell_qty 0 if not len(sell_arr) else sell_arr[:, 2].sum(), # buy_count len(buy_arr), # sell_count len(sell_arr) ]) if jh.is_collecting_data(): store_trade_into_db(exchange, symbol, generated) else: self.storage[key].append(generated) self.temp_storage[key].flush() self.temp_storage[key].append(trade)
def add_orderbook(self, exchange: str, symbol: str, asks: list, bids: list): """ :param exchange: :param symbol: :param asks: :param bids: """ key = jh.key(exchange, symbol) self.temp_storage[key]['asks'] = asks self.temp_storage[key]['bids'] = bids # generate new numpy formatted orderbook if it is # either the first time, or that it has passed # 1000 milliseconds since the last time if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now( ) - self.temp_storage[key]['last_updated_timestamp'] >= 1000: self.temp_storage[key]['last_updated_timestamp'] = jh.now() formatted_orderbook = self.format_orderbook(exchange, symbol) if jh.is_collecting_data(): store_orderbook_into_db(exchange, symbol, formatted_orderbook) else: self.storage[key].append(formatted_orderbook)
def add_candle(self, candle: np.ndarray, exchange: str, symbol: str, timeframe: str, with_execution: bool = True, with_generation: bool = True) -> None: if jh.is_collecting_data(): # make sure it's a complete (and not a forming) candle if jh.now_to_timestamp() >= (candle[0] + 60000): store_candle_into_db(exchange, symbol, candle) return arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe) if jh.is_live(): self.update_position(exchange, symbol, candle) # ignore new candle at the time of execution because it messes # the count of candles without actually having an impact if candle[0] >= jh.now(): return # initial if len(arr) == 0: arr.append(candle) # if it's new, add elif candle[0] > arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to be executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr.append(candle) # generate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution) # if it's the last candle again, update elif candle[0] == arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to get executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr[-1] = candle # regenerate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution) # past candles will be ignored (dropped) elif candle[0] < arr[-1][0]: return
def add_ticker(self, ticker: np.ndarray, exchange: str, symbol: str): key = jh.key(exchange, symbol) # only process once per second if len(self.storage[key][:]) == 0 or jh.now() - self.storage[key][-1][0] >= 1000: self.storage[key].append(ticker) if jh.is_collecting_data(): store_ticker_into_db(exchange, symbol, ticker) return
def info(msg: str) -> None: from jesse.store import store store.logs.info.append({'time': jh.now_to_timestamp(), 'message': msg}) if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}') if jh.is_live(): msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {str(msg)}" logging.info(msg)
def info(msg): from jesse.store import store store.logs.info.append({'time': jh.now(), 'message': msg}) if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta')) if jh.is_live(): msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg) import logging logging.info(msg)
def error(msg): from jesse.store import store if jh.is_live() and jh.get_config('env.notifications.events.errors', True): notify('ERROR:\n{}'.format(msg)) if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red')) store.logs.errors.append({'time': jh.now(), 'message': msg}) if jh.is_live(): msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg) import logging logging.error(msg)
def error(msg: str) -> None: msg = str(msg) from jesse.store import store if jh.is_live() and jh.get_config('env.notifications.events.errors', True): notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}") notify(f'ERROR:\n{msg}') if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(jh.color(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}', 'red')) store.logs.errors.append({'time': jh.now_to_timestamp(), 'message': msg}) if jh.is_live() or jh.is_optimizing(): msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}" logging.error(msg)
def _init_main_logger(): session_id = jh.get_session_id() jh.make_directory('storage/logs/live-mode') jh.make_directory('storage/logs/backtest-mode') jh.make_directory('storage/logs/optimize-mode') jh.make_directory('storage/logs/collect-mode') if jh.is_live(): filename = f'storage/logs/live-mode/{session_id}.txt' elif jh.is_collecting_data(): filename = f'storage/logs/collect-mode/{session_id}.txt' elif jh.is_optimizing(): filename = f'storage/logs/optimize-mode/{session_id}.txt' elif jh.is_backtesting(): filename = f'storage/logs/backtest-mode/{session_id}.txt' else: filename = 'storage/logs/etc.txt' new_logger = logging.getLogger(jh.app_mode()) new_logger.setLevel(logging.INFO) new_logger.addHandler(logging.FileHandler(filename, mode='w')) LOGGERS[jh.app_mode()] = new_logger
def test_is_collecting_data(): assert jh.is_collecting_data() is False
def install_routes() -> None: considering_candles = set() # when importing market data, considering_candles is all we need if jh.is_collecting_data(): for r in router.market_data: considering_candles.add((r.exchange, r.symbol)) config['app']['considering_candles'] = tuple(considering_candles) return # validate routes for duplicates: # each exchange-symbol pair can be traded only once. for r in router.routes: considering_candles.add((r.exchange, r.symbol)) exchange = r.exchange symbol = r.symbol count = sum(ro.exchange == exchange and ro.symbol == symbol for ro in router.routes) if count != 1: raise InvalidRoutes( 'each exchange-symbol pair can be traded only once. \nMore info: https://docs.jesse.trade/docs/routes.html#trading-multiple-routes' ) # check to make sure if trading more than one route, they all have the same quote # currency because otherwise we cannot calculate the correct performance metrics first_routes_quote = jh.quote_asset(router.routes[0].symbol) for r in router.routes: if jh.quote_asset(r.symbol) != first_routes_quote: raise InvalidRoutes( 'All trading routes must have the same quote asset.') trading_exchanges = set() trading_timeframes = set() trading_symbols = set() for r in router.routes: trading_exchanges.add(r.exchange) trading_timeframes.add(r.timeframe) trading_symbols.add(r.symbol) considering_exchanges = trading_exchanges.copy() considering_timeframes = trading_timeframes.copy() considering_symbols = trading_symbols.copy() for e in router.extra_candles: considering_candles.add((e['exchange'], e['symbol'])) considering_exchanges.add(e['exchange']) considering_symbols.add(e['symbol']) considering_timeframes.add(e['timeframe']) # 1m must be present at all times considering_timeframes.add('1m') config['app']['considering_candles'] = tuple(considering_candles) config['app']['considering_exchanges'] = tuple(considering_exchanges) config['app']['considering_symbols'] = tuple(considering_symbols) config['app']['considering_timeframes'] = tuple(considering_timeframes) config['app']['trading_exchanges'] = tuple(trading_exchanges) config['app']['trading_symbols'] = tuple(trading_symbols) config['app']['trading_timeframes'] = tuple(trading_timeframes)
def register_custom_exception_handler() -> None: import sys import threading import traceback import logging from jesse.services import logger as jesse_logger import click from jesse import exceptions log_format = "%(message)s" os.makedirs('./storage/logs', exist_ok=True) if jh.is_livetrading(): logging.basicConfig(filename='storage/logs/live-trade.txt', level=logging.INFO, filemode='w', format=log_format) elif jh.is_paper_trading(): logging.basicConfig(filename='storage/logs/paper-trade.txt', level=logging.INFO, filemode='w', format=log_format) elif jh.is_collecting_data(): logging.basicConfig(filename='storage/logs/collect.txt', level=logging.INFO, filemode='w', format=log_format) elif jh.is_optimizing(): logging.basicConfig(filename='storage/logs/optimize.txt', level=logging.INFO, filemode='w', format=log_format) else: logging.basicConfig(level=logging.INFO) # main thread def handle_exception(exc_type, exc_value, exc_traceback) -> None: if issubclass(exc_type, KeyboardInterrupt): sys.excepthook(exc_type, exc_value, exc_traceback) return # handle Breaking exceptions if exc_type in [ exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print('\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error(f'{exc_type.__name__}: {exc_value}') if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback)) else: print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print('\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt', 'red')) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt', 'red')) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt', 'red')) sys.excepthook = handle_exception # other threads if jh.python_version() >= (3, 8): def handle_thread_exception(args) -> None: if args.exc_type == SystemExit: return # handle Breaking exceptions if args.exc_type in [ exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{args.exc_type.__name__}: {args.exc_value}', 'yellow')) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error( f'{args.exc_type.__name__}: { args.exc_value}') if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(args.exc_type, args.exc_value, args.exc_traceback)) else: print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{args.exc_type.__name__}: {args.exc_value}', 'yellow')) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt', 'red')) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt', 'red')) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt', 'red')) threading.excepthook = handle_thread_exception
def handle_thread_exception(args): """ :param args: :return: """ if args.exc_type == SystemExit: return # handle Breaking exceptions if args.exc_type in [ exceptions.ConfigException, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow') ) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error( '{}: {}'.format(args.exc_type.__name__, args.exc_value) ) if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(args.exc_type, args.exc_value, args.exc_traceback)) else: print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow') ) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/paper-trade.txt' ), 'red' ) ) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/live-trade.txt' ), 'red' ) ) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/collect.txt' ), 'red' ) )
def add_candle(self, candle: np.ndarray, exchange: str, symbol: str, timeframe: str, with_execution=True, with_generation=True, is_forming_candle=False): """ :param candle: :param exchange: :param symbol: :param timeframe: :param with_execution: :param with_generation: :param is_forming_candle: :return: """ if jh.is_collecting_data(): # make sure it's a complete (and not a forming) candle if jh.now() >= (candle[0] + 60000): store_candle_into_db(exchange, symbol, candle) return arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe) if jh.is_live(): self.update_position(exchange, symbol, candle) # initial if len(arr) == 0: arr.append(candle) # if it's new, add elif candle[0] > arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to be executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr.append(candle) # generate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution, is_forming_candle) # if it's the last candle again, update elif candle[0] == arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to get executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr[-1] = candle # regenerate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution, is_forming_candle) # past candles will be ignored (dropped) elif candle[0] < arr[-1][0]: return