def handle_exception(exc_type, exc_value, exc_traceback) -> None:
if issubclass(exc_type, KeyboardInterrupt):
sys.excepthook(exc_type, exc_value, exc_traceback)
return
# handle Breaking exceptions
if exc_type in [
exceptions.InvalidConfig, exceptions.RouteNotFound,
exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase
]:
click.clear()
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color('{}: {}'.format(exc_type.__name__, exc_value),
'yellow'))
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error('{}: {}'.format(exc_type.__name__, exc_value))
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:",
exc_info=(exc_type, exc_value, exc_traceback))
else:
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color('{}: {}'.format(exc_type.__name__, exc_value),
'yellow'))
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/paper-trade.txt'), 'red'))
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/live-trade.txt'), 'red'))
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/collect.txt'), 'red'))
def install_routes():
"""
:return:
"""
considering_candles = set()
# when importing market data, considering_candles is all we need
if jh.is_collecting_data():
for r in router.market_data:
considering_candles.add((r.exchange, r.symbol))
config['app']['considering_candles'] = tuple(considering_candles)
return
# validate routes for duplicates:
# each exchange-symbol pair can be traded only once.
for r in router.routes:
considering_candles.add((r.exchange, r.symbol))
exchange = r.exchange
symbol = r.symbol
count = 0
for ro in router.routes:
if ro.exchange == exchange and ro.symbol == symbol:
count += 1
if count != 1:
raise InvalidRoutes(
'each exchange-symbol pair can be traded only once. \nMore info: https://docs.jesse.trade/docs/routes.html#trading-multiple-routes')
trading_exchanges = set()
trading_timeframes = set()
trading_symbols = set()
for r in router.routes:
trading_exchanges.add(r.exchange)
trading_timeframes.add(r.timeframe)
trading_symbols.add(r.symbol)
considering_exchanges = trading_exchanges.copy()
considering_timeframes = trading_timeframes.copy()
considering_symbols = trading_symbols.copy()
for e in router.extra_candles:
considering_candles.add((e[0], e[1]))
considering_exchanges.add(e[0])
considering_symbols.add(e[1])
considering_timeframes.add(e[2])
# 1m must be present at all times
considering_timeframes.add('1m')
config['app']['considering_candles'] = tuple(considering_candles)
config['app']['considering_exchanges'] = tuple(considering_exchanges)
config['app']['considering_symbols'] = tuple(considering_symbols)
config['app']['considering_timeframes'] = tuple(considering_timeframes)
config['app']['trading_exchanges'] = tuple(trading_exchanges)
config['app']['trading_symbols'] = tuple(trading_symbols)
config['app']['trading_timeframes'] = tuple(trading_timeframes)
def error(msg: str) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
# error logs should be logged as info logs as well
info(msg)
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
# notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify_urgently(f"ERROR:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data() or jh.is_live():
sync_publish('error_log', log_dict)
store.logs.errors.append(log_dict)
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.error(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'error')
def info(msg: str, send_notification=False) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
store.logs.info.append(log_dict)
if jh.is_collecting_data() or jh.is_live():
sync_publish('info_log', log_dict)
if jh.is_live() or (jh.is_backtesting() and jh.is_debugging()):
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.info(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'info')
if send_notification:
notify(msg)
def add_trade(self, trade: np.ndarray, exchange: str, symbol: str) -> None:
key = jh.key(exchange, symbol)
if not len(self.temp_storage[key]
) or trade[0] - self.temp_storage[key][0][0] < 1000:
self.temp_storage[key].append(trade)
else:
arr = self.temp_storage[key]
buy_arr = np.array(list(filter(lambda x: x[3] == 1, arr)))
sell_arr = np.array(list(filter(lambda x: x[3] == 0, arr)))
generated = np.array([
# timestamp
arr[0][0],
# price (weighted average)
(arr[:][:, 1] * arr[:][:, 2]).sum() / arr[:][:, 2].sum(),
# buy_qty
0 if not len(buy_arr) else buy_arr[:, 2].sum(),
# sell_qty
0 if not len(sell_arr) else sell_arr[:, 2].sum(),
# buy_count
len(buy_arr),
# sell_count
len(sell_arr)
])
if jh.is_collecting_data():
store_trade_into_db(exchange, symbol, generated)
else:
self.storage[key].append(generated)
self.temp_storage[key].flush()
self.temp_storage[key].append(trade)
def add_orderbook(self, exchange: str, symbol: str, asks: list,
bids: list):
"""
:param exchange:
:param symbol:
:param asks:
:param bids:
"""
key = jh.key(exchange, symbol)
self.temp_storage[key]['asks'] = asks
self.temp_storage[key]['bids'] = bids
# generate new numpy formatted orderbook if it is
# either the first time, or that it has passed
# 1000 milliseconds since the last time
if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now(
) - self.temp_storage[key]['last_updated_timestamp'] >= 1000:
self.temp_storage[key]['last_updated_timestamp'] = jh.now()
formatted_orderbook = self.format_orderbook(exchange, symbol)
if jh.is_collecting_data():
store_orderbook_into_db(exchange, symbol, formatted_orderbook)
else:
self.storage[key].append(formatted_orderbook)
def add_candle(self,
candle: np.ndarray,
exchange: str,
symbol: str,
timeframe: str,
with_execution: bool = True,
with_generation: bool = True) -> None:
if jh.is_collecting_data():
# make sure it's a complete (and not a forming) candle
if jh.now_to_timestamp() >= (candle[0] + 60000):
store_candle_into_db(exchange, symbol, candle)
return
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe)
if jh.is_live():
self.update_position(exchange, symbol, candle)
# ignore new candle at the time of execution because it messes
# the count of candles without actually having an impact
if candle[0] >= jh.now():
return
# initial
if len(arr) == 0:
arr.append(candle)
# if it's new, add
elif candle[0] > arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to be executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr.append(candle)
# generate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# if it's the last candle again, update
elif candle[0] == arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to get executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr[-1] = candle
# regenerate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# past candles will be ignored (dropped)
elif candle[0] < arr[-1][0]:
return
def add_ticker(self, ticker: np.ndarray, exchange: str, symbol: str):
key = jh.key(exchange, symbol)
# only process once per second
if len(self.storage[key][:]) == 0 or jh.now() - self.storage[key][-1][0] >= 1000:
self.storage[key].append(ticker)
if jh.is_collecting_data():
store_ticker_into_db(exchange, symbol, ticker)
return
def info(msg: str) -> None:
from jesse.store import store
store.logs.info.append({'time': jh.now_to_timestamp(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}')
if jh.is_live():
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {str(msg)}"
logging.info(msg)
def info(msg):
from jesse.store import store
store.logs.info.append({'time': jh.now(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta'))
if jh.is_live():
msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.info(msg)
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red'))
store.logs.errors.append({'time': jh.now(), 'message': msg})
if jh.is_live():
msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.error(msg)
def error(msg: str) -> None:
msg = str(msg)
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}', 'red'))
store.logs.errors.append({'time': jh.now_to_timestamp(), 'message': msg})
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logging.error(msg)
def _init_main_logger():
session_id = jh.get_session_id()
jh.make_directory('storage/logs/live-mode')
jh.make_directory('storage/logs/backtest-mode')
jh.make_directory('storage/logs/optimize-mode')
jh.make_directory('storage/logs/collect-mode')
if jh.is_live():
filename = f'storage/logs/live-mode/{session_id}.txt'
elif jh.is_collecting_data():
filename = f'storage/logs/collect-mode/{session_id}.txt'
elif jh.is_optimizing():
filename = f'storage/logs/optimize-mode/{session_id}.txt'
elif jh.is_backtesting():
filename = f'storage/logs/backtest-mode/{session_id}.txt'
else:
filename = 'storage/logs/etc.txt'
new_logger = logging.getLogger(jh.app_mode())
new_logger.setLevel(logging.INFO)
new_logger.addHandler(logging.FileHandler(filename, mode='w'))
LOGGERS[jh.app_mode()] = new_logger
def test_is_collecting_data():
assert jh.is_collecting_data() is False
def install_routes() -> None:
considering_candles = set()
# when importing market data, considering_candles is all we need
if jh.is_collecting_data():
for r in router.market_data:
considering_candles.add((r.exchange, r.symbol))
config['app']['considering_candles'] = tuple(considering_candles)
return
# validate routes for duplicates:
# each exchange-symbol pair can be traded only once.
for r in router.routes:
considering_candles.add((r.exchange, r.symbol))
exchange = r.exchange
symbol = r.symbol
count = sum(ro.exchange == exchange and ro.symbol == symbol
for ro in router.routes)
if count != 1:
raise InvalidRoutes(
'each exchange-symbol pair can be traded only once. \nMore info: https://docs.jesse.trade/docs/routes.html#trading-multiple-routes'
)
# check to make sure if trading more than one route, they all have the same quote
# currency because otherwise we cannot calculate the correct performance metrics
first_routes_quote = jh.quote_asset(router.routes[0].symbol)
for r in router.routes:
if jh.quote_asset(r.symbol) != first_routes_quote:
raise InvalidRoutes(
'All trading routes must have the same quote asset.')
trading_exchanges = set()
trading_timeframes = set()
trading_symbols = set()
for r in router.routes:
trading_exchanges.add(r.exchange)
trading_timeframes.add(r.timeframe)
trading_symbols.add(r.symbol)
considering_exchanges = trading_exchanges.copy()
considering_timeframes = trading_timeframes.copy()
considering_symbols = trading_symbols.copy()
for e in router.extra_candles:
considering_candles.add((e['exchange'], e['symbol']))
considering_exchanges.add(e['exchange'])
considering_symbols.add(e['symbol'])
considering_timeframes.add(e['timeframe'])
# 1m must be present at all times
considering_timeframes.add('1m')
config['app']['considering_candles'] = tuple(considering_candles)
config['app']['considering_exchanges'] = tuple(considering_exchanges)
config['app']['considering_symbols'] = tuple(considering_symbols)
config['app']['considering_timeframes'] = tuple(considering_timeframes)
config['app']['trading_exchanges'] = tuple(trading_exchanges)
config['app']['trading_symbols'] = tuple(trading_symbols)
config['app']['trading_timeframes'] = tuple(trading_timeframes)
def register_custom_exception_handler() -> None:
import sys
import threading
import traceback
import logging
from jesse.services import logger as jesse_logger
import click
from jesse import exceptions
log_format = "%(message)s"
os.makedirs('./storage/logs', exist_ok=True)
if jh.is_livetrading():
logging.basicConfig(filename='storage/logs/live-trade.txt',
level=logging.INFO,
filemode='w',
format=log_format)
elif jh.is_paper_trading():
logging.basicConfig(filename='storage/logs/paper-trade.txt',
level=logging.INFO,
filemode='w',
format=log_format)
elif jh.is_collecting_data():
logging.basicConfig(filename='storage/logs/collect.txt',
level=logging.INFO,
filemode='w',
format=log_format)
elif jh.is_optimizing():
logging.basicConfig(filename='storage/logs/optimize.txt',
level=logging.INFO,
filemode='w',
format=log_format)
else:
logging.basicConfig(level=logging.INFO)
# main thread
def handle_exception(exc_type, exc_value, exc_traceback) -> None:
if issubclass(exc_type, KeyboardInterrupt):
sys.excepthook(exc_type, exc_value, exc_traceback)
return
# handle Breaking exceptions
if exc_type in [
exceptions.InvalidConfig, exceptions.RouteNotFound,
exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase
]:
click.clear()
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print('\n', jh.color('Uncaught Exception:', 'red'),
jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow'))
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error(f'{exc_type.__name__}: {exc_value}')
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:",
exc_info=(exc_type, exc_value, exc_traceback))
else:
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print('\n', jh.color('Uncaught Exception:', 'red'),
jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow'))
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt',
'red'))
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt',
'red'))
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt',
'red'))
sys.excepthook = handle_exception
# other threads
if jh.python_version() >= (3, 8):
def handle_thread_exception(args) -> None:
if args.exc_type == SystemExit:
return
# handle Breaking exceptions
if args.exc_type in [
exceptions.InvalidConfig, exceptions.RouteNotFound,
exceptions.InvalidRoutes,
exceptions.CandleNotFoundInDatabase
]:
click.clear()
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color(f'{args.exc_type.__name__}: {args.exc_value}',
'yellow'))
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error(
f'{args.exc_type.__name__}: { args.exc_value}')
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:",
exc_info=(args.exc_type, args.exc_value,
args.exc_traceback))
else:
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color(f'{args.exc_type.__name__}: {args.exc_value}',
'yellow'))
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt',
'red'))
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt',
'red'))
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt',
'red'))
threading.excepthook = handle_thread_exception
def handle_thread_exception(args):
"""
:param args:
:return:
"""
if args.exc_type == SystemExit:
return
# handle Breaking exceptions
if args.exc_type in [
exceptions.ConfigException, exceptions.RouteNotFound, exceptions.InvalidRoutes,
exceptions.CandleNotFoundInDatabase
]:
click.clear()
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n',
jh.color('Uncaught Exception:', 'red'),
jh.color('{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow')
)
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error(
'{}: {}'.format(args.exc_type.__name__, args.exc_value)
)
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:", exc_info=(args.exc_type, args.exc_value, args.exc_traceback))
else:
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n',
jh.color('Uncaught Exception:', 'red'),
jh.color('{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow')
)
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'.format(
'storage/logs/paper-trade.txt'
),
'red'
)
)
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'.format(
'storage/logs/live-trade.txt'
),
'red'
)
)
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'.format(
'storage/logs/collect.txt'
),
'red'
)
)
def add_candle(self,
candle: np.ndarray,
exchange: str,
symbol: str,
timeframe: str,
with_execution=True,
with_generation=True,
is_forming_candle=False):
"""
:param candle:
:param exchange:
:param symbol:
:param timeframe:
:param with_execution:
:param with_generation:
:param is_forming_candle:
:return:
"""
if jh.is_collecting_data():
# make sure it's a complete (and not a forming) candle
if jh.now() >= (candle[0] + 60000):
store_candle_into_db(exchange, symbol, candle)
return
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe)
if jh.is_live():
self.update_position(exchange, symbol, candle)
# initial
if len(arr) == 0:
arr.append(candle)
# if it's new, add
elif candle[0] > arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to be executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr.append(candle)
# generate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution,
is_forming_candle)
# if it's the last candle again, update
elif candle[0] == arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to get executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr[-1] = candle
# regenerate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution,
is_forming_candle)
# past candles will be ignored (dropped)
elif candle[0] < arr[-1][0]:
return
