def test_get_current_and_past_ticker():
set_up()
# add 4 tickers
t1 = np.array([jh.now_to_timestamp(), 1, 2, 3, 4], dtype=np.float64)
t2 = np.array([jh.now_to_timestamp() + 1000, 2, 2, 3, 4], dtype=np.float64)
t3 = np.array([jh.now_to_timestamp() + 2000, 3, 2, 3, 4], dtype=np.float64)
t4 = np.array([jh.now_to_timestamp() + 3000, 4, 2, 3, 4], dtype=np.float64)
store.tickers.add_ticker(t1, 'Sandbox', 'BTC-USD')
store.app.time += 1000
store.tickers.add_ticker(t2, 'Sandbox', 'BTC-USD')
store.app.time += 1000
store.tickers.add_ticker(t3, 'Sandbox', 'BTC-USD')
store.app.time += 1000
store.tickers.add_ticker(t4, 'Sandbox', 'BTC-USD')
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTC-USD'),
np.array([t1, t2, t3, t4]))
# get the previous one
np.testing.assert_equal(
store.tickers.get_past_ticker('Sandbox', 'BTC-USD', 1), t3)
# get current
np.testing.assert_equal(
store.tickers.get_current_ticker('Sandbox', 'BTC-USD'), t4)
def test_can_log_error_by_firing_event():
set_up()
# fire first error event
logger.error('first error!!!!!')
first_logged_error = {'time': jh.now_to_timestamp(), 'message': 'first error!!!!!'}
assert store.logs.errors == [first_logged_error]
# fire second error event
logger.error('second error!!!!!')
second_logged_error = {'time': jh.now_to_timestamp(), 'message': 'second error!!!!!'}
assert store.logs.errors == [first_logged_error, second_logged_error]
def info(msg: str) -> None:
from jesse.store import store
store.logs.info.append({'time': jh.now_to_timestamp(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print('[{}]: {}'.format(jh.timestamp_to_time(jh.now_to_timestamp()),
msg))
if jh.is_live():
msg = '[INFO | {}] '.format(
jh.timestamp_to_time(jh.now_to_timestamp())[:19]) + str(msg)
import logging
logging.info(msg)
def __init__(self, attributes: dict = None, **kwargs) -> None:
Model.__init__(self, attributes=attributes, **kwargs)
if attributes is None:
attributes = {}
for a, value in attributes.items():
setattr(self, a, value)
if self.created_at is None:
self.created_at = jh.now_to_timestamp()
if jh.is_live():
from jesse.store import store
self.session_id = store.app.session_id
self.save(force_insert=True)
if jh.is_live():
self.notify_submission()
if jh.is_debuggable('order_submission'):
txt = f'{"QUEUED" if self.is_queued else "SUBMITTED"} order: {self.symbol}, {self.type}, {self.side}, {self.qty}'
if self.price:
txt += f', ${round(self.price, 2)}'
logger.info(txt)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_submission(self)
def execute(self, silent=False) -> None:
if self.is_canceled or self.is_executed:
return
self.executed_at = jh.now_to_timestamp()
self.status = order_statuses.EXECUTED
if jh.is_live():
self.save()
if not silent:
txt = f'EXECUTED order: {self.symbol}, {self.type}, {self.side}, {self.qty}'
if self.price:
txt += f', ${round(self.price, 2)}'
# log
if jh.is_debuggable('order_execution'):
logger.info(txt)
# notify
if jh.is_live():
self.broadcast()
if config['env']['notifications']['events']['executed_orders']:
notify(txt)
p = selectors.get_position(self.exchange, self.symbol)
if p:
p._on_executed_order(self)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_execution(self)
def cancel(self, silent=False) -> None:
if self.is_canceled or self.is_executed:
return
self.canceled_at = jh.now_to_timestamp()
self.status = order_statuses.CANCELED
if jh.is_live():
self.save()
if not silent:
txt = f'CANCELED order: {self.symbol}, {self.type}, {self.side}, {self.qty}'
if self.price:
txt += f', ${round(self.price, 2)}'
if jh.is_debuggable('order_cancellation'):
logger.info(txt)
if jh.is_live():
self.broadcast()
if config['env']['notifications']['events'][
'cancelled_orders']:
notify(txt)
# handle exchange balance
e = selectors.get_exchange(self.exchange)
e.on_order_cancellation(self)
def test_risk_percentage():
set_up(zero_fee=True)
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 12,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTC-USD',
'opened_at': jh.now_to_timestamp(),
'closed_at': jh.now_to_timestamp()
})
def __init__(self, attributes=None, **kwargs)-> None:
Model.__init__(self, attributes=attributes, **kwargs)
if attributes is None:
attributes = {}
for a in attributes:
setattr(self, a, attributes[a])
if self.created_at is None:
self.created_at = jh.now_to_timestamp()
if jh.is_live() and config['env']['notifications']['events']['submitted_orders']:
self.notify_submission()
if jh.is_debuggable('order_submission'):
logger.info(
'{} order: {}, {}, {}, {}, ${}'.format(
'QUEUED' if self.is_queued else 'SUBMITTED',
self.symbol, self.type, self.side, self.qty,
round(self.price, 2)
)
)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_submission(self)
def _open(self, qty, price, change_balance=True):
if self.is_open:
raise OpenPositionError(
'an already open position cannot be opened')
# if change_balance:
# size = abs(qty) * price
# if self.exchange:
# self.exchange.decrease_margin_balance(size)
self.entry_price = price
self.exit_price = None
self.qty = qty
self.opened_at = jh.now_to_timestamp()
info_text = 'OPENED {} position: {}, {}, {}, ${}'.format(
self.type, self.exchange_name, self.symbol, self.qty,
round(self.entry_price, 2))
if jh.is_debuggable('position_opened'):
logger.info(info_text)
if jh.is_live(
) and config['env']['notifications']['events']['updated_position']:
notifier.notify(info_text)
def test_trade_size():
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 20,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTC-USD',
'opened_at': jh.now_to_timestamp(),
'closed_at': jh.now_to_timestamp()
})
assert trade.size == 10
def store_orderbook_into_db(exchange: str, symbol: str, orderbook: np.ndarray):
"""
:param exchange:
:param symbol:
:param orderbook:
"""
d = {
'id': jh.generate_unique_id(),
'timestamp': jh.now_to_timestamp(),
'data': orderbook.dumps(),
'symbol': symbol,
'exchange': exchange,
}
def async_save():
Orderbook.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'orderbook: {}-{}-{}: [{}, {}], [{}, {}]'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
# best ask
orderbook[0][0][0], orderbook[0][0][1],
# best bid
orderbook[1][0][0], orderbook[1][0][1]
),
'magenta'
)
)
# async call
threading.Thread(target=async_save).start()
def info(msg: str, send_notification=False) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
store.logs.info.append(log_dict)
if jh.is_collecting_data() or jh.is_live():
sync_publish('info_log', log_dict)
if jh.is_live() or (jh.is_backtesting() and jh.is_debugging()):
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.info(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'info')
if send_notification:
notify(msg)
def error(msg: str) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
# error logs should be logged as info logs as well
info(msg)
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
# notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify_urgently(f"ERROR:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data() or jh.is_live():
sync_publish('error_log', log_dict)
store.logs.errors.append(log_dict)
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.error(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'error')
def __init__(self, iterations: int, population_size: int, solution_len: int,
charset: str = '()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw',
fitness_goal: float = 1,
options: Dict[str, Union[bool, Any]] = None) -> None:
self.started_index = 0
self.start_time = jh.now_to_timestamp()
self.population = []
self.iterations = iterations
self.population_size = population_size
self.solution_len = solution_len
self.charset = charset
self.fitness_goal = fitness_goal
self.cpu_cores = 0
if options is None:
self.options = {}
else:
self.options = options
os.makedirs('./storage/temp/optimize', exist_ok=True)
self.temp_path = f"./storage/temp/optimize/{self.options['strategy_name']}-{self.options['exchange']}-{self.options['symbol']}-{self.options['timeframe']}-{self.options['start_date']}-{self.options['finish_date']}.pickle"
if fitness_goal > 1 or fitness_goal < 0:
raise ValueError('fitness scores must be between 0 and 1')
# if temp file exists, load data to resume previous session
if jh.file_exists(self.temp_path):
if click.confirm('Previous session detected. Do you want to resume?', default=True):
self.load_progress()
def _close(self, close_price: float) -> None:
if self.is_open is False:
raise EmptyPosition('The position is already closed.')
# just to prevent confusion
close_qty = abs(self.qty)
estimated_profit = jh.estimate_PNL(close_qty, self.entry_price,
close_price, self.type)
entry = self.entry_price
trade_type = self.type
self.exit_price = close_price
if self.exchange:
self.exchange.add_realized_pnl(estimated_profit)
self.exchange.temp_reduced_amount[jh.base_asset(
self.symbol)] += abs(close_qty * close_price)
self.qty = 0
self.entry_price = None
self.closed_at = jh.now_to_timestamp()
if not jh.is_unit_testing():
info_text = 'CLOSED {} position: {}, {}, {}. PNL: ${}, Balance: ${}, entry: {}, exit: {}'.format(
trade_type, self.exchange_name, self.symbol,
self.strategy.name, round(estimated_profit, 2),
jh.format_currency(
round(self.exchange.wallet_balance(self.symbol),
2)), entry, close_price)
if jh.is_debuggable('position_closed'):
logger.info(info_text)
if jh.is_live(
) and config['env']['notifications']['events']['updated_position']:
notifier.notify(info_text)
def execute(self):
if self.is_canceled or self.is_executed:
return
self.executed_at = jh.now_to_timestamp()
self.status = order_statuses.EXECUTED
# log
if jh.is_debuggable('order_execution'):
logger.info('EXECUTED order: {}, {}, {}, {}, ${}'.format(
self.symbol, self.type, self.side, self.qty,
round(self.price, 2)))
# notify
if jh.is_live(
) and config['env']['notifications']['events']['executed_orders']:
notify('EXECUTED order: {}, {}, {}, {}, {}'.format(
self.symbol, self.type, self.side, self.qty,
round(self.price, 2)))
p = selectors.get_position(self.exchange, self.symbol)
if p:
p._on_executed_order(self)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_execution(self)
def broadcast_error_without_logging(msg: str):
msg = str(msg)
sync_publish('error_log', {
'id': jh.generate_unique_id(),
'timestamp': jh.now_to_timestamp(),
'message': msg
})
def test_can_log_info_by_firing_event():
set_up()
# fire first info event
logger.info('first info!!!!!')
first_logged_info = {'time': jh.now_to_timestamp(), 'message': 'first info!!!!!'}
assert store.logs.info == [first_logged_info]
# fire second info event
logger.info('second info!!!!!')
second_logged_info = {
'time': jh.now_to_timestamp(),
'message': 'second info!!!!!'
}
assert store.logs.info == [first_logged_info, second_logged_info]
def test_risk_percentage():
no_fee()
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 12,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTC-USD',
'opened_at': jh.now_to_timestamp(),
'closed_at': jh.now_to_timestamp()
})
assert trade.risk_percentage == round((((10 - 5) / 1) * 10), 2)
def test_R():
no_fee()
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 12,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTC-USD',
'opened_at': jh.now_to_timestamp(),
'closed_at': jh.now_to_timestamp()
})
assert trade.risk_reward_ratio == 2
def add_candle(self,
candle: np.ndarray,
exchange: str,
symbol: str,
timeframe: str,
with_execution: bool = True,
with_generation: bool = True) -> None:
if jh.is_collecting_data():
# make sure it's a complete (and not a forming) candle
if jh.now_to_timestamp() >= (candle[0] + 60000):
store_candle_into_db(exchange, symbol, candle)
return
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe)
if jh.is_live():
self.update_position(exchange, symbol, candle)
# ignore new candle at the time of execution because it messes
# the count of candles without actually having an impact
if candle[0] >= jh.now():
return
# initial
if len(arr) == 0:
arr.append(candle)
# if it's new, add
elif candle[0] > arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to be executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr.append(candle)
# generate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# if it's the last candle again, update
elif candle[0] == arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to get executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr[-1] = candle
# regenerate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# past candles will be ignored (dropped)
elif candle[0] < arr[-1][0]:
return
def add_orderbook(self, exchange: str, symbol: str, asks: list, bids: list) -> None:
key = jh.key(exchange, symbol)
self.temp_storage[key]['asks'] = asks
self.temp_storage[key]['bids'] = bids
# generate new numpy formatted orderbook if it is
# either the first time, or that it has passed
# 1000 milliseconds since the last time
if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now_to_timestamp() - self.temp_storage[key][
'last_updated_timestamp'] >= 1000:
self.temp_storage[key]['last_updated_timestamp'] = jh.now_to_timestamp()
formatted_orderbook = self.format_orderbook(exchange, symbol)
if jh.is_collecting_data():
store_orderbook_into_db(exchange, symbol, formatted_orderbook)
else:
self.storage[key].append(formatted_orderbook)
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(
jh.color(
'[{}]: {}'.format(jh.timestamp_to_time(jh.now_to_timestamp()),
msg), 'red'))
store.logs.errors.append({'time': jh.now_to_timestamp(), 'message': msg})
if jh.is_live() or jh.is_optimizing():
msg = '[ERROR | {}] '.format(
jh.timestamp_to_time(jh.now_to_timestamp())[:19]) + str(msg)
import logging
logging.error(msg)
def add_ticker(self, ticker: np.ndarray, exchange: str, symbol: str) -> None:
key = jh.key(exchange, symbol)
# only process once per second
if len(self.storage[key][:]) == 0 or jh.now_to_timestamp() - self.storage[key][-1][0] >= 1000:
self.storage[key].append(ticker)
if jh.is_collecting_data():
store_ticker_into_db(exchange, symbol, ticker)
return
def test_pnl_with_fee():
# set fee (0.20%)
config['env']['exchanges']['Sandbox']['fee'] = 0.002
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 20,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTC-USD',
'opened_at': jh.now_to_timestamp(),
'closed_at': jh.now_to_timestamp()
})
assert trade.fee == 0.06
assert trade.pnl == 9.94
def info(msg: str) -> None:
from jesse.store import store
store.logs.info.append({'time': jh.now_to_timestamp(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}')
if jh.is_live():
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {str(msg)}"
logging.info(msg)
def test_can_add_new_ticker():
set_up()
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTC-USD'),
np.zeros((0, 5)))
# add first ticker
t1 = np.array([jh.now_to_timestamp(), 1, 2, 3, 4], dtype=np.float64)
store.tickers.add_ticker(t1, 'Sandbox', 'BTC-USD')
np.testing.assert_equal(
store.tickers.get_tickers('Sandbox', 'BTC-USD')[0], t1)
# fake 1 second
store.app.time += 1000
# add second ticker
t2 = np.array([jh.now_to_timestamp() + 1, 11, 22, 33, 44],
dtype=np.float64)
store.tickers.add_ticker(t2, 'Sandbox', 'BTC-USD')
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTC-USD'),
np.array([t1, t2]))
def test_cancel_order():
set_up()
order = Order({
'id': jh.generate_unique_id(),
'exchange': 'Sandbox',
'symbol': 'BTC-USDT',
'type': order_types.LIMIT,
'price': 129.33,
'qty': 10.2041,
'side': sides.BUY,
'status': order_statuses.ACTIVE,
'created_at': jh.now_to_timestamp(),
})
assert order.is_canceled is False
order.cancel()
assert order.is_canceled is True
assert order.canceled_at == jh.now_to_timestamp()
def test_execute_order():
set_up_without_fee()
order = Order({
'id': jh.generate_unique_id(),
'symbol': 'BTC-USDT',
'exchange': exchange.name,
'type': order_types.LIMIT,
'price': 129.33,
'qty': 10.2041,
'side': sides.BUY,
'status': order_statuses.ACTIVE,
'created_at': jh.now_to_timestamp(),
})
assert order.is_executed is False
assert order.executed_at is None
order.execute()
assert order.is_executed is True
assert order.executed_at == jh.now_to_timestamp()
def livetrade():
# TODO: for now, we assume that we trade on one exchange only. Later, we need to support for more than one exchange at a time
# sum up balance of all trading exchanges
starting_balance = 0
current_balance = 0
for e in store.exchanges.storage:
starting_balance += store.exchanges.storage[e].starting_assets[
jh.app_currency()]
current_balance += store.exchanges.storage[e].assets[jh.app_currency()]
starting_balance = round(starting_balance, 2)
current_balance = round(current_balance, 2)
# short trades summary
if len(store.completed_trades.trades):
df = pd.DataFrame.from_records(
[t.to_dict() for t in store.completed_trades.trades])
total = len(df)
winning_trades = len(df.loc[df['PNL'] > 0])
losing_trades = len(df.loc[df['PNL'] < 0])
pnl = round(df['PNL'].sum(), 2)
pnl_perc = round((pnl / starting_balance) * 100, 2)
else:
pnl, pnl_perc, total, winning_trades, losing_trades = 0, 0, 0, 0, 0
routes = [{
'exchange': r.exchange,
'symbol': r.symbol,
'timeframe': r.timeframe,
'strategy': r.strategy_name
} for r in router.routes]
return {
'session_id': store.app.session_id,
'started_at': str(store.app.starting_time),
'current_time': str(jh.now_to_timestamp()),
'started_balance': str(starting_balance),
'current_balance': str(current_balance),
'debug_mode': str(config['app']['debug_mode']),
'paper_mode': str(jh.is_paper_trading()),
'count_error_logs': str(len(store.logs.errors)),
'count_info_logs': str(len(store.logs.info)),
'count_active_orders': str(store.orders.count_all_active_orders()),
'open_positions': str(store.positions.count_open_positions()),
'pnl': str(pnl),
'pnl_perc': str(pnl_perc),
'count_trades': str(total),
'count_winning_trades': str(winning_trades),
'count_losing_trades': str(losing_trades),
'routes': routes
}
