def test_get_current_and_past_ticker():
set_up()
# add 4 tickers
t1 = np.array([jh.now(), 1, 2, 3, 4], dtype=np.float64)
t2 = np.array([jh.now() + 1000, 2, 2, 3, 4], dtype=np.float64)
t3 = np.array([jh.now() + 2000, 3, 2, 3, 4], dtype=np.float64)
t4 = np.array([jh.now() + 3000, 4, 2, 3, 4], dtype=np.float64)
store.tickers.add_ticker(t1, 'Sandbox', 'BTCUSD')
store.app.time += 1000
store.tickers.add_ticker(t2, 'Sandbox', 'BTCUSD')
store.app.time += 1000
store.tickers.add_ticker(t3, 'Sandbox', 'BTCUSD')
store.app.time += 1000
store.tickers.add_ticker(t4, 'Sandbox', 'BTCUSD')
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'),
np.array([t1, t2, t3, t4]))
# get the previous one
np.testing.assert_equal(
store.tickers.get_past_ticker('Sandbox', 'BTCUSD', 1), t3)
# get current
np.testing.assert_equal(
store.tickers.get_current_ticker('Sandbox', 'BTCUSD'), t4)
def add_orderbook(self, exchange: str, symbol: str, asks: list,
bids: list):
"""
:param exchange:
:param symbol:
:param asks:
:param bids:
"""
key = jh.key(exchange, symbol)
self.temp_storage[key]['asks'] = asks
self.temp_storage[key]['bids'] = bids
# generate new numpy formatted orderbook if it is
# either the first time, or that it has passed
# 1000 milliseconds since the last time
if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now(
) - self.temp_storage[key]['last_updated_timestamp'] >= 1000:
self.temp_storage[key]['last_updated_timestamp'] = jh.now()
formatted_orderbook = self.format_orderbook(exchange, symbol)
if jh.is_collecting_data():
store_orderbook_into_db(exchange, symbol, formatted_orderbook)
else:
self.storage[key].append(formatted_orderbook)
def get_config(client_config: dict, has_live=False) -> dict:
from jesse.services.db import database
database.open_connection()
from jesse.models.Option import Option
try:
o = Option.get(Option.type == 'config')
# merge it with client's config (because it could include new keys added),
# update it in the database, and then return it
data = jh.merge_dicts(client_config, json.loads(o.json))
# make sure the list of BACKTEST exchanges is up to date
from jesse.modes.import_candles_mode.drivers import drivers
for k in list(data['backtest']['exchanges'].keys()):
if k not in drivers:
del data['backtest']['exchanges'][k]
# make sure the list of LIVE exchanges is up to date
if has_live:
from jesse_live.info import SUPPORTED_EXCHANGES_NAMES
live_exchanges = list(sorted(SUPPORTED_EXCHANGES_NAMES))
for k in list(data['live']['exchanges'].keys()):
if k not in live_exchanges:
del data['live']['exchanges'][k]
# fix the settlement_currency of exchanges
for k, e in data['live']['exchanges'].items():
e['settlement_currency'] = jh.get_settlement_currency_from_exchange(
e['name'])
for k, e in data['backtest']['exchanges'].items():
e['settlement_currency'] = jh.get_settlement_currency_from_exchange(
e['name'])
o.updated_at = jh.now()
o.save()
except peewee.DoesNotExist:
# if not found, that means it's the first time. Store in the DB and
# then return what was sent from the client side without changing it
o = Option({
'id': jh.generate_unique_id(),
'updated_at': jh.now(),
'type': 'config',
'json': json.dumps(client_config)
})
o.save(force_insert=True)
data = client_config
database.close_connection()
return {'data': data}
def info(msg):
from jesse.store import store
store.logs.info.append({'time': jh.now(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta'))
if jh.is_live():
msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.info(msg)
def test_can_log_info_by_firing_event():
set_up()
# fire first info event
logger.info('first info!!!!!')
first_logged_info = {'time': jh.now(), 'message': 'first info!!!!!'}
assert store.logs.info == [first_logged_info]
# fire second info event
logger.info('second info!!!!!')
second_logged_info = {'time': jh.now(), 'message': 'second info!!!!!'}
assert store.logs.info == [first_logged_info, second_logged_info]
def test_can_log_error_by_firing_event():
set_up()
# fire first error event
logger.error('first error!!!!!')
first_logged_error = {'time': jh.now(), 'message': 'first error!!!!!'}
assert store.logs.errors == [first_logged_error]
# fire second error event
logger.error('second error!!!!!')
second_logged_error = {'time': jh.now(), 'message': 'second error!!!!!'}
assert store.logs.errors == [first_logged_error, second_logged_error]
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red'))
store.logs.errors.append({'time': jh.now(), 'message': msg})
if jh.is_live():
msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.error(msg)
def _open(self, qty, price, change_balance=True):
if self.is_open:
raise OpenPositionError(
'an already open position cannot be opened')
if change_balance:
size = abs(qty) * price
if self.exchange:
self.exchange.decrease_balance(self, size)
self.entry_price = price
self.exit_price = None
self.qty = qty
self.opened_at = jh.now()
info_text = 'OPENED {} position: {}, {}, {}, ${}'.format(
self.type, self.exchange_name, self.symbol, self.qty,
round(self.entry_price, 2))
if jh.is_debuggable('position_opened'):
logger.info(info_text)
if jh.is_live(
) and config['env']['notifications']['events']['updated_position']:
notifier.notify(info_text)
def _close(self, close_price):
if self.is_open is False:
raise EmptyPosition('The position is already closed.')
# just to prevent confusion
close_qty = abs(self.qty)
estimated_profit = jh.estimate_PNL(close_qty, self.entry_price,
close_price, self.type)
entry = self.entry_price
trade_type = self.type
self.exit_price = close_price
if self.exchange:
self.exchange.increase_balance(
self, close_qty * self.entry_price + estimated_profit)
self.qty = 0
self.entry_price = None
self.closed_at = jh.now()
info_text = 'CLOSED {} position: {}, {}. PNL: ${}, entry: {}, exit: {}'.format(
trade_type, self.exchange_name, self.symbol,
round(estimated_profit, 2), entry, close_price)
if jh.is_debuggable('position_closed'):
logger.info(info_text)
if jh.is_live(
) and config['env']['notifications']['events']['updated_position']:
notifier.notify(info_text)
def cancel(self):
"""
:return:
"""
if self.is_canceled or self.is_executed:
return
self.canceled_at = jh.now()
self.status = order_statuses.CANCELED
if jh.is_debuggable('order_cancellation'):
logger.info(
'CANCELED order: {}, {}, {}, {}, ${}'.format(
self.symbol, self.type, self.side, self.qty, round(self.price, 2)
)
)
# notify
if jh.is_live() and config['env']['notifications']['events']['cancelled_orders']:
notify(
'CANCELED order: {}, {}, {}, {}, {}'.format(
self.symbol, self.type, self.side, self.qty, round(self.price, 2)
)
)
p = selectors.get_position(self.exchange, self.symbol)
if p:
p._on_canceled_order(self)
def execute(self):
if self.is_canceled or self.is_executed:
return
self.executed_at = jh.now()
self.status = order_statuses.EXECUTED
# log
if jh.is_debuggable('order_execution'):
logger.info('EXECUTED order: {}, {}, {}, {}, ${}'.format(
self.symbol, self.type, self.side, self.qty,
round(self.price, 2)))
# notify
if jh.is_live(
) and config['env']['notifications']['events']['executed_orders']:
notify('EXECUTED order: {}, {}, {}, {}, {}'.format(
self.symbol, self.type, self.side, self.qty,
round(self.price, 2)))
p = selectors.get_position(self.exchange, self.symbol)
if p:
p._on_executed_order(self)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_execution(self)
def test_trade_size():
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 20,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTCUSD',
'opened_at': jh.now(),
'closed_at': jh.now()
})
assert trade.size == 10
def get_starting_time(self, symbol: str) -> int:
formatted_symbol = symbol.replace('USD', 'PERP')
end_timestamp = jh.now()
start_timestamp = end_timestamp - (86400_000 * 365 * 8)
payload = {
'resolution': 86400,
'start_time': start_timestamp / 1000,
'end_time': end_timestamp / 1000,
}
response = requests.get(
f'https://ftx.com/api/markets/{formatted_symbol}/candles',
params=payload)
self._handle_errors(response)
data = response.json()['result']
# since the first timestamp doesn't include all the 1m
# candles, let's start since the second day then
first_timestamp = int(data[0]['time'])
# second_timestamp:
return first_timestamp + 60_000 * 1440
def save_daily_portfolio_balance() -> None:
balances = []
# add exchange balances
for key, e in store.exchanges.storage.items():
balances.append(e.assets[jh.app_currency()])
# store daily_balance of assets into database
if jh.is_livetrading():
for asset_key, asset_value in e.assets.items():
store_daily_balance_into_db({
'id':
jh.generate_unique_id(),
'timestamp':
jh.now(),
'identifier':
jh.get_config('env.identifier', 'main'),
'exchange':
e.name,
'asset':
asset_key,
'balance':
asset_value,
})
# add open position values
for key, pos in store.positions.storage.items():
if pos.is_open:
balances.append(pos.pnl)
total = sum(balances)
store.app.daily_balance.append(total)
logger.info('Saved daily portfolio balance: {}'.format(round(total, 2)))
def store_orderbook_into_db(exchange: str, symbol: str, orderbook: np.ndarray):
d = {
'id': jh.generate_unique_id(),
'timestamp': jh.now(),
'data': orderbook.dumps(),
'symbol': symbol,
'exchange': exchange,
}
def async_save():
Orderbook.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'orderbook: {}-{}-{}: [{}, {}], [{}, {}]'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
# best ask
orderbook[0][0][0], orderbook[0][0][1],
# best bid
orderbook[1][0][0], orderbook[1][0][1]
),
'magenta'
)
)
# async call
threading.Thread(target=async_save).start()
def __init__(
self,
iterations,
population_size,
solution_len,
charset='()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw',
fitness_goal=1,
options=None):
"""
:param iterations: int
:param population_size: int
:param solution_len: int
:param charset: str default= '()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw'
which is 40-119 (len=80)
:param fitness_goal:
"""
# used for naming the files related to this session
self.session_id = str(jh.now())
self.started_index = 0
self.start_time = jh.now()
self.population = []
self.iterations = iterations
self.population_size = population_size
self.solution_len = solution_len
self.charset = charset
self.fitness_goal = fitness_goal
if options is None:
self.options = {}
else:
self.options = options
os.makedirs('./storage/temp/optimize', exist_ok=True)
self.temp_path = './storage/temp/optimize/{}-{}-{}-{}.pickle'.format(
self.options['strategy_name'], self.options['exchange'],
self.options['symbol'], self.options['timeframe'])
if fitness_goal > 1 or fitness_goal < 0:
raise ValueError('fitness scores must be between 0 and 1')
# if temp file exists, load data to resume previous session
if jh.file_exists(self.temp_path):
if click.confirm(
'Previous session detected. Do you want to resume?',
default=True):
self.load_progress()
def test_PNL_percentage():
no_fee()
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 12,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTCUSD',
'opened_at': jh.now(),
'closed_at': jh.now()
})
assert trade.PNL_percentage == 20
def add_candle(self,
candle: np.ndarray,
exchange: str,
symbol: str,
timeframe: str,
with_execution: bool = True,
with_generation: bool = True) -> None:
if jh.is_collecting_data():
# make sure it's a complete (and not a forming) candle
if jh.now_to_timestamp() >= (candle[0] + 60000):
store_candle_into_db(exchange, symbol, candle)
return
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe)
if jh.is_live():
self.update_position(exchange, symbol, candle)
# ignore new candle at the time of execution because it messes
# the count of candles without actually having an impact
if candle[0] >= jh.now():
return
# initial
if len(arr) == 0:
arr.append(candle)
# if it's new, add
elif candle[0] > arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to be executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr.append(candle)
# generate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# if it's the last candle again, update
elif candle[0] == arr[-1][0]:
# in paper mode, check to see if the new candle causes any active orders to get executed
if with_execution and jh.is_paper_trading():
self.simulate_order_execution(exchange, symbol, timeframe,
candle)
arr[-1] = candle
# regenerate other timeframes
if with_generation and timeframe == '1m':
self.generate_bigger_timeframes(candle, exchange, symbol,
with_execution)
# past candles will be ignored (dropped)
elif candle[0] < arr[-1][0]:
return
def test_cancel_order():
order = Order({
'id': jh.generate_unique_id(),
'symbol': 'BTCUSD',
'type': order_types.LIMIT,
'price': 129.33,
'qty': 10.2041,
'side': sides.BUY,
'status': order_statuses.ACTIVE,
'created_at': jh.now(),
})
assert order.is_canceled is False
order.cancel()
assert order.is_canceled is True
assert order.canceled_at == jh.now()
def add_ticker(self, ticker: np.ndarray, exchange: str, symbol: str):
key = jh.key(exchange, symbol)
# only process once per second
if len(self.storage[key][:]) == 0 or jh.now() - self.storage[key][-1][0] >= 1000:
self.storage[key].append(ticker)
if jh.is_collecting_data():
store_ticker_into_db(exchange, symbol, ticker)
return
def test_can_add_new_ticker():
set_up()
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'),
np.zeros((0, 5)))
# add first ticker
t1 = np.array([jh.now(), 1, 2, 3, 4], dtype=np.float64)
store.tickers.add_ticker(t1, 'Sandbox', 'BTCUSD')
np.testing.assert_equal(
store.tickers.get_tickers('Sandbox', 'BTCUSD')[0], t1)
# fake 1 second
store.app.time += 1000
# add second ticker
t2 = np.array([jh.now() + 1, 11, 22, 33, 44], dtype=np.float64)
store.tickers.add_ticker(t2, 'Sandbox', 'BTCUSD')
np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'),
np.array([t1, t2]))
def test_PNL_with_fee():
# set fee (0.20%)
config['env']['exchanges']['Sandbox']['fee'] = 0.002
trade = CompletedTrade({
'type': 'long',
'exchange': 'Sandbox',
'entry_price': 10,
'exit_price': 20,
'take_profit_at': 20,
'stop_loss_at': 5,
'qty': 1,
'orders': [],
'symbol': 'BTCUSD',
'opened_at': jh.now(),
'closed_at': jh.now()
})
assert trade.fee == 0.06
assert trade.PNL == 9.94
def log_exchange_message(exchange, message):
# if the type of message is not str, convert it to str
if not isinstance(message, str):
message = str(message)
formatted_time = jh.timestamp_to_time(jh.now())[:19]
message = f'[{formatted_time} - {exchange}]: ' + message
if 'exchange-streams' not in LOGGERS:
create_disposable_logger('exchange-streams')
LOGGERS['exchange-streams'].info(message)
def livetrade():
"""
:return:
"""
# sum up balance of all trading exchanges
starting_balance = 0
current_balance = 0
for e in store.exchanges.storage:
starting_balance += store.exchanges.storage[e].starting_balance
current_balance += store.exchanges.storage[e].balance
starting_balance = round(starting_balance, 2)
current_balance = round(current_balance, 2)
arr = [[
'started/current balance', '{}/{}'.format(starting_balance,
current_balance)
], ['started at',
jh.get_arrow(store.app.starting_time).humanize()],
['current time',
jh.timestamp_to_time(jh.now())[:19]],
[
'errors/info', '{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
], ['active orders',
store.orders.count_all_active_orders()],
['open positions',
store.positions.count_open_positions()]]
# short trades summary
if len(store.completed_trades.trades):
df = pd.DataFrame.from_records(
[t.to_dict() for t in store.completed_trades.trades])
total = len(df)
winning_trades = df.loc[df['PNL'] > 0]
losing_trades = df.loc[df['PNL'] < 0]
pnl = round(df['PNL'].sum(), 2)
pnl_percentage = round((pnl / starting_balance) * 100, 2)
arr.append([
'total/winning/losing trades',
'{}/{}/{}'.format(total, len(winning_trades), len(losing_trades))
])
arr.append(['PNL (%)', '${} ({}%)'.format(pnl, pnl_percentage)])
if config['app']['debug_mode']:
arr.append(['debug mode', config['app']['debug_mode']])
if config['app']['is_test_driving']:
arr.append(['Test Drive', config['app']['is_test_driving']])
return arr
def test_execute_order():
set_up_without_fee()
order = Order({
'id': jh.generate_unique_id(),
'symbol': 'BTCUSDT',
'exchange': exchange.name,
'type': order_types.LIMIT,
'price': 129.33,
'qty': 10.2041,
'side': sides.BUY,
'status': order_statuses.ACTIVE,
'created_at': jh.now(),
})
assert order.is_executed is False
assert order.executed_at is None
order.execute()
assert order.is_executed is True
assert order.executed_at == jh.now()
def log_optimize_mode(message):
# if the type of message is not str, convert it to str
if not isinstance(message, str):
message = str(message)
formatted_time = jh.timestamp_to_time(jh.now())[:19]
message = f'[{formatted_time}]: ' + message
file_name = 'optimize-mode'
if file_name not in LOGGERS:
create_logger_file(file_name)
LOGGERS[file_name].info(message)
def get_candles(exchange: str, symbol: str, timeframe: str):
from jesse.services.db import database
database.open_connection()
from jesse.services.candle import generate_candle_from_one_minutes
from jesse.models.utils import fetch_candles_from_db
symbol = symbol.upper()
num_candles = 210
one_min_count = jh.timeframe_to_one_minutes(timeframe)
finish_date = jh.now(force_fresh=True)
start_date = finish_date - (num_candles * one_min_count * 60_000)
# fetch 1m candles from database
candles = np.array(
fetch_candles_from_db(exchange, symbol, start_date, finish_date))
# if there are no candles in the database, return []
if candles.size == 0:
database.close_connection()
return []
# leave out first candles until the timestamp of the first candle is the beginning of the timeframe
timeframe_duration = one_min_count * 60_000
while candles[0][0] % timeframe_duration != 0:
candles = candles[1:]
# generate bigger candles from 1m candles
if timeframe != '1m':
generated_candles = []
for i in range(len(candles)):
if (i + 1) % one_min_count == 0:
bigger_candle = generate_candle_from_one_minutes(
timeframe, candles[(i - (one_min_count - 1)):(i + 1)],
True)
generated_candles.append(bigger_candle)
candles = generated_candles
database.close_connection()
return [{
'time': int(c[0] / 1000),
'open': c[1],
'close': c[2],
'high': c[3],
'low': c[4],
'volume': c[5],
} for c in candles]
def update_config(client_config: dict):
from jesse.services.db import database
database.open_connection()
from jesse.models.Option import Option
# at this point there must already be one option record for "config" existing, so:
o = Option.get(Option.type == 'config')
o.json = json.dumps(client_config)
o.updated_at = jh.now()
o.save()
database.close_connection()
def __init__(self, attributes=None):
# id generated by Jesse for database usage
self.id = ''
# id generated by market, used in live-trade mode
self.exchange_id = ''
# some exchanges might require even further info
self.vars = {}
self.symbol = ''
self.exchange = ''
self.side = ''
self.type = ''
self.flag = ''
self.qty = 0
self.price = 0
self.status = order_statuses.ACTIVE
self.created_at = None
self.executed_at = None
self.canceled_at = None
self.role = None
if attributes is None:
attributes = {}
for a in attributes:
setattr(self, a, attributes[a])
if self.created_at is None:
self.created_at = jh.now()
p = selectors.get_position(self.exchange, self.symbol)
if p:
if jh.is_live(
) and config['env']['notifications']['events']['submitted_orders']:
self.notify_submission()
if jh.is_debuggable('order_submission'):
logger.info('{} order: {}, {}, {}, {}, ${}'.format(
'QUEUED' if self.is_queued else 'SUBMITTED', self.symbol,
self.type, self.side, self.qty, round(self.price, 2)))
p._on_opened_order(self)
# handle exchange balance for ordered asset
e = selectors.get_exchange(self.exchange)
e.on_order_submission(self)
def positions():
"""
:return:
"""
array = []
# headers
array.append([
'type', 'strategy', 'symbol', 'opened at', 'qty', 'entry',
'current price', 'PNL (%)'
])
for p in store.positions.storage:
pos = store.positions.storage[p]
if pos.pnl_percentage > 0:
pnl_color = 'green'
elif pos.pnl_percentage < 0:
pnl_color = 'red'
else:
pnl_color = 'black'
if pos.type == 'long':
type_color = 'green'
elif pos.type == 'short':
type_color = 'red'
else:
type_color = 'black'
array.append([
jh.color(pos.type, type_color),
pos.strategy.name,
pos.symbol,
'' if pos.is_close else '{} ago'.format(
jh.readable_duration((jh.now() - pos.opened_at) / 1000, 3)),
pos.qty if abs(pos.qty) > 0 else None,
pos.entry_price,
pos.current_price,
'' if pos.is_close else '{} ({}%)'.format(
jh.color(str(round(pos.pnl, 2)), pnl_color),
jh.color(str(round(pos.pnl_percentage, 4)), pnl_color)),
])
return array
