def set_routes(self, routes):
self.routes = []
for r in routes:
# validate strategy
import jesse.helpers as jh
from jesse import exceptions
strategy_name = r[3]
if jh.is_unit_testing():
exists = jh.file_exists(
'jesse/strategies/{}/__init__.py'.format(strategy_name))
else:
exists = jh.file_exists(
'strategies/{}/__init__.py'.format(strategy_name))
if not exists:
raise exceptions.InvalidRoutes(
'A strategy with the name of "{}" could not be found.'.
format(r[3]))
# validate timeframe
timeframe = r[2]
if timeframe not in [
'1m', '3m', '5m', '15m', '30m', '1h', '2h', '3h', '4h',
'6h', '8h', '1D'
]:
raise exceptions.InvalidRoutes(
'Timeframe "{}" is invalid. Supported timeframes are 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 6h, 8h, 1D'
.format(timeframe))
self.routes.append(Route(*r))
def set_routes(self, routes: List[Any]) -> None:
self._reset()
self.routes = []
for r in routes:
# validate strategy
strategy_name = r[3]
if jh.is_unit_testing():
exists = jh.file_exists(sys.path[0] + '/jesse/strategies/{}/__init__.py'.format(strategy_name))
else:
exists = jh.file_exists('strategies/{}/__init__.py'.format(strategy_name))
if not exists:
raise exceptions.InvalidRoutes(
'A strategy with the name of "{}" could not be found.'.format(r[3]))
# validate timeframe
route_timeframe = r[2]
all_timeframes = [timeframe for timeframe in jh.class_iter(timeframes)]
if route_timeframe not in all_timeframes:
raise exceptions.InvalidRoutes(
'Timeframe "{}" is invalid. Supported timeframes are {}'.format(
route_timeframe, ', '.join(all_timeframes))
)
self.routes.append(Route(*r))
def set_routes(self, routes: List[Any]) -> None:
self._reset()
self.routes = []
for r in routes:
# validate strategy that the strategy file exists (if sent as a string)
if isinstance(r["strategy"], str):
strategy_name = r["strategy"]
if jh.is_unit_testing():
path = sys.path[0]
# live plugin
if path.endswith('jesse-live'):
strategies_dir = f'{sys.path[0]}/tests/strategies'
# main framework
else:
strategies_dir = f'{sys.path[0]}/jesse/strategies'
exists = jh.file_exists(
f"{strategies_dir}/{strategy_name}/__init__.py")
else:
exists = jh.file_exists(
f'strategies/{strategy_name}/__init__.py')
else:
exists = True
if not exists and isinstance(r["strategy"], str):
raise exceptions.InvalidRoutes(
f'A strategy with the name of "{r["strategy"]}" could not be found.'
)
self.routes.append(
Route(r["exchange"], r["symbol"], r["timeframe"],
r["strategy"], None))
def _close(self, close_price: float) -> None:
if self.is_open is False:
raise EmptyPosition('The position is already closed.')
# just to prevent confusion
close_qty = abs(self.qty)
estimated_profit = jh.estimate_PNL(close_qty, self.entry_price,
close_price, self.type)
entry = self.entry_price
trade_type = self.type
self.exit_price = close_price
if self.exchange:
self.exchange.add_realized_pnl(estimated_profit)
self.exchange.temp_reduced_amount[jh.base_asset(
self.symbol)] += abs(close_qty * close_price)
self.qty = 0
self.entry_price = None
self.closed_at = jh.now_to_timestamp()
if not jh.is_unit_testing():
info_text = 'CLOSED {} position: {}, {}, {}. PNL: ${}, Balance: ${}, entry: {}, exit: {}'.format(
trade_type, self.exchange_name, self.symbol,
self.strategy.name, round(estimated_profit, 2),
jh.format_currency(
round(self.exchange.wallet_balance(self.symbol),
2)), entry, close_price)
if jh.is_debuggable('position_closed'):
logger.info(info_text)
if jh.is_live(
) and config['env']['notifications']['events']['updated_position']:
notifier.notify(info_text)
def set_routes(self, routes: List[Any]) -> None:
self._reset()
self.routes = []
for r in routes:
# validate strategy
strategy_name = r[3]
if jh.is_unit_testing():
exists = jh.file_exists(sys.path[0] + '/jesse/strategies/{}/__init__.py'.format(strategy_name))
else:
exists = jh.file_exists('strategies/{}/__init__.py'.format(strategy_name))
if not exists:
raise exceptions.InvalidRoutes(
'A strategy with the name of "{}" could not be found.'.format(r[3]))
# validate timeframe
timeframe = r[2]
if timeframe not in ['1m', '3m', '5m', '15m', '30m', '45m', '1h', '2h', '3h', '4h', '6h', '8h', '12h', '1D', '3D', '1W'']:
raise exceptions.InvalidRoutes(
'Timeframe "{}" is invalid. Supported timeframes are 1m, 3m, 5m, 15m, 30m, 45m, 1h, 2h, 3h, 4h, 6h, 8h, 12h, 1D, 3D, 1W'.format(
timeframe)
)
self.routes.append(Route(*r))
def open_connection(self) -> None:
if not jh.is_jesse_project() or jh.is_unit_testing():
return
# if it's not None, then we already have a connection
if self.db is not None:
return
options = {
"keepalives": 1,
"keepalives_idle": 60,
"keepalives_interval": 10,
"keepalives_count": 5
}
self.db = PostgresqlExtDatabase(
ENV_VALUES['POSTGRES_NAME'],
user=ENV_VALUES['POSTGRES_USERNAME'],
password=ENV_VALUES['POSTGRES_PASSWORD'],
host=ENV_VALUES['POSTGRES_HOST'],
port=int(ENV_VALUES['POSTGRES_PORT']),
sslmode=ENV_VALUES.get('POSTGRES_SSLMODE', 'disable'),
**options)
# connect to the database
self.db.connect()
def initiate(self, routes: list, extra_routes: list = None):
if extra_routes is None:
extra_routes = []
self.set_routes(routes)
self.set_extra_candles(extra_routes)
from jesse.store import store
store.reset(force_install_routes=jh.is_unit_testing())
def set_routes(self, routes: List[Any]) -> None:
self._reset()
self.routes = []
for r in routes:
# validate strategy
strategy_name = r[3]
if jh.is_unit_testing():
exists = jh.file_exists(
f"{sys.path[0]}/jesse/strategies/{strategy_name}/__init__.py"
)
else:
exists = jh.file_exists(
f'strategies/{strategy_name}/__init__.py')
if not exists:
raise exceptions.InvalidRoutes(
f'A strategy with the name of "{r[3]}" could not be found.'
)
# validate timeframe
route_timeframe = r[2]
all_timeframes = [
timeframe for timeframe in jh.class_iter(timeframes)
]
if route_timeframe not in all_timeframes:
raise exceptions.InvalidRoutes(
f'Timeframe "{route_timeframe}" is invalid. Supported timeframes are {", ".join(all_timeframes)}'
)
self.routes.append(Route(*r))
def cancel_all_orders(self, symbol):
orders = filter(lambda o: o.is_new,
store.orders.get_orders(self.name, symbol))
for o in orders:
o.cancel()
if not jh.is_unit_testing():
store.orders.storage[f'{self.name}-{symbol}'].clear()
def _check(self) -> None:
"""Based on the newly updated info, check if we should take action or not"""
if not self._is_initiated:
self._is_initiated = True
if jh.is_live() and jh.is_debugging():
logger.info(
f'Executing {self.name}-{self.exchange}-{self.symbol}-{self.timeframe}'
)
# for caution to make sure testing on livetrade won't bleed your account
if jh.is_test_driving() and store.completed_trades.count >= 2:
logger.info('Maximum allowed trades in test-drive mode is reached')
return
if self._open_position_orders != [] and self.is_close and self.should_cancel(
):
self._execute_cancel()
# make sure order cancellation response is received via WS
if jh.is_live():
# sleep a little until cancel is received via WS
sleep(0.1)
# just in case, sleep some more if necessary
for _ in range(20):
if store.orders.count_active_orders(
self.exchange, self.symbol) == 0:
break
logger.info('sleeping 0.2 more seconds...')
sleep(0.2)
# If it's still not cancelled, something is wrong. Handle cancellation failure
if store.orders.count_active_orders(self.exchange,
self.symbol) != 0:
raise exceptions.ExchangeNotResponding(
'The exchange did not respond as expected')
if self.position.is_open:
self._update_position()
if jh.is_backtesting() or jh.is_unit_testing():
store.orders.execute_pending_market_orders()
if self.position.is_close and self._open_position_orders == []:
should_short = self.should_short()
should_long = self.should_long()
# validation
if should_short and should_long:
raise exceptions.ConflictingRules(
'should_short and should_long should not be true at the same time.'
)
if should_long:
self._execute_long()
elif should_short:
self._execute_short()
def sync_publish(event: str, msg):
if jh.is_unit_testing():
raise EnvironmentError('sync_publish() should be NOT called during testing. There must be something wrong')
sync_redis.publish(
f"{ENV_VALUES['APP_PORT']}:channel:1", json.dumps({
'id': os.getpid(),
'event': f'{jh.app_mode()}.{event}',
'data': msg
}, ignore_nan=True, cls=NpEncoder)
)
def cancel_all_orders(self, symbol):
"""
:param symbol:
"""
orders = filter(lambda o: o.is_new,
store.orders.get_orders(self.name, symbol))
for o in orders:
o.cancel()
if not jh.is_unit_testing():
store.orders.storage['{}-{}'.format(self.name, symbol)].clear()
def process_status(pid=None) -> str:
if jh.is_unit_testing():
raise EnvironmentError('process_status() is not meant to be called in unit tests')
if pid is None:
pid = jh.get_pid()
key = f"{ENV_VALUES['APP_PORT']}|process-status:{pid}"
res: str = jh.str_or_none(sync_redis.get(key))
if res is None:
raise ValueError(f'No value exists in Redis for process ID of: {pid}')
return jh.string_after_character(res, ':')
def reset(self, force_install_routes=False):
"""resets all the states within the store
Keyword Arguments:
force_install_routes {bool} -- used for unit_testing (default: {False})
"""
if not jh.is_unit_testing() or force_install_routes:
install_routes()
self.app = AppState()
self.orders = OrdersState()
self.completed_trades = CompletedTrades()
self.logs = LogsState()
self.exchanges = ExchangesState()
self.candles = CandlesState()
self.positions = PositionsState()
self.tickers = TickersState()
self.trades = TradesState()
self.orderbooks = OrderbookState()
def _execute_cancel(self):
"""
cancels everything so that the strategy can keep looking for new trades.
"""
# validation
if self.position.is_open:
raise Exception('cannot cancel orders when position is still open. there must be a bug somewhere.')
logger.info('cancel all remaining orders to prepare for a fresh start...')
self.broker.cancel_all_orders()
self._reset()
self._broadcast('route-canceled')
self.on_cancel()
if not jh.is_unit_testing() and not jh.is_live():
store.orders.storage['{}-{}'.format(self.exchange, self.symbol)].clear()
def pnl_percentage(self) -> float:
"""
Alias for self.roi
"""
return self.roi
@property
def roi(self) -> float:
"""
Return on Investment in percentage
More at: https://www.binance.com/en/support/faq/5b9ad93cb4854f5990b9fb97c03cfbeb
"""
return self.pnl / self.total_cost * 100
@property
def total_cost(self) -> float:
"""
How much we paid to open this position (currently does not include fees, should we?!)
"""
return self.entry_price * abs(self.qty) / self.leverage
@property
def holding_period(self) -> int:
"""How many SECONDS has it taken for the trade to be done."""
return (self.closed_at - self.opened_at) / 1000
if not jh.is_unit_testing():
# create the table
CompletedTrade.create_table()
def run(
debug_mode,
user_config: dict,
routes: List[Dict[str, str]],
extra_routes: List[Dict[str, str]],
start_date: str,
finish_date: str,
candles: dict = None,
chart: bool = False,
tradingview: bool = False,
full_reports: bool = False,
csv: bool = False,
json: bool = False
) -> None:
if not jh.is_unit_testing():
# at every second, we check to see if it's time to execute stuff
status_checker = Timeloop()
@status_checker.job(interval=timedelta(seconds=1))
def handle_time():
if process_status() != 'started':
raise exceptions.Termination
status_checker.start()
from jesse.config import config, set_config
config['app']['trading_mode'] = 'backtest'
# debug flag
config['app']['debug_mode'] = debug_mode
# inject config
if not jh.is_unit_testing():
set_config(user_config)
# set routes
router.initiate(routes, extra_routes)
store.app.set_session_id()
register_custom_exception_handler()
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
candles = load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
sync_publish('general_info', {
'session_id': jh.get_session_id(),
'debug_mode': str(config['app']['debug_mode']),
})
# candles info
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
sync_publish('candles_info', stats.candles_info(candles[key]['candles']))
# routes info
sync_publish('routes_info', stats.routes(router.routes))
# run backtest simulation
simulator(candles, run_silently=jh.should_execute_silently())
# hyperparameters (if any)
if not jh.should_execute_silently():
sync_publish('hyperparameters', stats.hyperparameters(router.routes))
if not jh.should_execute_silently():
if store.completed_trades.count > 0:
sync_publish('metrics', report.portfolio_metrics())
routes_count = len(router.routes)
more = f"-and-{routes_count - 1}-more" if routes_count > 1 else ""
study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}"
store_logs(study_name, json, tradingview, csv)
if chart:
charts.portfolio_vs_asset_returns(study_name)
sync_publish('equity_curve', charts.equity_curve())
# QuantStats' report
if full_reports:
price_data = []
# load close candles for Buy and hold and calculate pct_change
for index, c in enumerate(config['app']['considering_candles']):
exchange, symbol = c[0], c[1]
if exchange in config['app']['trading_exchanges'] and symbol in config['app']['trading_symbols']:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.close
).where(
Candle.timestamp.between(jh.date_to_timestamp(start_date),
jh.date_to_timestamp(finish_date) - 60000),
Candle.exchange == exchange,
Candle.symbol == symbol
).order_by(Candle.timestamp.asc()).tuples()
candles = np.array(candles_tuple)
timestamps = candles[:, 0]
price_data.append(candles[:, 1])
price_data = np.transpose(price_data)
price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample(
'D').mean()
price_pct_change = price_df.pct_change(1).fillna(0)
bh_daily_returns_all_routes = price_pct_change.mean(1)
quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name)
else:
sync_publish('equity_curve', None)
sync_publish('metrics', None)
# close database connection
from jesse.services.db import database
database.close_connection()
def test_is_unit_testing():
assert jh.is_unit_testing() is True
def portfolio_vs_asset_returns(study_name: str = None) -> str:
if jh.is_unit_testing():
return 'charts'
register_matplotlib_converters()
trades = store.completed_trades.trades
# create a plot figure
plt.figure(figsize=(26, 16))
# daily balance
plt.subplot(2, 1, 1)
start_date = datetime.fromtimestamp(store.app.starting_time / 1000)
date_list = [
start_date + timedelta(days=x)
for x in range(len(store.app.daily_balance))
]
plt.xlabel('date')
plt.ylabel('balance')
if study_name:
plt.title(f'Portfolio Daily Return - {study_name}')
else:
plt.title('Portfolio Daily Return')
plt.plot(date_list, store.app.daily_balance)
# price change%
plt.subplot(2, 1, 2)
price_dict = {}
for r in router.routes:
key = jh.key(r.exchange, r.symbol)
price_dict[key] = {'indexes': {}, 'prices': []}
dates = []
prices = []
candles = store.candles.get_candles(r.exchange, r.symbol, '1m')
max_timeframe = jh.max_timeframe(
config['app']['considering_timeframes'])
pre_candles_count = jh.timeframe_to_one_minutes(
max_timeframe) * jh.get_config('env.data.warmup_candles_num', 210)
for i, c in enumerate(candles):
# do not plot prices for required_initial_candles period
if i < pre_candles_count:
continue
dates.append(datetime.fromtimestamp(c[0] / 1000))
prices.append(c[2])
# save index of the price instead of the actual price
price_dict[key]['indexes'][str(int(c[0]))] = len(prices) - 1
# price => %returns
price_returns = pd.Series(prices).pct_change(1) * 100
cumsum_returns = np.cumsum(price_returns)
if len(router.routes) == 1:
plt.plot(dates, cumsum_returns, label=r.symbol, c='grey')
else:
plt.plot(dates, cumsum_returns, label=r.symbol)
price_dict[key]['prices'] = cumsum_returns
# buy and sell plots
buy_x = []
buy_y = []
sell_x = []
sell_y = []
for index, t in enumerate(trades):
key = jh.key(t.exchange, t.symbol)
# dirty fix for an issue with last trade being an open trade at the end of backtest
if index == len(trades) - 1 and store.app.total_open_trades > 0:
continue
if t.type == 'long':
# Buy
if str(int(t.opened_at)) in price_dict[key]['indexes']:
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.opened_at / 1000))
# Sell: only generate data point if this trade wasn't after the last candle (open position at end)
if str(int(t.closed_at)) in price_dict[key]['indexes']:
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.closed_at / 1000))
elif t.type == 'short':
# Buy: only generate data point if this trade wasn't after the last candle (open position at end)
if str(int(t.closed_at)) in price_dict[key]['indexes']:
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.closed_at / 1000))
# Sell
if str(int(t.opened_at)) in price_dict[key]['indexes']:
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.opened_at / 1000))
plt.plot(buy_x, np.array(buy_y) * 0.99, '^', color='blue', markersize=7)
plt.plot(sell_x, np.array(sell_y) * 1.01, 'v', color='red', markersize=7)
plt.xlabel('date')
plt.ylabel('price change %')
plt.title('Asset Daily Return')
plt.legend(loc='upper left')
# store final result
# make sure directories exist
os.makedirs('./storage/charts', exist_ok=True)
file_path = f'storage/charts/{jh.get_session_id()}.png'.replace(":", "-")
plt.savefig(file_path)
return file_path
import os
import sys
# fix directory issue
sys.path.insert(0, os.getcwd())
ENV_VALUES = {}
if jh.is_jesse_project():
# load env
load_dotenv()
# create and expose ENV_VALUES
ENV_VALUES = dotenv_values('.env')
if jh.is_unit_testing():
ENV_VALUES['POSTGRES_HOST'] = '127.0.0.1'
ENV_VALUES['POSTGRES_NAME'] = 'jesse_db'
ENV_VALUES['POSTGRES_PORT'] = '5432'
ENV_VALUES['POSTGRES_USERNAME'] = '******'
ENV_VALUES['POSTGRES_PASSWORD'] = '******'
ENV_VALUES['REDIS_HOST'] = 'localhost'
ENV_VALUES['REDIS_PORT'] = '6379'
ENV_VALUES['REDIS_DB'] = 0
ENV_VALUES['REDIS_PASSWORD'] = ''
# validation for existence of .env file
if len(list(ENV_VALUES.keys())) == 0:
jh.error(
'.env file is missing from within your local project. '
'This usually happens when you\'re in the wrong directory. '
def _log_position_update(self, order: Order, role: str):
"""
A log can be either about opening, adding, reducing, or closing the position.
Arguments:
order {order} -- the order object
"""
if role == order_roles.OPEN_POSITION:
self.trade = CompletedTrade()
self.trade.orders = [order]
self.trade.timeframe = self.timeframe
self.trade.id = order.id
self.trade.strategy_name = self.name
self.trade.exchange = order.exchange
self.trade.symbol = order.symbol
self.trade.type = trade_types.LONG if order.side == sides.BUY else trade_types.SHORT
self.trade.qty = order.qty
self.trade.opened_at = jh.now()
self.trade.entry_candle_timestamp = self.current_candle[0]
elif role == order_roles.INCREASE_POSITION:
self.trade.orders.append(order)
self.trade.qty += order.qty
elif role == order_roles.REDUCE_POSITION:
self.trade.orders.append(order)
self.trade.qty += order.qty
elif role == order_roles.CLOSE_POSITION:
self.trade.exit_candle_timestamp = self.current_candle[0]
self.trade.orders.append(order)
# calculate average stop-loss price
sum_price = 0
sum_qty = 0
if self._log_stop_loss is not None:
for l in self._log_stop_loss:
sum_qty += abs(l[0])
sum_price += abs(l[0]) * l[1]
self.trade.stop_loss_at = sum_price / sum_qty
else:
self.trade.stop_loss_at = np.nan
# calculate average take-profit price
sum_price = 0
sum_qty = 0
if self._log_take_profit is not None:
for l in self._log_take_profit:
sum_qty += abs(l[0])
sum_price += abs(l[0]) * l[1]
self.trade.take_profit_at = sum_price / sum_qty
else:
self.trade.take_profit_at = np.nan
# calculate average entry_price price
sum_price = 0
sum_qty = 0
for l in self.trade.orders:
if not l.is_executed:
continue
if jh.side_to_type(l.side) != self.trade.type:
continue
sum_qty += abs(l.qty)
sum_price += abs(l.qty) * l.price
self.trade.entry_price = sum_price / sum_qty
# calculate average exit_price
sum_price = 0
sum_qty = 0
for l in self.trade.orders:
if not l.is_executed:
continue
if jh.side_to_type(l.side) == self.trade.type:
continue
sum_qty += abs(l.qty)
sum_price += abs(l.qty) * l.price
self.trade.exit_price = sum_price / sum_qty
self.trade.closed_at = jh.now()
self.trade.qty = pydash.sum_by(
filter(lambda o: o.side == jh.type_to_side(self.trade.type),
self.trade.orders), lambda o: abs(o.qty))
if not jh.is_unit_testing():
store.orders.storage['{}-{}'.format(self.exchange,
self.symbol)].clear()
store.completed_trades.add_trade(self.trade)
self.trade = None
self.trades_count += 1
