def test_max_timeframe():
assert jh.max_timeframe(['1m', '3m']) == '3m'
assert jh.max_timeframe(['3m', '5m']) == '5m'
assert jh.max_timeframe(['15m', '5m']) == '15m'
assert jh.max_timeframe(['30m', '15m']) == '30m'
assert jh.max_timeframe(['30m', '1h']) == '1h'
assert jh.max_timeframe(['1h', '2h']) == '2h'
assert jh.max_timeframe(['2h', '3h']) == '3h'
assert jh.max_timeframe(['4h', '3h']) == '4h'
assert jh.max_timeframe(['6h', '4h']) == '6h'
assert jh.max_timeframe(['6h', '1D']) == '1D'
def portfolio_vs_asset_returns():
register_matplotlib_converters()
trades = store.completed_trades.trades
# create a plot figure
plt.figure(figsize=(26, 16))
# daily balance
plt.subplot(2, 1, 1)
start_date = datetime.fromtimestamp(store.app.starting_time / 1000)
date_list = [start_date + timedelta(days=x) for x in range(len(store.app.daily_balance))]
plt.xlabel('date')
plt.ylabel('balance')
plt.title('Portfolio Daily Return')
plt.plot(date_list, store.app.daily_balance)
# price change%
plt.subplot(2, 1, 2)
price_dict = {}
for r in router.routes:
key = jh.key(r.exchange, r.symbol)
price_dict[key] = {
'indexes': {},
'prices': []
}
dates = []
prices = []
candles = store.candles.get_candles(r.exchange, r.symbol, '1m')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
pre_candles_count = jh.timeframe_to_one_minutes(max_timeframe) * 210
for i, c in enumerate(candles):
# do not plot prices for required_initial_candles period
if i < pre_candles_count:
continue
dates.append(datetime.fromtimestamp(c[0] / 1000))
prices.append(c[2])
# save index of the price instead of the actual price
price_dict[key]['indexes'][str(int(c[0]))] = len(prices)-1
# price => %returns
price_returns = pd.Series(prices).pct_change(1) * 100
cumsum_returns = np.cumsum(price_returns)
if len(router.routes) == 1:
plt.plot(dates, cumsum_returns, label=r.symbol, c='grey')
else:
plt.plot(dates, cumsum_returns, label=r.symbol)
price_dict[key]['prices'] = cumsum_returns
# buy and sell plots
buy_x = []
buy_y = []
sell_x = []
sell_y = []
for t in trades:
key = jh.key(t.exchange, t.symbol)
if t.type == 'long':
buy_x.append(datetime.fromtimestamp(t.opened_at / 1000))
sell_x.append(datetime.fromtimestamp(t.closed_at / 1000))
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.opened_at))]]
)
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.closed_at))]]
)
elif t.type == 'short':
buy_x.append(datetime.fromtimestamp(t.closed_at / 1000))
sell_x.append(datetime.fromtimestamp(t.opened_at / 1000))
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.closed_at))]]
)
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.opened_at))]]
)
plt.plot(buy_x, buy_y, '.', color='green')
plt.plot(sell_x, sell_y, '.', color='red')
plt.xlabel('date')
plt.ylabel('price change %')
plt.title('Asset Daily Return')
plt.legend(loc='upper left')
# store final result
mode = config['app']['trading_mode']
if mode == 'backtest':
mode = 'BT'
if mode == 'livetrade':
mode = 'LT'
if mode == 'papertrade':
mode = 'PT'
# make sure directories exist
os.makedirs('./storage/charts', exist_ok=True)
file_path = 'storage/charts/{}-{}.png'.format(
mode, str(arrow.utcnow())[0:19]
).replace(":", "-")
plt.savefig(file_path)
print(
'Chart output saved at:\n{}'.format(file_path)
)
def portfolio_vs_asset_returns(study_name: str) -> None:
register_matplotlib_converters()
trades = store.completed_trades.trades
# create a plot figure
plt.figure(figsize=(26, 16))
# daily balance
plt.subplot(2, 1, 1)
start_date = datetime.fromtimestamp(store.app.starting_time / 1000)
date_list = [
start_date + timedelta(days=x)
for x in range(len(store.app.daily_balance))
]
plt.xlabel('date')
plt.ylabel('balance')
plt.title(f'Portfolio Daily Return - {study_name}')
plt.plot(date_list, store.app.daily_balance)
# price change%
plt.subplot(2, 1, 2)
price_dict = {}
for r in router.routes:
key = jh.key(r.exchange, r.symbol)
price_dict[key] = {'indexes': {}, 'prices': []}
dates = []
prices = []
candles = store.candles.get_candles(r.exchange, r.symbol, '1m')
max_timeframe = jh.max_timeframe(
config['app']['considering_timeframes'])
pre_candles_count = jh.timeframe_to_one_minutes(
max_timeframe) * jh.get_config('env.data.warmup_candles_num', 210)
for i, c in enumerate(candles):
# do not plot prices for required_initial_candles period
if i < pre_candles_count:
continue
dates.append(datetime.fromtimestamp(c[0] / 1000))
prices.append(c[2])
# save index of the price instead of the actual price
price_dict[key]['indexes'][str(int(c[0]))] = len(prices) - 1
# price => %returns
price_returns = pd.Series(prices).pct_change(1) * 100
cumsum_returns = np.cumsum(price_returns)
if len(router.routes) == 1:
plt.plot(dates, cumsum_returns, label=r.symbol, c='grey')
else:
plt.plot(dates, cumsum_returns, label=r.symbol)
price_dict[key]['prices'] = cumsum_returns
# buy and sell plots
buy_x = []
buy_y = []
sell_x = []
sell_y = []
for index, t in enumerate(trades):
key = jh.key(t.exchange, t.symbol)
# dirty fix for an issue with last trade being an open trade at the end of backtest
if index == len(trades) - 1 and store.app.total_open_trades > 0:
continue
if t.type == 'long':
#Buy
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.opened_at / 1000))
#Sell
if str(int(t.closed_at)) in price_dict[key][
'indexes']: #only generate data point if this trade wasn't after the last candle (open position at end)
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.closed_at / 1000))
elif t.type == 'short':
#Buy
if str(int(t.closed_at)) in price_dict[key][
'indexes']: #only generate data point if this trade wasn't after the last candle (open position at end)
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.closed_at / 1000))
#Sell
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.opened_at / 1000))
plt.plot(buy_x, np.array(buy_y) * 0.99, '^', color='blue', markersize=7)
plt.plot(sell_x, np.array(sell_y) * 1.01, 'v', color='red', markersize=7)
plt.xlabel('date')
plt.ylabel('price change %')
plt.title('Asset Daily Return')
plt.legend(loc='upper left')
# store final result
mode = config['app']['trading_mode']
if mode == 'backtest':
mode = 'BT'
if mode == 'livetrade':
mode = 'LT'
if mode == 'papertrade':
mode = 'PT'
now = str(arrow.utcnow())[0:19]
# make sure directories exist
os.makedirs('./storage/charts', exist_ok=True)
file_path = f'storage/charts/{mode}-{now}-{study_name}.png'.replace(
":", "-")
plt.savefig(file_path)
print(f'\nChart output saved at:\n{file_path}')
def load_required_candles(exchange: str, symbol: str, start_date_str: str,
finish_date_str: str):
"""
loads initial candles that required before executing strategies.
210 for the biggest timeframe and more for the rest
"""
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
short_candles_count = jh.get_config(
'env.data.warmup_candles_num',
210) * jh.timeframe_to_one_minutes(max_timeframe)
pre_finish_date = start_date - 60_000
pre_start_date = pre_finish_date - short_candles_count * 60_000
# make sure starting from the beginning of the day instead
pre_start_date = jh.get_arrow(pre_start_date).floor(
'day').int_timestamp * 1000
# update candles_count to count from the beginning of the day instead
short_candles_count = int((pre_finish_date - pre_start_date) / 60_000)
key = jh.key(exchange, symbol)
cache_key = '{}-{}-{}'.format(jh.timestamp_to_date(pre_start_date),
jh.timestamp_to_date(pre_finish_date), key)
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = tuple(
Candle.select(Candle.timestamp, Candle.open, Candle.close,
Candle.high, Candle.low, Candle.volume).where(
Candle.timestamp.between(pre_start_date,
pre_finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples())
# cache it for near future calls
cache.set_value(cache_key,
candles_tuple,
expire_seconds=60 * 60 * 24 * 7)
candles = np.array(candles_tuple)
if len(candles) < short_candles_count + 1:
first_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).limit(1).tuples())
if not len(first_existing_candle):
raise CandleNotFoundInDatabase(
'No candle for {} {} is present in the database. Try importing candles.'
.format(exchange, symbol))
first_existing_candle = first_existing_candle[0][0]
last_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.desc()).limit(1).tuples())[0][0]
first_backtestable_timestamp = first_existing_candle + (
pre_finish_date - pre_start_date) + (60_000 * 1440)
# if first backtestable timestamp is in the future, that means we have some but not enough candles
if first_backtestable_timestamp > jh.today():
raise CandleNotFoundInDatabase(
'Not enough candle for {} {} is present in the database. Jesse requires "210 * biggest_timeframe" warm-up candles. '
'Try importing more candles from an earlier date.'.format(
exchange, symbol))
raise CandleNotFoundInDatabase(
'Not enough candles for {} {} exists to run backtest from {} => {}. \n'
'First available date is {}\n'
'Last available date is {}'.format(
exchange,
symbol,
start_date_str,
finish_date_str,
jh.timestamp_to_date(first_backtestable_timestamp),
jh.timestamp_to_date(last_existing_candle),
))
return candles
def _isolated_backtest(config: dict,
routes: List[Dict[str, str]],
extra_routes: List[Dict[str, str]],
candles: dict,
run_silently: bool = True,
hyperparameters: dict = None) -> dict:
from jesse.services.validators import validate_routes
from jesse.modes.backtest_mode import simulator
from jesse.config import config as jesse_config, reset_config
from jesse.routes import router
from jesse.store import store
from jesse.config import set_config
from jesse.services import metrics
from jesse.services import required_candles
import jesse.helpers as jh
jesse_config['app']['trading_mode'] = 'backtest'
# inject (formatted) configuration values
set_config(_format_config(config))
# set routes
router.initiate(routes, extra_routes)
# register_custom_exception_handler()
validate_routes(router)
# TODO: further validate routes and allow only one exchange
# TODO: validate the name of the exchange in the config and the route? or maybe to make sure it's a supported exchange
# initiate candle store
store.candles.init_storage(5000)
# divide candles into warm_up_candles and trading_candles and then inject warm_up_candles
max_timeframe = jh.max_timeframe(
jesse_config['app']['considering_timeframes'])
warm_up_num = config['warm_up_candles'] * jh.timeframe_to_one_minutes(
max_timeframe)
trading_candles = candles
if warm_up_num != 0:
for c in jesse_config['app']['considering_candles']:
key = jh.key(c[0], c[1])
# update trading_candles
trading_candles[key]['candles'] = candles[key]['candles'][
warm_up_num:]
# inject warm-up candles
required_candles.inject_required_candles_to_store(
candles[key]['candles'][:warm_up_num], c[0], c[1])
# run backtest simulation
simulator(trading_candles, run_silently, hyperparameters)
result = {
'metrics': {
'total': 0,
'win_rate': 0,
'net_profit_percentage': 0
},
'charts': None,
'logs': None,
}
if store.completed_trades.count > 0:
# add metrics
result['metrics'] = metrics.trades(store.completed_trades.trades,
store.app.daily_balance)
# add charts
result['charts'] = charts.portfolio_vs_asset_returns()
# add logs
result['logs'] = store.logs.info
# reset store and config so rerunning would be flawlessly possible
reset_config()
store.reset()
return result
