def tradingview_logs(study_name=None, mode=None, now=None):
tv_text = '//@version=4\nstrategy("{}", overlay=true, initial_capital=10000, commission_type=strategy.commission.percent, commission_value=0.2)\n'.format(
study_name)
for i, t in enumerate(store.completed_trades.trades[::-1][:]):
tv_text += '\n'
for j, o in enumerate(t.orders):
when = "time_close == {}".format(int(o.executed_at))
if int(o.executed_at) % (jh.timeframe_to_one_minutes(t.timeframe) *
60_000) != 0:
when = "time_close >= {} and time_close - {} < {}" \
.format(int(o.executed_at),
int(o.executed_at) + jh.timeframe_to_one_minutes(t.timeframe) * 60_000,
jh.timeframe_to_one_minutes(t.timeframe) * 60_000)
if j == len(t.orders) - 1:
tv_text += 'strategy.close("{}", when = {})\n'.format(i, when)
else:
tv_text += 'strategy.order("{}", {}, {}, {}, when = {})\n'.format(
i, 1 if t.type == 'long' else 0, abs(o.qty), o.price, when)
path = 'storage/trading-view-pine-editor/{}-{}.txt'.format(mode,
now).replace(
":", "-")
os.makedirs('./storage/trading-view-pine-editor', exist_ok=True)
with open(path, 'w+') as outfile:
outfile.write(tv_text)
print('Pine-editor output saved at: \n{}'.format(path))
def generate_candle_from_one_minutes(timeframe,
candles: np.ndarray,
accept_forming_candles=False):
"""
:param timeframe:
:param candles:
:param accept_forming_candles:
:return:
"""
if len(candles) == 0:
raise ValueError('No candles were passed')
if not accept_forming_candles and len(
candles) != jh.timeframe_to_one_minutes(timeframe):
raise ValueError(
'Sent only {} candles but {} is required to create a "{}" candle.'.
format(len(candles), jh.timeframe_to_one_minutes(timeframe),
timeframe))
return np.array([
candles[0][0],
candles[0][1],
candles[-1][2],
candles[:, 3].max(),
candles[:, 4].min(),
candles[:, 5].sum(),
])
def test_timeframe_to_one_minutes():
assert jh.timeframe_to_one_minutes('1m') == 1
assert jh.timeframe_to_one_minutes('3m') == 3
assert jh.timeframe_to_one_minutes('5m') == 5
assert jh.timeframe_to_one_minutes('15m') == 15
assert jh.timeframe_to_one_minutes('30m') == 30
assert jh.timeframe_to_one_minutes('1h') == 60
assert jh.timeframe_to_one_minutes('2h') == 60 * 2
assert jh.timeframe_to_one_minutes('3h') == 60 * 3
assert jh.timeframe_to_one_minutes('4h') == 60 * 4
assert jh.timeframe_to_one_minutes('1D') == 60 * 24
def tradingview_logs(study_name: str) -> None:
starting_balance = 0
for e in store.exchanges.storage:
starting_balance += store.exchanges.storage[e].starting_assets[
jh.app_currency()]
tv_text = f'//@version=4\nstrategy("{study_name}", overlay=true, initial_capital={starting_balance}, commission_type=strategy.commission.percent, commission_value=0.2)\n'
for i, t in enumerate(store.completed_trades.trades[::-1][:]):
tv_text += '\n'
for j, o in enumerate(t.orders):
when = f"time_close == {int(o.executed_at)}"
if int(o.executed_at) % (jh.timeframe_to_one_minutes(t.timeframe) *
60_000) != 0:
when = f"time_close >= {int(o.executed_at)} and time_close - {int(o.executed_at) + jh.timeframe_to_one_minutes(t.timeframe) * 60_000} < {jh.timeframe_to_one_minutes(t.timeframe) * 60_000}"
if j == len(t.orders) - 1:
tv_text += f'strategy.close("{i}", when = {when})\n'
else:
tv_text += f'strategy.order("{i}", {1 if t.type == "long" else 0}, {abs(o.qty)}, {o.price}, when = {when})\n'
path = f'storage/trading-view-pine-editor/{study_name}.txt'
os.makedirs('./storage/trading-view-pine-editor', exist_ok=True)
with open(path, 'w+') as outfile:
outfile.write(tv_text)
print(f'\nPine-editor output saved at: \n{path}')
def inject_required_candles_to_store(candles: np.ndarray, exchange: str,
symbol: str):
"""
generate and add required candles to the candle store
"""
# batch add 1m candles:
store.candles.batch_add_candle(candles,
exchange,
symbol,
'1m',
with_generation=False)
# loop to generate, and add candles (without execution)
for i in range(len(candles)):
for timeframe in config['app']['considering_timeframes']:
# skip 1m. already added
if timeframe == '1m':
continue
num = jh.timeframe_to_one_minutes(timeframe)
if (i + 1) % num == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe, candles[(i - (num - 1)):(i + 1)], True)
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
def generate_bigger_timeframes(self, candle: np.ndarray, exchange: str,
symbol: str, with_execution: bool,
is_forming_candle: bool):
if not jh.is_live():
return
for timeframe in config['app']['considering_timeframes']:
# skip '1m'
if timeframe == '1m':
continue
last_candle = self.get_current_candle(exchange, symbol, timeframe)
generate_from_count = int((candle[0] - last_candle[0]) / 60_000)
required_for_complete_candle = jh.timeframe_to_one_minutes(
timeframe)
short_candles = self.get_candles(exchange, symbol,
'1m')[-1 - generate_from_count:]
if generate_from_count == (required_for_complete_candle -
1) and not is_forming_candle:
is_forming_candle = False
else:
is_forming_candle = True
# update latest candle
generated_candle = generate_candle_from_one_minutes(
timeframe, short_candles, True)
self.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution,
with_generation=False,
is_forming_candle=is_forming_candle)
def forming_estimation(self, exchange, symbol, timeframe):
long_key = jh.key(exchange, symbol, timeframe)
short_key = jh.key(exchange, symbol, '1m')
required_1m_to_complete_count = jh.timeframe_to_one_minutes(timeframe)
current_1m_count = len(self.get_storage(exchange, symbol, '1m'))
dif = current_1m_count % required_1m_to_complete_count
return dif, long_key, short_key
def store_logs(tradingview=False):
# store trades
mode = config['app']['trading_mode']
if mode == 'backtest':
mode = 'BT'
if mode == 'livetrade':
mode = 'LT'
if mode == 'papertrade':
mode = 'PT'
now = str(arrow.utcnow())[0:19]
study_name = '{}-{}'.format(mode, now)
path = './storage/logs/trades/{}-{}.json'.format(mode, now).replace(":", "-")
trades_json = {'trades': []}
for t in store.completed_trades.trades:
trades_json['trades'].append(t.toJSON())
os.makedirs('./storage/logs/trades', exist_ok=True)
with open(path, 'w+') as outfile:
json.dump(trades_json, outfile)
# store output for TradingView.com's pine-editor
if tradingview:
tv_text = '//@version=4\nstrategy("{}", overlay=true, initial_capital=10000, commission_type=strategy.commission.percent, commission_value=0.2)\n'.format(study_name)
for i, t in enumerate(store.completed_trades.trades[::-1][:]):
tv_text += '\n'
for j, o in enumerate(t.orders):
tv_text += '\nnext_timestamp{}{} = time + {}'.format(
i, j,
jh.timeframe_to_one_minutes(t.timeframe) * 60_000
)
tv_text += '\nwhen{}{} = {} >= time and {} < next_timestamp{}{}'.format(
i, j,
o.executed_at,
o.executed_at,
i, j
)
# if it's last order, it is the closing order
if j == len(t.orders) - 1:
tv_text += '\nstrategy.close("{}", when = when{}{}, comment = "close trade with {} {}")\n'.format(
i, i, j, o.side, o.type,
)
else:
tv_text += '\nstrategy.order("{}", strategy.{}, {}, {}, comment = "{}", when = when{}{} )\n'.format(
i, t.type, abs(o.qty), o.price, o.side + " " + o.type, i, j
)
path = 'storage/trading-view-pine-editor/{}-{}.txt'.format(mode, now).replace(":", "-")
os.makedirs('./storage/trading-view-pine-editor', exist_ok=True)
with open(path, 'w+') as outfile:
outfile.write(tv_text)
print('Pine-editor output saved at: \n{}'.format(path))
def init_storage(self, bucket_size: int = 1000) -> None:
for c in config['app']['considering_candles']:
exchange, symbol = c[0], c[1]
# initiate the '1m' timeframes
key = jh.key(exchange, symbol, timeframes.MINUTE_1)
self.storage[key] = DynamicNumpyArray((bucket_size, 6))
for timeframe in config['app']['considering_timeframes']:
key = jh.key(exchange, symbol, timeframe)
# ex: 1440 / 60 + 1 (reserve one for forming candle)
total_bigger_timeframe = int((bucket_size / jh.timeframe_to_one_minutes(timeframe)) + 1)
self.storage[key] = DynamicNumpyArray((total_bigger_timeframe, 6))
def get_candles(exchange: str, symbol: str, timeframe: str):
from jesse.services.db import database
database.open_connection()
from jesse.services.candle import generate_candle_from_one_minutes
from jesse.models.utils import fetch_candles_from_db
symbol = symbol.upper()
num_candles = 210
one_min_count = jh.timeframe_to_one_minutes(timeframe)
finish_date = jh.now(force_fresh=True)
start_date = finish_date - (num_candles * one_min_count * 60_000)
# fetch 1m candles from database
candles = np.array(
fetch_candles_from_db(exchange, symbol, start_date, finish_date))
# if there are no candles in the database, return []
if candles.size == 0:
database.close_connection()
return []
# leave out first candles until the timestamp of the first candle is the beginning of the timeframe
timeframe_duration = one_min_count * 60_000
while candles[0][0] % timeframe_duration != 0:
candles = candles[1:]
# generate bigger candles from 1m candles
if timeframe != '1m':
generated_candles = []
for i in range(len(candles)):
if (i + 1) % one_min_count == 0:
bigger_candle = generate_candle_from_one_minutes(
timeframe, candles[(i - (one_min_count - 1)):(i + 1)],
True)
generated_candles.append(bigger_candle)
candles = generated_candles
database.close_connection()
return [{
'time': int(c[0] / 1000),
'open': c[1],
'close': c[2],
'high': c[3],
'low': c[4],
'volume': c[5],
} for c in candles]
def generate_candle_from_one_minutes(
timeframe: str,
candles: np.ndarray,
accept_forming_candles: bool = False) -> np.ndarray:
if len(candles) == 0:
raise ValueError('No candles were passed')
if not accept_forming_candles and len(
candles) != jh.timeframe_to_one_minutes(timeframe):
raise ValueError(
f'Sent only {len(candles)} candles but {jh.timeframe_to_one_minutes(timeframe)} is required to create a "{timeframe}" candle.'
)
return np.array([
candles[0][0],
candles[0][1],
candles[-1][2],
candles[:, 3].max(),
candles[:, 4].min(),
candles[:, 5].sum(),
])
def portfolio_vs_asset_returns(study_name: str) -> None:
register_matplotlib_converters()
trades = store.completed_trades.trades
# create a plot figure
plt.figure(figsize=(26, 16))
# daily balance
plt.subplot(2, 1, 1)
start_date = datetime.fromtimestamp(store.app.starting_time / 1000)
date_list = [
start_date + timedelta(days=x)
for x in range(len(store.app.daily_balance))
]
plt.xlabel('date')
plt.ylabel('balance')
plt.title(f'Portfolio Daily Return - {study_name}')
plt.plot(date_list, store.app.daily_balance)
# price change%
plt.subplot(2, 1, 2)
price_dict = {}
for r in router.routes:
key = jh.key(r.exchange, r.symbol)
price_dict[key] = {'indexes': {}, 'prices': []}
dates = []
prices = []
candles = store.candles.get_candles(r.exchange, r.symbol, '1m')
max_timeframe = jh.max_timeframe(
config['app']['considering_timeframes'])
pre_candles_count = jh.timeframe_to_one_minutes(
max_timeframe) * jh.get_config('env.data.warmup_candles_num', 210)
for i, c in enumerate(candles):
# do not plot prices for required_initial_candles period
if i < pre_candles_count:
continue
dates.append(datetime.fromtimestamp(c[0] / 1000))
prices.append(c[2])
# save index of the price instead of the actual price
price_dict[key]['indexes'][str(int(c[0]))] = len(prices) - 1
# price => %returns
price_returns = pd.Series(prices).pct_change(1) * 100
cumsum_returns = np.cumsum(price_returns)
if len(router.routes) == 1:
plt.plot(dates, cumsum_returns, label=r.symbol, c='grey')
else:
plt.plot(dates, cumsum_returns, label=r.symbol)
price_dict[key]['prices'] = cumsum_returns
# buy and sell plots
buy_x = []
buy_y = []
sell_x = []
sell_y = []
for index, t in enumerate(trades):
key = jh.key(t.exchange, t.symbol)
# dirty fix for an issue with last trade being an open trade at the end of backtest
if index == len(trades) - 1 and store.app.total_open_trades > 0:
continue
if t.type == 'long':
#Buy
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.opened_at / 1000))
#Sell
if str(int(t.closed_at)) in price_dict[key][
'indexes']: #only generate data point if this trade wasn't after the last candle (open position at end)
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.closed_at / 1000))
elif t.type == 'short':
#Buy
if str(int(t.closed_at)) in price_dict[key][
'indexes']: #only generate data point if this trade wasn't after the last candle (open position at end)
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.closed_at))]])
# add datetime
buy_x.append(datetime.fromtimestamp(t.closed_at / 1000))
#Sell
# add price change%
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(
int(t.opened_at))]])
# add datetime
sell_x.append(datetime.fromtimestamp(t.opened_at / 1000))
plt.plot(buy_x, np.array(buy_y) * 0.99, '^', color='blue', markersize=7)
plt.plot(sell_x, np.array(sell_y) * 1.01, 'v', color='red', markersize=7)
plt.xlabel('date')
plt.ylabel('price change %')
plt.title('Asset Daily Return')
plt.legend(loc='upper left')
# store final result
mode = config['app']['trading_mode']
if mode == 'backtest':
mode = 'BT'
if mode == 'livetrade':
mode = 'LT'
if mode == 'papertrade':
mode = 'PT'
now = str(arrow.utcnow())[0:19]
# make sure directories exist
os.makedirs('./storage/charts', exist_ok=True)
file_path = f'storage/charts/{mode}-{now}-{study_name}.png'.replace(
":", "-")
plt.savefig(file_path)
print(f'\nChart output saved at:\n{file_path}')
def _isolated_backtest(config: dict,
routes: List[Dict[str, str]],
extra_routes: List[Dict[str, str]],
candles: dict,
run_silently: bool = True,
hyperparameters: dict = None) -> dict:
from jesse.services.validators import validate_routes
from jesse.modes.backtest_mode import simulator
from jesse.config import config as jesse_config, reset_config
from jesse.routes import router
from jesse.store import store
from jesse.config import set_config
from jesse.services import metrics
from jesse.services import required_candles
import jesse.helpers as jh
jesse_config['app']['trading_mode'] = 'backtest'
# inject (formatted) configuration values
set_config(_format_config(config))
# set routes
router.initiate(routes, extra_routes)
# register_custom_exception_handler()
validate_routes(router)
# TODO: further validate routes and allow only one exchange
# TODO: validate the name of the exchange in the config and the route? or maybe to make sure it's a supported exchange
# initiate candle store
store.candles.init_storage(5000)
# divide candles into warm_up_candles and trading_candles and then inject warm_up_candles
max_timeframe = jh.max_timeframe(
jesse_config['app']['considering_timeframes'])
warm_up_num = config['warm_up_candles'] * jh.timeframe_to_one_minutes(
max_timeframe)
trading_candles = candles
if warm_up_num != 0:
for c in jesse_config['app']['considering_candles']:
key = jh.key(c[0], c[1])
# update trading_candles
trading_candles[key]['candles'] = candles[key]['candles'][
warm_up_num:]
# inject warm-up candles
required_candles.inject_required_candles_to_store(
candles[key]['candles'][:warm_up_num], c[0], c[1])
# run backtest simulation
simulator(trading_candles, run_silently, hyperparameters)
result = {
'metrics': {
'total': 0,
'win_rate': 0,
'net_profit_percentage': 0
},
'charts': None,
'logs': None,
}
if store.completed_trades.count > 0:
# add metrics
result['metrics'] = metrics.trades(store.completed_trades.trades,
store.app.daily_balance)
# add charts
result['charts'] = charts.portfolio_vs_asset_returns()
# add logs
result['logs'] = store.logs.info
# reset store and config so rerunning would be flawlessly possible
reset_config()
store.reset()
return result
def get_candles(exchange: str, symbol: str, timeframe: str, start_date: str,
finish_date: str) -> np.ndarray:
"""
Returns candles from the database in numpy format
:param exchange: str
:param symbol: str
:param timeframe: str
:param start_date: str
:param finish_date: str
:return: np.ndarray
"""
exchange = exchange.title()
symbol = symbol.upper()
import arrow
import jesse.helpers as jh
from jesse.models import Candle
from jesse.exceptions import CandleNotFoundInDatabase
from jesse.services.candle import generate_candle_from_one_minutes
start_date = jh.arrow_to_timestamp(arrow.get(start_date, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(arrow.get(finish_date,
'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date,
finish_date), Candle.exchange == exchange,
Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples()
candles = np.array(tuple(candles_tuple))
# validate that there are enough candles for selected period
if len(candles) == 0 or candles[-1][0] != finish_date or candles[0][
0] != start_date:
raise CandleNotFoundInDatabase(
f'Not enough candles for {symbol}. Try running "jesse import-candles"'
)
if timeframe == '1m':
return candles
generated_candles = []
for i in range(len(candles)):
num = jh.timeframe_to_one_minutes(timeframe)
if (i + 1) % num == 0:
generated_candles.append(
generate_candle_from_one_minutes(
timeframe, candles[(i - (num - 1)):(i + 1)], True))
return np.array(generated_candles)
def portfolio_vs_asset_returns():
register_matplotlib_converters()
trades = store.completed_trades.trades
# create a plot figure
plt.figure(figsize=(26, 16))
# daily balance
plt.subplot(2, 1, 1)
start_date = datetime.fromtimestamp(store.app.starting_time / 1000)
date_list = [start_date + timedelta(days=x) for x in range(len(store.app.daily_balance))]
plt.xlabel('date')
plt.ylabel('balance')
plt.title('Portfolio Daily Return')
plt.plot(date_list, store.app.daily_balance)
# price change%
plt.subplot(2, 1, 2)
price_dict = {}
for r in router.routes:
key = jh.key(r.exchange, r.symbol)
price_dict[key] = {
'indexes': {},
'prices': []
}
dates = []
prices = []
candles = store.candles.get_candles(r.exchange, r.symbol, '1m')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
pre_candles_count = jh.timeframe_to_one_minutes(max_timeframe) * 210
for i, c in enumerate(candles):
# do not plot prices for required_initial_candles period
if i < pre_candles_count:
continue
dates.append(datetime.fromtimestamp(c[0] / 1000))
prices.append(c[2])
# save index of the price instead of the actual price
price_dict[key]['indexes'][str(int(c[0]))] = len(prices)-1
# price => %returns
price_returns = pd.Series(prices).pct_change(1) * 100
cumsum_returns = np.cumsum(price_returns)
if len(router.routes) == 1:
plt.plot(dates, cumsum_returns, label=r.symbol, c='grey')
else:
plt.plot(dates, cumsum_returns, label=r.symbol)
price_dict[key]['prices'] = cumsum_returns
# buy and sell plots
buy_x = []
buy_y = []
sell_x = []
sell_y = []
for t in trades:
key = jh.key(t.exchange, t.symbol)
if t.type == 'long':
buy_x.append(datetime.fromtimestamp(t.opened_at / 1000))
sell_x.append(datetime.fromtimestamp(t.closed_at / 1000))
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.opened_at))]]
)
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.closed_at))]]
)
elif t.type == 'short':
buy_x.append(datetime.fromtimestamp(t.closed_at / 1000))
sell_x.append(datetime.fromtimestamp(t.opened_at / 1000))
# add price change%
buy_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.closed_at))]]
)
sell_y.append(
price_dict[key]['prices'][price_dict[key]['indexes'][str(int(t.opened_at))]]
)
plt.plot(buy_x, buy_y, '.', color='green')
plt.plot(sell_x, sell_y, '.', color='red')
plt.xlabel('date')
plt.ylabel('price change %')
plt.title('Asset Daily Return')
plt.legend(loc='upper left')
# store final result
mode = config['app']['trading_mode']
if mode == 'backtest':
mode = 'BT'
if mode == 'livetrade':
mode = 'LT'
if mode == 'papertrade':
mode = 'PT'
# make sure directories exist
os.makedirs('./storage/charts', exist_ok=True)
file_path = 'storage/charts/{}-{}.png'.format(
mode, str(arrow.utcnow())[0:19]
).replace(":", "-")
plt.savefig(file_path)
print(
'Chart output saved at:\n{}'.format(file_path)
)
def load_required_candles(exchange: str, symbol: str, start_date_str: str,
finish_date_str: str):
"""
loads initial candles that required before executing strategies.
210 for the biggest timeframe and more for the rest
"""
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
short_candles_count = jh.get_config(
'env.data.warmup_candles_num',
210) * jh.timeframe_to_one_minutes(max_timeframe)
pre_finish_date = start_date - 60_000
pre_start_date = pre_finish_date - short_candles_count * 60_000
# make sure starting from the beginning of the day instead
pre_start_date = jh.get_arrow(pre_start_date).floor(
'day').int_timestamp * 1000
# update candles_count to count from the beginning of the day instead
short_candles_count = int((pre_finish_date - pre_start_date) / 60_000)
key = jh.key(exchange, symbol)
cache_key = '{}-{}-{}'.format(jh.timestamp_to_date(pre_start_date),
jh.timestamp_to_date(pre_finish_date), key)
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = tuple(
Candle.select(Candle.timestamp, Candle.open, Candle.close,
Candle.high, Candle.low, Candle.volume).where(
Candle.timestamp.between(pre_start_date,
pre_finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples())
# cache it for near future calls
cache.set_value(cache_key,
candles_tuple,
expire_seconds=60 * 60 * 24 * 7)
candles = np.array(candles_tuple)
if len(candles) < short_candles_count + 1:
first_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).limit(1).tuples())
if not len(first_existing_candle):
raise CandleNotFoundInDatabase(
'No candle for {} {} is present in the database. Try importing candles.'
.format(exchange, symbol))
first_existing_candle = first_existing_candle[0][0]
last_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.desc()).limit(1).tuples())[0][0]
first_backtestable_timestamp = first_existing_candle + (
pre_finish_date - pre_start_date) + (60_000 * 1440)
# if first backtestable timestamp is in the future, that means we have some but not enough candles
if first_backtestable_timestamp > jh.today():
raise CandleNotFoundInDatabase(
'Not enough candle for {} {} is present in the database. Jesse requires "210 * biggest_timeframe" warm-up candles. '
'Try importing more candles from an earlier date.'.format(
exchange, symbol))
raise CandleNotFoundInDatabase(
'Not enough candles for {} {} exists to run backtest from {} => {}. \n'
'First available date is {}\n'
'Last available date is {}'.format(
exchange,
symbol,
start_date_str,
finish_date_str,
jh.timestamp_to_date(first_backtestable_timestamp),
jh.timestamp_to_date(last_existing_candle),
))
return candles
def simulator(
candles: dict, run_silently: bool, hyperparameters: dict = None
) -> None:
begin_time_track = time.time()
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
try:
store.app.starting_time = first_candles_set[0][0]
except IndexError:
raise IndexError('Check your "warm_up_candles" config value')
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
# if the r.strategy is str read it from file
if isinstance(r.strategy_name, str):
StrategyClass = jh.get_strategy_class(r.strategy_name)
# else it is a class object so just use it
else:
StrategyClass = r.strategy_name
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# read the dna from strategy's dna() and use it for injecting inject hyperparameters
# first convert DNS string into hyperparameters
if len(r.strategy.dna()) > 0 and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.strategy.dna())
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
progressbar = Progressbar(length, step=60)
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
# until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False,
with_generation=False)
# update progressbar
if not run_silently and i % 60 == 0:
progressbar.update()
sync_publish('progressbar', {
'current': progressbar.current,
'estimated_remaining_seconds': progressbar.estimated_remaining_seconds
})
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(store.candles.get_current_candle(r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not run_silently:
# print executed time for the backtest session
finish_time_track = time.time()
sync_publish('alert', {
'message': f'Successfully executed backtest simulation in: {round(finish_time_track - begin_time_track, 2)} seconds',
'type': 'success'
})
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def simulator(candles: Dict[str, Dict[str, Union[str, np.ndarray]]],
hyperparameters=None) -> None:
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# inject hyper parameters (used for optimize_mode)
# convert DNS string into hyperparameters
if r.dna and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.dna)
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(
previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def simulator(candles, hyper_parameters=None):
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['trading_exchanges'][0],
config['app']['trading_symbols'][0])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
# convert DNS string into hyper_parameters
if r.dna and hyper_parameters is None:
hyper_parameters = jh.dna_to_hp(StrategyClass.hyper_parameters(),
r.dna)
r.strategy = StrategyClass()
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# init few objects that couldn't be initiated in Strategy __init__
r.strategy._init_objects()
# inject hyper parameters (used for optimize_mode)
if hyper_parameters is not None:
r.strategy.hp = hyper_parameters
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
_save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
_save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
# now that backtest is finished, add finishing balance
_save_daily_portfolio_balance()
