def test_get_candles_including_forming():
set_up()
candles_to_add = fake_range_candle(14)
store.candles.batch_add_candle(candles_to_add, 'Sandbox', 'BTCUSD', '1m')
store.candles.add_candle(
generate_candle_from_one_minutes('5m', candles_to_add[0:5], False),
'Sandbox', 'BTCUSD', '5m')
store.candles.add_candle(
generate_candle_from_one_minutes('5m', candles_to_add[5:10], False),
'Sandbox', 'BTCUSD', '5m')
assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '5m')) == 3
assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '1m')) == 14
candles = store.candles.get_candles('Sandbox', 'BTCUSD', '5m')
assert candles[0][0] == candles_to_add[0][0]
assert candles[-1][2] == candles_to_add[13][2]
assert candles[-1][0] == candles_to_add[10][0]
# add third one while still a forming candle. Now since
# we already have forming, get_candles() must not
# append another forming candle to the end.
store.candles.add_candle(
generate_candle_from_one_minutes('5m', candles_to_add[10:14], True),
'Sandbox', 'BTCUSD', '5m')
assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '5m')) == 3
assert candles[-1][2] == candles_to_add[13][2]
assert candles[-1][0] == candles_to_add[10][0]
def get_current_candle(self, exchange, symbol, timeframe) -> np.ndarray:
# no need to worry for forming candles when timeframe == 1m
if timeframe == '1m':
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, '1m')
if len(arr) == 0:
return np.zeros((0, 6))
else:
return arr[-1]
# other timeframes
dif, long_key, short_key = self.forming_estimation(
exchange, symbol, timeframe)
long_count = len(self.get_storage(exchange, symbol, timeframe))
short_count = len(self.get_storage(exchange, symbol, '1m'))
# complete candle
if dif == 0:
if long_count == 0:
return np.zeros((0, 6))
else:
return self.storage[long_key][-1]
# generate forming
else:
return generate_candle_from_one_minutes(
timeframe,
self.storage[short_key][short_count - dif:short_count], True)
def generate_bigger_timeframes(self, candle: np.ndarray, exchange: str,
symbol: str, with_execution: bool,
is_forming_candle: bool):
if not jh.is_live():
return
for timeframe in config['app']['considering_timeframes']:
# skip '1m'
if timeframe == '1m':
continue
last_candle = self.get_current_candle(exchange, symbol, timeframe)
generate_from_count = int((candle[0] - last_candle[0]) / 60_000)
required_for_complete_candle = jh.timeframe_to_one_minutes(
timeframe)
short_candles = self.get_candles(exchange, symbol,
'1m')[-1 - generate_from_count:]
if generate_from_count == (required_for_complete_candle -
1) and not is_forming_candle:
is_forming_candle = False
else:
is_forming_candle = True
# update latest candle
generated_candle = generate_candle_from_one_minutes(
timeframe, short_candles, True)
self.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution,
with_generation=False,
is_forming_candle=is_forming_candle)
def inject_required_candles_to_store(candles: np.ndarray, exchange: str,
symbol: str):
"""
generate and add required candles to the candle store
"""
# batch add 1m candles:
store.candles.batch_add_candle(candles,
exchange,
symbol,
'1m',
with_generation=False)
# loop to generate, and add candles (without execution)
for i in range(len(candles)):
for timeframe in config['app']['considering_timeframes']:
# skip 1m. already added
if timeframe == '1m':
continue
num = jh.timeframe_to_one_minutes(timeframe)
if (i + 1) % num == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe, candles[(i - (num - 1)):(i + 1)], True)
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
def get_candles(self, exchange: str, symbol: str,
timeframe: str) -> np.ndarray:
# no need to worry for forming candles when timeframe == 1m
if timeframe == '1m':
arr: DynamicNumpyArray = self.get_storage(exchange, symbol, '1m')
if len(arr) == 0:
return np.zeros((0, 6))
else:
return arr[:]
# other timeframes
dif, long_key, short_key = self.forming_estimation(
exchange, symbol, timeframe)
long_count = len(self.get_storage(exchange, symbol, timeframe))
short_count = len(self.get_storage(exchange, symbol, '1m'))
if dif == 0 and long_count == 0:
return np.zeros((0, 6))
# complete candle
if dif == 0 or self.storage[long_key][:long_count][-1][
0] == self.storage[short_key][short_count - dif][0]:
return self.storage[long_key][:long_count]
# generate forming
else:
return np.concatenate(
(self.storage[long_key][:long_count],
np.array((generate_candle_from_one_minutes(
timeframe,
self.storage[short_key][short_count - dif:short_count],
True), ))),
axis=0)
def generate_bigger_timeframes(self, candle: np.ndarray, exchange: str,
symbol: str, with_execution: bool) -> None:
if not jh.is_live():
return
for timeframe in config['app']['considering_timeframes']:
# skip '1m'
if timeframe == '1m':
continue
last_candle = self.get_current_candle(exchange, symbol, timeframe)
generate_from_count = int((candle[0] - last_candle[0]) / 60_000)
short_candles = self.get_candles(exchange, symbol,
'1m')[-1 - generate_from_count:]
# update latest candle
generated_candle = generate_candle_from_one_minutes(
timeframe, short_candles, accept_forming_candles=True)
self.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution,
with_generation=False)
def test_get_past_candle():
set_up()
candles_to_add = fake_range_candle(14)
store.candles.batch_add_candle(candles_to_add, 'Sandbox', 'BTCUSD', '1m')
store.candles.add_candle(
generate_candle_from_one_minutes('5m', candles_to_add[0:5]), 'Sandbox',
'BTCUSD', '5m')
store.candles.add_candle(
generate_candle_from_one_minutes('5m', candles_to_add[5:10]),
'Sandbox', 'BTCUSD', '5m')
assert len(store.candles.get_candles('Sandbox', 'BTCUSD', '1m')) == 14
# get_past_candle() should act as if there are 3 candles (last one being the forming)
np.testing.assert_equal(
store.candles.get_past_candle('Sandbox', 'BTCUSD', '5m', 1),
store.candles.storage[jh.key('Sandbox', 'BTCUSD', '5m')][1])
np.testing.assert_equal(
store.candles.get_past_candle('Sandbox', 'BTCUSD', '5m', 0),
generate_candle_from_one_minutes('5m', candles_to_add[10:], True))
def get_candles(exchange: str, symbol: str, timeframe: str):
from jesse.services.db import database
database.open_connection()
from jesse.services.candle import generate_candle_from_one_minutes
from jesse.models.utils import fetch_candles_from_db
symbol = symbol.upper()
num_candles = 210
one_min_count = jh.timeframe_to_one_minutes(timeframe)
finish_date = jh.now(force_fresh=True)
start_date = finish_date - (num_candles * one_min_count * 60_000)
# fetch 1m candles from database
candles = np.array(
fetch_candles_from_db(exchange, symbol, start_date, finish_date))
# if there are no candles in the database, return []
if candles.size == 0:
database.close_connection()
return []
# leave out first candles until the timestamp of the first candle is the beginning of the timeframe
timeframe_duration = one_min_count * 60_000
while candles[0][0] % timeframe_duration != 0:
candles = candles[1:]
# generate bigger candles from 1m candles
if timeframe != '1m':
generated_candles = []
for i in range(len(candles)):
if (i + 1) % one_min_count == 0:
bigger_candle = generate_candle_from_one_minutes(
timeframe, candles[(i - (one_min_count - 1)):(i + 1)],
True)
generated_candles.append(bigger_candle)
candles = generated_candles
database.close_connection()
return [{
'time': int(c[0] / 1000),
'open': c[1],
'close': c[2],
'high': c[3],
'low': c[4],
'volume': c[5],
} for c in candles]
def simulator(candles: Dict[str, Dict[str, Union[str, np.ndarray]]],
hyperparameters=None) -> None:
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# inject hyper parameters (used for optimize_mode)
# convert DNS string into hyperparameters
if r.dna and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.dna)
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(
previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def simulator(
candles: dict, run_silently: bool, hyperparameters: dict = None
) -> None:
begin_time_track = time.time()
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
try:
store.app.starting_time = first_candles_set[0][0]
except IndexError:
raise IndexError('Check your "warm_up_candles" config value')
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
# if the r.strategy is str read it from file
if isinstance(r.strategy_name, str):
StrategyClass = jh.get_strategy_class(r.strategy_name)
# else it is a class object so just use it
else:
StrategyClass = r.strategy_name
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# read the dna from strategy's dna() and use it for injecting inject hyperparameters
# first convert DNS string into hyperparameters
if len(r.strategy.dna()) > 0 and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.strategy.dna())
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
progressbar = Progressbar(length, step=60)
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
# until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False,
with_generation=False)
# update progressbar
if not run_silently and i % 60 == 0:
progressbar.update()
sync_publish('progressbar', {
'current': progressbar.current,
'estimated_remaining_seconds': progressbar.estimated_remaining_seconds
})
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(store.candles.get_current_candle(r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not run_silently:
# print executed time for the backtest session
finish_time_track = time.time()
sync_publish('alert', {
'message': f'Successfully executed backtest simulation in: {round(finish_time_track - begin_time_track, 2)} seconds',
'type': 'success'
})
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def get_candles(exchange: str, symbol: str, timeframe: str, start_date: str,
finish_date: str) -> np.ndarray:
"""
Returns candles from the database in numpy format
:param exchange: str
:param symbol: str
:param timeframe: str
:param start_date: str
:param finish_date: str
:return: np.ndarray
"""
exchange = exchange.title()
symbol = symbol.upper()
import arrow
import jesse.helpers as jh
from jesse.models import Candle
from jesse.exceptions import CandleNotFoundInDatabase
from jesse.services.candle import generate_candle_from_one_minutes
start_date = jh.arrow_to_timestamp(arrow.get(start_date, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(arrow.get(finish_date,
'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date,
finish_date), Candle.exchange == exchange,
Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples()
candles = np.array(tuple(candles_tuple))
# validate that there are enough candles for selected period
if len(candles) == 0 or candles[-1][0] != finish_date or candles[0][
0] != start_date:
raise CandleNotFoundInDatabase(
f'Not enough candles for {symbol}. Try running "jesse import-candles"'
)
if timeframe == '1m':
return candles
generated_candles = []
for i in range(len(candles)):
num = jh.timeframe_to_one_minutes(timeframe)
if (i + 1) % num == 0:
generated_candles.append(
generate_candle_from_one_minutes(
timeframe, candles[(i - (num - 1)):(i + 1)], True))
return np.array(generated_candles)
def simulator(candles, hyper_parameters=None):
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['trading_exchanges'][0],
config['app']['trading_symbols'][0])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
# convert DNS string into hyper_parameters
if r.dna and hyper_parameters is None:
hyper_parameters = jh.dna_to_hp(StrategyClass.hyper_parameters(),
r.dna)
r.strategy = StrategyClass()
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# init few objects that couldn't be initiated in Strategy __init__
r.strategy._init_objects()
# inject hyper parameters (used for optimize_mode)
if hyper_parameters is not None:
r.strategy.hp = hyper_parameters
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
_save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
_save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
# now that backtest is finished, add finishing balance
_save_daily_portfolio_balance()
