def load_candles(
start_date_str: str,
finish_date_str: str) -> Dict[str, Dict[str, Union[str, np.ndarray]]]:
start_date = jh.date_to_timestamp(start_date_str)
finish_date = jh.date_to_timestamp(finish_date_str) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError("Can't load candle data from the future!")
# load and add required warm-up candles for backtest
if jh.is_backtesting():
for c in config['app']['considering_candles']:
required_candles.inject_required_candles_to_store(
required_candles.load_required_candles(c[0], c[1],
start_date_str,
finish_date_str), c[0],
c[1])
# download candles for the duration of the backtest
candles = {}
for c in config['app']['considering_candles']:
exchange, symbol = c[0], c[1]
key = jh.key(exchange, symbol)
cache_key = '{}-{}-'.format(start_date_str, finish_date_str) + key
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date, finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples()
# validate that there are enough candles for selected period
required_candles_count = (finish_date - start_date) / 60_000
if len(candles_tuple) == 0 or candles_tuple[-1][
0] != finish_date or candles_tuple[0][0] != start_date:
raise exceptions.CandleNotFoundInDatabase(
'Not enough candles for {}. Try running "jesse import-candles"'
.format(symbol))
elif len(candles_tuple) != required_candles_count + 1:
raise exceptions.CandleNotFoundInDatabase(
'There are missing candles between {} => {}'.format(
start_date_str, finish_date_str))
# cache it for near future calls
cache.set_value(cache_key,
tuple(candles_tuple),
expire_seconds=60 * 60 * 24 * 7)
candles[key] = {
'exchange': exchange,
'symbol': symbol,
'candles': np.array(candles_tuple)
}
return candles
def load_required_candles(exchange: str, symbol: str, start_date_str: str,
finish_date_str: str):
"""
loads initial candles that required before executing strategies.
210 for the biggest timeframe and more for the rest
"""
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
short_candles_count = jh.get_config(
'env.data.warmup_candles_num',
210) * jh.timeframe_to_one_minutes(max_timeframe)
pre_finish_date = start_date - 60_000
pre_start_date = pre_finish_date - short_candles_count * 60_000
# make sure starting from the beginning of the day instead
pre_start_date = jh.get_arrow(pre_start_date).floor(
'day').int_timestamp * 1000
# update candles_count to count from the beginning of the day instead
short_candles_count = int((pre_finish_date - pre_start_date) / 60_000)
key = jh.key(exchange, symbol)
cache_key = '{}-{}-{}'.format(jh.timestamp_to_date(pre_start_date),
jh.timestamp_to_date(pre_finish_date), key)
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = tuple(
Candle.select(Candle.timestamp, Candle.open, Candle.close,
Candle.high, Candle.low, Candle.volume).where(
Candle.timestamp.between(pre_start_date,
pre_finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples())
# cache it for near future calls
cache.set_value(cache_key,
candles_tuple,
expire_seconds=60 * 60 * 24 * 7)
candles = np.array(candles_tuple)
if len(candles) < short_candles_count + 1:
first_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).limit(1).tuples())
if not len(first_existing_candle):
raise CandleNotFoundInDatabase(
'No candle for {} {} is present in the database. Try importing candles.'
.format(exchange, symbol))
first_existing_candle = first_existing_candle[0][0]
last_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.desc()).limit(1).tuples())[0][0]
first_backtestable_timestamp = first_existing_candle + (
pre_finish_date - pre_start_date) + (60_000 * 1440)
# if first backtestable timestamp is in the future, that means we have some but not enough candles
if first_backtestable_timestamp > jh.today():
raise CandleNotFoundInDatabase(
'Not enough candle for {} {} is present in the database. Jesse requires "210 * biggest_timeframe" warm-up candles. '
'Try importing more candles from an earlier date.'.format(
exchange, symbol))
raise CandleNotFoundInDatabase(
'Not enough candles for {} {} exists to run backtest from {} => {}. \n'
'First available date is {}\n'
'Last available date is {}'.format(
exchange,
symbol,
start_date_str,
finish_date_str,
jh.timestamp_to_date(first_backtestable_timestamp),
jh.timestamp_to_date(last_existing_candle),
))
return candles
