def get_training_and_testing_candles(start_date_str: str, finish_date_str: str) -> Tuple[
Dict[str, Dict[str, Union[Union[str, ndarray], Any]]], Dict[str, Dict[str, Union[Union[str, ndarray], Any]]]]:
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# Load candles (first try cache, then database)
candles = load_candles(start_date_str, finish_date_str)
# divide into training(85%) and testing(15%) sets
training_candles = {}
testing_candles = {}
days_diff = jh.date_diff_in_days(jh.timestamp_to_arrow(start_date), jh.timestamp_to_arrow(finish_date))
divider_index = int(days_diff * 0.85) * 1440
for key in candles:
training_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][0:divider_index],
}
testing_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][divider_index:],
}
return training_candles, testing_candles
def get_training_and_testing_candles(start_date_str: str,
finish_date_str: str):
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# Load candles (first try cache, then database)
from jesse.modes.backtest_mode import load_candles
candles = load_candles(start_date_str, finish_date_str)
# divide into training(85%) and testing(15%) sets
training_candles = {}
testing_candles = {}
days_diff = jh.date_diff_in_days(jh.get_arrow(start_date),
jh.get_arrow(finish_date))
divider_index = int(days_diff * 0.85) * 1440
for key in candles:
training_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][0:divider_index],
}
testing_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][divider_index:],
}
return training_candles, testing_candles
def load_candles(start_date_str: str, finish_date_str: str):
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError("Can't load data of the future!")
# load and add required warm-up candles for backtest
if jh.is_backtesting():
for c in config['app']['considering_candles']:
required_candles.inject_required_candles_to_store(
required_candles.load_required_candles(c[0], c[1], start_date_str, finish_date_str),
c[0],
c[1]
)
# download candles for the duration of the backtest
candles = {}
for exchange in config['app']['considering_exchanges']:
for symbol in config['app']['considering_symbols']:
key = jh.key(exchange, symbol)
cache_key = '{}-{}-'.format(start_date_str, finish_date_str) + key
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low,
Candle.volume
).where(
Candle.timestamp.between(start_date, finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol
).order_by(Candle.timestamp.asc()).tuples()
# validate that there are enough candles for selected period
required_candles_count = (finish_date - start_date) / 60_000
if len(candles_tuple) == 0 or candles_tuple[-1][0] != finish_date or candles_tuple[0][0] != start_date:
raise exceptions.CandleNotFoundInDatabase(
'Not enough candles for {}. Try running "jesse import-candles"'.format(symbol))
elif len(candles_tuple) != required_candles_count + 1:
raise exceptions.CandleNotFoundInDatabase('There are missing candles between {} => {}'.format(
start_date_str, finish_date_str
))
# cache it for near future calls
cache.set_value(cache_key, tuple(candles_tuple), expire_seconds=60 * 60 * 24 * 7)
candles[key] = {
'exchange': exchange,
'symbol': symbol,
'candles': np.array(candles_tuple)
}
return candles
def test_arrow_to_timestamp():
arrow_time = arrow.get('2015-08-01')
assert jh.arrow_to_timestamp(arrow_time) == 1438387200000
def run(exchange: str,
symbol: str,
start_date_str: str,
skip_confirmation=False):
try:
start_timestamp = jh.arrow_to_timestamp(
arrow.get(start_date_str, 'YYYY-MM-DD'))
except:
raise ValueError(
'start_date must be a string representing a date before today. ex: 2020-01-17'
)
# more start_date validations
today = arrow.utcnow().floor('day').int_timestamp * 1000
if start_timestamp == today:
raise ValueError(
"Today's date is not accepted. start_date must be a string a representing date BEFORE today."
)
elif start_timestamp > today:
raise ValueError(
"Future's date is not accepted. start_date must be a string a representing date BEFORE today."
)
click.clear()
symbol = symbol.upper()
until_date = arrow.utcnow().floor('day')
start_date = arrow.get(start_timestamp / 1000)
days_count = jh.date_diff_in_days(start_date, until_date)
candles_count = days_count * 1440
exchange = exchange.title()
try:
driver: CandleExchange = drivers[exchange]()
except KeyError:
raise ValueError('{} is not a supported exchange'.format(exchange))
loop_length = int(candles_count / driver.count) + 1
# ask for confirmation
if not skip_confirmation:
click.confirm(
'Importing {} days candles from "{}" for "{}". Duplicates will be skipped. All good?'
.format(days_count, exchange, symbol),
abort=True,
default=True)
with click.progressbar(length=loop_length,
label='Importing candles...') as progressbar:
for _ in range(candles_count):
temp_start_timestamp = start_date.int_timestamp * 1000
temp_end_timestamp = temp_start_timestamp + (driver.count -
1) * 60000
# to make sure it won't try to import candles from the future! LOL
if temp_start_timestamp > jh.now_to_timestamp():
break
# prevent duplicates calls to boost performance
count = Candle.select().where(
Candle.timestamp.between(temp_start_timestamp,
temp_end_timestamp),
Candle.symbol == symbol, Candle.exchange == exchange).count()
already_exists = count == driver.count
if not already_exists:
# it's today's candles if temp_end_timestamp < now
if temp_end_timestamp > jh.now_to_timestamp():
temp_end_timestamp = arrow.utcnow().floor(
'minute').int_timestamp * 1000 - 60000
# fetch from market
candles = driver.fetch(symbol, temp_start_timestamp)
if not len(candles):
click.clear()
first_existing_timestamp = driver.get_starting_time(symbol)
# if driver can't provide accurate get_starting_time()
if first_existing_timestamp is None:
raise CandleNotFoundInExchange(
'No candles exists in the market for this day: {} \n'
'Try another start_date'.format(
jh.timestamp_to_time(temp_start_timestamp)
[:10], ))
# handle when there's missing candles during the period
if temp_start_timestamp > first_existing_timestamp:
# see if there are candles for the same date for the backup exchange,
# if so, get those, if not, download from that exchange.
driver.init_backup_exchange()
if driver.backup_exchange is not None:
candles = _get_candles_from_backup_exchange(
exchange, driver.backup_exchange, symbol,
temp_start_timestamp, temp_end_timestamp)
else:
if not skip_confirmation:
print(
jh.color(
'No candle exists in the market for {}\n'.
format(
jh.timestamp_to_time(
temp_start_timestamp)[:10]),
'yellow'))
click.confirm(
'First present candle is since {}. Would you like to continue?'
.format(
jh.timestamp_to_time(
first_existing_timestamp)[:10]),
abort=True,
default=True)
run(
exchange, symbol,
jh.timestamp_to_time(first_existing_timestamp)
[:10], True)
return
# fill absent candles (if there's any)
candles = _fill_absent_candles(candles, temp_start_timestamp,
temp_end_timestamp)
# store in the database
if skip_confirmation:
_insert_to_database(candles)
else:
threading.Thread(target=_insert_to_database,
args=[candles]).start()
# add as much as driver's count to the temp_start_time
start_date = start_date.shift(minutes=driver.count)
progressbar.update(1)
# sleep so that the exchange won't get angry at us
if not already_exists:
time.sleep(driver.sleep_time)
def get_candles(exchange: str, symbol: str, timeframe: str, start_date: str,
finish_date: str) -> np.ndarray:
"""
Returns candles from the database in numpy format
:param exchange: str
:param symbol: str
:param timeframe: str
:param start_date: str
:param finish_date: str
:return: np.ndarray
"""
exchange = exchange.title()
symbol = symbol.upper()
import arrow
import jesse.helpers as jh
from jesse.models import Candle
from jesse.exceptions import CandleNotFoundInDatabase
from jesse.services.candle import generate_candle_from_one_minutes
start_date = jh.arrow_to_timestamp(arrow.get(start_date, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(arrow.get(finish_date,
'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date,
finish_date), Candle.exchange == exchange,
Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples()
candles = np.array(tuple(candles_tuple))
# validate that there are enough candles for selected period
if len(candles) == 0 or candles[-1][0] != finish_date or candles[0][
0] != start_date:
raise CandleNotFoundInDatabase(
f'Not enough candles for {symbol}. Try running "jesse import-candles"'
)
if timeframe == '1m':
return candles
generated_candles = []
for i in range(len(candles)):
num = jh.timeframe_to_one_minutes(timeframe)
if (i + 1) % num == 0:
generated_candles.append(
generate_candle_from_one_minutes(
timeframe, candles[(i - (num - 1)):(i + 1)], True))
return np.array(generated_candles)
def get_training_and_testing_candles(start_date_str: str,
finish_date_str: str):
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
candles = {}
for exchange in config['app']['considering_exchanges']:
for symbol in config['app']['considering_symbols']:
key = jh.key(exchange, symbol)
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date, finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples()
candles[key] = {
'exchange': exchange,
'symbol': symbol,
'candles': np.array(candles_tuple)
}
# validate that there are enough candles for selected period
if len(
candles[key]['candles']
) == 0 or candles[key]['candles'][-1][0] != finish_date or candles[
key]['candles'][0][0] != start_date:
raise Breaker(
'Not enough candles for {}. Try running "jesse import-candles"'
.format(symbol))
# divide into training(85%) and testing(15%) sets
training_candles = {}
testing_candles = {}
days_diff = jh.date_diff_in_days(jh.get_arrow(start_date),
jh.get_arrow(finish_date))
divider_index = int(days_diff * 0.85) * 1440
for key in candles:
training_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][0:divider_index],
}
testing_candles[key] = {
'exchange': candles[key]['exchange'],
'symbol': candles[key]['symbol'],
'candles': candles[key]['candles'][divider_index:],
}
return training_candles, testing_candles
def load_required_candles(exchange: str, symbol: str, start_date_str: str,
finish_date_str: str):
"""
loads initial candles that required before executing strategies.
210 for the biggest timeframe and more for the rest
"""
start_date = jh.arrow_to_timestamp(arrow.get(start_date_str, 'YYYY-MM-DD'))
finish_date = jh.arrow_to_timestamp(
arrow.get(finish_date_str, 'YYYY-MM-DD')) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError('Can\'t backtest the future!')
max_timeframe = jh.max_timeframe(config['app']['considering_timeframes'])
short_candles_count = jh.get_config(
'env.data.warmup_candles_num',
210) * jh.timeframe_to_one_minutes(max_timeframe)
pre_finish_date = start_date - 60_000
pre_start_date = pre_finish_date - short_candles_count * 60_000
# make sure starting from the beginning of the day instead
pre_start_date = jh.get_arrow(pre_start_date).floor(
'day').int_timestamp * 1000
# update candles_count to count from the beginning of the day instead
short_candles_count = int((pre_finish_date - pre_start_date) / 60_000)
key = jh.key(exchange, symbol)
cache_key = '{}-{}-{}'.format(jh.timestamp_to_date(pre_start_date),
jh.timestamp_to_date(pre_finish_date), key)
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = tuple(
Candle.select(Candle.timestamp, Candle.open, Candle.close,
Candle.high, Candle.low, Candle.volume).where(
Candle.timestamp.between(pre_start_date,
pre_finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples())
# cache it for near future calls
cache.set_value(cache_key,
candles_tuple,
expire_seconds=60 * 60 * 24 * 7)
candles = np.array(candles_tuple)
if len(candles) < short_candles_count + 1:
first_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).limit(1).tuples())
if not len(first_existing_candle):
raise CandleNotFoundInDatabase(
'No candle for {} {} is present in the database. Try importing candles.'
.format(exchange, symbol))
first_existing_candle = first_existing_candle[0][0]
last_existing_candle = tuple(
Candle.select(Candle.timestamp).where(
Candle.exchange == exchange, Candle.symbol == symbol).order_by(
Candle.timestamp.desc()).limit(1).tuples())[0][0]
first_backtestable_timestamp = first_existing_candle + (
pre_finish_date - pre_start_date) + (60_000 * 1440)
# if first backtestable timestamp is in the future, that means we have some but not enough candles
if first_backtestable_timestamp > jh.today():
raise CandleNotFoundInDatabase(
'Not enough candle for {} {} is present in the database. Jesse requires "210 * biggest_timeframe" warm-up candles. '
'Try importing more candles from an earlier date.'.format(
exchange, symbol))
raise CandleNotFoundInDatabase(
'Not enough candles for {} {} exists to run backtest from {} => {}. \n'
'First available date is {}\n'
'Last available date is {}'.format(
exchange,
symbol,
start_date_str,
finish_date_str,
jh.timestamp_to_date(first_backtestable_timestamp),
jh.timestamp_to_date(last_existing_candle),
))
return candles
def run(exchange: str,
symbol: str,
start_date_str: str,
skip_confirmation: bool = False,
mode: str = 'candles') -> None:
config['app']['trading_mode'] = mode
# first, create and set session_id
store.app.set_session_id()
register_custom_exception_handler()
# close database connection
from jesse.services.db import database
database.open_connection()
# at every second, we check to see if it's time to execute stuff
status_checker = Timeloop()
@status_checker.job(interval=timedelta(seconds=1))
def handle_time():
if process_status() != 'started':
raise exceptions.Termination
status_checker.start()
try:
start_timestamp = jh.arrow_to_timestamp(
arrow.get(start_date_str, 'YYYY-MM-DD'))
except:
raise ValueError(
'start_date must be a string representing a date before today. ex: 2020-01-17'
)
# more start_date validations
today = arrow.utcnow().floor('day').int_timestamp * 1000
if start_timestamp == today:
raise ValueError(
"Today's date is not accepted. start_date must be a string a representing date BEFORE today."
)
elif start_timestamp > today:
raise ValueError(
"Future's date is not accepted. start_date must be a string a representing date BEFORE today."
)
# We just call this to throw a exception in case of a symbol without dash
jh.quote_asset(symbol)
click.clear()
symbol = symbol.upper()
until_date = arrow.utcnow().floor('day')
start_date = arrow.get(start_timestamp / 1000)
days_count = jh.date_diff_in_days(start_date, until_date)
candles_count = days_count * 1440
try:
driver: CandleExchange = drivers[exchange]()
except KeyError:
raise ValueError(f'{exchange} is not a supported exchange')
except TypeError:
raise FileNotFoundError('You are missing the "plugins.py" file')
loop_length = int(candles_count / driver.count) + 1
# ask for confirmation
if not skip_confirmation:
click.confirm(
f'Importing {days_count} days candles from "{exchange}" for "{symbol}". Duplicates will be skipped. All good?',
abort=True,
default=True)
progressbar = Progressbar(loop_length)
for i in range(candles_count):
temp_start_timestamp = start_date.int_timestamp * 1000
temp_end_timestamp = temp_start_timestamp + (driver.count - 1) * 60000
# to make sure it won't try to import candles from the future! LOL
if temp_start_timestamp > jh.now_to_timestamp():
break
# prevent duplicates calls to boost performance
count = Candle.select().where(
Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp),
Candle.symbol == symbol, Candle.exchange == exchange).count()
already_exists = count == driver.count
if not already_exists:
# it's today's candles if temp_end_timestamp < now
if temp_end_timestamp > jh.now_to_timestamp():
temp_end_timestamp = arrow.utcnow().floor(
'minute').int_timestamp * 1000 - 60000
# fetch from market
candles = driver.fetch(symbol, temp_start_timestamp)
# check if candles have been returned and check those returned start with the right timestamp.
# Sometimes exchanges just return the earliest possible candles if the start date doesn't exist.
if not len(candles) or arrow.get(
candles[0]['timestamp'] / 1000) > start_date:
click.clear()
first_existing_timestamp = driver.get_starting_time(symbol)
# if driver can't provide accurate get_starting_time()
if first_existing_timestamp is None:
raise CandleNotFoundInExchange(
f'No candles exists in the market for this day: {jh.timestamp_to_time(temp_start_timestamp)[:10]} \n'
'Try another start_date')
# handle when there's missing candles during the period
if temp_start_timestamp > first_existing_timestamp:
# see if there are candles for the same date for the backup exchange,
# if so, get those, if not, download from that exchange.
if driver.backup_exchange is not None:
candles = _get_candles_from_backup_exchange(
exchange, driver.backup_exchange, symbol,
temp_start_timestamp, temp_end_timestamp)
else:
temp_start_time = jh.timestamp_to_time(
temp_start_timestamp)[:10]
temp_existing_time = jh.timestamp_to_time(
first_existing_timestamp)[:10]
sync_publish(
'alert', {
'message':
f'No candle exists in the market for {temp_start_time}. So '
f'Jesse started importing since the first existing date which is {temp_existing_time}',
'type':
'success'
})
run(exchange, symbol,
jh.timestamp_to_time(first_existing_timestamp)[:10],
True)
return
# fill absent candles (if there's any)
candles = _fill_absent_candles(candles, temp_start_timestamp,
temp_end_timestamp)
# store in the database
if skip_confirmation:
store_candles(candles)
else:
threading.Thread(target=store_candles, args=[candles]).start()
# add as much as driver's count to the temp_start_time
start_date = start_date.shift(minutes=driver.count)
progressbar.update()
sync_publish(
'progressbar', {
'current':
progressbar.current,
'estimated_remaining_seconds':
progressbar.estimated_remaining_seconds
})
# sleep so that the exchange won't get angry at us
if not already_exists:
time.sleep(driver.sleep_time)
# stop the status_checker time loop
status_checker.stop()
sync_publish(
'alert', {
'message':
f'Successfully imported candles since {jh.timestamp_to_date(start_timestamp)} until today ({days_count} days). ',
'type': 'success'
})
# if it is to skip, then it's being called from another process hence we should leave the database be
if not skip_confirmation:
# close database connection
from jesse.services.db import database
database.close_connection()
