def get_starting_time(self, symbol: str):
# hard-code few common symbols
if symbol == 'BTCUSD':
return jh.date_to_timestamp('2015-08-01')
elif symbol == 'ETHUSD':
return jh.date_to_timestamp('2016-01-01')
payload = {
'sort': 1,
'limit': 5000,
}
response = requests.get(self.endpoint +
'/trade:1D:t{}/hist'.format(symbol),
params=payload)
if response.status_code != 200:
raise Exception(response.content)
data = response.json()
# wrong symbol entered
if not len(data):
raise exceptions.SymbolNotFound(
"No candle exists for {} in Bitfinex. You're probably misspelling the symbol name."
.format(symbol))
first_timestamp = int(data[0][0])
second_timestamp = first_timestamp + 60_000 * 1440
return second_timestamp
def get_starting_time(self, symbol: str):
dashless_symbol = jh.dashless_symbol(symbol)
# hard-code few common symbols
if symbol == 'BTC-USD':
return jh.date_to_timestamp('2015-08-01')
elif symbol == 'ETH-USD':
return jh.date_to_timestamp('2016-01-01')
payload = {
'sort': 1,
'limit': 5000,
}
response = requests.get(f"{self.endpoint}/trade:1D:t{dashless_symbol}/hist", params=payload)
if response.status_code != 200:
raise Exception(response.content)
data = response.json()
# wrong symbol entered
if not len(data):
raise exceptions.SymbolNotFound(
f"No candle exists for {symbol} in Bitfinex. You're probably misspelling the symbol name."
)
# since the first timestamp doesn't include all the 1m
# candles, let's start since the second day then
first_timestamp = int(data[0][0])
second_timestamp = first_timestamp + 60_000 * 1440
return second_timestamp
def run(start_date: str,
finish_date: str,
candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None,
chart: bool = False,
tradingview: bool = False,
full_reports: bool = False,
csv: bool = False,
json: bool = False) -> None:
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
print('loading candles...')
candles = load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
# print candles table
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
table.key_value(stats.candles(candles[key]['candles']),
'candles',
alignments=('left', 'right'))
print('\n')
# print routes table
table.multi_value(stats.routes(router.routes))
print('\n')
# print guidance for debugging candles
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print(
' Symbol | timestamp | open | close | high | low | volume'
)
# run backtest simulation
simulator(candles)
if not jh.should_execute_silently():
# print trades metrics
if store.completed_trades.count > 0:
change = []
# calcualte market change
for e in router.routes:
if e.strategy is None:
return
first = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(start_date),
Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
last = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(finish_date) -
60000, Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
change.append(
((last.close - first.close) / first.close) * 100.0)
data = report.portfolio_metrics()
data.append(
['Market Change',
str(round(np.average(change), 2)) + "%"])
print('\n')
table.key_value(data, 'Metrics', alignments=('left', 'right'))
print('\n')
# save logs
store_logs(json, tradingview, csv)
if chart:
charts.portfolio_vs_asset_returns()
# QuantStats' report
if full_reports:
quantstats.quantstats_tearsheet()
else:
print(jh.color('No trades were made.', 'yellow'))
def load_candles(
start_date_str: str,
finish_date_str: str) -> Dict[str, Dict[str, Union[str, np.ndarray]]]:
start_date = jh.date_to_timestamp(start_date_str)
finish_date = jh.date_to_timestamp(finish_date_str) - 60000
# validate
if start_date == finish_date:
raise ValueError('start_date and finish_date cannot be the same.')
if start_date > finish_date:
raise ValueError('start_date cannot be bigger than finish_date.')
if finish_date > arrow.utcnow().int_timestamp * 1000:
raise ValueError("Can't load candle data from the future!")
# load and add required warm-up candles for backtest
if jh.is_backtesting():
for c in config['app']['considering_candles']:
required_candles.inject_required_candles_to_store(
required_candles.load_required_candles(c[0], c[1],
start_date_str,
finish_date_str), c[0],
c[1])
# download candles for the duration of the backtest
candles = {}
for c in config['app']['considering_candles']:
exchange, symbol = c[0], c[1]
key = jh.key(exchange, symbol)
cache_key = '{}-{}-'.format(start_date_str, finish_date_str) + key
cached_value = cache.get_value(cache_key)
# if cache exists
if cached_value:
candles_tuple = cached_value
# not cached, get and cache for later calls in the next 5 minutes
else:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high,
Candle.low, Candle.volume).where(
Candle.timestamp.between(start_date, finish_date),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples()
# validate that there are enough candles for selected period
required_candles_count = (finish_date - start_date) / 60_000
if len(candles_tuple) == 0 or candles_tuple[-1][
0] != finish_date or candles_tuple[0][0] != start_date:
raise exceptions.CandleNotFoundInDatabase(
'Not enough candles for {}. Try running "jesse import-candles"'
.format(symbol))
elif len(candles_tuple) != required_candles_count + 1:
raise exceptions.CandleNotFoundInDatabase(
'There are missing candles between {} => {}'.format(
start_date_str, finish_date_str))
# cache it for near future calls
cache.set_value(cache_key,
tuple(candles_tuple),
expire_seconds=60 * 60 * 24 * 7)
candles[key] = {
'exchange': exchange,
'symbol': symbol,
'candles': np.array(candles_tuple)
}
return candles
def run(
debug_mode,
user_config: dict,
routes: List[Dict[str, str]],
extra_routes: List[Dict[str, str]],
start_date: str,
finish_date: str,
candles: dict = None,
chart: bool = False,
tradingview: bool = False,
full_reports: bool = False,
csv: bool = False,
json: bool = False
) -> None:
if not jh.is_unit_testing():
# at every second, we check to see if it's time to execute stuff
status_checker = Timeloop()
@status_checker.job(interval=timedelta(seconds=1))
def handle_time():
if process_status() != 'started':
raise exceptions.Termination
status_checker.start()
from jesse.config import config, set_config
config['app']['trading_mode'] = 'backtest'
# debug flag
config['app']['debug_mode'] = debug_mode
# inject config
if not jh.is_unit_testing():
set_config(user_config)
# set routes
router.initiate(routes, extra_routes)
store.app.set_session_id()
register_custom_exception_handler()
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
candles = load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
sync_publish('general_info', {
'session_id': jh.get_session_id(),
'debug_mode': str(config['app']['debug_mode']),
})
# candles info
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
sync_publish('candles_info', stats.candles_info(candles[key]['candles']))
# routes info
sync_publish('routes_info', stats.routes(router.routes))
# run backtest simulation
simulator(candles, run_silently=jh.should_execute_silently())
# hyperparameters (if any)
if not jh.should_execute_silently():
sync_publish('hyperparameters', stats.hyperparameters(router.routes))
if not jh.should_execute_silently():
if store.completed_trades.count > 0:
sync_publish('metrics', report.portfolio_metrics())
routes_count = len(router.routes)
more = f"-and-{routes_count - 1}-more" if routes_count > 1 else ""
study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}"
store_logs(study_name, json, tradingview, csv)
if chart:
charts.portfolio_vs_asset_returns(study_name)
sync_publish('equity_curve', charts.equity_curve())
# QuantStats' report
if full_reports:
price_data = []
# load close candles for Buy and hold and calculate pct_change
for index, c in enumerate(config['app']['considering_candles']):
exchange, symbol = c[0], c[1]
if exchange in config['app']['trading_exchanges'] and symbol in config['app']['trading_symbols']:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.close
).where(
Candle.timestamp.between(jh.date_to_timestamp(start_date),
jh.date_to_timestamp(finish_date) - 60000),
Candle.exchange == exchange,
Candle.symbol == symbol
).order_by(Candle.timestamp.asc()).tuples()
candles = np.array(candles_tuple)
timestamps = candles[:, 0]
price_data.append(candles[:, 1])
price_data = np.transpose(price_data)
price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample(
'D').mean()
price_pct_change = price_df.pct_change(1).fillna(0)
bh_daily_returns_all_routes = price_pct_change.mean(1)
quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name)
else:
sync_publish('equity_curve', None)
sync_publish('metrics', None)
# close database connection
from jesse.services.db import database
database.close_connection()
def test_date_to_timestamp():
assert jh.date_to_timestamp('2015-08-01') == 1438387200000
def run(start_date: str,
finish_date: str,
candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None,
chart: bool = False,
tradingview: bool = False,
full_reports: bool = False,
csv: bool = False,
json: bool = False) -> None:
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
print('loading candles...')
candles = load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
# print candles table
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
table.key_value(stats.candles(candles[key]['candles']),
'candles',
alignments=('left', 'right'))
print('\n')
# print routes table
table.multi_value(stats.routes(router.routes))
print('\n')
# print guidance for debugging candles
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print(
' Symbol | timestamp | open | close | high | low | volume'
)
# run backtest simulation
simulator(candles)
if not jh.should_execute_silently():
# print trades metrics
if store.completed_trades.count > 0:
change = []
# calcualte market change
for e in router.routes:
if e.strategy is None:
return
first = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(start_date),
Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
last = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(finish_date) -
60000, Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
change.append(
((last.close - first.close) / first.close) * 100.0)
data = report.portfolio_metrics()
data.append(
['Market Change', f"{str(round(np.average(change), 2))}%"])
print('\n')
table.key_value(data, 'Metrics', alignments=('left', 'right'))
print('\n')
# save logs
more = ""
routes_count = len(router.routes)
if routes_count > 1:
more = f"-and-{routes_count-1}-more"
study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}"
store_logs(study_name, json, tradingview, csv)
if chart:
charts.portfolio_vs_asset_returns(study_name)
# QuantStats' report
if full_reports:
price_data = []
# load close candles for Buy and hold and calculate pct_change
for index, c in enumerate(
config['app']['considering_candles']):
exchange, symbol = c[0], c[1]
if exchange in config['app'][
'trading_exchanges'] and symbol in config['app'][
'trading_symbols']:
# fetch from database
candles_tuple = Candle.select(
Candle.timestamp, Candle.close).where(
Candle.timestamp.between(
jh.date_to_timestamp(start_date),
jh.date_to_timestamp(finish_date) - 60000),
Candle.exchange == exchange,
Candle.symbol == symbol).order_by(
Candle.timestamp.asc()).tuples()
candles = np.array(candles_tuple)
timestamps = candles[:, 0]
price_data.append(candles[:, 1])
price_data = np.transpose(price_data)
price_df = pd.DataFrame(price_data,
index=pd.to_datetime(timestamps,
unit="ms"),
dtype=float).resample('D').mean()
price_pct_change = price_df.pct_change(1).fillna(0)
bh_daily_returns_all_routes = price_pct_change.mean(1)
quantstats.quantstats_tearsheet(bh_daily_returns_all_routes,
study_name)
else:
print(jh.color('No trades were made.', 'yellow'))
