def get_all_data(st: datetime, config):
# Get all data which we will use to reconstruct the order book
all_ob_start_time = st - datetime.timedelta(seconds=config.orderbook_window)
all_ob_end_time = st
all_ob_data = DataLoader().load_split_data(config.real_root, all_ob_start_time, all_ob_end_time, config.product)
# Assume orderbook_window > sampling_window, and therefore filter already loaded ob data
all_sample_start_time = st - datetime.timedelta(seconds=config.sampling_window)
all_sample_end_time = st
all_sampling_data = map(lambda x: DataSplitter.get_between(x, all_sample_start_time, all_sample_end_time),
all_ob_data)
# Get future data
all_future_data_start_time = st
all_future_data_end_time = st + datetime.timedelta(seconds=config.sampling_window)
all_future_data = DataLoader().load_split_data(config.real_root, all_future_data_start_time,
all_future_data_end_time, config.product)
return all_ob_data, all_sampling_data, all_future_data
def get_lyapunov_exponent_over_time(trades, st, et, step_minutes,
window_minutes):
num_steps = ((et - st).total_seconds() / 60) / step_minutes
lyap_exps = []
times = []
for i in range(0, int(num_steps)):
iter_st = st + datetime.timedelta(minutes=step_minutes * i)
iter_et = iter_st + datetime.timedelta(minutes=window_minutes)
window = DataSplitter.get_between(trades, iter_st, iter_et)
prices = np.asarray(window['price'].dropna(), dtype=np.float32)
if len(prices) == 0:
continue
lyap_exp = nolds.lyap_r(prices)
if lyap_exp > 0:
lyap_exps.append(lyap_exp)
times.append(iter_et)
else:
pass
return times, lyap_exps
def get_hurst_exponent_over_time(trades, st, et, step_minutes,
window_minutes):
num_steps = ((et - st).total_seconds() / 60) / step_minutes
hurst_exps = []
times = []
for i in range(0, int(num_steps)):
iter_st = st + datetime.timedelta(minutes=step_minutes * i)
iter_et = iter_st + datetime.timedelta(minutes=window_minutes)
window = DataSplitter.get_between(trades, iter_st, iter_et)
prices = np.asarray(window['price'].dropna(), dtype=np.float32)
if len(prices) == 0:
continue
hurst_exp = nolds.hurst_rs(prices)
# hurst_exp = nolds.dfa(prices) - 1
print(hurst_exp)
if 0 < hurst_exp < 1:
hurst_exps.append(hurst_exp)
times.append(iter_st)
else:
pass
return times, hurst_exps
def backtest_mode(st: datetime.datetime = None):
all_data_st = take_secs(st, max(config.orderbook_window, config.sampling_window))
all_data_et = add_secs(st, config.num_predictions * config.interval)
all_data = DataLoader.load_split_data(config.real_root, all_data_st, all_data_et, config.product)
validate_future = None
previous_backtest = None
current_backtest = None
sim_future = None
sim_success = False
sim_st = None
for i in range(0, config.num_predictions):
logger.info("Iteration " + str(i))
sim_st = add_secs(st, config.interval * i)
sim_et = add_secs(sim_st, config.simulation_window)
ob_st = take_secs(sim_st, config.orderbook_window)
ob_et = sim_st
sam_st = take_secs(sim_st, config.sampling_window)
sam_et = sim_st
try:
logger.info("Gathering data for simulation at: " + sim_st.isoformat())
all_sampling_data = map(lambda x: DataSplitter.get_between(x, sam_st, sam_et),
all_data)
all_future_data = map(lambda x: DataSplitter.get_between(x, sim_st, sim_et),
all_data)
previous_backtest = current_backtest
current_backtest = Backtest(config, sim_st, all_sampling_data, all_future_data)
except Exception as e:
logger.error("Error occurred when gathering data: " + str(e))
current_backtest = None
# Initiate simulation prep synchronously
prep_success = current_backtest.prepare_simulation()
# Wait for previous simulation to finish
sim_future, sim_success = wait_on_simulation(sim_future, sim_st, sim_success)
# Wait for previous validation to finish
wait_on_validation(validate_future)
# Set off validation for previous iteration
validate_future = run_validation_async(previous_backtest, sim_success)
# Run this current iteration's simulation async
if current_backtest is not None and prep_success:
sim_future = current_backtest.run_simulation()
# Wait for previous validation to finish
wait_on_validation(validate_future)
sim_future, sim_success = wait_on_simulation(sim_future, sim_st, sim_success)
if sim_success:
logger.info("Starting final validation")
current_backtest.evaluate_simulation(prog_start)
def test_orders_per_minute_windowed(self):
product = "LTC-USD"
root = "/Users/jamesprince/project-data/data/consolidated-feed/"
st = datetime.datetime(2018, 5, 17, 0, 0, 0)
et = datetime.datetime(2018, 5, 17, 23, 59, 59)
feed_df = DataLoader.load_feed(root + product + "/", st, et, product)
orders = DataSplitter.get_orders(feed_df)
limit_orders = DataSplitter.get_limit_orders(orders)
market_orders = DataSplitter.get_market_orders(orders)
trades = DataSplitter.get_trades(feed_df)
cancels = DataSplitter.get_cancellations(feed_df)
print("Total limit orders: " + str(len(limit_orders)))
print("Total market orders: " + str(len(market_orders)))
print("Total trades: " + str(len(trades)))
print("Total cancels: " + str(len(cancels)))
# total_vol = trades['remaining_size'].sum()
# print("Total traded volume: " + str(total_vol))
window_minutes = 60
step_minutes = 5
times = []
num_limit_orders = []
num_market_orders = []
num_trades = []
num_cancels = []
traded_vols = []
for i in range(0, int((24 * 60) / step_minutes - 1)):
window_st = st + datetime.timedelta(seconds=i * step_minutes * 60)
window_et = window_st + datetime.timedelta(seconds=window_minutes *
60)
limit_orders_this_window = DataSplitter.get_between(
limit_orders, window_st, window_et)
market_orders_this_window = DataSplitter.get_between(
market_orders, window_st, window_et)
trades_this_window = DataSplitter.get_between(
trades, window_st, window_et)
cancels_this_window = DataSplitter.get_between(
cancels, window_st, window_et)
times.append(window_st)
num_limit_orders.append(len(limit_orders_this_window))
num_market_orders.append(len(market_orders_this_window))
num_trades.append(len(trades_this_window))
num_cancels.append(len(cancels_this_window))
# vol_this_window = trades_this_window['remaining_size'].sum()
# traded_vols.append(vol_this_window)
Statistics.plot_metric_daily_comparison(times, num_limit_orders,
num_cancels, "LTC-USD", st,
step_minutes, window_minutes,
"Limit Orders", "Cancels")
Statistics.plot_metric_daily(times, num_limit_orders, "LTC-USD", st,
step_minutes, window_minutes,
"Limit Orders")
Statistics.plot_metric_daily(times, num_market_orders, "LTC-USD", st,
step_minutes, window_minutes,
"Market Orders")
Statistics.plot_metric_daily(times, num_trades, "LTC-USD", st,
step_minutes, window_minutes, "Trades")
Statistics.plot_metric_daily(times, num_cancels, "LTC-USD", st,
step_minutes, window_minutes, "Cancels")
Statistics.plot_metric_daily(times, traded_vols, "LTC-USD", st,
step_minutes, window_minutes,
"Traded Volume")