def test_get_spread(self):
ob_state_df = OrderBookCreator().load_orderbook_state(
"/Users/jamesprince/project-data/new-nimue-backup/orderbook/BTC-USD/2018-05-16T18:05:08.067228.json"
)
print(ob_state_df)
spread = OrderBookCreator.get_spread(ob_state_df)
print(spread)
def test_plot_orderbook(self):
ob_state_df = OrderBookCreator().load_orderbook_state(
"/Users/jamesprince/project-data/data/raw-orderbooks/LTC-USD/2018-05-17T03:35:25.600377.json"
)
xwindow = 4
OrderBookCreator.plot_orderbook(ob_state_df,
xwindow,
log_y_scale=False)
def orderbook_mode(st: datetime.datetime = None):
closest_ob_state, closest_ob_state_str = OrderBookCreator.locate_closest_ob_state(config.orderbook_output_root, st)
orders_df, trades_df, cancels_df = DataLoader.load_split_data(config.real_root, closest_ob_state,
st, config.product)
ob_state_path = config.root_path + closest_ob_state_str
ob_state = OrderBookCreator.load_orderbook_state(ob_state_path)
orderbook = OrderBookCreator.get_orderbook(orders_df, trades_df, cancels_df, ob_state)
output_file = "/Users/jamesprince/project-data/orderbook-" + st.isoformat() + ".csv"
OrderBookCreator.orderbook_to_file(orderbook, output_file)
logger.info("Orderbook saved to: " + output_file)
def get_test_orderbook_evolutor(self):
feed_data = DataLoader.load_split_data(self.real_root, self.st,
self.et, self.product)
orderbook_root = "/Users/jamesprince/project-data/data/raw-orderbooks/" + self.product + "/"
# Have to go an hour back because order book file names are UTC+1
closest_ob_time, closest_ob_file = OrderBookCreator.locate_closest_ob_state(
orderbook_root, self.st + datetime.timedelta(hours=1))
self.feed_start_time = closest_ob_time - datetime.timedelta(hours=1)
# closest_ob_path = orderbook_root + closest_ob_file
closest_ob_path = orderbook_root + "2018-05-17T01:58:45.628278.json"
start_seq, ob_state_df = OrderBookCreator().load_orderbook_state(
closest_ob_path)
# ob_start_state_df = OrderBookCreator().get_orderbook(*feed_data, ob_state_df)
orderbook = OrderBookEvolutor(ob_state_df, self.st, start_seq)
print("order book seq: " + str(start_seq))
return orderbook
def prepare_simulation(self):
try:
ob_state_seq, ob_state = reconstruct_orderbook(self.config, self.sim_st, self.logger)
# Save orderbook
orderbook_path = self.config.orderbook_output_root + self.orderbook_window_end_time.isoformat() + ".csv"
OrderBookCreator.orderbook_to_file(ob_state, orderbook_path)
params_path = self.params_root \
+ self.sim_st.time().isoformat() + ".json"
if self.config.use_cached_params and os.path.isfile(params_path):
self.logger.info("Params file exists, therefore we're using it! " + params_path)
else:
self.logger.info("Not using params cache" + "\nGenerating params...")
# Get parameters
orders_df, trades_df, cancels_df = self.all_sampling_data
# TODO: massive duplication in the arguments
params_ob_state_seq, params_ob_state = reconstruct_orderbook(self.config,
self.sampling_window_start_time,
self.logger)
feed_df = DataLoader.load_feed(self.config.real_root, self.sampling_window_start_time,
self.sim_st, self.config.product)
params = Sample.generate_sim_params(orders_df, trades_df, cancels_df, feed_df, params_ob_state,
params_ob_state_seq, self.sim_st)
# Save params (that can be reused!)
Writer.json_to_file(params, params_path)
self.logger.info("Permanent parameters saved to: " + params_path)
# Generate .conf file
sim_config = self.generate_config_dict(orderbook_path, params_path)
sim_config_string = SimConfig.generate_config_string(sim_config)
self.save_config(sim_config_string)
return True
except Exception as e:
self.logger.error(
"Simulation preparation failed, skipping, at: " + self.sim_st.isoformat() + "\nError was\n" + str(e))
return False
def test_can_reconstruct(self):
st = datetime.datetime(2018, 5, 16, 18, 0, 0)
et = datetime.datetime(2018, 5, 16, 18, 10, 0)
all_data = DataLoader.load_split_data(
"/Users/jamesprince/project-data/data/consolidated-feed/", st, et,
"BTC-USD")
print(all_data[1]['taker_order_id'].unique())
print(all_data[1]['maker_order_id'].unique())
ob_state_df = OrderBookCreator().load_orderbook_state(
"/Users/jamesprince/project-data/data/raw-orderbooks/BTC-USD/2018-05-16T18:05:08.067228.json"
)
ob_final = OrderBookCreator().get_orderbook(*all_data, ob_state_df)
print(ob_final)
print(ob_final['size'].apply(pd.to_numeric).sum())
OrderBookCreator.orderbook_to_file(
ob_final,
"/Users/jamesprince/project-data/control/aligned-orderbooks/2018-05-16T18:10:00.csv"
)
def test_ob_valid(self):
ob_state_df = OrderBookCreator().load_orderbook_state(
"/Users/jamesprince/project-data/new-nimue-backup/orderbook/BTC-USD/2018-05-16T18:05:08.067228.json"
)
OrderBookCreator.check_ob_valid(ob_state_df)
def test_can_locate_files(self):
time = datetime.datetime(2018, 5, 16, 18, 0, 0)
OrderBookCreator.locate_closest_ob_state(
"/Users/jamesprince/project-data/new-nimue-backup/orderbook/BTC-USD/",
time)
def test_can_load_data(self):
df = OrderBookCreator().load_orderbook_state(
"/Users/jamesprince/project-data/gdax/test-orderbook.json")
print(df)
assert not df.empty