def extend_sample(self, day_diff: int, ob_ref: OrderBook):
for msg in self.initial_messages:
m_ = msg.copy()
m_.time += day_diff
if not ob_ref.is_valid_msg(m_):
continue
yield from self._yield_n_copies(m_)
prev_loop = 0
prev_diff = None
diff_top_of_book = _TopOfBook(0, 0)
for loop_count, msg in _endless_copies(self.mixed_messages):
if loop_count > prev_loop:
prev_loop = loop_count
prev_diff = diff_top_of_book
diff_top_of_book = _TopOfBook(ob_ref.ask, ob_ref.bid) - \
self.initial_top_of_book
msg.time += day_diff + self.time_diff * loop_count
if msg.id != 0:
msg.id += self.id_diff * loop_count * self.n_duplicates
msg.price = msg.price + \
(prev_diff if msg.fake else diff_top_of_book).get(msg.direction)
if not ob_ref.is_valid_msg(msg):
continue
# Delete all possible conflicts from the book first
yield from self._handle_conflicts(ob_ref.conflicts(msg), msg)
yield from self._yield_n_copies(msg)
def load_single_day(self, day):
"""Loads messages for a single day"""
self._new_book(day)
self.last_multiple = 1
day_diff = day - self.start_timestamp
max_time = day + EOD
for message in self.extender.extend_sample(day_diff, self.order_book):
if message.time > max_time:
break
self._handle_message(message)
# Flushing out remaining messages in buffer
if len(self.message_buffer) > 0:
self._save_buffer()
eod_clear = OrderBook(self.instrument)
eod_clear.last_time = max_time
eod_clear.last_sod_offset = self.sod_offset_counter
# save an empty order_book at the end of the day
save_order_book(eod_clear, self.interval)
logger.info(
'best bid volume=%d\tbest ask volume=%d',
sum(o.qty for o in self.order_book.bid_book[self.order_book.bid]),
sum(o.qty for o in self.order_book.ask_book[self.order_book.ask]))
def test_delete_order(self):
orderbook = OrderBook("testInstrument")
order_to_be_removed = Order(1, 10, 10, 1)
orderbook.id_map[1] = order_to_be_removed
orderbook.bid_book[10].append(order_to_be_removed)
# Deleting order with id 1
delete_message = Message(1, MessageType.DELETE, 1, 10, 10, 1)
orderbook.send(delete_message)
self.assertEqual(len(orderbook.bid_book[10]), 0)
def test_has_order(self):
orderbook = OrderBook("testInstrument")
orderbook.send(Message(1, MessageType.NEW_ORDER, 11, 10, 2, -1))
orderbook.send(Message(2, MessageType.NEW_ORDER, 12, 10, 1, 1))
self.assertTrue(
orderbook.has_order(Message(3, MessageType.DELETE, 11, 10, 2, -1)))
self.assertFalse(
orderbook.has_order(Message(3, MessageType.DELETE, 11, 10, 2, 1)))
self.assertFalse(
orderbook.has_order(Message(3, MessageType.DELETE, 12, 10, 2, 1)))
self.assertTrue(
orderbook.has_order(Message(3, MessageType.DELETE, 12, 10, 1, 1)))
def test_modify_order(self):
orderbook = OrderBook("testInstrument")
order_to_be_modified = Order(1, 10, 10, 1)
orderbook.id_map[1] = order_to_be_modified
orderbook.bid_book[10].append(order_to_be_modified)
order = Order(2, 10, 10, 1)
orderbook.id_map[2] = order
orderbook.bid_book[10].append(order)
self.assertEqual(len(orderbook.bid_book[10]), 2)
self.assertEqual(orderbook.bid_book[10][0].id, 1)
# Removing 8 shares from the order
modify_message = Message(1, MessageType.MODIFY, 1, 8, 10, 1)
orderbook.send(modify_message)
# Ordering has not changed in the orderbook
modified_order = orderbook.bid_book[10][0]
self.assertEqual(modified_order.id, 1)
self.assertEqual(modified_order.qty, 2)
# Adding 8 shares to the order
modify_message = Message(2, MessageType.MODIFY, 1, -8, 10, 1)
orderbook.send(modify_message)
# Ordering has changed
modified_order = orderbook.bid_book[10][1]
self.assertEqual(modified_order.id, 1)
self.assertEqual(modified_order.qty, 10)
def load(**kwargs):
"""Loads the given messages file into the database.
Args:
message_file: File object containing sample messages as csv
ob_file_path: optional path to the order book csv data
top_of_book: optional values for the best bid and ask before the sample
interval: number of nanoseconds between order book snapshots
instrument: string representing the name of the instrument
duplicate: number of times to duplicate messages
extend: number of days to extend the sample over
start_time: datetime representing the first day of the sample
Note: at least one of `top_of_book` or `ob_file_path` must be provided
"""
start_time = kwargs['start_time'].replace(tzinfo=TZ)
start_timestamp = int(start_time.timestamp() * 10**9) # in nanoseconds
instrument = kwargs['instrument']
interval = check_interval(kwargs['interval'], instrument)
upsert_order_book(OrderBook(instrument), interval)
initial_top_of_book = parse_top_of_book(kwargs['ob_file_path'],
kwargs['top_of_book'])
extender = Extender(kwargs['message_file'], start_timestamp,
int(kwargs['duplicate']), initial_top_of_book)
loader = _Loader(extender, start_timestamp, instrument, interval)
for day in weekdays(start_timestamp, int(kwargs['extend'])):
loader.load_single_day(day)
def test_backfill(self):
"""Run with `-b` to silence prints generated in Extender"""
start = _datetime_ts(2019, 10, 21)
sample = []
extended = []
with open('tests/messages_sample.fixture', 'r') as f:
e = Extender(f, start, 1, (1356500, 1356300))
sample_count = 9
extended_count = (9 + 4) * 2
ob = OrderBook("SPY")
for m in e.extend_sample(0, ob):
if len(sample) < sample_count:
sample.append(m)
elif len(extended) < extended_count:
extended.append(m)
else:
break
ob.send(m)
new_orders = [m for m in extended
if m.message_type == MessageType.NEW_ORDER]
self.assertEqual(11, len(new_orders))
id_map = defaultdict(list)
sample_ids = set(m.id for m in sample if m.id > 0)
extended_ids = set(m.id for m in extended if m.id > 0)
for m in sample:
id_map[m.id].append(m)
for m in extended:
id_map[m.id].append(m)
for id_ in extended_ids.difference(sample_ids):
# All ids generated in the extended set have a NEW
self.assertTrue(
any(m.message_type ==
MessageType.NEW_ORDER for m in id_map[id_])
)
for id_ in sample_ids.intersection(extended_ids):
# All ids from the sample set have a DELETE
self.assertTrue(
any(m.message_type == MessageType.DELETE for m in id_map[id_])
)
def test_conflict(self):
orderbook = OrderBook("testInstrument")
new_order = Message(1, MessageType.NEW_ORDER, 11, 10, 2, -1)
new_order_2 = Message(2, MessageType.NEW_ORDER, 12, 10, 1, 1)
orderbook.send(new_order)
orderbook.send(new_order_2)
bids = list(
orderbook.conflicts(
Message(3, MessageType.NEW_ORDER, 13, 10, 1, -1)))
asks = list(
orderbook.conflicts(Message(3, MessageType.NEW_ORDER, 13, 10, 2,
1)))
self.assertListEqual(bids, [Order(12, 10, 1, 1)])
self.assertListEqual(asks, [Order(11, 10, 2, -1)])
empty_bids = list(
orderbook.conflicts(Message(3, MessageType.DELETE, 13, 10, 1, 1)))
empty_asks = list(
orderbook.conflicts(Message(3, MessageType.DELETE, 13, 10, 2, -1)))
self.assertListEqual(empty_bids, [])
self.assertListEqual(empty_asks, [])
def test_new_order(self):
orderbook = OrderBook("testInstrument")
new_order = Message(1, MessageType.NEW_ORDER, 11, 10, 1, -1)
new_order_2 = Message(1, MessageType.NEW_ORDER, 12, 10, 1, -1)
orderbook.send(new_order)
orderbook.send(new_order_2)
self.assertEqual(len(orderbook.ask_book[1]), 2)
# Making sure they are ordered correctly
actual_order_1 = orderbook.ask_book[1][0]
actual_order_2 = orderbook.ask_book[1][1]
self.assertEqual(actual_order_1.price, 1)
self.assertEqual(actual_order_1.qty, 10)
self.assertEqual(actual_order_1.id, 11)
self.assertEqual(actual_order_2.price, 1)
self.assertEqual(actual_order_2.qty, 10)
self.assertEqual(actual_order_2.id, 12)
def _new_book(self, day):
self.order_book = OrderBook(self.instrument)
self.order_book.last_time = day
self.order_book.last_sod_offset = day
self.sod_offset_counter = day
class _Loader:
def __init__(self, extender, start_timestamp, instrument, interval):
self.extender = extender
self.start_timestamp = start_timestamp
self.instrument = instrument
self.interval = interval
self.order_book = None
self.sod_offset_counter = 0
self.message_buffer = []
self.last_multiple = 0
def _new_book(self, day):
self.order_book = OrderBook(self.instrument)
self.order_book.last_time = day
self.order_book.last_sod_offset = day
self.sod_offset_counter = day
def _save_buffer(self):
save_messages(self.message_buffer, self.instrument)
self.message_buffer.clear()
def _handle_message(self, message):
message.sod_offset = self.sod_offset_counter
current_multiple = message.time // self.interval
if current_multiple > self.last_multiple:
logger.debug(str(self.order_book))
self.last_multiple = current_multiple
save_order_book(self.order_book, self.interval)
self.message_buffer.append(message)
if len(self.message_buffer) > MESSAGE_BATCH_SIZE:
self._save_buffer()
self.order_book.send(message)
self.sod_offset_counter += 1
def load_single_day(self, day):
"""Loads messages for a single day"""
self._new_book(day)
self.last_multiple = 1
day_diff = day - self.start_timestamp
max_time = day + EOD
for message in self.extender.extend_sample(day_diff, self.order_book):
if message.time > max_time:
break
self._handle_message(message)
# Flushing out remaining messages in buffer
if len(self.message_buffer) > 0:
self._save_buffer()
eod_clear = OrderBook(self.instrument)
eod_clear.last_time = max_time
eod_clear.last_sod_offset = self.sod_offset_counter
# save an empty order_book at the end of the day
save_order_book(eod_clear, self.interval)
logger.info(
'best bid volume=%d\tbest ask volume=%d',
sum(o.qty for o in self.order_book.bid_book[self.order_book.bid]),
sum(o.qty for o in self.order_book.ask_book[self.order_book.ask]))