def test_add_order_book_snapshots_adds_to_container(self):
# Arrange
data = BacktestDataContainer()
snapshot1 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
timestamp_ns=0,
)
snapshot2 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1551.15, 0.51], [1581.00, 1.20]],
asks=[[1553.15, 1.51], [1583.00, 2.20]],
timestamp_ns=1_000_000_000,
)
# Act
data.add_order_book_data([snapshot2, snapshot1]) # <-- reverse order
# Assert
assert ClientId("BINANCE") in data.clients
assert ETHUSDT_BINANCE.id in data.books
assert data.order_book_data == [snapshot1, snapshot2] # <-- sorted
def test_add_order_book_snapshots_adds_to_container(self):
# Arrange
engine = BacktestEngine()
engine.add_instrument(ETHUSDT_BINANCE)
snapshot1 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
timestamp_origin_ns=0,
timestamp_ns=0,
)
snapshot2 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1551.15, 0.51], [1581.00, 1.20]],
asks=[[1553.15, 1.51], [1583.00, 2.20]],
timestamp_origin_ns=1_000_000_000,
timestamp_ns=1_000_000_000,
)
# Act
engine.add_order_book_data([snapshot2, snapshot1]) # <-- reverse order
def test_with_mix_of_stream_data_produces_correct_stream_of_data(self):
# Assert
data = BacktestDataContainer()
data.add_instrument(ETHUSDT_BINANCE)
snapshot1 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
timestamp_ns=0,
)
data_type = DataType(str, metadata={"news_wire": "hacks"})
generic_data1 = [
GenericData(data_type, data="AAPL hacked", timestamp_ns=0),
GenericData(data_type, data="AMZN hacked", timestamp_ns=500_000),
GenericData(data_type, data="NFLX hacked", timestamp_ns=1_000_000),
GenericData(data_type, data="MSFT hacked", timestamp_ns=2_000_000),
]
snapshot2 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1551.15, 0.51], [1581.00, 1.20]],
asks=[[1553.15, 1.51], [1583.00, 2.20]],
timestamp_ns=1_000_000,
)
data.add_generic_data(ClientId("NEWS_CLIENT"), generic_data1)
data.add_order_book_data([snapshot1, snapshot2])
producer = BacktestDataProducer(data=data, logger=self.logger)
producer.setup(producer.min_timestamp_ns, producer.max_timestamp_ns)
# Act
streamed_data = []
while producer.has_data:
streamed_data.append(producer.next())
# Assert
timestamps = [x.timestamp_ns for x in streamed_data]
assert timestamps == [0, 0, 500000, 1000000, 1000000, 2000000]
assert producer.min_timestamp_ns == 0
assert producer.max_timestamp_ns == 2_000_000
assert producer.min_timestamp == pd.Timestamp(
"1970-01-01 00:00:00.000000+0000", tz="UTC")
assert producer.max_timestamp == pd.Timestamp(
"1970-01-01 00:00:00.002000+0000", tz="UTC")
def test_apply(empty_l2_book, clock):
snapshot = OrderBookSnapshot(
instrument_id=empty_l2_book.instrument_id,
level=OrderBookLevel.L2,
bids=[[150.0, 0.51]],
asks=[[160.0, 1.51]],
timestamp_origin_ns=0,
timestamp_ns=0,
)
empty_l2_book.apply_snapshot(snapshot)
assert empty_l2_book.best_ask_price() == 160
delta = OrderBookDelta(
instrument_id=TestStubs.audusd_id(),
level=OrderBookLevel.L2,
delta_type=OrderBookDeltaType.ADD,
order=Order(
155.0,
672.45,
OrderSide.SELL,
"4a25c3f6-76e7-7584-c5a3-4ec84808e240",
),
timestamp_origin_ns=clock.timestamp(),
timestamp_ns=clock.timestamp(),
)
empty_l2_book.apply(delta)
assert empty_l2_book.best_ask_price() == 155
def _handle_market_snapshot(selection, instrument, timestamp_ns):
updates = []
# Check we only have one of [best bets / depth bets / all bets]
bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"]
ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"]
assert len(bid_keys) <= 1
assert len(ask_keys) <= 1
# OrderBook Snapshot
# TODO Clean this crap up
if bid_keys[0] == "atb":
bids = selection.get("atb", [])
else:
bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])]
if ask_keys[0] == "atl":
asks = selection.get("atl", [])
else:
asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])]
snapshot = OrderBookSnapshot(
level=OrderBookLevel.L2,
instrument_id=instrument.id,
bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks],
asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids],
timestamp_ns=timestamp_ns,
)
updates.append(snapshot)
if "trd" in selection:
updates.extend(
_handle_market_trades(runner=selection,
instrument=instrument,
timestamp_ns=timestamp_ns))
return updates
def test_order_book_when_order_book_exists_returns_expected(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=BookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
ts_event_ns=0,
ts_recv_ns=0,
)
order_book = L2OrderBook(
instrument_id=ETHUSDT_BINANCE.id,
price_precision=2,
size_precision=8,
)
order_book.apply_snapshot(snapshot)
self.cache.add_order_book(order_book)
# Act
result = self.cache.order_book(ETHUSDT_BINANCE.id)
# Assert
self.assertEqual(order_book, result)
def test_from_dict_returns_expected_tick(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=AUDUSD,
level=BookLevel.L2,
bids=[[1010, 2], [1009, 1]],
asks=[[1020, 2], [1021, 1]],
ts_event_ns=0,
ts_recv_ns=0,
)
# Act
result = OrderBookSnapshot.from_dict(
OrderBookSnapshot.to_dict(snapshot))
# Assert
assert result == snapshot
def test_orderbook_snapshot(empty_l2_book):
snapshot = OrderBookSnapshot(
instrument_id=empty_l2_book.instrument_id,
level=OrderBookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
timestamp_ns=0,
)
empty_l2_book.apply_snapshot(snapshot)
assert empty_l2_book.best_bid_price() == 1580.0
assert empty_l2_book.best_ask_price() == 1552.15
def build_market_snapshot_messages(
raw, instrument_provider: BetfairInstrumentProvider
) -> List[OrderBookSnapshot]:
updates = []
for market in raw.get("mc", []):
# Market status events
# market_definition = market.get("marketDefinition", {})
# TODO - Need to handle instrument status = CLOSED here
# Orderbook snapshots
if market.get("img") is True:
market_id = market["id"]
for (selection_id, handicap), selections in itertools.groupby(
market.get("rc", []), lambda x: (x["id"], x.get("hc"))):
for selection in list(selections):
kw = dict(
market_id=market_id,
selection_id=str(selection_id),
handicap=str(handicap or "0.0"),
)
instrument = instrument_provider.get_betting_instrument(
**kw)
# Check we only have one of [best bets / depth bets / all bets]
bid_keys = [k for k in B_BID_KINDS if k in selection
] or ["atb"]
ask_keys = [k for k in B_ASK_KINDS if k in selection
] or ["atl"]
assert len(bid_keys) <= 1
assert len(ask_keys) <= 1
# TODO Clean this crap up
if bid_keys[0] == "atb":
bids = selection.get("atb", [])
else:
bids = [(p, v)
for _, p, v in selection.get(bid_keys[0], [])]
if ask_keys[0] == "atl":
asks = selection.get("atl", [])
else:
asks = [(p, v)
for _, p, v in selection.get(ask_keys[0], [])]
snapshot = OrderBookSnapshot(
level=OrderBookLevel.L2,
instrument_id=instrument.id,
bids=[(price_to_probability(p, OrderSide.BUY), v)
for p, v in asks],
asks=[(price_to_probability(p, OrderSide.SELL), v)
for p, v in bids],
timestamp_ns=millis_to_nanos(raw["pt"]),
)
updates.append(snapshot)
return updates
def test_handle_order_book_snapshot_sends_to_data_engine(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1000, 1]],
asks=[[1001, 1]],
timestamp_ns=0,
)
# Act
self.client._handle_data_py(snapshot)
# Assert
self.assertEqual(1, self.data_engine.data_count)
def test_to_dict_returns_expected_dict(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=AUDUSD,
level=BookLevel.L2,
bids=[[1010, 2], [1009, 1]],
asks=[[1020, 2], [1021, 1]],
ts_event_ns=0,
ts_recv_ns=0,
)
# Act
result = OrderBookSnapshot.to_dict(snapshot)
# Assert
assert result == {
"type": "OrderBookSnapshot",
"instrument_id": "AUD/USD.SIM",
"level": "L2",
"bids": "[[1010, 2], [1009, 1]]",
"asks": "[[1020, 2], [1021, 1]]",
"ts_event_ns": 0,
"ts_recv_ns": 0,
}
def test_repr(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=AUDUSD,
level=OrderBookLevel.L2,
bids=[[1010, 2], [1009, 1]],
asks=[[1020, 2], [1021, 1]],
timestamp_ns=0,
)
# Act
# Assert
assert (
repr(snapshot)
== "OrderBookSnapshot('AUD/USD.SIM', bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], timestamp_ns=0)"
)
def test_process_order_book_snapshot_when_one_subscriber_then_sends_to_registered_handler(
self,
):
# Arrange
self.data_engine.register_client(self.binance_client)
self.binance_client.connect()
self.data_engine.process(
ETHUSDT_BINANCE
) # <-- add necessary instrument for test
handler = []
subscribe = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(
OrderBook,
{
"InstrumentId": ETHUSDT_BINANCE.id,
"Level": OrderBookLevel.L2,
"Depth": 25,
"Interval": 0, # Streaming
},
),
handler=handler.append,
command_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
self.data_engine.execute(subscribe)
snapshot = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=OrderBookLevel.L2,
bids=[[1000, 1]],
asks=[[1001, 1]],
timestamp_origin_ns=0,
timestamp_ns=0,
)
# Act
self.data_engine.process(snapshot)
# Assert
assert self.data_engine.subscribed_order_books == [ETHUSDT_BINANCE.id]
assert handler[0].instrument_id == ETHUSDT_BINANCE.id
assert type(handler[0]) == L2OrderBook
def _build_order_book_snapshot(values):
# First value is a CLEAR message, which we ignore
return OrderBookSnapshot(
instrument_id=InstrumentId.from_str(values[1]["instrument_id"]),
level=BookLevelParser.from_str_py(values[1]["level"]),
bids=[
(order["order_price"], order["order_size"])
for order in data[1:]
if order["order_side"] == "BUY"
],
asks=[
(order["order_price"], order["order_size"])
for order in data[1:]
if order["order_side"] == "SELL"
],
ts_event_ns=data[1]["ts_event_ns"],
ts_recv_ns=data[1]["ts_recv_ns"],
)
def _handle_market_snapshot(selection, instrument, timestamp_ns):
updates = []
# Check we only have one of [best bets / depth bets / all bets]
bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"]
ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"]
assert len(bid_keys) <= 1
assert len(ask_keys) <= 1
# OrderBook Snapshot
# TODO Clean this crap up
if bid_keys[0] == "atb":
bids = selection.get("atb", [])
else:
bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])]
if ask_keys[0] == "atl":
asks = selection.get("atl", [])
else:
asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])]
snapshot = OrderBookSnapshot(
level=OrderBookLevel.L2,
instrument_id=instrument.id,
bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks],
asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids],
timestamp_ns=timestamp_ns,
)
updates.append(snapshot)
# Trade Ticks
for price, volume in selection.get("trd", []):
trade_id = hash_json((timestamp_ns, price, volume))
tick = TradeTick(
instrument_id=instrument.id,
price=Price(price_to_probability(price, force=True)),
size=Quantity(volume, precision=4),
side=OrderSide.BUY,
match_id=TradeMatchId(trade_id),
timestamp_ns=timestamp_ns,
)
updates.append(tick)
return updates
def test_hash_str_and_repr(self):
# Arrange
snapshot = OrderBookSnapshot(
instrument_id=AUDUSD,
level=BookLevel.L2,
bids=[[1010, 2], [1009, 1]],
asks=[[1020, 2], [1021, 1]],
ts_event_ns=0,
ts_recv_ns=0,
)
# Act, Assert
assert isinstance(hash(snapshot), int)
assert (
str(snapshot) ==
"OrderBookSnapshot('AUD/USD.SIM', level=L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_recv_ns=0)"
)
assert (
repr(snapshot) ==
"OrderBookSnapshot('AUD/USD.SIM', level=L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_recv_ns=0)"
)
def order_book_snapshot(
instrument_id=None,
bid_price=10,
ask_price=15,
bid_levels=3,
ask_levels=3,
bid_volume=10,
ask_volume=10,
level=OrderBookLevel.L2,
) -> OrderBookSnapshot:
err = "Too many levels generated; orders will be in cross. Increase bid/ask spread or reduce number of levels"
assert bid_price < ask_price, err
return OrderBookSnapshot(
instrument_id=instrument_id or TestStubs.audusd_id(),
level=level,
bids=[(bid_price - i, bid_volume * (1 + i))
for i in range(bid_levels)],
asks=[(ask_price + i, ask_volume * (1 + i))
for i in range(ask_levels)],
timestamp_ns=0,
)
def test_process_order_book_when_multiple_subscribers_then_sends_to_registered_handlers(
self, ):
# Arrange
self.data_engine.register_client(self.binance_client)
self.binance_client.connect()
self.data_engine.process(
ETHUSDT_BINANCE) # <-- add necessary instrument for test
handler1 = []
subscribe1 = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(
OrderBook,
{
"instrument_id": ETHUSDT_BINANCE.id,
"level": BookLevel.L2,
"depth": 25,
"interval": 0, # Streaming
},
),
handler=handler1.append,
command_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
handler2 = []
subscribe2 = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(
OrderBook,
{
"instrument_id": ETHUSDT_BINANCE.id,
"level": BookLevel.L2,
"depth": 25,
"interval": 0, # Streaming
},
),
handler=handler2.append,
command_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
self.data_engine.execute(subscribe1)
self.data_engine.execute(subscribe2)
snapshot = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=BookLevel.L2,
bids=[[1000, 1]],
asks=[[1001, 1]],
ts_event_ns=0,
ts_recv_ns=0,
)
# Act
self.data_engine.process(snapshot)
# Assert
cached_book = self.data_engine.cache.order_book(ETHUSDT_BINANCE.id)
assert self.data_engine.subscribed_order_books == [ETHUSDT_BINANCE.id]
assert type(cached_book) == L2OrderBook
assert cached_book.instrument_id == ETHUSDT_BINANCE.id
assert handler1[0] == cached_book
assert handler2[0] == cached_book
def test_with_mix_of_stream_data_produces_correct_stream_of_data(self):
# Assert
snapshot1 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=BookLevel.L2,
bids=[[1550.15, 0.51], [1580.00, 1.20]],
asks=[[1552.15, 1.51], [1582.00, 2.20]],
ts_event_ns=0,
ts_recv_ns=0,
)
data_type = DataType(MyData, metadata={"news_wire": "hacks"})
generic_data1 = [
GenericData(
data_type,
data=MyData("AAPL hacked"),
),
GenericData(
data_type,
data=MyData("AMZN hacked", 500_000, 500_000),
),
GenericData(
data_type,
data=MyData("NFLX hacked", 1_000_000, 1_000_000),
),
GenericData(
data_type,
data=MyData("MSFT hacked", 2_000_000, 2_000_000),
),
]
snapshot2 = OrderBookSnapshot(
instrument_id=ETHUSDT_BINANCE.id,
level=BookLevel.L2,
bids=[[1551.15, 0.51], [1581.00, 1.20]],
asks=[[1553.15, 1.51], [1583.00, 2.20]],
ts_event_ns=1_000_000,
ts_recv_ns=1_000_000,
)
producer = BacktestDataProducer(
logger=self.logger,
instruments=[ETHUSDT_BINANCE],
generic_data=generic_data1,
order_book_data=[snapshot1, snapshot2],
)
producer.setup(producer.min_timestamp_ns, producer.max_timestamp_ns)
# Act
streamed_data = []
while producer.has_data:
streamed_data.append(producer.next())
# Assert
timestamps = [x.ts_recv_ns for x in streamed_data]
assert timestamps == [0, 0, 500000, 1000000, 1000000, 2000000]
assert producer.min_timestamp_ns == 0
assert producer.max_timestamp_ns == 2_000_000
assert producer.min_timestamp == pd.Timestamp(
"1970-01-01 00:00:00.000000+0000", tz="UTC")
assert producer.max_timestamp == pd.Timestamp(
"1970-01-01 00:00:00.002000+0000", tz="UTC")
