def test_add_order_book_snapshots_adds_to_container(self): # Arrange data = BacktestDataContainer() snapshot1 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], timestamp_ns=0, ) snapshot2 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1551.15, 0.51], [1581.00, 1.20]], asks=[[1553.15, 1.51], [1583.00, 2.20]], timestamp_ns=1_000_000_000, ) # Act data.add_order_book_data([snapshot2, snapshot1]) # <-- reverse order # Assert assert ClientId("BINANCE") in data.clients assert ETHUSDT_BINANCE.id in data.books assert data.order_book_data == [snapshot1, snapshot2] # <-- sorted
def test_add_order_book_snapshots_adds_to_container(self): # Arrange engine = BacktestEngine() engine.add_instrument(ETHUSDT_BINANCE) snapshot1 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], timestamp_origin_ns=0, timestamp_ns=0, ) snapshot2 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1551.15, 0.51], [1581.00, 1.20]], asks=[[1553.15, 1.51], [1583.00, 2.20]], timestamp_origin_ns=1_000_000_000, timestamp_ns=1_000_000_000, ) # Act engine.add_order_book_data([snapshot2, snapshot1]) # <-- reverse order
def test_with_mix_of_stream_data_produces_correct_stream_of_data(self): # Assert data = BacktestDataContainer() data.add_instrument(ETHUSDT_BINANCE) snapshot1 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], timestamp_ns=0, ) data_type = DataType(str, metadata={"news_wire": "hacks"}) generic_data1 = [ GenericData(data_type, data="AAPL hacked", timestamp_ns=0), GenericData(data_type, data="AMZN hacked", timestamp_ns=500_000), GenericData(data_type, data="NFLX hacked", timestamp_ns=1_000_000), GenericData(data_type, data="MSFT hacked", timestamp_ns=2_000_000), ] snapshot2 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1551.15, 0.51], [1581.00, 1.20]], asks=[[1553.15, 1.51], [1583.00, 2.20]], timestamp_ns=1_000_000, ) data.add_generic_data(ClientId("NEWS_CLIENT"), generic_data1) data.add_order_book_data([snapshot1, snapshot2]) producer = BacktestDataProducer(data=data, logger=self.logger) producer.setup(producer.min_timestamp_ns, producer.max_timestamp_ns) # Act streamed_data = [] while producer.has_data: streamed_data.append(producer.next()) # Assert timestamps = [x.timestamp_ns for x in streamed_data] assert timestamps == [0, 0, 500000, 1000000, 1000000, 2000000] assert producer.min_timestamp_ns == 0 assert producer.max_timestamp_ns == 2_000_000 assert producer.min_timestamp == pd.Timestamp( "1970-01-01 00:00:00.000000+0000", tz="UTC") assert producer.max_timestamp == pd.Timestamp( "1970-01-01 00:00:00.002000+0000", tz="UTC")
def test_apply(empty_l2_book, clock): snapshot = OrderBookSnapshot( instrument_id=empty_l2_book.instrument_id, level=OrderBookLevel.L2, bids=[[150.0, 0.51]], asks=[[160.0, 1.51]], timestamp_origin_ns=0, timestamp_ns=0, ) empty_l2_book.apply_snapshot(snapshot) assert empty_l2_book.best_ask_price() == 160 delta = OrderBookDelta( instrument_id=TestStubs.audusd_id(), level=OrderBookLevel.L2, delta_type=OrderBookDeltaType.ADD, order=Order( 155.0, 672.45, OrderSide.SELL, "4a25c3f6-76e7-7584-c5a3-4ec84808e240", ), timestamp_origin_ns=clock.timestamp(), timestamp_ns=clock.timestamp(), ) empty_l2_book.apply(delta) assert empty_l2_book.best_ask_price() == 155
def _handle_market_snapshot(selection, instrument, timestamp_ns): updates = [] # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # OrderBook Snapshot # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=timestamp_ns, ) updates.append(snapshot) if "trd" in selection: updates.extend( _handle_market_trades(runner=selection, instrument=instrument, timestamp_ns=timestamp_ns)) return updates
def test_order_book_when_order_book_exists_returns_expected(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=BookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], ts_event_ns=0, ts_recv_ns=0, ) order_book = L2OrderBook( instrument_id=ETHUSDT_BINANCE.id, price_precision=2, size_precision=8, ) order_book.apply_snapshot(snapshot) self.cache.add_order_book(order_book) # Act result = self.cache.order_book(ETHUSDT_BINANCE.id) # Assert self.assertEqual(order_book, result)
def test_from_dict_returns_expected_tick(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=AUDUSD, level=BookLevel.L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_event_ns=0, ts_recv_ns=0, ) # Act result = OrderBookSnapshot.from_dict( OrderBookSnapshot.to_dict(snapshot)) # Assert assert result == snapshot
def test_orderbook_snapshot(empty_l2_book): snapshot = OrderBookSnapshot( instrument_id=empty_l2_book.instrument_id, level=OrderBookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], timestamp_ns=0, ) empty_l2_book.apply_snapshot(snapshot) assert empty_l2_book.best_bid_price() == 1580.0 assert empty_l2_book.best_ask_price() == 1552.15
def build_market_snapshot_messages( raw, instrument_provider: BetfairInstrumentProvider ) -> List[OrderBookSnapshot]: updates = [] for market in raw.get("mc", []): # Market status events # market_definition = market.get("marketDefinition", {}) # TODO - Need to handle instrument status = CLOSED here # Orderbook snapshots if market.get("img") is True: market_id = market["id"] for (selection_id, handicap), selections in itertools.groupby( market.get("rc", []), lambda x: (x["id"], x.get("hc"))): for selection in list(selections): kw = dict( market_id=market_id, selection_id=str(selection_id), handicap=str(handicap or "0.0"), ) instrument = instrument_provider.get_betting_instrument( **kw) # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection ] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection ] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=millis_to_nanos(raw["pt"]), ) updates.append(snapshot) return updates
def test_handle_order_book_snapshot_sends_to_data_engine(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1000, 1]], asks=[[1001, 1]], timestamp_ns=0, ) # Act self.client._handle_data_py(snapshot) # Assert self.assertEqual(1, self.data_engine.data_count)
def test_to_dict_returns_expected_dict(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=AUDUSD, level=BookLevel.L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_event_ns=0, ts_recv_ns=0, ) # Act result = OrderBookSnapshot.to_dict(snapshot) # Assert assert result == { "type": "OrderBookSnapshot", "instrument_id": "AUD/USD.SIM", "level": "L2", "bids": "[[1010, 2], [1009, 1]]", "asks": "[[1020, 2], [1021, 1]]", "ts_event_ns": 0, "ts_recv_ns": 0, }
def test_repr(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=AUDUSD, level=OrderBookLevel.L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], timestamp_ns=0, ) # Act # Assert assert ( repr(snapshot) == "OrderBookSnapshot('AUD/USD.SIM', bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], timestamp_ns=0)" )
def test_process_order_book_snapshot_when_one_subscriber_then_sends_to_registered_handler( self, ): # Arrange self.data_engine.register_client(self.binance_client) self.binance_client.connect() self.data_engine.process( ETHUSDT_BINANCE ) # <-- add necessary instrument for test handler = [] subscribe = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType( OrderBook, { "InstrumentId": ETHUSDT_BINANCE.id, "Level": OrderBookLevel.L2, "Depth": 25, "Interval": 0, # Streaming }, ), handler=handler.append, command_id=self.uuid_factory.generate(), timestamp_ns=self.clock.timestamp_ns(), ) self.data_engine.execute(subscribe) snapshot = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=OrderBookLevel.L2, bids=[[1000, 1]], asks=[[1001, 1]], timestamp_origin_ns=0, timestamp_ns=0, ) # Act self.data_engine.process(snapshot) # Assert assert self.data_engine.subscribed_order_books == [ETHUSDT_BINANCE.id] assert handler[0].instrument_id == ETHUSDT_BINANCE.id assert type(handler[0]) == L2OrderBook
def _build_order_book_snapshot(values): # First value is a CLEAR message, which we ignore return OrderBookSnapshot( instrument_id=InstrumentId.from_str(values[1]["instrument_id"]), level=BookLevelParser.from_str_py(values[1]["level"]), bids=[ (order["order_price"], order["order_size"]) for order in data[1:] if order["order_side"] == "BUY" ], asks=[ (order["order_price"], order["order_size"]) for order in data[1:] if order["order_side"] == "SELL" ], ts_event_ns=data[1]["ts_event_ns"], ts_recv_ns=data[1]["ts_recv_ns"], )
def _handle_market_snapshot(selection, instrument, timestamp_ns): updates = [] # Check we only have one of [best bets / depth bets / all bets] bid_keys = [k for k in B_BID_KINDS if k in selection] or ["atb"] ask_keys = [k for k in B_ASK_KINDS if k in selection] or ["atl"] assert len(bid_keys) <= 1 assert len(ask_keys) <= 1 # OrderBook Snapshot # TODO Clean this crap up if bid_keys[0] == "atb": bids = selection.get("atb", []) else: bids = [(p, v) for _, p, v in selection.get(bid_keys[0], [])] if ask_keys[0] == "atl": asks = selection.get("atl", []) else: asks = [(p, v) for _, p, v in selection.get(ask_keys[0], [])] snapshot = OrderBookSnapshot( level=OrderBookLevel.L2, instrument_id=instrument.id, bids=[(price_to_probability(p, OrderSide.BUY), v) for p, v in asks], asks=[(price_to_probability(p, OrderSide.SELL), v) for p, v in bids], timestamp_ns=timestamp_ns, ) updates.append(snapshot) # Trade Ticks for price, volume in selection.get("trd", []): trade_id = hash_json((timestamp_ns, price, volume)) tick = TradeTick( instrument_id=instrument.id, price=Price(price_to_probability(price, force=True)), size=Quantity(volume, precision=4), side=OrderSide.BUY, match_id=TradeMatchId(trade_id), timestamp_ns=timestamp_ns, ) updates.append(tick) return updates
def test_hash_str_and_repr(self): # Arrange snapshot = OrderBookSnapshot( instrument_id=AUDUSD, level=BookLevel.L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_event_ns=0, ts_recv_ns=0, ) # Act, Assert assert isinstance(hash(snapshot), int) assert ( str(snapshot) == "OrderBookSnapshot('AUD/USD.SIM', level=L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_recv_ns=0)" ) assert ( repr(snapshot) == "OrderBookSnapshot('AUD/USD.SIM', level=L2, bids=[[1010, 2], [1009, 1]], asks=[[1020, 2], [1021, 1]], ts_recv_ns=0)" )
def order_book_snapshot( instrument_id=None, bid_price=10, ask_price=15, bid_levels=3, ask_levels=3, bid_volume=10, ask_volume=10, level=OrderBookLevel.L2, ) -> OrderBookSnapshot: err = "Too many levels generated; orders will be in cross. Increase bid/ask spread or reduce number of levels" assert bid_price < ask_price, err return OrderBookSnapshot( instrument_id=instrument_id or TestStubs.audusd_id(), level=level, bids=[(bid_price - i, bid_volume * (1 + i)) for i in range(bid_levels)], asks=[(ask_price + i, ask_volume * (1 + i)) for i in range(ask_levels)], timestamp_ns=0, )
def test_process_order_book_when_multiple_subscribers_then_sends_to_registered_handlers( self, ): # Arrange self.data_engine.register_client(self.binance_client) self.binance_client.connect() self.data_engine.process( ETHUSDT_BINANCE) # <-- add necessary instrument for test handler1 = [] subscribe1 = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType( OrderBook, { "instrument_id": ETHUSDT_BINANCE.id, "level": BookLevel.L2, "depth": 25, "interval": 0, # Streaming }, ), handler=handler1.append, command_id=self.uuid_factory.generate(), timestamp_ns=self.clock.timestamp_ns(), ) handler2 = [] subscribe2 = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType( OrderBook, { "instrument_id": ETHUSDT_BINANCE.id, "level": BookLevel.L2, "depth": 25, "interval": 0, # Streaming }, ), handler=handler2.append, command_id=self.uuid_factory.generate(), timestamp_ns=self.clock.timestamp_ns(), ) self.data_engine.execute(subscribe1) self.data_engine.execute(subscribe2) snapshot = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=BookLevel.L2, bids=[[1000, 1]], asks=[[1001, 1]], ts_event_ns=0, ts_recv_ns=0, ) # Act self.data_engine.process(snapshot) # Assert cached_book = self.data_engine.cache.order_book(ETHUSDT_BINANCE.id) assert self.data_engine.subscribed_order_books == [ETHUSDT_BINANCE.id] assert type(cached_book) == L2OrderBook assert cached_book.instrument_id == ETHUSDT_BINANCE.id assert handler1[0] == cached_book assert handler2[0] == cached_book
def test_with_mix_of_stream_data_produces_correct_stream_of_data(self): # Assert snapshot1 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=BookLevel.L2, bids=[[1550.15, 0.51], [1580.00, 1.20]], asks=[[1552.15, 1.51], [1582.00, 2.20]], ts_event_ns=0, ts_recv_ns=0, ) data_type = DataType(MyData, metadata={"news_wire": "hacks"}) generic_data1 = [ GenericData( data_type, data=MyData("AAPL hacked"), ), GenericData( data_type, data=MyData("AMZN hacked", 500_000, 500_000), ), GenericData( data_type, data=MyData("NFLX hacked", 1_000_000, 1_000_000), ), GenericData( data_type, data=MyData("MSFT hacked", 2_000_000, 2_000_000), ), ] snapshot2 = OrderBookSnapshot( instrument_id=ETHUSDT_BINANCE.id, level=BookLevel.L2, bids=[[1551.15, 0.51], [1581.00, 1.20]], asks=[[1553.15, 1.51], [1583.00, 2.20]], ts_event_ns=1_000_000, ts_recv_ns=1_000_000, ) producer = BacktestDataProducer( logger=self.logger, instruments=[ETHUSDT_BINANCE], generic_data=generic_data1, order_book_data=[snapshot1, snapshot2], ) producer.setup(producer.min_timestamp_ns, producer.max_timestamp_ns) # Act streamed_data = [] while producer.has_data: streamed_data.append(producer.next()) # Assert timestamps = [x.ts_recv_ns for x in streamed_data] assert timestamps == [0, 0, 500000, 1000000, 1000000, 2000000] assert producer.min_timestamp_ns == 0 assert producer.max_timestamp_ns == 2_000_000 assert producer.min_timestamp == pd.Timestamp( "1970-01-01 00:00:00.000000+0000", tz="UTC") assert producer.max_timestamp == pd.Timestamp( "1970-01-01 00:00:00.002000+0000", tz="UTC")