class TestBetfairDataClient: def setup(self): # Fixture Setup self.loop = asyncio.get_event_loop() self.loop.set_debug(True) self.clock = LiveClock() self.uuid_factory = UUIDFactory() self.trader_id = TestStubs.trader_id() self.uuid = UUID4() self.venue = BETFAIR_VENUE self.account_id = AccountId(self.venue.value, "001") # Setup logging self.logger = LiveLogger(loop=self.loop, clock=self.clock, level_stdout=LogLevel.ERROR) self._log = LoggerAdapter("TestBetfairExecutionClient", self.logger) self.msgbus = MessageBus( trader_id=self.trader_id, clock=self.clock, logger=self.logger, ) self.cache = TestStubs.cache() self.cache.add_instrument(BetfairTestStubs.betting_instrument()) self.portfolio = Portfolio( msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) self.data_engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) self.betfair_client = BetfairTestStubs.betfair_client( loop=self.loop, logger=self.logger) self.instrument_provider = BetfairTestStubs.instrument_provider( betfair_client=self.betfair_client) # Add a subset of instruments instruments = [ ins for ins in INSTRUMENTS if ins.market_id in BetfairDataProvider.market_ids() ] self.instrument_provider.add_bulk(instruments) self.client = BetfairDataClient( loop=self.loop, client=self.betfair_client, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, instrument_provider=self.instrument_provider, market_filter={}, ) self.data_engine.register_client(self.client) # Re-route exec engine messages through `handler` self.messages = [] def handler(x, endpoint): self.messages.append(x) if endpoint == "execute": self.data_engine.execute(x) elif endpoint == "process": self.data_engine.process(x) elif endpoint == "response": self.data_engine.response(x) self.msgbus.deregister( endpoint="DataEngine.execute", handler=self.data_engine.execute) # type: ignore self.msgbus.register( endpoint="DataEngine.execute", handler=partial(handler, endpoint="execute") # type: ignore ) self.msgbus.deregister( endpoint="DataEngine.process", handler=self.data_engine.process) # type: ignore self.msgbus.register( endpoint="DataEngine.process", handler=partial(handler, endpoint="process") # type: ignore ) self.msgbus.deregister( endpoint="DataEngine.response", handler=self.data_engine.response) # type: ignore self.msgbus.register( endpoint="DataEngine.response", handler=partial(handler, endpoint="response") # type: ignore ) @pytest.mark.asyncio @patch( "nautilus_trader.adapters.betfair.data.BetfairDataClient._post_connect_heartbeat" ) @patch( "nautilus_trader.adapters.betfair.data.BetfairMarketStreamClient.connect" ) @patch("nautilus_trader.adapters.betfair.client.core.BetfairClient.connect" ) async def test_connect(self, mock_client_connect, mock_stream_connect, mock_post_connect_heartbeat): await self.client._connect() def test_subscriptions(self): self.client.subscribe_trade_ticks(BetfairTestStubs.instrument_id()) self.client.subscribe_instrument_status_updates( BetfairTestStubs.instrument_id()) self.client.subscribe_instrument_close_prices( BetfairTestStubs.instrument_id()) def test_market_heartbeat(self): self.client._on_market_update(BetfairStreaming.mcm_HEARTBEAT()) def test_stream_latency(self): logs = [] self.logger.register_sink(logs.append) self.client.start() self.client._on_market_update(BetfairStreaming.mcm_latency()) warning, degrading, degraded = logs[2:] assert warning["level"] == "WRN" assert warning["msg"] == "Stream unhealthy, waiting for recover" assert degraded["msg"] == "DEGRADED." def test_stream_con_true(self): logs = [] self.logger.register_sink(logs.append) self.client._on_market_update(BetfairStreaming.mcm_con_true()) (warning, ) = logs assert warning["level"] == "WRN" assert ( warning["msg"] == "Conflated stream - consuming data too slow (data received is delayed)" ) @pytest.mark.asyncio async def test_market_sub_image_market_def(self): update = BetfairStreaming.mcm_SUB_IMAGE() self.client._on_market_update(update) result = [type(event).__name__ for event in self.messages] expected = ["InstrumentStatusUpdate"] * 7 + ["OrderBookSnapshot"] * 7 assert result == expected # Check prices are probabilities result = set( float(order[0]) for ob_snap in self.messages if isinstance(ob_snap, OrderBookSnapshot) for order in ob_snap.bids + ob_snap.asks) expected = set([ 0.0010204, 0.0076923, 0.0217391, 0.0238095, 0.1724138, 0.2173913, 0.3676471, 0.3937008, 0.4587156, 0.5555556, ]) assert result == expected def test_market_sub_image_no_market_def(self): self.client._on_market_update( BetfairStreaming.mcm_SUB_IMAGE_no_market_def()) result = Counter([type(event).__name__ for event in self.messages]) expected = Counter({ "InstrumentStatusUpdate": 270, "OrderBookSnapshot": 270, "InstrumentClosePrice": 22, }) assert result == expected def test_market_resub_delta(self): self.client._on_market_update(BetfairStreaming.mcm_RESUB_DELTA()) result = [type(event).__name__ for event in self.messages] expected = ["InstrumentStatusUpdate"] * 12 + ["OrderBookDeltas"] * 269 assert result == expected def test_market_update(self): self.client._on_market_update(BetfairStreaming.mcm_UPDATE()) result = [type(event).__name__ for event in self.messages] expected = ["OrderBookDeltas"] * 1 assert result == expected result = [d.action for d in self.messages[0].deltas] expected = [BookAction.UPDATE, BookAction.DELETE] assert result == expected # Ensure order prices are coming through as probability update_op = self.messages[0].deltas[0] assert update_op.order.price == 0.212766 def test_market_update_md(self): self.client._on_market_update(BetfairStreaming.mcm_UPDATE_md()) result = [type(event).__name__ for event in self.messages] expected = ["InstrumentStatusUpdate"] * 2 assert result == expected def test_market_update_live_image(self): self.client._on_market_update(BetfairStreaming.mcm_live_IMAGE()) result = [type(event).__name__ for event in self.messages] expected = (["OrderBookSnapshot"] + ["TradeTick"] * 13 + ["OrderBookSnapshot"] + ["TradeTick"] * 17) assert result == expected def test_market_update_live_update(self): self.client._on_market_update(BetfairStreaming.mcm_live_UPDATE()) result = [type(event).__name__ for event in self.messages] expected = ["TradeTick", "OrderBookDeltas"] assert result == expected def test_market_bsp(self): # Setup update = BetfairStreaming.mcm_BSP() provider = self.client.instrument_provider() for mc in update[0]["mc"]: market_def = {**mc["marketDefinition"], "marketId": mc["id"]} instruments = make_instruments(market_definition=market_def, currency="GBP") provider.add_bulk(instruments) for update in update: self.client._on_market_update(update) result = Counter([type(event).__name__ for event in self.messages]) expected = { "TradeTick": 95, "BSPOrderBookDelta": 30, "InstrumentStatusUpdate": 9, "OrderBookSnapshot": 8, "OrderBookDeltas": 2, } assert result == expected @pytest.mark.asyncio async def test_request_search_instruments(self): req = DataType( type=InstrumentSearch, metadata={"event_type_id": "7"}, ) self.client.request(req, self.uuid) await asyncio.sleep(0) resp = self.messages[0] assert len(resp.data.instruments) == 6800 def test_orderbook_repr(self): self.client._on_market_update(BetfairStreaming.mcm_live_IMAGE()) ob_snap = self.messages[14] ob = L2OrderBook(InstrumentId(Symbol("1"), BETFAIR_VENUE), 5, 5) ob.apply_snapshot(ob_snap) print(ob.pprint()) assert ob.best_ask_price() == 0.5882353 assert ob.best_bid_price() == 0.5847953 def test_orderbook_updates(self): order_books = {} for raw_update in BetfairStreaming.market_updates(): for update in on_market_update( update=raw_update, instrument_provider=self.client.instrument_provider(), ): if len(order_books) > 1 and update.instrument_id != list( order_books)[1]: continue print(update) if isinstance(update, OrderBookSnapshot): order_books[update.instrument_id] = L2OrderBook( instrument_id=update.instrument_id, price_precision=4, size_precision=4, ) order_books[update.instrument_id].apply_snapshot(update) elif isinstance(update, OrderBookDeltas): order_books[update.instrument_id].apply_deltas(update) elif isinstance(update, TradeTick): pass else: raise KeyError book = order_books[list(order_books)[0]] expected = """bids price asks -------- ------- --------- 0.8621 [932.64] 0.8547 [1275.83] 0.8475 [151.96] [147.79] 0.8403 [156.74] 0.8333 [11.19] 0.8197""" result = book.pprint() assert result == expected def test_instrument_opening_events(self): updates = BetfairDataProvider.raw_market_updates() messages = on_market_update( instrument_provider=self.client.instrument_provider(), update=updates[0]) assert len(messages) == 2 assert (isinstance(messages[0], InstrumentStatusUpdate) and messages[0].status == InstrumentStatus.PRE_OPEN) assert (isinstance(messages[1], InstrumentStatusUpdate) and messages[0].status == InstrumentStatus.PRE_OPEN) def test_instrument_in_play_events(self): events = [ msg for update in BetfairDataProvider.raw_market_updates() for msg in on_market_update( instrument_provider=self.client.instrument_provider(), update=update) if isinstance(msg, InstrumentStatusUpdate) ] assert len(events) == 14 result = [ev.status for ev in events] expected = [ InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PRE_OPEN.value, InstrumentStatus.PAUSE.value, InstrumentStatus.PAUSE.value, InstrumentStatus.OPEN.value, InstrumentStatus.OPEN.value, InstrumentStatus.PAUSE.value, InstrumentStatus.PAUSE.value, InstrumentStatus.CLOSED.value, InstrumentStatus.CLOSED.value, ] assert result == expected def test_instrument_closing_events(self): updates = BetfairDataProvider.raw_market_updates() messages = on_market_update( instrument_provider=self.client.instrument_provider(), update=updates[-1], ) assert len(messages) == 4 assert (isinstance(messages[0], InstrumentStatusUpdate) and messages[0].status == InstrumentStatus.CLOSED) assert isinstance( messages[1], InstrumentClosePrice) and messages[1].close_price == 1.0000 assert (isinstance(messages[1], InstrumentClosePrice) and messages[1].close_type == InstrumentCloseType.EXPIRED) assert (isinstance(messages[2], InstrumentStatusUpdate) and messages[2].status == InstrumentStatus.CLOSED) assert isinstance( messages[3], InstrumentClosePrice) and messages[3].close_price == 0.0 assert (isinstance(messages[3], InstrumentClosePrice) and messages[3].close_type == InstrumentCloseType.EXPIRED) def test_betfair_ticker(self): self.client._on_market_update(BetfairStreaming.mcm_UPDATE_tv()) ticker: BetfairTicker = self.messages[1] assert ticker.last_traded_price == Price.from_str("0.3174603") assert ticker.traded_volume == Quantity.from_str("364.45") def test_betfair_orderbook(self): book = L2OrderBook( instrument_id=BetfairTestStubs.instrument_id(), price_precision=2, size_precision=2, ) for update in BetfairDataProvider.raw_market_updates(): for message in on_market_update( instrument_provider=self.instrument_provider, update=update): try: if isinstance(message, OrderBookSnapshot): book.apply_snapshot(message) elif isinstance(message, OrderBookDeltas): book.apply_deltas(message) elif isinstance(message, OrderBookDelta): book.apply_delta(message) elif isinstance(message, (Ticker, TradeTick, InstrumentStatusUpdate, InstrumentClosePrice)): pass else: raise NotImplementedError(str(type(message))) book.check_integrity() except Exception as ex: print(str(type(ex)) + " " + str(ex))
class TestLiveDataEngine: def setup(self): # Fixture Setup self.loop = asyncio.get_event_loop() self.loop.set_debug(True) self.clock = LiveClock() self.uuid_factory = UUIDFactory() self.logger = Logger(self.clock) self.trader_id = TestStubs.trader_id() self.msgbus = MessageBus( trader_id=self.trader_id, clock=self.clock, logger=self.logger, ) self.cache = TestStubs.cache() self.portfolio = Portfolio( msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) def teardown(self): self.engine.dispose() @pytest.mark.asyncio async def test_start_when_loop_not_running_logs(self): # Arrange, Act self.engine.start() # Assert assert True # No exceptions raised self.engine.stop() @pytest.mark.asyncio async def test_message_qsize_at_max_blocks_on_put_data_command(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) subscribe = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType(QuoteTick), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.execute(subscribe) self.engine.execute(subscribe) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 1 assert self.engine.command_count == 0 @pytest.mark.asyncio async def test_message_qsize_at_max_blocks_on_send_request(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) handler = [] request = DataRequest( client_id=ClientId("RANDOM"), data_type=DataType( QuoteTick, metadata={ "instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")), "from_datetime": None, "to_datetime": None, "limit": 1000, }, ), callback=handler.append, request_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.request(request) self.engine.request(request) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 1 assert self.engine.command_count == 0 @pytest.mark.asyncio async def test_message_qsize_at_max_blocks_on_receive_response(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) response = DataResponse( client_id=ClientId("BINANCE"), data_type=DataType(QuoteTick), data=[], correlation_id=self.uuid_factory.generate(), response_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.response(response) self.engine.response(response) # Add over max size await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 1 assert self.engine.command_count == 0 @pytest.mark.asyncio async def test_data_qsize_at_max_blocks_on_put_data(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) data = Data(1_000_000_000, 1_000_000_000) # Act self.engine.process(data) self.engine.process(data) # Add over max size await asyncio.sleep(0.1) # Assert assert self.engine.data_qsize() == 1 assert self.engine.data_count == 0 def test_get_event_loop_returns_expected_loop(self): # Arrange, Act loop = self.engine.get_event_loop() # Assert assert loop == self.loop @pytest.mark.asyncio async def test_start(self): # Arrange, Act self.engine.start() await asyncio.sleep(0.1) # Assert assert self.engine.is_running # Tear Down self.engine.stop() @pytest.mark.asyncio async def test_kill_when_running_and_no_messages_on_queues(self): # Arrange, Act self.engine.start() await asyncio.sleep(0) self.engine.kill() # Assert assert self.engine.is_stopped @pytest.mark.asyncio async def test_kill_when_not_running_with_messages_on_queue(self): # Arrange, Act self.engine.kill() # Assert assert self.engine.data_qsize() == 0 @pytest.mark.asyncio async def test_execute_command_processes_message(self): # Arrange self.engine.start() subscribe = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType(QuoteTick), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.execute(subscribe) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 0 assert self.engine.command_count == 1 # Tear Down self.engine.stop() @pytest.mark.asyncio async def test_send_request_processes_message(self): # Arrange self.engine.start() handler = [] request = DataRequest( client_id=ClientId("RANDOM"), data_type=DataType( QuoteTick, metadata={ "instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")), "from_datetime": None, "to_datetime": None, "limit": 1000, }, ), callback=handler.append, request_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.request(request) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 0 assert self.engine.request_count == 1 # Tear Down self.engine.stop() @pytest.mark.asyncio async def test_receive_response_processes_message(self): # Arrange self.engine.start() response = DataResponse( client_id=ClientId("BINANCE"), data_type=DataType(QuoteTick), data=[], correlation_id=self.uuid_factory.generate(), response_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.response(response) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 0 assert self.engine.response_count == 1 # Tear Down self.engine.stop() @pytest.mark.asyncio async def test_process_data_processes_data(self): # Arrange self.engine.start() # Act tick = TestStubs.trade_tick_5decimal() # Act self.engine.process(tick) await asyncio.sleep(0.1) # Assert assert self.engine.data_qsize() == 0 assert self.engine.data_count == 1 # Tear Down self.engine.stop()