class TestBetfairDataClient:
def setup(self):
# Fixture Setup
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.clock = LiveClock()
self.uuid_factory = UUIDFactory()
self.trader_id = TestStubs.trader_id()
self.uuid = UUID4()
self.venue = BETFAIR_VENUE
self.account_id = AccountId(self.venue.value, "001")
# Setup logging
self.logger = LiveLogger(loop=self.loop,
clock=self.clock,
level_stdout=LogLevel.ERROR)
self._log = LoggerAdapter("TestBetfairExecutionClient", self.logger)
self.msgbus = MessageBus(
trader_id=self.trader_id,
clock=self.clock,
logger=self.logger,
)
self.cache = TestStubs.cache()
self.cache.add_instrument(BetfairTestStubs.betting_instrument())
self.portfolio = Portfolio(
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
self.data_engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
self.betfair_client = BetfairTestStubs.betfair_client(
loop=self.loop, logger=self.logger)
self.instrument_provider = BetfairTestStubs.instrument_provider(
betfair_client=self.betfair_client)
# Add a subset of instruments
instruments = [
ins for ins in INSTRUMENTS
if ins.market_id in BetfairDataProvider.market_ids()
]
self.instrument_provider.add_bulk(instruments)
self.client = BetfairDataClient(
loop=self.loop,
client=self.betfair_client,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
instrument_provider=self.instrument_provider,
market_filter={},
)
self.data_engine.register_client(self.client)
# Re-route exec engine messages through `handler`
self.messages = []
def handler(x, endpoint):
self.messages.append(x)
if endpoint == "execute":
self.data_engine.execute(x)
elif endpoint == "process":
self.data_engine.process(x)
elif endpoint == "response":
self.data_engine.response(x)
self.msgbus.deregister(
endpoint="DataEngine.execute",
handler=self.data_engine.execute) # type: ignore
self.msgbus.register(
endpoint="DataEngine.execute",
handler=partial(handler, endpoint="execute") # type: ignore
)
self.msgbus.deregister(
endpoint="DataEngine.process",
handler=self.data_engine.process) # type: ignore
self.msgbus.register(
endpoint="DataEngine.process",
handler=partial(handler, endpoint="process") # type: ignore
)
self.msgbus.deregister(
endpoint="DataEngine.response",
handler=self.data_engine.response) # type: ignore
self.msgbus.register(
endpoint="DataEngine.response",
handler=partial(handler, endpoint="response") # type: ignore
)
@pytest.mark.asyncio
@patch(
"nautilus_trader.adapters.betfair.data.BetfairDataClient._post_connect_heartbeat"
)
@patch(
"nautilus_trader.adapters.betfair.data.BetfairMarketStreamClient.connect"
)
@patch("nautilus_trader.adapters.betfair.client.core.BetfairClient.connect"
)
async def test_connect(self, mock_client_connect, mock_stream_connect,
mock_post_connect_heartbeat):
await self.client._connect()
def test_subscriptions(self):
self.client.subscribe_trade_ticks(BetfairTestStubs.instrument_id())
self.client.subscribe_instrument_status_updates(
BetfairTestStubs.instrument_id())
self.client.subscribe_instrument_close_prices(
BetfairTestStubs.instrument_id())
def test_market_heartbeat(self):
self.client._on_market_update(BetfairStreaming.mcm_HEARTBEAT())
def test_stream_latency(self):
logs = []
self.logger.register_sink(logs.append)
self.client.start()
self.client._on_market_update(BetfairStreaming.mcm_latency())
warning, degrading, degraded = logs[2:]
assert warning["level"] == "WRN"
assert warning["msg"] == "Stream unhealthy, waiting for recover"
assert degraded["msg"] == "DEGRADED."
def test_stream_con_true(self):
logs = []
self.logger.register_sink(logs.append)
self.client._on_market_update(BetfairStreaming.mcm_con_true())
(warning, ) = logs
assert warning["level"] == "WRN"
assert (
warning["msg"] ==
"Conflated stream - consuming data too slow (data received is delayed)"
)
@pytest.mark.asyncio
async def test_market_sub_image_market_def(self):
update = BetfairStreaming.mcm_SUB_IMAGE()
self.client._on_market_update(update)
result = [type(event).__name__ for event in self.messages]
expected = ["InstrumentStatusUpdate"] * 7 + ["OrderBookSnapshot"] * 7
assert result == expected
# Check prices are probabilities
result = set(
float(order[0]) for ob_snap in self.messages
if isinstance(ob_snap, OrderBookSnapshot)
for order in ob_snap.bids + ob_snap.asks)
expected = set([
0.0010204,
0.0076923,
0.0217391,
0.0238095,
0.1724138,
0.2173913,
0.3676471,
0.3937008,
0.4587156,
0.5555556,
])
assert result == expected
def test_market_sub_image_no_market_def(self):
self.client._on_market_update(
BetfairStreaming.mcm_SUB_IMAGE_no_market_def())
result = Counter([type(event).__name__ for event in self.messages])
expected = Counter({
"InstrumentStatusUpdate": 270,
"OrderBookSnapshot": 270,
"InstrumentClosePrice": 22,
})
assert result == expected
def test_market_resub_delta(self):
self.client._on_market_update(BetfairStreaming.mcm_RESUB_DELTA())
result = [type(event).__name__ for event in self.messages]
expected = ["InstrumentStatusUpdate"] * 12 + ["OrderBookDeltas"] * 269
assert result == expected
def test_market_update(self):
self.client._on_market_update(BetfairStreaming.mcm_UPDATE())
result = [type(event).__name__ for event in self.messages]
expected = ["OrderBookDeltas"] * 1
assert result == expected
result = [d.action for d in self.messages[0].deltas]
expected = [BookAction.UPDATE, BookAction.DELETE]
assert result == expected
# Ensure order prices are coming through as probability
update_op = self.messages[0].deltas[0]
assert update_op.order.price == 0.212766
def test_market_update_md(self):
self.client._on_market_update(BetfairStreaming.mcm_UPDATE_md())
result = [type(event).__name__ for event in self.messages]
expected = ["InstrumentStatusUpdate"] * 2
assert result == expected
def test_market_update_live_image(self):
self.client._on_market_update(BetfairStreaming.mcm_live_IMAGE())
result = [type(event).__name__ for event in self.messages]
expected = (["OrderBookSnapshot"] + ["TradeTick"] * 13 +
["OrderBookSnapshot"] + ["TradeTick"] * 17)
assert result == expected
def test_market_update_live_update(self):
self.client._on_market_update(BetfairStreaming.mcm_live_UPDATE())
result = [type(event).__name__ for event in self.messages]
expected = ["TradeTick", "OrderBookDeltas"]
assert result == expected
def test_market_bsp(self):
# Setup
update = BetfairStreaming.mcm_BSP()
provider = self.client.instrument_provider()
for mc in update[0]["mc"]:
market_def = {**mc["marketDefinition"], "marketId": mc["id"]}
instruments = make_instruments(market_definition=market_def,
currency="GBP")
provider.add_bulk(instruments)
for update in update:
self.client._on_market_update(update)
result = Counter([type(event).__name__ for event in self.messages])
expected = {
"TradeTick": 95,
"BSPOrderBookDelta": 30,
"InstrumentStatusUpdate": 9,
"OrderBookSnapshot": 8,
"OrderBookDeltas": 2,
}
assert result == expected
@pytest.mark.asyncio
async def test_request_search_instruments(self):
req = DataType(
type=InstrumentSearch,
metadata={"event_type_id": "7"},
)
self.client.request(req, self.uuid)
await asyncio.sleep(0)
resp = self.messages[0]
assert len(resp.data.instruments) == 6800
def test_orderbook_repr(self):
self.client._on_market_update(BetfairStreaming.mcm_live_IMAGE())
ob_snap = self.messages[14]
ob = L2OrderBook(InstrumentId(Symbol("1"), BETFAIR_VENUE), 5, 5)
ob.apply_snapshot(ob_snap)
print(ob.pprint())
assert ob.best_ask_price() == 0.5882353
assert ob.best_bid_price() == 0.5847953
def test_orderbook_updates(self):
order_books = {}
for raw_update in BetfairStreaming.market_updates():
for update in on_market_update(
update=raw_update,
instrument_provider=self.client.instrument_provider(),
):
if len(order_books) > 1 and update.instrument_id != list(
order_books)[1]:
continue
print(update)
if isinstance(update, OrderBookSnapshot):
order_books[update.instrument_id] = L2OrderBook(
instrument_id=update.instrument_id,
price_precision=4,
size_precision=4,
)
order_books[update.instrument_id].apply_snapshot(update)
elif isinstance(update, OrderBookDeltas):
order_books[update.instrument_id].apply_deltas(update)
elif isinstance(update, TradeTick):
pass
else:
raise KeyError
book = order_books[list(order_books)[0]]
expected = """bids price asks
-------- ------- ---------
0.8621 [932.64]
0.8547 [1275.83]
0.8475 [151.96]
[147.79] 0.8403
[156.74] 0.8333
[11.19] 0.8197"""
result = book.pprint()
assert result == expected
def test_instrument_opening_events(self):
updates = BetfairDataProvider.raw_market_updates()
messages = on_market_update(
instrument_provider=self.client.instrument_provider(),
update=updates[0])
assert len(messages) == 2
assert (isinstance(messages[0], InstrumentStatusUpdate)
and messages[0].status == InstrumentStatus.PRE_OPEN)
assert (isinstance(messages[1], InstrumentStatusUpdate)
and messages[0].status == InstrumentStatus.PRE_OPEN)
def test_instrument_in_play_events(self):
events = [
msg for update in BetfairDataProvider.raw_market_updates()
for msg in on_market_update(
instrument_provider=self.client.instrument_provider(),
update=update) if isinstance(msg, InstrumentStatusUpdate)
]
assert len(events) == 14
result = [ev.status for ev in events]
expected = [
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PRE_OPEN.value,
InstrumentStatus.PAUSE.value,
InstrumentStatus.PAUSE.value,
InstrumentStatus.OPEN.value,
InstrumentStatus.OPEN.value,
InstrumentStatus.PAUSE.value,
InstrumentStatus.PAUSE.value,
InstrumentStatus.CLOSED.value,
InstrumentStatus.CLOSED.value,
]
assert result == expected
def test_instrument_closing_events(self):
updates = BetfairDataProvider.raw_market_updates()
messages = on_market_update(
instrument_provider=self.client.instrument_provider(),
update=updates[-1],
)
assert len(messages) == 4
assert (isinstance(messages[0], InstrumentStatusUpdate)
and messages[0].status == InstrumentStatus.CLOSED)
assert isinstance(
messages[1],
InstrumentClosePrice) and messages[1].close_price == 1.0000
assert (isinstance(messages[1], InstrumentClosePrice)
and messages[1].close_type == InstrumentCloseType.EXPIRED)
assert (isinstance(messages[2], InstrumentStatusUpdate)
and messages[2].status == InstrumentStatus.CLOSED)
assert isinstance(
messages[3],
InstrumentClosePrice) and messages[3].close_price == 0.0
assert (isinstance(messages[3], InstrumentClosePrice)
and messages[3].close_type == InstrumentCloseType.EXPIRED)
def test_betfair_ticker(self):
self.client._on_market_update(BetfairStreaming.mcm_UPDATE_tv())
ticker: BetfairTicker = self.messages[1]
assert ticker.last_traded_price == Price.from_str("0.3174603")
assert ticker.traded_volume == Quantity.from_str("364.45")
def test_betfair_orderbook(self):
book = L2OrderBook(
instrument_id=BetfairTestStubs.instrument_id(),
price_precision=2,
size_precision=2,
)
for update in BetfairDataProvider.raw_market_updates():
for message in on_market_update(
instrument_provider=self.instrument_provider,
update=update):
try:
if isinstance(message, OrderBookSnapshot):
book.apply_snapshot(message)
elif isinstance(message, OrderBookDeltas):
book.apply_deltas(message)
elif isinstance(message, OrderBookDelta):
book.apply_delta(message)
elif isinstance(message,
(Ticker, TradeTick, InstrumentStatusUpdate,
InstrumentClosePrice)):
pass
else:
raise NotImplementedError(str(type(message)))
book.check_integrity()
except Exception as ex:
print(str(type(ex)) + " " + str(ex))
class TestLiveDataEngine:
def setup(self):
# Fixture Setup
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.clock = LiveClock()
self.uuid_factory = UUIDFactory()
self.logger = Logger(self.clock)
self.trader_id = TestStubs.trader_id()
self.msgbus = MessageBus(
trader_id=self.trader_id,
clock=self.clock,
logger=self.logger,
)
self.cache = TestStubs.cache()
self.portfolio = Portfolio(
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
def teardown(self):
self.engine.dispose()
@pytest.mark.asyncio
async def test_start_when_loop_not_running_logs(self):
# Arrange, Act
self.engine.start()
# Assert
assert True # No exceptions raised
self.engine.stop()
@pytest.mark.asyncio
async def test_message_qsize_at_max_blocks_on_put_data_command(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
subscribe = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(QuoteTick),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.execute(subscribe)
self.engine.execute(subscribe)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
@pytest.mark.asyncio
async def test_message_qsize_at_max_blocks_on_send_request(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
@pytest.mark.asyncio
async def test_message_qsize_at_max_blocks_on_receive_response(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
response = DataResponse(
client_id=ClientId("BINANCE"),
data_type=DataType(QuoteTick),
data=[],
correlation_id=self.uuid_factory.generate(),
response_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.response(response)
self.engine.response(response) # Add over max size
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
@pytest.mark.asyncio
async def test_data_qsize_at_max_blocks_on_put_data(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
data = Data(1_000_000_000, 1_000_000_000)
# Act
self.engine.process(data)
self.engine.process(data) # Add over max size
await asyncio.sleep(0.1)
# Assert
assert self.engine.data_qsize() == 1
assert self.engine.data_count == 0
def test_get_event_loop_returns_expected_loop(self):
# Arrange, Act
loop = self.engine.get_event_loop()
# Assert
assert loop == self.loop
@pytest.mark.asyncio
async def test_start(self):
# Arrange, Act
self.engine.start()
await asyncio.sleep(0.1)
# Assert
assert self.engine.is_running
# Tear Down
self.engine.stop()
@pytest.mark.asyncio
async def test_kill_when_running_and_no_messages_on_queues(self):
# Arrange, Act
self.engine.start()
await asyncio.sleep(0)
self.engine.kill()
# Assert
assert self.engine.is_stopped
@pytest.mark.asyncio
async def test_kill_when_not_running_with_messages_on_queue(self):
# Arrange, Act
self.engine.kill()
# Assert
assert self.engine.data_qsize() == 0
@pytest.mark.asyncio
async def test_execute_command_processes_message(self):
# Arrange
self.engine.start()
subscribe = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(QuoteTick),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.execute(subscribe)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.command_count == 1
# Tear Down
self.engine.stop()
@pytest.mark.asyncio
async def test_send_request_processes_message(self):
# Arrange
self.engine.start()
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.request_count == 1
# Tear Down
self.engine.stop()
@pytest.mark.asyncio
async def test_receive_response_processes_message(self):
# Arrange
self.engine.start()
response = DataResponse(
client_id=ClientId("BINANCE"),
data_type=DataType(QuoteTick),
data=[],
correlation_id=self.uuid_factory.generate(),
response_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.response(response)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.response_count == 1
# Tear Down
self.engine.stop()
@pytest.mark.asyncio
async def test_process_data_processes_data(self):
# Arrange
self.engine.start()
# Act
tick = TestStubs.trade_tick_5decimal()
# Act
self.engine.process(tick)
await asyncio.sleep(0.1)
# Assert
assert self.engine.data_qsize() == 0
assert self.engine.data_count == 1
# Tear Down
self.engine.stop()
