def instrument_list(mock_load_markets_metadata,
loop: asyncio.AbstractEventLoop):
"""Prefill `INSTRUMENTS` cache for tests"""
global INSTRUMENTS
# Setup
logger = LiveLogger(loop=loop,
clock=LiveClock(),
level_stdout=LogLevel.ERROR)
client = BetfairTestStubs.betfair_client(loop=loop, logger=logger)
logger = LiveLogger(loop=loop,
clock=LiveClock(),
level_stdout=LogLevel.DEBUG)
instrument_provider = BetfairInstrumentProvider(client=client,
logger=logger,
market_filter={})
# Load instruments
market_ids = BetfairDataProvider.market_ids()
catalog = {
r["marketId"]: r
for r in BetfairResponses.betting_list_market_catalogue()["result"]
if r["marketId"] in market_ids
}
mock_load_markets_metadata.return_value = catalog
t = loop.create_task(
instrument_provider.load_all_async(
market_filter={"market_id": market_ids}))
loop.run_until_complete(t)
# Fill INSTRUMENTS global cache
INSTRUMENTS.extend(instrument_provider.list_all())
assert INSTRUMENTS
def setup(self):
# Fixture Setup
self.loop = asyncio.get_event_loop()
self.clock = LiveClock()
self.logger = LiveLogger(loop=self.loop, clock=self.clock)
self.client = BetfairTestStubs.betfair_client(loop=self.loop,
logger=self.logger)
self.provider = BetfairInstrumentProvider(
client=self.client,
logger=TestComponentStubs.logger(),
)
def setup(self):
# Fixture Setup
self.loop = asyncio.get_event_loop()
self.clock = LiveClock()
self.logger = LiveLogger(loop=self.loop, clock=self.clock)
self.client = BetfairTestStubs.betfair_client(loop=self.loop,
logger=self.logger)
self.provider = BetfairInstrumentProvider(
client=self.client,
logger=BetfairTestStubs.live_logger(BetfairTestStubs.clock()),
market_filter=None,
)
def get_cached_betfair_instrument_provider(
client: BetfairClient,
logger: Logger,
market_filter: tuple,
) -> BetfairInstrumentProvider:
"""
Cache and return a BetfairInstrumentProvider.
If a cached provider already exists, then that cached provider will be returned.
Parameters
----------
client : BinanceHttpClient
The client for the instrument provider.
logger : Logger
The logger for the instrument provider.
market_filter : tuple
The market filter to load into the instrument provider.
Returns
-------
BinanceInstrumentProvider
"""
LoggerAdapter(
"BetfairFactory",
logger).warning("Creating new instance of BetfairInstrumentProvider")
return BetfairInstrumentProvider(client=client,
logger=logger,
market_filter=dict(market_filter))
def test_byte_reader_parser(self):
def block_parser(block: bytes, instrument_provider):
for raw in block.split(b"\\n"):
ts, line = raw.split(b" - ")
state = {
"ts_init":
int(pd.Timestamp(ts.decode(), tz="UTC").to_datetime64())
}
line = line.strip().replace(b"b'", b"")
orjson.loads(line)
for obj in BetfairTestStubs.parse_betfair(
line, instrument_provider=instrument_provider):
values = obj.to_dict(obj)
values["ts_init"] = state["ts_init"]
yield obj.from_dict(values)
provider = BetfairInstrumentProvider.from_instruments(
[BetfairTestStubs.betting_instrument()])
block = BetfairDataProvider.badly_formatted_log()
reader = ByteReader(
block_parser=partial(block_parser, instrument_provider=provider),
instrument_provider=provider,
)
data = list(reader.parse(block=block))
result = [pd.Timestamp(d.ts_init).isoformat() for d in data]
expected = ["2021-06-29T06:03:14.528000"]
assert result == expected
def _loaded_data_into_catalog(self):
self.instrument_provider = BetfairInstrumentProvider.from_instruments([])
process_files(
glob_path=PACKAGE_ROOT + "/data/1.166564490.bz2",
reader=BetfairTestStubs.betfair_reader(instrument_provider=self.instrument_provider),
instrument_provider=self.instrument_provider,
catalog=self.catalog,
)
def test_text_reader(self):
provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(provider) # type: TextReader
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/betfair/1.166811431.bz2")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 22692
def betfair_reader(instrument_provider=None, **kwargs):
instrument_provider = instrument_provider or BetfairInstrumentProvider.from_instruments(
[])
reader = TextReader(
line_parser=partial(BetfairTestStubs.parse_betfair,
instrument_provider=instrument_provider),
instrument_provider=instrument_provider,
instrument_provider_update=historical_instrument_provider_loader,
**kwargs,
)
return reader
def make_betfair_reader(instrument_provider=None, line_preprocessor=None) -> TextReader:
from nautilus_trader.adapters.betfair.providers import BetfairInstrumentProvider
instrument_provider = instrument_provider or BetfairInstrumentProvider.from_instruments([])
return TextReader(
# Use the standard `on_market_update` betfair parser that the adapter uses
line_preprocessor=line_preprocessor,
line_parser=partial(line_parser, instrument_provider=instrument_provider),
instrument_provider_update=historical_instrument_provider_loader,
instrument_provider=instrument_provider,
)
def betfair_feed_parsed(market_id="1.166564490", folder="data"):
instrument_provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(
instrument_provider=instrument_provider)
files = make_raw_files(
glob_path=f"{PACKAGE_ROOT}/{folder}/{market_id}*")
data = []
for rf in files:
for block in rf.iter():
data.extend(reader.parse(block=block))
return data
def data_loader():
instrument_provider = BetfairInstrumentProvider.from_instruments([])
parser = TextParser(
parser=lambda x, state: on_market_update(
instrument_provider=instrument_provider, update=orjson.loads(x)),
instrument_provider_update=historical_instrument_provider_loader,
)
return DataLoader(
path=TEST_DATA_DIR,
parser=parser,
glob_pattern="1.166564490*",
instrument_provider=instrument_provider,
)
def build_market_snapshot_messages(
raw, instrument_provider: BetfairInstrumentProvider
) -> List[OrderBookSnapshot]:
updates = []
for market in raw.get("mc", []):
# Market status events
# market_definition = market.get("marketDefinition", {})
# TODO - Need to handle instrument status = CLOSED here
# Orderbook snapshots
if market.get("img") is True:
market_id = market["id"]
for (selection_id, handicap), selections in itertools.groupby(
market.get("rc", []), lambda x: (x["id"], x.get("hc"))):
for selection in list(selections):
kw = dict(
market_id=market_id,
selection_id=str(selection_id),
handicap=str(handicap or "0.0"),
)
instrument = instrument_provider.get_betting_instrument(
**kw)
# Check we only have one of [best bets / depth bets / all bets]
bid_keys = [k for k in B_BID_KINDS if k in selection
] or ["atb"]
ask_keys = [k for k in B_ASK_KINDS if k in selection
] or ["atl"]
assert len(bid_keys) <= 1
assert len(ask_keys) <= 1
# TODO Clean this crap up
if bid_keys[0] == "atb":
bids = selection.get("atb", [])
else:
bids = [(p, v)
for _, p, v in selection.get(bid_keys[0], [])]
if ask_keys[0] == "atl":
asks = selection.get("atl", [])
else:
asks = [(p, v)
for _, p, v in selection.get(ask_keys[0], [])]
snapshot = OrderBookSnapshot(
level=OrderBookLevel.L2,
instrument_id=instrument.id,
bids=[(price_to_probability(p, OrderSide.BUY), v)
for p, v in asks],
asks=[(price_to_probability(p, OrderSide.SELL), v)
for p, v in bids],
timestamp_ns=millis_to_nanos(raw["pt"]),
)
updates.append(snapshot)
return updates
def instrument_provider(betfair_client) -> BetfairInstrumentProvider:
mock.patch(
"betfairlightweight.endpoints.navigation.Navigation.list_navigation",
return_value=BetfairTestStubs.navigation(),
)
mock.patch(
"betfairlightweight.endpoints.betting.Betting.resp_market_catalogue",
return_value=BetfairTestStubs.market_catalogue(),
)
return BetfairInstrumentProvider(
client=betfair_client,
logger=BetfairTestStubs.live_logger(BetfairTestStubs.clock()),
market_filter={"event_type_name": "Tennis"},
load_all=False,
)
def _loaded_data_into_catalog(self):
self.instrument_provider = BetfairInstrumentProvider.from_instruments(
[])
result = process_files(
glob_path=PACKAGE_ROOT + "/data/1.166564490*.bz2",
reader=BetfairTestStubs.betfair_reader(
instrument_provider=self.instrument_provider),
instrument_provider=self.instrument_provider,
catalog=self.catalog,
)
assert result
data = (self.catalog.instruments(as_nautilus=True) +
self.catalog.instrument_status_updates(as_nautilus=True) +
self.catalog.trade_ticks(as_nautilus=True) +
self.catalog.order_book_deltas(as_nautilus=True) +
self.catalog.tickers(as_nautilus=True))
return data
def __init__(
self,
loop: asyncio.AbstractEventLoop,
client: BetfairClient,
base_currency: Currency,
msgbus: MessageBus,
cache: Cache,
clock: LiveClock,
logger: Logger,
market_filter: Dict,
instrument_provider: BetfairInstrumentProvider,
):
super().__init__(
loop=loop,
client_id=ClientId(BETFAIR_VENUE.value),
venue=BETFAIR_VENUE,
oms_type=OMSType.NETTING,
account_type=AccountType.BETTING,
base_currency=base_currency,
instrument_provider=instrument_provider
or BetfairInstrumentProvider(
client=client, logger=logger, filters=market_filter),
msgbus=msgbus,
cache=cache,
clock=clock,
logger=logger,
)
self._client: BetfairClient = client
self.stream = BetfairOrderStreamClient(
client=self._client,
logger=logger,
message_handler=self.handle_order_stream_update,
)
self.venue_order_id_to_client_order_id: Dict[VenueOrderId,
ClientOrderId] = {}
self.pending_update_order_client_ids: Set[Tuple[ClientOrderId,
VenueOrderId]] = set()
self.published_executions: Dict[ClientOrderId,
TradeId] = defaultdict(list)
self._set_account_id(AccountId(BETFAIR_VENUE.value,
"001")) # TODO(cs): Temporary
AccountFactory.register_calculated_account(BETFAIR_VENUE.value)
def test_data_loader_json_betting_parser():
instrument_provider = BetfairInstrumentProvider.from_instruments([])
parser = TextParser(
parser=lambda x, state: on_market_update(
instrument_provider=instrument_provider, update=orjson.loads(x)),
instrument_provider_update=historical_instrument_provider_loader,
)
loader = DataLoader(
path=TEST_DATA_DIR,
parser=parser,
glob_pattern="**.bz2",
instrument_provider=instrument_provider,
)
assert len(loader.path) == 3
data = [x for y in loader.run() for x in y]
assert len(data) == 30829
def test_load_text_betfair(self):
# Arrange
instrument_provider = BetfairInstrumentProvider.from_instruments([])
# Act
files = process_files(
glob_path=f"{TEST_DATA_DIR}/**.bz2",
reader=BetfairTestStubs.betfair_reader(instrument_provider=instrument_provider),
catalog=self.catalog,
instrument_provider=instrument_provider,
)
# Assert
assert files == {
TEST_DATA_DIR + "/1.166564490.bz2": 2908,
TEST_DATA_DIR + "/betfair/1.180305278.bz2": 17085,
TEST_DATA_DIR + "/betfair/1.166811431.bz2": 22692,
}
def test_load_dask_distributed_client(self):
# Arrange
from distributed import Client
instrument_provider = BetfairInstrumentProvider.from_instruments([])
with Client(processes=False, threads_per_worker=1) as c:
tasks = process_files(
glob_path=f"{TEST_DATA_DIR}/1.166564490*",
reader=make_betfair_reader(instrument_provider),
catalog=self.catalog,
instrument_provider=instrument_provider,
)
# Act
results = c.gather(c.compute(tasks))
# Assert
expected = {TEST_DATA + "/1.166564490.bz2": 2908}
assert results == expected
def __init__(
self,
loop: asyncio.AbstractEventLoop,
client: BetfairClient,
account_id: AccountId,
base_currency: Currency,
msgbus: MessageBus,
cache: Cache,
clock: LiveClock,
logger: Logger,
market_filter: Dict,
instrument_provider: BetfairInstrumentProvider,
):
super().__init__(
loop=loop,
client_id=ClientId(BETFAIR_VENUE.value),
instrument_provider=instrument_provider
or BetfairInstrumentProvider(client=client, logger=logger, market_filter=market_filter),
venue_type=VenueType.EXCHANGE,
account_id=account_id,
account_type=AccountType.BETTING,
base_currency=base_currency,
msgbus=msgbus,
cache=cache,
clock=clock,
logger=logger,
config={"name": "BetfairExecClient"},
)
self._client: BetfairClient = client
self.stream = BetfairOrderStreamClient(
client=self._client,
logger=logger,
message_handler=self.handle_order_stream_update,
)
self.venue_order_id_to_client_order_id: Dict[VenueOrderId, ClientOrderId] = {}
self.pending_update_order_client_ids: Set[Tuple[ClientOrderId, VenueOrderId]] = set()
self.published_executions: Dict[ClientOrderId, ExecutionId] = defaultdict(list)
AccountFactory.register_calculated_account(account_id.issuer)
def __init__(
self,
loop: asyncio.AbstractEventLoop,
client: BetfairClient,
msgbus: MessageBus,
cache: Cache,
clock: LiveClock,
logger: Logger,
market_filter: Dict,
instrument_provider: Optional[BetfairInstrumentProvider] = None,
strict_handling: bool = False,
):
super().__init__(
loop=loop,
client_id=ClientId(BETFAIR_VENUE.value),
venue=BETFAIR_VENUE,
instrument_provider=instrument_provider
or BetfairInstrumentProvider(
client=client, logger=logger, filters=market_filter),
msgbus=msgbus,
cache=cache,
clock=clock,
logger=logger,
)
self._client = client
self._stream = BetfairMarketStreamClient(
client=self._client,
logger=logger,
message_handler=self.on_market_update,
)
self.subscription_status = SubscriptionStatus.UNSUBSCRIBED
# Subscriptions
self._subscribed_instrument_ids: Set[InstrumentId] = set()
self._strict_handling = strict_handling
self._subscribed_market_ids: Set[InstrumentId] = set()
def build_market_update_messages( # noqa TODO: cyclomatic complexity 14
raw, instrument_provider: BetfairInstrumentProvider
) -> List[Union[OrderBookOperation, TradeTick]]:
updates = []
for market in raw.get("mc", []):
market_id = market["id"]
for runner in market.get("rc", []):
kw = dict(
market_id=market_id,
selection_id=str(runner["id"]),
handicap=str(runner.get("hc") or "0.0"),
)
instrument = instrument_provider.get_betting_instrument(**kw)
if not instrument:
continue
operations = []
for side in B_SIDE_KINDS:
for upd in runner.get(side, []):
# TODO - Fix this crap
if len(upd) == 3:
_, price, volume = upd
else:
price, volume = upd
operations.append(
OrderBookOperation(
op_type=OrderBookOperationType.DELETE
if volume == 0 else OrderBookOperationType.UPDATE,
order=Order(
price=price_to_probability(
price, side=B2N_MARKET_STREAM_SIDE[side]),
volume=volume,
side=B2N_MARKET_STREAM_SIDE[side],
),
timestamp_ns=millis_to_nanos(raw["pt"]),
))
ob_update = OrderBookOperations(
level=OrderBookLevel.L2,
instrument_id=instrument.id,
ops=operations,
timestamp_ns=millis_to_nanos(raw["pt"]),
)
updates.append(ob_update)
for price, volume in runner.get("trd", []):
# Betfair doesn't publish trade ids, so we make our own
# TODO - should we use clk here?
trade_id = hash_json(data=(
raw["pt"],
market_id,
str(runner["id"]),
str(runner.get("hc", "0.0")),
price,
volume,
))
trade_tick = TradeTick(
instrument_id=instrument.id,
price=Price(price_to_probability(price)),
size=Quantity(volume, precision=4),
side=OrderSide.BUY,
match_id=TradeMatchId(trade_id),
timestamp_ns=millis_to_nanos(raw["pt"]),
)
updates.append(trade_tick)
if market.get("marketDefinition", {}).get("status") == "CLOSED":
for runner in market["marketDefinition"]["runners"]:
kw = dict(
market_id=market_id,
selection_id=str(runner["id"]),
handicap=str(runner.get("hc") or "0.0"),
)
instrument = instrument_provider.get_betting_instrument(**kw)
assert instrument
# TODO - handle market closed
# on_market_status()
if runner["status"] == "LOSER":
# TODO - handle closing valuation = 0
pass
elif runner["status"] == "WINNER":
# TODO handle closing valuation = 1
pass
if (market.get("marketDefinition", {}).get("inPlay")
and not market.get("marketDefinition",
{}).get("status") == "CLOSED"):
for selection in market["marketDefinition"]["runners"]:
kw = dict(
market_id=market_id,
selection_id=str(selection["id"]),
handicap=str(
selection.get("hc", selection.get("handicap"))
or "0.0"),
)
instrument = instrument_provider.get_betting_instrument(**kw)
assert instrument
# TODO - handle instrument status IN_PLAY
return updates
def instrument_provider(betfair_client) -> BetfairInstrumentProvider:
return BetfairInstrumentProvider(
client=betfair_client,
logger=BetfairTestStubs.live_logger(BetfairTestStubs.clock()),
# market_filter={"event_type_name": "Tennis"},
)
class TestBetfairInstrumentProvider:
def setup(self):
# Fixture Setup
self.loop = asyncio.get_event_loop()
self.clock = LiveClock()
self.logger = LiveLogger(loop=self.loop, clock=self.clock)
self.client = BetfairTestStubs.betfair_client(loop=self.loop,
logger=self.logger)
self.provider = BetfairInstrumentProvider(
client=self.client,
logger=TestComponentStubs.logger(),
)
@pytest.mark.asyncio
async def test_load_markets(self):
markets = await load_markets(self.client, market_filter={})
assert len(markets) == 13227
markets = await load_markets(
self.client, market_filter={"event_type_name": "Basketball"})
assert len(markets) == 302
markets = await load_markets(
self.client, market_filter={"event_type_name": "Tennis"})
assert len(markets) == 1958
markets = await load_markets(
self.client, market_filter={"market_id": "1.177125728"})
assert len(markets) == 1
@pytest.mark.asyncio
async def test_load_markets_metadata(self):
markets = await load_markets(
self.client, market_filter={"event_type_name": "Basketball"})
market_metadata = await load_markets_metadata(client=self.client,
markets=markets)
assert isinstance(market_metadata, dict)
assert len(market_metadata) == 169
@pytest.mark.asyncio
async def test_make_instruments(self):
# Arrange
list_market_catalogue_data = {
m["marketId"]: m
for m in BetfairResponses.betting_list_market_catalogue()["result"]
if m["eventType"]["name"] == "Basketball"
}
# Act
instruments = [
instrument for metadata in list_market_catalogue_data.values()
for instrument in make_instruments(metadata, currency="GBP")
]
# Assert
assert len(instruments) == 30412
@pytest.mark.asyncio
async def test_load_all(self):
await self.provider.load_all_async({"event_type_name": "Tennis"})
assert len(self.provider.list_all()) == 4711
@pytest.mark.asyncio
async def test_list_all(self):
await self.provider.load_all_async(
market_filter={"event_type_name": "Basketball"})
instruments = self.provider.list_all()
assert len(instruments) == 23908
@pytest.mark.asyncio
async def test_search_instruments(self):
await self.provider.load_all_async(
market_filter={"event_type_name": "Basketball"})
instruments = self.provider.search_instruments(
instrument_filter={"market_type": "MATCH_ODDS"})
assert len(instruments) == 104
@pytest.mark.asyncio
async def test_get_betting_instrument(self):
await self.provider.load_all_async(
market_filter={"market_id": ["1.180678317"]})
kw = dict(
market_id="1.180678317",
selection_id="11313157",
handicap="0.0",
)
instrument = self.provider.get_betting_instrument(**kw)
assert instrument.market_id == "1.180678317"
# Test throwing warning
kw["handicap"] = "-1000"
instrument = self.provider.get_betting_instrument(**kw)
assert instrument is None
# Test already in self._subscribed_instruments
instrument = self.provider.get_betting_instrument(**kw)
assert instrument is None
def test_market_update_runner_removed(self):
update = BetfairStreaming.market_definition_runner_removed()
# Setup
market_def = update["mc"][0]["marketDefinition"]
market_def["marketId"] = update["mc"][0]["id"]
instruments = make_instruments(
market_definition=update["mc"][0]["marketDefinition"],
currency="GBP")
self.provider.add_bulk(instruments)
results = []
for data in on_market_update(instrument_provider=self.provider,
update=update):
results.append(data)
result = [r.status for r in results[:8]]
expected = [InstrumentStatus.PRE_OPEN] * 7 + [InstrumentStatus.CLOSED]
assert result == expected
