def test_byte_reader_parser(self):
def block_parser(block: bytes, instrument_provider):
for raw in block.split(b"\\n"):
ts, line = raw.split(b" - ")
state = {
"ts_init":
int(pd.Timestamp(ts.decode(), tz="UTC").to_datetime64())
}
line = line.strip().replace(b"b'", b"")
orjson.loads(line)
for obj in BetfairTestStubs.parse_betfair(
line, instrument_provider=instrument_provider):
values = obj.to_dict(obj)
values["ts_init"] = state["ts_init"]
yield obj.from_dict(values)
provider = BetfairInstrumentProvider.from_instruments(
[BetfairTestStubs.betting_instrument()])
block = BetfairDataProvider.badly_formatted_log()
reader = ByteReader(
block_parser=partial(block_parser, instrument_provider=provider),
instrument_provider=provider,
)
data = list(reader.parse(block=block))
result = [pd.Timestamp(d.ts_init).isoformat() for d in data]
expected = ["2021-06-29T06:03:14.528000"]
assert result == expected
def _loaded_data_into_catalog(self):
self.instrument_provider = BetfairInstrumentProvider.from_instruments([])
process_files(
glob_path=PACKAGE_ROOT + "/data/1.166564490.bz2",
reader=BetfairTestStubs.betfair_reader(instrument_provider=self.instrument_provider),
instrument_provider=self.instrument_provider,
catalog=self.catalog,
)
def test_text_reader(self):
provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(provider) # type: TextReader
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/betfair/1.166811431.bz2")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 22692
def betfair_reader(instrument_provider=None, **kwargs):
instrument_provider = instrument_provider or BetfairInstrumentProvider.from_instruments(
[])
reader = TextReader(
line_parser=partial(BetfairTestStubs.parse_betfair,
instrument_provider=instrument_provider),
instrument_provider=instrument_provider,
instrument_provider_update=historical_instrument_provider_loader,
**kwargs,
)
return reader
def make_betfair_reader(instrument_provider=None, line_preprocessor=None) -> TextReader:
from nautilus_trader.adapters.betfair.providers import BetfairInstrumentProvider
instrument_provider = instrument_provider or BetfairInstrumentProvider.from_instruments([])
return TextReader(
# Use the standard `on_market_update` betfair parser that the adapter uses
line_preprocessor=line_preprocessor,
line_parser=partial(line_parser, instrument_provider=instrument_provider),
instrument_provider_update=historical_instrument_provider_loader,
instrument_provider=instrument_provider,
)
def data_loader():
instrument_provider = BetfairInstrumentProvider.from_instruments([])
parser = TextParser(
parser=lambda x, state: on_market_update(
instrument_provider=instrument_provider, update=orjson.loads(x)),
instrument_provider_update=historical_instrument_provider_loader,
)
return DataLoader(
path=TEST_DATA_DIR,
parser=parser,
glob_pattern="1.166564490*",
instrument_provider=instrument_provider,
)
def betfair_feed_parsed(market_id="1.166564490", folder="data"):
instrument_provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(
instrument_provider=instrument_provider)
files = make_raw_files(
glob_path=f"{PACKAGE_ROOT}/{folder}/{market_id}*")
data = []
for rf in files:
for block in rf.iter():
data.extend(reader.parse(block=block))
return data
def _loaded_data_into_catalog(self):
self.instrument_provider = BetfairInstrumentProvider.from_instruments(
[])
result = process_files(
glob_path=PACKAGE_ROOT + "/data/1.166564490*.bz2",
reader=BetfairTestStubs.betfair_reader(
instrument_provider=self.instrument_provider),
instrument_provider=self.instrument_provider,
catalog=self.catalog,
)
assert result
data = (self.catalog.instruments(as_nautilus=True) +
self.catalog.instrument_status_updates(as_nautilus=True) +
self.catalog.trade_ticks(as_nautilus=True) +
self.catalog.order_book_deltas(as_nautilus=True) +
self.catalog.tickers(as_nautilus=True))
return data
def test_data_loader_json_betting_parser():
instrument_provider = BetfairInstrumentProvider.from_instruments([])
parser = TextParser(
parser=lambda x, state: on_market_update(
instrument_provider=instrument_provider, update=orjson.loads(x)),
instrument_provider_update=historical_instrument_provider_loader,
)
loader = DataLoader(
path=TEST_DATA_DIR,
parser=parser,
glob_pattern="**.bz2",
instrument_provider=instrument_provider,
)
assert len(loader.path) == 3
data = [x for y in loader.run() for x in y]
assert len(data) == 30829
def test_load_text_betfair(self):
# Arrange
instrument_provider = BetfairInstrumentProvider.from_instruments([])
# Act
files = process_files(
glob_path=f"{TEST_DATA_DIR}/**.bz2",
reader=BetfairTestStubs.betfair_reader(instrument_provider=instrument_provider),
catalog=self.catalog,
instrument_provider=instrument_provider,
)
# Assert
assert files == {
TEST_DATA_DIR + "/1.166564490.bz2": 2908,
TEST_DATA_DIR + "/betfair/1.180305278.bz2": 17085,
TEST_DATA_DIR + "/betfair/1.166811431.bz2": 22692,
}
def test_load_dask_distributed_client(self):
# Arrange
from distributed import Client
instrument_provider = BetfairInstrumentProvider.from_instruments([])
with Client(processes=False, threads_per_worker=1) as c:
tasks = process_files(
glob_path=f"{TEST_DATA_DIR}/1.166564490*",
reader=make_betfair_reader(instrument_provider),
catalog=self.catalog,
instrument_provider=instrument_provider,
)
# Act
results = c.gather(c.compute(tasks))
# Assert
expected = {TEST_DATA + "/1.166564490.bz2": 2908}
assert results == expected