def test_partition_key_correctly_remapped(self):
# Arrange
instrument = TestInstrumentProvider.default_fx_ccy("AUD/USD")
tick = QuoteTick(
instrument_id=instrument.id,
bid=Price(10, 1),
ask=Price(11, 1),
bid_size=Quantity(10, 1),
ask_size=Quantity(10, 1),
ts_init=0,
ts_event=0,
)
tables = dicts_to_dataframes(split_and_serialize([tick]))
write_tables(catalog=self.catalog, tables=tables)
# Act
df = self.catalog.quote_ticks()
# Assert
assert len(df) == 1
assert self.fs.isdir(
"/root/data/quote_tick.parquet/instrument_id=AUD-USD.SIM/")
# Ensure we "unmap" the keys that we write the partition filenames as;
# this instrument_id should be AUD/USD not AUD-USD
assert df.iloc[0]["instrument_id"] == instrument.id.value
def test_nautilus_chunk_to_dataframes(self):
# Arrange, Act
data = self._loaded_data_into_catalog()
dfs = split_and_serialize(data)
result = {}
for cls in dfs:
for ins in dfs[cls]:
result[cls.__name__] = len(dfs[cls][ins])
# Assert
assert result == {
"BetfairTicker": 82,
"BettingInstrument": 2,
"InstrumentStatusUpdate": 1,
"OrderBookData": 1077,
"TradeTick": 114,
}
def test_split_and_serialize_generic_data_gets_correct_class(self):
# Arrange
TestPersistenceStubs.setup_news_event_persistence()
process_files(
glob_path=f"{TEST_DATA_DIR}/news_events.csv",
reader=CSVReader(block_parser=TestPersistenceStubs.news_event_parser),
catalog=self.catalog,
)
objs = self.catalog.generic_data(
cls=NewsEventData, filter_expr=ds.field("currency") == "USD", as_nautilus=True
)
# Act
split = split_and_serialize(objs)
# Assert
assert NewsEventData in split
assert None in split[NewsEventData]
assert len(split[NewsEventData][None]) == 22941
def test_write_parquet_determine_partitions_writes_instrument_id(
self,
):
# Arrange
quote = QuoteTick(
instrument_id=TestIdStubs.audusd_id(),
bid=Price.from_str("0.80"),
ask=Price.from_str("0.81"),
bid_size=Quantity.from_int(1000),
ask_size=Quantity.from_int(1000),
ts_event=0,
ts_init=0,
)
chunk = [quote]
tables = dicts_to_dataframes(split_and_serialize(chunk))
# Act
write_tables(catalog=self.catalog, tables=tables)
# Assert
files = self.fs.ls("/root/data/quote_tick.parquet")
expected = "/root/data/quote_tick.parquet/instrument_id=AUD-USD.SIM"
assert expected in files