def test_serialize_and_deserialize_account_state(self, event):
serialized = ParquetSerializer.serialize(event)
[deserialized] = ParquetSerializer.deserialize(cls=AccountState,
chunk=serialized)
# Assert
assert deserialized == event
write_objects(catalog=self.catalog, chunk=[event])
def test_serialize_and_deserialize_order_book_snapshot(self):
book = TestStubs.order_book_snapshot()
serialized = ParquetSerializer.serialize(book)
deserialized = ParquetSerializer.deserialize(cls=OrderBookSnapshot,
chunk=serialized)
# Assert
assert deserialized == [book]
write_objects(catalog=self.catalog, chunk=[book])
def test_serialize_and_deserialize_instruments(self, instrument):
serialized = ParquetSerializer.serialize(instrument)
assert serialized
deserialized = ParquetSerializer.deserialize(cls=type(instrument),
chunk=[serialized])
# Assert
assert deserialized == [instrument]
write_objects(catalog=self.catalog, chunk=[instrument])
df = self.catalog.instruments()
assert len(df) == 1
def test_serialize_and_deserialize_trading_state_changed(self):
event = TestStubs.event_trading_state_changed()
serialized = ParquetSerializer.serialize(event)
[deserialized] = ParquetSerializer.deserialize(cls=TradingStateChanged,
chunk=[serialized])
# Assert
assert deserialized == event
write_objects(catalog=self.catalog, chunk=[event])
def _test_serialization(self, obj: Any):
cls = type(obj)
serialized = ParquetSerializer.serialize(obj)
if not isinstance(serialized, list):
serialized = [serialized]
deserialized = ParquetSerializer.deserialize(cls=cls, chunk=serialized)
# Assert
expected = obj
if isinstance(deserialized, list) and not isinstance(expected, list):
expected = [expected]
assert deserialized == expected
write_objects(catalog=self.catalog, chunk=[obj])
df = self.catalog._query(cls=cls)
assert len(df) == 1
nautilus = self.catalog._query(cls=cls, as_dataframe=False)[0]
assert nautilus.ts_init == 0
return True
def split_and_serialize(objs: List) -> Dict[type, Dict[Optional[str], List]]:
"""
Given a list of Nautilus `objs`; serialize and split into dictionaries per type / instrument ID.
"""
# Split objects into their respective tables
values: Dict[type, Dict[str, List]] = {}
for obj in objs:
cls = get_cls_table(type(obj))
if isinstance(obj, GenericData):
cls = obj.data_type.type
if cls not in values:
values[cls] = {}
for data in maybe_list(ParquetSerializer.serialize(obj)):
instrument_id = data.get("instrument_id", None)
if instrument_id not in values[cls]:
values[cls][instrument_id] = []
values[cls][instrument_id].append(data)
return values
def write(self, obj: object):
assert obj is not None
cls = obj.__class__
if isinstance(obj, GenericData):
cls = obj.data_type.type
table = get_cls_table(cls).__name__
if table not in self._writers:
print(f"Can't find writer for cls: {cls}")
return
writer = self._writers[table]
serialized = ParquetSerializer.serialize(obj)
if isinstance(serialized, dict):
serialized = [serialized]
data = list_dicts_to_dict_lists(
serialized,
keys=self._schemas[cls].names,
)
data = list(data.values())
batch = pa.record_batch(data, schema=self._schemas[cls])
writer.write_batch(batch)
self.check_flush()
def test_serialize_and_deserialize_order_book_deltas(self):
kw = {
"instrument_id": "AUD/USD.SIM",
"ts_event": 0,
"ts_init": 0,
"book_type": "L2_MBP",
}
deltas = OrderBookDeltas(
instrument_id=TestStubs.audusd_id(),
book_type=BookType.L2_MBP,
deltas=[
OrderBookDelta.from_dict({
"action": "ADD",
"order_side": "BUY",
"order_price": 8.0,
"order_size": 30.0,
"order_id": "e0364f94-8fcb-0262-cbb3-075c51ee4917",
**kw,
}),
OrderBookDelta.from_dict({
"action": "ADD",
"order_side": "SELL",
"order_price": 15.0,
"order_size": 10.0,
"order_id": "cabec174-acc6-9204-9ebf-809da3896daf",
**kw,
}),
],
ts_event=0,
ts_init=0,
)
serialized = ParquetSerializer.serialize(deltas)
deserialized = ParquetSerializer.deserialize(cls=OrderBookDeltas,
chunk=serialized)
# Assert
assert deserialized == [deltas]
write_objects(catalog=self.catalog, chunk=[deltas])
def test_serialize_and_deserialize_order_book_delta(self):
delta = OrderBookDelta(
instrument_id=TestStubs.audusd_id(),
book_type=BookType.L2_MBP,
action=BookAction.CLEAR,
order=None,
ts_event=0,
ts_init=0,
)
serialized = ParquetSerializer.serialize(delta)
[deserialized] = ParquetSerializer.deserialize(cls=OrderBookDelta,
chunk=serialized)
# Assert
expected = OrderBookDeltas(
instrument_id=TestStubs.audusd_id(),
book_type=BookType.L2_MBP,
deltas=[delta],
ts_event=0,
ts_init=0,
)
assert deserialized == expected
write_objects(catalog=self.catalog, chunk=[delta])
def test_serialize_and_deserialize_order_book_deltas_grouped(self):
kw = {
"instrument_id": "AUD/USD.SIM",
"ts_event": 0,
"ts_init": 0,
"book_type": "L2_MBP",
}
deltas = [
{
"action": "ADD",
"order_side": "SELL",
"order_price": 0.9901,
"order_size": 327.25,
"order_id": "1",
},
{
"action": "CLEAR",
"order_side": None,
"order_price": None,
"order_size": None,
"order_id": None,
},
{
"action": "ADD",
"order_side": "SELL",
"order_price": 0.98039,
"order_size": 27.91,
"order_id": "2",
},
{
"action": "ADD",
"order_side": "SELL",
"order_price": 0.97087,
"order_size": 14.43,
"order_id": "3",
},
]
deltas = OrderBookDeltas(
instrument_id=TestStubs.audusd_id(),
book_type=BookType.L2_MBP,
deltas=[OrderBookDelta.from_dict({
**kw,
**d
}) for d in deltas],
ts_event=0,
ts_init=0,
)
serialized = ParquetSerializer.serialize(deltas)
[deserialized] = ParquetSerializer.deserialize(cls=OrderBookDeltas,
chunk=serialized)
# Assert
assert deserialized == deltas
write_objects(catalog=self.catalog, chunk=[deserialized])
assert [d.action for d in deserialized.deltas] == [
BookAction.ADD,
BookAction.CLEAR,
BookAction.ADD,
BookAction.ADD,
]