def streaming_backtest_runner(
run_config_id: str,
engine: BacktestEngine,
data_configs: List[BacktestDataConfig],
batch_size_bytes: Optional[int] = None,
):
config = data_configs[0]
catalog: DataCatalog = config.catalog()
data_client_ids = _extract_generic_data_client_id(
data_configs=data_configs)
for data in batch_files(
catalog=catalog,
data_configs=data_configs,
target_batch_size_bytes=batch_size_bytes,
):
engine.clear_data()
for data in groupby_datatype(data):
if data["type"] in data_client_ids:
# Generic data - manually re-add client_id as it gets lost in the streaming join
data.update(
{"client_id": ClientId(data_client_ids[data["type"]])})
data["data"] = [
GenericData(data_type=DataType(data["type"]), data=d)
for d in data["data"]
]
_load_engine_data(engine=engine, data=data)
engine.run_streaming(run_config_id=run_config_id)
engine.end_streaming()
def generic_data(self, cls, filter_expr=None, as_nautilus=False, **kwargs):
data = self._query(cls=cls,
filter_expr=filter_expr,
as_dataframe=not as_nautilus,
**kwargs)
if as_nautilus:
if data is None:
return []
return [GenericData(data_type=DataType(cls), data=d) for d in data]
return data
def test_add_generic_data_adds_to_engine(self, capsys):
# Arrange
engine = BacktestEngine()
data_type = DataType(MyData, metadata={"news_wire": "hacks"})
generic_data1 = [
GenericData(data_type, MyData("AAPL hacked")),
GenericData(
data_type,
MyData("AMZN hacked", 1000, 1000),
),
GenericData(
data_type,
MyData("NFLX hacked", 3000, 3000),
),
GenericData(
data_type,
MyData("MSFT hacked", 2000, 2000),
),
]
generic_data2 = [
GenericData(
data_type,
MyData("FB hacked", 1500, 1500),
),
]
# Act
engine.add_generic_data(ClientId("NEWS_CLIENT"), generic_data1)
engine.add_generic_data(ClientId("NEWS_CLIENT"), generic_data2)
# Assert
log = "".join(capsys.readouterr())
assert "Added 4 MyData GenericData elements." in log
assert "Added 1 MyData GenericData element." in log
def test_data_instantiation(self):
# Arrange, Act
data_type = DataType(NewsEvent, {"publisher": "NEWS_WIRE"})
data = NewsEvent(
impact=NewsImpact.HIGH,
name="Unemployment Rate",
currency=USD,
ts_event=0,
ts_init=0,
)
generic_data = GenericData(data_type, data)
# Assert
assert generic_data.data_type == data_type
assert generic_data.data == data
def test_handle_data_sends_to_data_engine(self):
# Arrange
data_type = DataType(NewsEvent, {"Type": "NEWS_WIRE"})
data = NewsEvent(
impact=NewsImpact.HIGH,
name="Unemployment Rate",
currency=USD,
ts_event=0,
ts_init=0,
)
generic_data = GenericData(data_type, data)
# Act
self.client._handle_data_py(generic_data)
# Assert
assert self.data_engine.data_count == 1