def test_equality_when_types_not_equal_returns_false(self):
# Arrange
data_type1 = DataType(type=str)
data_type2 = DataType(type=int)
# Act, Assert
assert data_type1 != data_type2
def test_equality_when_definitions_equal_returns_false(self):
# Arrange
data_type1 = DataType(type=str, metadata={"category": 1})
data_type2 = DataType(type=str, metadata={"category": 1})
# Act, Assert
assert data_type1 == data_type2
def test_equality_when_types_equal_returns_true(self):
# Arrange
data_type1 = DataType(type=str)
data_type2 = DataType(type=str)
# Act, Assert
assert data_type1 == data_type2
def test_data_messages_when_client_id_and_venue_none_raise_value_error(
self):
# Arrange, Act , Assert
with pytest.raises(ValueError) as ex:
Subscribe(
client_id=None,
venue=None,
data_type=DataType(str, {"type": "newswire"}),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
Unsubscribe(
client_id=None,
venue=None,
data_type=DataType(str, {"type": "newswire"}),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
handler = []
DataRequest(
client_id=None,
venue=None,
data_type=DataType(QuoteTick),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
DataResponse(
client_id=None,
venue=None,
data_type=DataType(QuoteTick),
data=[],
correlation_id=self.uuid_factory.generate(),
response_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
def test_venue_data_response_message_str_and_repr(self):
# Arrange, Act
correlation_id = self.uuid_factory.generate()
response_id = self.uuid_factory.generate()
instrument_id = InstrumentId(Symbol("AUD/USD"), IDEALPRO)
response = DataResponse(
client_id=ClientId("IB"),
venue=Venue("IDEALPRO"),
data_type=DataType(QuoteTick,
metadata={"instrument_id": instrument_id}),
data=[],
correlation_id=correlation_id,
response_id=response_id,
ts_init=self.clock.timestamp_ns(),
)
# Assert
assert (
str(response) ==
"DataResponse(QuoteTick{'instrument_id': InstrumentId('AUD/USD.IDEALPRO')})"
)
assert repr(response) == (
f"DataResponse("
f"client_id=IB, "
f"venue=IDEALPRO, "
f"data_type=QuoteTick{{'instrument_id': InstrumentId('AUD/USD.IDEALPRO')}}, "
f"correlation_id={correlation_id}, "
f"id={response_id})")
def test_venue_data_request_message_str_and_repr(self):
# Arrange, Act
handler = [].append
request_id = self.uuid_factory.generate()
request = DataRequest(
client_id=None,
venue=BINANCE,
data_type=DataType(
TradeTick,
metadata={ # str data type is invalid
"instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime": None,
"to_datetime": None,
"limit": 1000,
},
),
callback=handler,
request_id=request_id,
ts_init=self.clock.timestamp_ns(),
)
# Assert
assert (
str(request) ==
"DataRequest(TradeTick{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000})" # noqa
)
assert repr(request) == (
f"DataRequest("
f"client_id=None, "
f"venue=BINANCE, "
f"data_type=TradeTick{{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000}}, "
f"callback={repr(handler)}, "
f"id={request_id})")
async def test_message_qsize_at_max_blocks_on_put_data_command(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
subscribe = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(QuoteTick),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.execute(subscribe)
self.engine.execute(subscribe)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
async def test_send_request_processes_message(self):
# Arrange
self.engine.start()
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.request_count == 1
# Tear Down
self.engine.stop()
def streaming_backtest_runner(
run_config_id: str,
engine: BacktestEngine,
data_configs: List[BacktestDataConfig],
batch_size_bytes: Optional[int] = None,
):
config = data_configs[0]
catalog: DataCatalog = config.catalog()
data_client_ids = _extract_generic_data_client_id(
data_configs=data_configs)
for data in batch_files(
catalog=catalog,
data_configs=data_configs,
target_batch_size_bytes=batch_size_bytes,
):
engine.clear_data()
for data in groupby_datatype(data):
if data["type"] in data_client_ids:
# Generic data - manually re-add client_id as it gets lost in the streaming join
data.update(
{"client_id": ClientId(data_client_ids[data["type"]])})
data["data"] = [
GenericData(data_type=DataType(data["type"]), data=d)
for d in data["data"]
]
_load_engine_data(engine=engine, data=data)
engine.run_streaming(run_config_id=run_config_id)
engine.end_streaming()
def test_publish_data_sends_to_subscriber(self):
# Arrange
actor = MockActor()
actor.register_base(
trader_id=self.trader_id,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
handler = []
self.msgbus.subscribe(
topic="data*",
handler=handler.append,
)
# Act
data = Data(
ts_event=self.clock.timestamp_ns(),
ts_init=self.clock.timestamp_ns(),
)
actor.publish_data(data_type=DataType(Data), data=data)
# Assert
assert data in handler
def test_data_type_comparison(self):
# Arrange, Act
data_type1 = DataType(str, {
"type": "NEWS_WIRE",
"topic": "Earthquake"
})
data_type2 = DataType(str, {"type": "NEWS_WIRE", "topic": "Flood"})
data_type3 = DataType(int, {
"type": "FED_DATA",
"topic": "NonFarmPayroll"
})
# Assert
assert data_type1 <= data_type1
assert data_type1 < data_type2
assert data_type2 > data_type1
assert data_type1 >= data_type3
def test_data_type_as_key_in_dict(self):
# Arrange, Act
data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"})
hash_map = {data_type: []}
# Assert
assert data_type in hash_map
def test_data_type_equality_and_hash(self):
# Arrange, Act
data_type1 = DataType(str, {
"type": "NEWS_WIRE",
"topic": "Earthquake"
})
data_type2 = DataType(str, {"type": "NEWS_WIRE", "topic": "Flood"})
data_type3 = DataType(int, {
"type": "FED_DATA",
"topic": "NonFarmPayroll"
})
# Assert
assert data_type1 == data_type1
assert data_type1 != data_type2
assert data_type1 != data_type2
assert data_type1 != data_type3
assert isinstance(hash(data_type1), int)
async def test_request_search_instruments(self):
req = DataType(
type=InstrumentSearch,
metadata={"event_type_id": "7"},
)
self.client.request(req, self.uuid)
await asyncio.sleep(0)
resp = self.messages[0]
assert len(resp.data.instruments) == 6800
def generic_data(self, cls, filter_expr=None, as_nautilus=False, **kwargs):
data = self._query(cls=cls,
filter_expr=filter_expr,
as_dataframe=not as_nautilus,
**kwargs)
if as_nautilus:
if data is None:
return []
return [GenericData(data_type=DataType(cls), data=d) for d in data]
return data
def test_hash_str_repr(self):
# Arrange
data_type = DataType(type=str, metadata={"category": 1, "code": 0})
# Act, Assert
assert isinstance(hash(data_type), int)
assert str(data_type) == "str{'category': 1, 'code': 0}"
assert repr(
data_type
) == "DataType(type=str, metadata={'category': 1, 'code': 0})"
def test_data_type_instantiation(self):
# Arrange, Act
data_type = DataType(str, {"type": "NEWS_WIRE"})
# Assert
assert data_type.type == str
assert data_type.metadata == {"type": "NEWS_WIRE"}
assert data_type.topic == "str.type=NEWS_WIRE"
assert str(data_type) == "str{'type': 'NEWS_WIRE'}"
assert repr(
data_type) == "DataType(type=str, metadata={'type': 'NEWS_WIRE'})"
def test_data_type_instantiation_when_no_metadata(self):
# Arrange, Act
data_type = DataType(str)
# Assert
assert data_type.type == str
assert data_type.metadata == {}
assert data_type.topic == "str*"
assert str(data_type) == "str"
assert repr(
data_type) == "DataType(type=str, metadata={})" # noqa (P103??)
def test_data_type_instantiation_with_multiple_metadata(self):
# Arrange, Act
data_type = DataType(str, {"b": 2, "a": 1, "c": None})
# Assert
assert data_type.type == str
assert data_type.metadata == {"a": 1, "b": 2, "c": None}
assert data_type.topic == "str.b=2.a=1.c=*"
assert str(data_type) == "str{'b': 2, 'a': 1, 'c': None}"
assert repr(
data_type
) == "DataType(type=str, metadata={'b': 2, 'a': 1, 'c': None})"
async def test_message_qsize_at_max_blocks_on_send_request(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
venue=None,
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
def test_data_instantiation(self):
# Arrange, Act
data_type = DataType(NewsEvent, {"publisher": "NEWS_WIRE"})
data = NewsEvent(
impact=NewsImpact.HIGH,
name="Unemployment Rate",
currency=USD,
ts_event=0,
ts_init=0,
)
generic_data = GenericData(data_type, data)
# Assert
assert generic_data.data_type == data_type
assert generic_data.data == data
def test_handle_data_response_sends_to_data_engine(self):
# Arrange
data_type = DataType(NewsEvent, {"Type": "NEWS_WIRE"})
data = NewsEvent(
impact=NewsImpact.HIGH,
name="Unemployment Rate",
currency=USD,
ts_event=0,
ts_init=0,
)
# Act
self.client._handle_data_response_py(data_type, data,
self.uuid_factory.generate())
# Assert
assert self.data_engine.response_count == 1
def test_handle_data_sends_to_data_engine(self):
# Arrange
data_type = DataType(NewsEvent, {"Type": "NEWS_WIRE"})
data = NewsEvent(
impact=NewsImpact.HIGH,
name="Unemployment Rate",
currency=USD,
ts_event=0,
ts_init=0,
)
generic_data = GenericData(data_type, data)
# Act
self.client._handle_data_py(generic_data)
# Assert
assert self.data_engine.data_count == 1
def test_data_command_str_and_repr(self):
# Arrange, Act
command_id = self.uuid_factory.generate()
command = Subscribe(
client_id=None,
venue=BINANCE,
data_type=DataType(str, {"type": "newswire"}),
command_id=command_id,
ts_init=self.clock.timestamp_ns(),
)
# Assert
assert str(command) == "Subscribe(str{'type': 'newswire'})"
assert repr(command) == (f"Subscribe("
f"client_id=None, "
f"venue=BINANCE, "
f"data_type=str{{'type': 'newswire'}}, "
f"id={command_id})")
async def _request_trade_ticks(
self,
instrument_id: InstrumentId,
from_datetime: pd.Timestamp,
to_datetime: pd.Timestamp,
limit: int,
correlation_id: UUID4,
):
instrument = self._instrument_provider.find(instrument_id)
if instrument is None:
self._log.error(
f"Cannot parse trades response: no instrument found for {instrument_id}.",
)
return
data = await self._http_client.get_trades(instrument_id.symbol.value)
# Limit trades data
if limit:
while len(data) > limit:
data.pop(0) # Pop left
ticks: List[TradeTick] = parse_trade_ticks_ws(
instrument=instrument,
data=data,
ts_init=self._clock.timestamp_ns(),
)
data_type = DataType(
type=TradeTick,
metadata={
"instrument_id": instrument_id,
"from_datetime": from_datetime,
"to_datetime": to_datetime,
},
)
self._handle_data_response(
data_type=data_type,
data=ticks,
correlation_id=correlation_id,
)
def test_unsubscribe_custom_data_with_client_id(self):
# Arrange
actor = MockActor()
actor.register_base(
trader_id=self.trader_id,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"})
actor.subscribe_data(data_type, ClientId("QUANDL"))
# Act
actor.unsubscribe_data(data_type, ClientId("QUANDL"))
# Assert
assert self.data_engine.command_count == 2
assert actor.msgbus.subscriptions() == []
def test_subscribe_custom_data(self):
# Arrange
actor = MockActor()
actor.register_base(
trader_id=self.trader_id,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"})
# Act
actor.subscribe_data(data_type)
# Assert
assert self.data_engine.command_count == 0
assert actor.msgbus.subscriptions(
)[0].topic == "data.str.type=NEWS_WIRE.topic=Earthquake"
def test_request_data_sends_request_to_data_engine(self):
# Arrange
actor = MockActor()
actor.register_base(
trader_id=self.trader_id,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
)
data_type = DataType(str, {
"type": "NEWS_WIRE",
"topic": "Earthquakes"
})
# Act
actor.request_data(ClientId("BLOOMBERG-01"), data_type)
# Assert
assert self.data_engine.request_count == 1
async def test_execute_command_processes_message(self):
# Arrange
self.engine.start()
subscribe = Subscribe(
client_id=ClientId(BINANCE.value),
data_type=DataType(QuoteTick),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.execute(subscribe)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.command_count == 1
# Tear Down
self.engine.stop()
def test_venue_data_command_str_and_repr(self):
# Arrange, Act
command_id = self.uuid_factory.generate()
command = Subscribe(
client_id=ClientId(BINANCE.value),
venue=BINANCE,
data_type=DataType(TradeTick, {"instrument_id": "BTCUSDT"}),
command_id=command_id,
ts_init=self.clock.timestamp_ns(),
)
# Assert
assert str(
command) == "Subscribe(TradeTick{'instrument_id': 'BTCUSDT'})"
assert repr(command) == (
f"Subscribe("
f"client_id=BINANCE, "
f"venue=BINANCE, "
f"data_type=TradeTick{{'instrument_id': 'BTCUSDT'}}, "
f"id={command_id})")
