def test_equality_when_types_not_equal_returns_false(self): # Arrange data_type1 = DataType(type=str) data_type2 = DataType(type=int) # Act, Assert assert data_type1 != data_type2
def test_equality_when_definitions_equal_returns_false(self): # Arrange data_type1 = DataType(type=str, metadata={"category": 1}) data_type2 = DataType(type=str, metadata={"category": 1}) # Act, Assert assert data_type1 == data_type2
def test_equality_when_types_equal_returns_true(self): # Arrange data_type1 = DataType(type=str) data_type2 = DataType(type=str) # Act, Assert assert data_type1 == data_type2
def test_data_messages_when_client_id_and_venue_none_raise_value_error( self): # Arrange, Act , Assert with pytest.raises(ValueError) as ex: Subscribe( client_id=None, venue=None, data_type=DataType(str, {"type": "newswire"}), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) assert ex.type == ValueError assert ex.match("Both `client_id` and `venue` were None") with pytest.raises(ValueError) as ex: Unsubscribe( client_id=None, venue=None, data_type=DataType(str, {"type": "newswire"}), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) assert ex.type == ValueError assert ex.match("Both `client_id` and `venue` were None") with pytest.raises(ValueError) as ex: handler = [] DataRequest( client_id=None, venue=None, data_type=DataType(QuoteTick), callback=handler.append, request_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) assert ex.type == ValueError assert ex.match("Both `client_id` and `venue` were None") with pytest.raises(ValueError) as ex: DataResponse( client_id=None, venue=None, data_type=DataType(QuoteTick), data=[], correlation_id=self.uuid_factory.generate(), response_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) assert ex.type == ValueError assert ex.match("Both `client_id` and `venue` were None")
def test_venue_data_response_message_str_and_repr(self): # Arrange, Act correlation_id = self.uuid_factory.generate() response_id = self.uuid_factory.generate() instrument_id = InstrumentId(Symbol("AUD/USD"), IDEALPRO) response = DataResponse( client_id=ClientId("IB"), venue=Venue("IDEALPRO"), data_type=DataType(QuoteTick, metadata={"instrument_id": instrument_id}), data=[], correlation_id=correlation_id, response_id=response_id, ts_init=self.clock.timestamp_ns(), ) # Assert assert ( str(response) == "DataResponse(QuoteTick{'instrument_id': InstrumentId('AUD/USD.IDEALPRO')})" ) assert repr(response) == ( f"DataResponse(" f"client_id=IB, " f"venue=IDEALPRO, " f"data_type=QuoteTick{{'instrument_id': InstrumentId('AUD/USD.IDEALPRO')}}, " f"correlation_id={correlation_id}, " f"id={response_id})")
def test_venue_data_request_message_str_and_repr(self): # Arrange, Act handler = [].append request_id = self.uuid_factory.generate() request = DataRequest( client_id=None, venue=BINANCE, data_type=DataType( TradeTick, metadata={ # str data type is invalid "instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")), "from_datetime": None, "to_datetime": None, "limit": 1000, }, ), callback=handler, request_id=request_id, ts_init=self.clock.timestamp_ns(), ) # Assert assert ( str(request) == "DataRequest(TradeTick{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000})" # noqa ) assert repr(request) == ( f"DataRequest(" f"client_id=None, " f"venue=BINANCE, " f"data_type=TradeTick{{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000}}, " f"callback={repr(handler)}, " f"id={request_id})")
async def test_message_qsize_at_max_blocks_on_put_data_command(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) subscribe = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType(QuoteTick), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.execute(subscribe) self.engine.execute(subscribe) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 1 assert self.engine.command_count == 0
async def test_send_request_processes_message(self): # Arrange self.engine.start() handler = [] request = DataRequest( client_id=ClientId("RANDOM"), data_type=DataType( QuoteTick, metadata={ "instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")), "from_datetime": None, "to_datetime": None, "limit": 1000, }, ), callback=handler.append, request_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.request(request) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 0 assert self.engine.request_count == 1 # Tear Down self.engine.stop()
def streaming_backtest_runner( run_config_id: str, engine: BacktestEngine, data_configs: List[BacktestDataConfig], batch_size_bytes: Optional[int] = None, ): config = data_configs[0] catalog: DataCatalog = config.catalog() data_client_ids = _extract_generic_data_client_id( data_configs=data_configs) for data in batch_files( catalog=catalog, data_configs=data_configs, target_batch_size_bytes=batch_size_bytes, ): engine.clear_data() for data in groupby_datatype(data): if data["type"] in data_client_ids: # Generic data - manually re-add client_id as it gets lost in the streaming join data.update( {"client_id": ClientId(data_client_ids[data["type"]])}) data["data"] = [ GenericData(data_type=DataType(data["type"]), data=d) for d in data["data"] ] _load_engine_data(engine=engine, data=data) engine.run_streaming(run_config_id=run_config_id) engine.end_streaming()
def test_publish_data_sends_to_subscriber(self): # Arrange actor = MockActor() actor.register_base( trader_id=self.trader_id, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) handler = [] self.msgbus.subscribe( topic="data*", handler=handler.append, ) # Act data = Data( ts_event=self.clock.timestamp_ns(), ts_init=self.clock.timestamp_ns(), ) actor.publish_data(data_type=DataType(Data), data=data) # Assert assert data in handler
def test_data_type_comparison(self): # Arrange, Act data_type1 = DataType(str, { "type": "NEWS_WIRE", "topic": "Earthquake" }) data_type2 = DataType(str, {"type": "NEWS_WIRE", "topic": "Flood"}) data_type3 = DataType(int, { "type": "FED_DATA", "topic": "NonFarmPayroll" }) # Assert assert data_type1 <= data_type1 assert data_type1 < data_type2 assert data_type2 > data_type1 assert data_type1 >= data_type3
def test_data_type_as_key_in_dict(self): # Arrange, Act data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"}) hash_map = {data_type: []} # Assert assert data_type in hash_map
def test_data_type_equality_and_hash(self): # Arrange, Act data_type1 = DataType(str, { "type": "NEWS_WIRE", "topic": "Earthquake" }) data_type2 = DataType(str, {"type": "NEWS_WIRE", "topic": "Flood"}) data_type3 = DataType(int, { "type": "FED_DATA", "topic": "NonFarmPayroll" }) # Assert assert data_type1 == data_type1 assert data_type1 != data_type2 assert data_type1 != data_type2 assert data_type1 != data_type3 assert isinstance(hash(data_type1), int)
async def test_request_search_instruments(self): req = DataType( type=InstrumentSearch, metadata={"event_type_id": "7"}, ) self.client.request(req, self.uuid) await asyncio.sleep(0) resp = self.messages[0] assert len(resp.data.instruments) == 6800
def generic_data(self, cls, filter_expr=None, as_nautilus=False, **kwargs): data = self._query(cls=cls, filter_expr=filter_expr, as_dataframe=not as_nautilus, **kwargs) if as_nautilus: if data is None: return [] return [GenericData(data_type=DataType(cls), data=d) for d in data] return data
def test_hash_str_repr(self): # Arrange data_type = DataType(type=str, metadata={"category": 1, "code": 0}) # Act, Assert assert isinstance(hash(data_type), int) assert str(data_type) == "str{'category': 1, 'code': 0}" assert repr( data_type ) == "DataType(type=str, metadata={'category': 1, 'code': 0})"
def test_data_type_instantiation(self): # Arrange, Act data_type = DataType(str, {"type": "NEWS_WIRE"}) # Assert assert data_type.type == str assert data_type.metadata == {"type": "NEWS_WIRE"} assert data_type.topic == "str.type=NEWS_WIRE" assert str(data_type) == "str{'type': 'NEWS_WIRE'}" assert repr( data_type) == "DataType(type=str, metadata={'type': 'NEWS_WIRE'})"
def test_data_type_instantiation_when_no_metadata(self): # Arrange, Act data_type = DataType(str) # Assert assert data_type.type == str assert data_type.metadata == {} assert data_type.topic == "str*" assert str(data_type) == "str" assert repr( data_type) == "DataType(type=str, metadata={})" # noqa (P103??)
def test_data_type_instantiation_with_multiple_metadata(self): # Arrange, Act data_type = DataType(str, {"b": 2, "a": 1, "c": None}) # Assert assert data_type.type == str assert data_type.metadata == {"a": 1, "b": 2, "c": None} assert data_type.topic == "str.b=2.a=1.c=*" assert str(data_type) == "str{'b': 2, 'a': 1, 'c': None}" assert repr( data_type ) == "DataType(type=str, metadata={'b': 2, 'a': 1, 'c': None})"
async def test_message_qsize_at_max_blocks_on_send_request(self): # Arrange self.msgbus.deregister(endpoint="DataEngine.execute", handler=self.engine.execute) self.msgbus.deregister(endpoint="DataEngine.process", handler=self.engine.process) self.msgbus.deregister(endpoint="DataEngine.request", handler=self.engine.request) self.msgbus.deregister(endpoint="DataEngine.response", handler=self.engine.response) self.engine = LiveDataEngine( loop=self.loop, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, config=LiveDataEngineConfig(qsize=1), ) handler = [] request = DataRequest( client_id=ClientId("RANDOM"), venue=None, data_type=DataType( QuoteTick, metadata={ "instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")), "from_datetime": None, "to_datetime": None, "limit": 1000, }, ), callback=handler.append, request_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.request(request) self.engine.request(request) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 1 assert self.engine.command_count == 0
def test_data_instantiation(self): # Arrange, Act data_type = DataType(NewsEvent, {"publisher": "NEWS_WIRE"}) data = NewsEvent( impact=NewsImpact.HIGH, name="Unemployment Rate", currency=USD, ts_event=0, ts_init=0, ) generic_data = GenericData(data_type, data) # Assert assert generic_data.data_type == data_type assert generic_data.data == data
def test_handle_data_response_sends_to_data_engine(self): # Arrange data_type = DataType(NewsEvent, {"Type": "NEWS_WIRE"}) data = NewsEvent( impact=NewsImpact.HIGH, name="Unemployment Rate", currency=USD, ts_event=0, ts_init=0, ) # Act self.client._handle_data_response_py(data_type, data, self.uuid_factory.generate()) # Assert assert self.data_engine.response_count == 1
def test_handle_data_sends_to_data_engine(self): # Arrange data_type = DataType(NewsEvent, {"Type": "NEWS_WIRE"}) data = NewsEvent( impact=NewsImpact.HIGH, name="Unemployment Rate", currency=USD, ts_event=0, ts_init=0, ) generic_data = GenericData(data_type, data) # Act self.client._handle_data_py(generic_data) # Assert assert self.data_engine.data_count == 1
def test_data_command_str_and_repr(self): # Arrange, Act command_id = self.uuid_factory.generate() command = Subscribe( client_id=None, venue=BINANCE, data_type=DataType(str, {"type": "newswire"}), command_id=command_id, ts_init=self.clock.timestamp_ns(), ) # Assert assert str(command) == "Subscribe(str{'type': 'newswire'})" assert repr(command) == (f"Subscribe(" f"client_id=None, " f"venue=BINANCE, " f"data_type=str{{'type': 'newswire'}}, " f"id={command_id})")
async def _request_trade_ticks( self, instrument_id: InstrumentId, from_datetime: pd.Timestamp, to_datetime: pd.Timestamp, limit: int, correlation_id: UUID4, ): instrument = self._instrument_provider.find(instrument_id) if instrument is None: self._log.error( f"Cannot parse trades response: no instrument found for {instrument_id}.", ) return data = await self._http_client.get_trades(instrument_id.symbol.value) # Limit trades data if limit: while len(data) > limit: data.pop(0) # Pop left ticks: List[TradeTick] = parse_trade_ticks_ws( instrument=instrument, data=data, ts_init=self._clock.timestamp_ns(), ) data_type = DataType( type=TradeTick, metadata={ "instrument_id": instrument_id, "from_datetime": from_datetime, "to_datetime": to_datetime, }, ) self._handle_data_response( data_type=data_type, data=ticks, correlation_id=correlation_id, )
def test_unsubscribe_custom_data_with_client_id(self): # Arrange actor = MockActor() actor.register_base( trader_id=self.trader_id, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"}) actor.subscribe_data(data_type, ClientId("QUANDL")) # Act actor.unsubscribe_data(data_type, ClientId("QUANDL")) # Assert assert self.data_engine.command_count == 2 assert actor.msgbus.subscriptions() == []
def test_subscribe_custom_data(self): # Arrange actor = MockActor() actor.register_base( trader_id=self.trader_id, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) data_type = DataType(str, {"type": "NEWS_WIRE", "topic": "Earthquake"}) # Act actor.subscribe_data(data_type) # Assert assert self.data_engine.command_count == 0 assert actor.msgbus.subscriptions( )[0].topic == "data.str.type=NEWS_WIRE.topic=Earthquake"
def test_request_data_sends_request_to_data_engine(self): # Arrange actor = MockActor() actor.register_base( trader_id=self.trader_id, msgbus=self.msgbus, cache=self.cache, clock=self.clock, logger=self.logger, ) data_type = DataType(str, { "type": "NEWS_WIRE", "topic": "Earthquakes" }) # Act actor.request_data(ClientId("BLOOMBERG-01"), data_type) # Assert assert self.data_engine.request_count == 1
async def test_execute_command_processes_message(self): # Arrange self.engine.start() subscribe = Subscribe( client_id=ClientId(BINANCE.value), data_type=DataType(QuoteTick), command_id=self.uuid_factory.generate(), ts_init=self.clock.timestamp_ns(), ) # Act self.engine.execute(subscribe) await asyncio.sleep(0.1) # Assert assert self.engine.message_qsize() == 0 assert self.engine.command_count == 1 # Tear Down self.engine.stop()
def test_venue_data_command_str_and_repr(self): # Arrange, Act command_id = self.uuid_factory.generate() command = Subscribe( client_id=ClientId(BINANCE.value), venue=BINANCE, data_type=DataType(TradeTick, {"instrument_id": "BTCUSDT"}), command_id=command_id, ts_init=self.clock.timestamp_ns(), ) # Assert assert str( command) == "Subscribe(TradeTick{'instrument_id': 'BTCUSDT'})" assert repr(command) == ( f"Subscribe(" f"client_id=BINANCE, " f"venue=BINANCE, " f"data_type=TradeTick{{'instrument_id': 'BTCUSDT'}}, " f"id={command_id})")