def test_venue_data_request_message_str_and_repr(self):
# Arrange, Act
handler = [].append
request_id = self.uuid_factory.generate()
request = DataRequest(
client_id=None,
venue=BINANCE,
data_type=DataType(
TradeTick,
metadata={ # str data type is invalid
"instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime": None,
"to_datetime": None,
"limit": 1000,
},
),
callback=handler,
request_id=request_id,
ts_init=self.clock.timestamp_ns(),
)
# Assert
assert (
str(request) ==
"DataRequest(TradeTick{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000})" # noqa
)
assert repr(request) == (
f"DataRequest("
f"client_id=None, "
f"venue=BINANCE, "
f"data_type=TradeTick{{'instrument_id': InstrumentId('SOMETHING.RANDOM'), 'from_datetime': None, 'to_datetime': None, 'limit': 1000}}, "
f"callback={repr(handler)}, "
f"id={request_id})")
def test_message_qsize_at_max_blocks_on_send_request(self):
self.data_engine = LiveDataEngine(loop=self.loop,
portfolio=self.portfolio,
clock=self.clock,
logger=self.logger,
config={"qsize": 1})
handler = []
request = DataRequest(
venue=Venue("RANDOM"),
data_type=QuoteTick,
metadata={
"Symbol": Symbol("SOMETHING", Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
callback=handler.append,
request_id=self.uuid_factory.generate(),
request_timestamp=self.clock.utc_now(),
)
# Act
self.data_engine.send(request)
self.data_engine.send(request)
# Assert
self.assertEqual(1, self.data_engine.message_qsize())
self.assertEqual(0, self.data_engine.command_count)
async def run_test():
# Arrange
self.data_engine.start() # Also starts client
await asyncio.sleep(0.3) # Allow engine message queue to start
handler = ObjectStorer()
request = DataRequest(
client_name=BINANCE.value,
data_type=DataType(
TradeTick,
metadata={
"InstrumentId": ETHUSDT,
"FromDateTime": None,
"ToDateTime": None,
"Limit": 100,
},
),
callback=handler.store,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request)
await asyncio.sleep(1)
# Assert
self.assertEqual(1, self.data_engine.response_count)
self.assertEqual(1, handler.count)
# Tear Down
self.data_engine.stop()
await self.data_engine.get_run_queue_task()
async def run_test():
# Arrange
self.data_engine.start()
handler = []
request = DataRequest(
venue=Venue("RANDOM"),
data_type=QuoteTick,
metadata={
"Symbol": Symbol("SOMETHING", Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
callback=handler.append,
request_id=self.uuid_factory.generate(),
request_timestamp=self.clock.utc_now(),
)
# Act
self.data_engine.send(request)
await asyncio.sleep(0.1)
# Assert
self.assertEqual(0, self.data_engine.message_qsize())
self.assertEqual(1, self.data_engine.request_count)
# Tear Down
self.data_engine.stop()
def test_send_request_when_no_data_clients_registered_does_nothing(self):
# Arrange
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request)
# Assert
self.assertEqual(1, self.data_engine.request_count)
def test_send_data_request_when_data_type_unrecognized_logs_and_does_nothing(self):
# Arrange
self.data_engine.register_client(self.binance_client)
handler = []
request = DataRequest(
client_id=ClientId(BINANCE.value),
data_type=DataType(
str,
metadata={ # str data type is invalid
"InstrumentId": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request)
# Assert
self.assertEqual(1, self.data_engine.request_count)
async def test_send_request_processes_message(self):
# Arrange
self.engine.start()
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 0
assert self.engine.request_count == 1
# Tear Down
self.engine.stop()
async def run_test():
# Arrange
self.engine.start()
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"InstrumentId":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"FromDateTime":
None,
"ToDateTime":
None,
"Limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.engine.send(request)
await asyncio.sleep(0.1)
# Assert
self.assertEqual(0, self.engine.message_qsize())
self.assertEqual(1, self.engine.request_count)
# Tear Down
self.engine.stop()
def test_send_data_request_when_data_type_unrecognized_logs_and_does_nothing(
self):
# Arrange
self.data_engine.register_client(self.binance_client)
handler = []
request = DataRequest(
venue=BINANCE,
data_type=str, # str data type is invalid
metadata={
"Symbol": Symbol("SOMETHING", Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
callback=handler.append,
request_id=self.uuid_factory.generate(),
request_timestamp=self.clock.utc_now(),
)
# Act
self.data_engine.send(request)
# Assert
self.assertEqual(1, self.data_engine.request_count)
def test_send_data_request_with_duplicate_ids_logs_and_does_not_handle_second(
self):
# Arrange
self.data_engine.register_client(self.binance_client)
self.data_engine.start()
handler = []
uuid = self.uuid_factory.generate() # We'll use this as a duplicate
request1 = DataRequest(
client_id=ClientId(BINANCE.value),
data_type=DataType(
QuoteTick,
metadata={ # str data type is invalid
"instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime": None,
"to_datetime": None,
"limit": 1000,
},
),
callback=handler.append,
request_id=uuid, # Duplicate
timestamp_ns=self.clock.timestamp_ns(),
)
request2 = DataRequest(
client_id=ClientId(BINANCE.value),
data_type=DataType(
QuoteTick,
metadata={ # str data type is invalid
"instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime": None,
"to_datetime": None,
"limit": 1000,
},
),
callback=handler.append,
request_id=uuid, # Duplicate
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request1)
self.data_engine.send(request2)
# Assert
self.assertEqual(2, self.data_engine.request_count)
def test_send_data_request_with_duplicate_ids_logs_and_does_not_handle_second(
self):
# Arrange
self.data_engine.register_client(self.binance_client)
handler = []
uuid = self.uuid_factory.generate() # We'll use this as a duplicate
request1 = DataRequest(
venue=BINANCE,
data_type=QuoteTick, # str data type is invalid
metadata={
"Symbol": Symbol("SOMETHING", Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
callback=handler.append,
request_id=uuid, # Duplicate
request_timestamp=self.clock.utc_now(),
)
request2 = DataRequest(
venue=BINANCE,
data_type=QuoteTick, # str data type is invalid
metadata={
"Symbol": Symbol("SOMETHING", Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
callback=handler.append,
request_id=uuid, # Duplicate
request_timestamp=self.clock.utc_now(),
)
# Act
self.data_engine.send(request1)
self.data_engine.send(request2)
# Assert
self.assertEqual(2, self.data_engine.request_count)
async def run_test():
# Arrange
with open(TEST_PATH + "instruments.json") as response:
instruments = json.load(response)
# Arrange
with open(TEST_PATH + "bars.json") as response:
bars = json.load(response)
self.mock_oanda.request.side_effect = [instruments, bars]
handler = ObjectStorer()
self.data_engine.start()
await asyncio.sleep(0.3)
bar_spec = BarSpecification(1, BarAggregation.MINUTE,
PriceType.MID)
bar_type = BarType(instrument_id=AUDUSD, bar_spec=bar_spec)
request = DataRequest(
client_id=ClientId(OANDA.value),
data_type=DataType(
Bar,
metadata={
"BarType": bar_type,
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
},
),
callback=handler.store,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request)
# Allow time for request to be sent, processed and response returned
await asyncio.sleep(1)
# Assert
self.assertEqual(1, self.data_engine.response_count)
self.assertEqual(1, handler.count)
# Final bar incomplete so becomes partial
self.assertEqual(99, len(handler.get_store()[0]))
# Tear Down
self.data_engine.stop()
self.data_engine.dispose()
def test_data_messages_when_client_id_and_venue_none_raise_value_error(
self):
# Arrange, Act , Assert
with pytest.raises(ValueError) as ex:
Subscribe(
client_id=None,
venue=None,
data_type=DataType(str, {"type": "newswire"}),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
Unsubscribe(
client_id=None,
venue=None,
data_type=DataType(str, {"type": "newswire"}),
command_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
handler = []
DataRequest(
client_id=None,
venue=None,
data_type=DataType(QuoteTick),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
with pytest.raises(ValueError) as ex:
DataResponse(
client_id=None,
venue=None,
data_type=DataType(QuoteTick),
data=[],
correlation_id=self.uuid_factory.generate(),
response_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
assert ex.type == ValueError
assert ex.match("Both `client_id` and `venue` were None")
async def test_message_qsize_at_max_blocks_on_send_request(self):
# Arrange
self.msgbus.deregister(endpoint="DataEngine.execute",
handler=self.engine.execute)
self.msgbus.deregister(endpoint="DataEngine.process",
handler=self.engine.process)
self.msgbus.deregister(endpoint="DataEngine.request",
handler=self.engine.request)
self.msgbus.deregister(endpoint="DataEngine.response",
handler=self.engine.response)
self.engine = LiveDataEngine(
loop=self.loop,
msgbus=self.msgbus,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config=LiveDataEngineConfig(qsize=1),
)
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
venue=None,
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
ts_init=self.clock.timestamp_ns(),
)
# Act
self.engine.request(request)
self.engine.request(request)
await asyncio.sleep(0.1)
# Assert
assert self.engine.message_qsize() == 1
assert self.engine.command_count == 0
async def run_test():
# Arrange
with open(TEST_PATH + "fetch_ohlcv.json") as response:
fetch_ohlcv = json.load(response)
self.mock_ccxt.fetch_ohlcv = fetch_ohlcv
self.data_engine.start() # Also starts client
await asyncio.sleep(0.3) # Allow engine message queue to start
handler = ObjectStorer()
bar_spec = BarSpecification(1, BarAggregation.MINUTE,
PriceType.LAST)
bar_type = BarType(instrument_id=ETHUSDT, bar_spec=bar_spec)
request = DataRequest(
client_name=BINANCE.value,
data_type=DataType(
Bar,
metadata={
"BarType": bar_type,
"FromDateTime": None,
"ToDateTime": None,
"Limit": 100,
},
),
callback=handler.store,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.data_engine.send(request)
await asyncio.sleep(0.3)
# Assert
self.assertEqual(1, self.data_engine.response_count)
self.assertEqual(1, handler.count)
self.assertEqual(100, len(handler.get_store()[0]))
# Tear Down
self.data_engine.stop()
await self.data_engine.get_run_queue_task()
def test_send_request_when_no_data_clients_registered_does_nothing(self):
# Arrange
handler = []
request = DataRequest(
provider="RANDOM",
data_type=DataType(QuoteTick, metadata={
"InstrumentId": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
}),
callback=handler.append,
request_id=self.uuid_factory.generate(),
request_timestamp=self.clock.utc_now(),
)
# Act
self.data_engine.send(request)
# Assert
self.assertEqual(1, self.data_engine.request_count)
def test_data_request_message_str_and_repr(self):
# Arrange
# Act
handler = [].append
request_id = self.uuid_factory.generate()
request = DataRequest(
client_id=ClientId(BINANCE.value),
data_type=DataType(
str,
metadata={ # str data type is invalid
"instrument_id": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime": None,
"to_datetime": None,
"limit": 1000,
},
),
callback=handler,
request_id=request_id,
timestamp_ns=self.clock.timestamp_ns(),
)
# Assert
self.assertEqual(
"DataRequest("
"<str> {'instrument_id': InstrumentId('SOMETHING.RANDOM'), "
"'from_datetime': None, 'to_datetime': None, 'limit': 1000})",
str(request),
)
self.assertEqual(
f"DataRequest("
f"client_id=BINANCE, "
f"data_type=<str> {{'instrument_id': InstrumentId('SOMETHING.RANDOM'), "
f"'from_datetime': None, "
f"'to_datetime': None, "
f"'limit': 1000}}, "
f"callback={repr(handler)}, "
f"id={request_id})",
repr(request),
)
def test_message_qsize_at_max_blocks_on_send_request(self):
# Arrange
self.engine = LiveDataEngine(
loop=self.loop,
portfolio=self.portfolio,
cache=self.cache,
clock=self.clock,
logger=self.logger,
config={"qsize": 1},
)
handler = []
request = DataRequest(
client_id=ClientId("RANDOM"),
data_type=DataType(
QuoteTick,
metadata={
"instrument_id":
InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"from_datetime":
None,
"to_datetime":
None,
"limit":
1000,
},
),
callback=handler.append,
request_id=self.uuid_factory.generate(),
timestamp_ns=self.clock.timestamp_ns(),
)
# Act
self.engine.send(request)
self.engine.send(request)
# Assert
self.assertEqual(1, self.engine.message_qsize())
self.assertEqual(0, self.engine.command_count)
def test_data_request_message_str_and_repr(self):
# Arrange
# Act
handler = [].append
request_id = self.uuid_factory.generate()
request = DataRequest(
provider=BINANCE.value,
data_type=DataType(str, metadata={ # str data type is invalid
"InstrumentId": InstrumentId(Symbol("SOMETHING"), Venue("RANDOM")),
"FromDateTime": None,
"ToDateTime": None,
"Limit": 1000,
}),
callback=handler,
request_id=request_id,
request_timestamp=self.clock.utc_now(),
)
# Assert
self.assertEqual(
"DataRequest("
"<str> {'InstrumentId': InstrumentId('SOMETHING.RANDOM'), "
"'FromDateTime': None, 'ToDateTime': None, 'Limit': 1000})",
str(request),
)
self.assertEqual(
f"DataRequest("
f"provider=BINANCE, "
f"data_type=<str> {{'InstrumentId': InstrumentId('SOMETHING.RANDOM'), "
f"'FromDateTime': None, "
f"'ToDateTime': None, "
f"'Limit': 1000}}, "
f"callback={repr(handler)}, "
f"id={request_id}, "
f"timestamp=1970-01-01 00:00:00+00:00)",
repr(request),
)
