def setUp(self) -> None: super().setUp() self.log_records = [] self.listening_task = None self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) self.data_source = NdaxAPIOrderBookDataSource(self.throttler, [self.trading_pair]) self.data_source.logger().setLevel(1) self.data_source.logger().addHandler(self) self.data_source._trading_pair_id_map.clear() self.mocking_assistant = NetworkMockingAssistant()
def __init__( self, throttler: Optional[AsyncThrottler] = None, shared_client: Optional[aiohttp.ClientSession] = None, trading_pairs: Optional[List[str]] = None, domain: Optional[str] = None, ): super().__init__( NdaxAPIOrderBookDataSource(throttler=throttler, shared_client=shared_client, trading_pairs=trading_pairs, domain=domain), trading_pairs, domain) self._domain = domain self._ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop() self._order_book_snapshot_stream: asyncio.Queue = asyncio.Queue() self._order_book_diff_stream: asyncio.Queue = asyncio.Queue() self._order_book_trade_stream: asyncio.Queue = asyncio.Queue() self._process_msg_deque_task: Optional[asyncio.Task] = None self._past_diffs_windows: Dict[str, Deque] = {} self._order_books: Dict[str, NdaxOrderBook] = {} self._saved_message_queues: Dict[str, Deque[NdaxOrderBookMessage]] = \ defaultdict(lambda: deque(maxlen=1000)) self._order_book_stream_listener_task: Optional[asyncio.Task] = None self._order_book_trade_listener_task: Optional[asyncio.Task] = None self._order_books_initialized_counter: int = 0
class NdaxAPIOrderBookDataSourceUnitTests(unittest.TestCase): # logging.Level required to receive logs from the data source logger level = 0 @classmethod def setUpClass(cls) -> None: super().setUpClass() cls.ev_loop = asyncio.get_event_loop() cls.base_asset = "COINALPHA" cls.quote_asset = "HBOT" cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" cls.instrument_id = 1 def setUp(self) -> None: super().setUp() self.log_records = [] self.listening_task = None self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) self.data_source = NdaxAPIOrderBookDataSource( throttler=self.throttler, trading_pairs=[self.trading_pair]) self.data_source.logger().setLevel(1) self.data_source.logger().addHandler(self) self.data_source._trading_pair_id_map.clear() self.mocking_assistant = NetworkMockingAssistant() def tearDown(self) -> None: self.listening_task and self.listening_task.cancel() super().tearDown() def handle(self, record): self.log_records.append(record) def simulate_trading_pair_ids_initialized(self): self.data_source._trading_pair_id_map.update( {self.trading_pair: self.instrument_id}) def _raise_exception(self, exception_class): raise exception_class def _is_logged(self, log_level: str, message: str) -> bool: return any( record.levelname == log_level and record.getMessage() == message for record in self.log_records) def _subscribe_level_2_response(self): resp = { "m": 1, "i": 2, "n": "SubscribeLevel2", "o": "[[93617617, 1, 1626788175000, 0, 37800.0, 1, 37750.0, 1, 0.015, 0],[93617617, 1, 1626788175000, 0, 37800.0, 1, 37751.0, 1, 0.015, 1]]" } return ujson.dumps(resp) def _orderbook_update_event(self): resp = { "m": 3, "i": 3, "n": "Level2UpdateEvent", "o": "[[93617618, 1, 1626788175001, 0, 37800.0, 1, 37740.0, 1, 0.015, 0]]" } return ujson.dumps(resp) @patch("aiohttp.ClientSession.get") def test_init_trading_pair_ids(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) mock_response: List[Any] = [{ "Product1Symbol": self.base_asset, "Product2Symbol": self.quote_asset, "InstrumentId": self.instrument_id, "SessionStatus": "Running" }, { "Product1Symbol": "ANOTHER_ACTIVE", "Product2Symbol": "MARKET", "InstrumentId": 2, "SessionStatus": "Running" }, { "Product1Symbol": "NOT_ACTIVE", "Product2Symbol": "MARKET", "InstrumentId": 3, "SessionStatus": "Stopped" }] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) self.ev_loop.run_until_complete( self.data_source.init_trading_pair_ids()) self.assertEqual(2, len(self.data_source._trading_pair_id_map)) self.assertEqual( 1, self.data_source._trading_pair_id_map[self.trading_pair]) self.assertEqual( 2, self.data_source._trading_pair_id_map["ANOTHER_ACTIVE-MARKET"]) @patch("aiohttp.ClientSession.get") def test_get_last_traded_prices(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) self.simulate_trading_pair_ids_initialized() mock_response: Dict[Any] = {"LastTradedPx": 1.0} self.mocking_assistant.add_http_response(mock_api, 200, mock_response) results = self.ev_loop.run_until_complete( asyncio.gather( self.data_source.get_last_traded_prices([self.trading_pair]))) results: Dict[str, Any] = results[0] self.assertEqual(results[self.trading_pair], mock_response["LastTradedPx"]) @patch("aiohttp.ClientSession.get") def test_fetch_trading_pairs(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) self.simulate_trading_pair_ids_initialized() mock_response: List[Any] = [{ "Product1Symbol": self.base_asset, "Product2Symbol": self.quote_asset, "InstrumentId": self.instrument_id, "SessionStatus": "Running" }, { "Product1Symbol": "ANOTHER_ACTIVE", "Product2Symbol": "MARKET", "InstrumentId": 2, "SessionStatus": "Running" }, { "Product1Symbol": "NOT_ACTIVE", "Product2Symbol": "MARKET", "InstrumentId": 3, "SessionStatus": "Stopped" }] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) self.mocking_assistant.add_http_response(mock_api, 200, mock_response) results: List[str] = self.ev_loop.run_until_complete( self.data_source.fetch_trading_pairs()) self.assertTrue(self.trading_pair in results) self.assertTrue("ANOTHER_ACTIVE-MARKET" in results) self.assertFalse("NOT_ACTIVE-MARKET" in results) @patch("aiohttp.ClientSession.get") def test_fetch_trading_pairs_with_error_status_in_response(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) mock_response = {} self.mocking_assistant.add_http_response(mock_api, 100, mock_response) result = self.ev_loop.run_until_complete( self.data_source.fetch_trading_pairs()) self.assertEqual(0, len(result)) @patch("aiohttp.ClientSession.get") def test_get_order_book_data(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) self.simulate_trading_pair_ids_initialized() mock_response: List[List[Any]] = [ # mdUpdateId, accountId, actionDateTime, actionType, lastTradePrice, orderId, price, productPairCode, quantity, side [ 93617617, 1, 1626788175416, 0, 37813.22, 1, 37750.6, 1, 0.014698, 0 ] ] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) results = self.ev_loop.run_until_complete( asyncio.gather( self.data_source.get_order_book_data(self.trading_pair))) result = results[0] self.assertTrue("data" in result) self.assertGreaterEqual(len(result["data"]), 0) self.assertEqual(NdaxOrderBookEntry(*mock_response[0]), result["data"][0]) @patch("aiohttp.ClientSession.get") def test_get_order_book_data_raises_exception_when_response_has_error_code( self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) self.simulate_trading_pair_ids_initialized() mock_response = {"Erroneous response"} self.mocking_assistant.add_http_response(mock_api, 100, mock_response) with self.assertRaises(IOError) as context: self.ev_loop.run_until_complete( self.data_source.get_order_book_data(self.trading_pair)) self.assertEqual( str(context.exception), f"Error fetching OrderBook for {self.trading_pair} " f"at {CONSTANTS.ORDER_BOOK_URL}. " f"HTTP {100}. Response: {mock_response}") @patch("aiohttp.ClientSession.get") def test_get_new_order_book(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) self.simulate_trading_pair_ids_initialized() mock_response: List[List[Any]] = [ # mdUpdateId, accountId, actionDateTime, actionType, lastTradePrice, orderId, price, productPairCode, quantity, side [93617617, 1, 1626788175416, 0, 37800.0, 1, 37750.0, 1, 0.015, 0], [93617617, 1, 1626788175416, 0, 37800.0, 1, 37751.0, 1, 0.015, 1] ] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) results = self.ev_loop.run_until_complete( asyncio.gather( self.data_source.get_new_order_book(self.trading_pair))) result: OrderBook = results[0] self.assertTrue(type(result) == OrderBook) self.assertEqual(result.snapshot_uid, 0) @patch("aiohttp.ClientSession.get") def test_get_instrument_ids(self, mock_api): self.mocking_assistant.configure_http_request_mock(mock_api) mock_response: List[Any] = [{ "Product1Symbol": self.base_asset, "Product2Symbol": self.quote_asset, "InstrumentId": self.instrument_id, "SessionStatus": "Running", }] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) results = self.ev_loop.run_until_complete( asyncio.gather(self.data_source.get_instrument_ids())) result: Dict[str, Any] = results[0] self.assertEqual( 1, self.data_source._trading_pair_id_map[self.trading_pair]) self.assertEqual(result[self.trading_pair], self.instrument_id) @patch( "hummingbot.connector.exchange.ndax.ndax_api_order_book_data_source.NdaxAPIOrderBookDataSource._sleep" ) @patch("aiohttp.ClientSession.get") def test_listen_for_snapshots_cancelled_when_fetching_snapshot( self, mock_api, mock_sleep): mock_api.side_effect = asyncio.CancelledError self.simulate_trading_pair_ids_initialized() msg_queue: asyncio.Queue = asyncio.Queue() with self.assertRaises(asyncio.CancelledError): self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_snapshots( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(self.listening_task) self.assertEqual(msg_queue.qsize(), 0) @patch( "hummingbot.connector.exchange.ndax.ndax_api_order_book_data_source.NdaxAPIOrderBookDataSource._sleep" ) @patch("aiohttp.ClientSession.get") def test_listen_for_snapshots_logs_exception_when_fetching_snapshot( self, mock_api, mock_sleep): # the queue and the division by zero error are used just to synchronize the test sync_queue = deque() sync_queue.append(1) self.simulate_trading_pair_ids_initialized() mock_api.side_effect = Exception mock_sleep.side_effect = lambda delay: 1 / 0 if len( sync_queue) == 0 else sync_queue.pop() msg_queue: asyncio.Queue = asyncio.Queue() with self.assertRaises(ZeroDivisionError): self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_snapshots( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(self.listening_task) self.assertEqual(msg_queue.qsize(), 0) self.assertTrue( self._is_logged( "ERROR", "Unexpected error occured listening for orderbook snapshots. Retrying in 5 secs..." )) @patch( "hummingbot.connector.exchange.ndax.ndax_api_order_book_data_source.NdaxAPIOrderBookDataSource._sleep" ) @patch("aiohttp.ClientSession.get") def test_listen_for_snapshots_successful(self, mock_api, mock_sleep): self.mocking_assistant.configure_http_request_mock(mock_api) # the queue and the division by zero error are used just to synchronize the test sync_queue = deque() sync_queue.append(1) mock_response: List[List[Any]] = [ # mdUpdateId, accountId, actionDateTime, actionType, lastTradePrice, orderId, price, productPairCode, quantity, side [93617617, 1, 1626788175416, 0, 37800.0, 1, 37750.0, 1, 0.015, 0], [93617617, 1, 1626788175416, 0, 37800.0, 1, 37751.0, 1, 0.015, 1], ] self.mocking_assistant.add_http_response(mock_api, 200, mock_response) self.simulate_trading_pair_ids_initialized() mock_sleep.side_effect = lambda delay: 1 / 0 if len( sync_queue) == 0 else sync_queue.pop() msg_queue: asyncio.Queue = asyncio.Queue() with self.assertRaises(ZeroDivisionError): self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_snapshots( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(self.listening_task) self.assertEqual(msg_queue.qsize(), 1) snapshot_msg: OrderBookMessage = msg_queue.get_nowait() self.assertEqual(snapshot_msg.update_id, 0) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) def test_listen_for_order_book_diffs_cancelled_when_subscribing( self, mock_ws): msg_queue: asyncio.Queue = asyncio.Queue() mock_ws.return_value = self.mocking_assistant.create_websocket_mock() mock_ws.return_value.send_json.side_effect = asyncio.CancelledError() self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, self._subscribe_level_2_response()) self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, self._orderbook_update_event()) self.simulate_trading_pair_ids_initialized() with self.assertRaises(asyncio.CancelledError): self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_diffs( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(self.listening_task) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) def test_listen_for_order_book_diffs_cancelled_when_listening( self, mock_ws): msg_queue: asyncio.Queue = asyncio.Queue() mock_ws.return_value = self.mocking_assistant.create_websocket_mock() mock_ws.return_value.receive.side_effect = lambda: ( self._raise_exception(asyncio.CancelledError)) self.simulate_trading_pair_ids_initialized() with self.assertRaises(asyncio.CancelledError): self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_diffs( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(self.listening_task) self.assertEqual(msg_queue.qsize(), 0) @patch("hummingbot.client.hummingbot_application.HummingbotApplication") @patch( "hummingbot.connector.exchange.ndax.ndax_api_order_book_data_source.NdaxAPIOrderBookDataSource._sleep" ) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) @patch("aiohttp.ClientSession.get", new_callable=AsyncMock) def test_listen_for_order_book_diffs_logs_exception( self, mock_api, mock_ws, *_): msg_queue: asyncio.Queue = asyncio.Queue() mock_ws.return_value = self.mocking_assistant.create_websocket_mock() mock_ws.return_value.close.return_value = None incomplete_resp = { "m": 1, "i": 2, } self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, ujson.dumps(incomplete_resp)) self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, self._orderbook_update_event()) self.simulate_trading_pair_ids_initialized() self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_diffs( self.ev_loop, msg_queue)) self.ev_loop.run_until_complete(msg_queue.get()) self.assertTrue( self._is_logged("NETWORK", "Unexpected error with WebSocket connection.")) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) def test_listen_for_order_book_diffs_successful(self, mock_ws): msg_queue: asyncio.Queue = asyncio.Queue() mock_ws.return_value = self.mocking_assistant.create_websocket_mock() mock_ws.return_value.send_json.return_value = None self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, self._subscribe_level_2_response()) self.mocking_assistant.add_websocket_aiohttp_message( mock_ws.return_value, self._orderbook_update_event()) self.simulate_trading_pair_ids_initialized() self.listening_task = self.ev_loop.create_task( self.data_source.listen_for_order_book_diffs( self.ev_loop, msg_queue)) first_msg = self.ev_loop.run_until_complete(msg_queue.get()) second_msg = self.ev_loop.run_until_complete(msg_queue.get()) self.assertTrue(first_msg.type == OrderBookMessageType.SNAPSHOT) self.assertTrue(second_msg.type == OrderBookMessageType.DIFF) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) def test_websocket_connection_creation_raises_cancel_exception( self, mock_ws): mock_ws.side_effect = asyncio.CancelledError with self.assertRaises(asyncio.CancelledError): asyncio.get_event_loop().run_until_complete( self.data_source._create_websocket_connection()) @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) def test_websocket_connection_creation_raises_exception_after_loging( self, mock_ws): mock_ws.side_effect = Exception with self.assertRaises(Exception): asyncio.get_event_loop().run_until_complete( self.data_source._create_websocket_connection()) self.assertTrue( self._is_logged( "NETWORK", "Unexpected error occurred during ndax WebSocket Connection ()" ))