def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.binance_client = MockBinanceClient(api_key="TEST_API_KEY")
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.data_source = BinanceAPIUserStreamDataSource(
binance_client=self.binance_client,
domain=self.domain,
throttler=self.throttler)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
def _create_user_stream_data_source(self) -> UserStreamTrackerDataSource:
return BinanceAPIUserStreamDataSource(
auth=self._auth,
trading_pairs=self._trading_pairs,
connector=self,
api_factory=self._web_assistants_factory,
domain=self.domain,
)
def data_source(self) -> UserStreamTrackerDataSource:
"""
Returns the instance of the data source that listens to the private user channel to receive updates from the
exchange. If the instance is not initialized it will be created.
:return: the user stream instance that is listening to user updates from the server using the private channel
"""
if not self._data_source:
self._data_source = BinanceAPIUserStreamDataSource(auth=self._auth, domain=self._domain, throttler=self._throttler)
return self._data_source
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.data_source = BinanceAPIUserStreamDataSource(
auth=BinanceAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider),
domain=self.domain,
throttler=self.throttler
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
def __init__(self,
binance_api_key: str,
binance_api_secret: str,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
):
self._domain = domain
self._binance_time_synchronizer = TimeSynchronizer()
super().__init__()
self._trading_required = trading_required
self._auth = BinanceAuth(
api_key=binance_api_key,
secret_key=binance_api_secret,
time_provider=self._binance_time_synchronizer)
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._api_factory = web_utils.build_api_factory(
throttler=self._throttler,
time_synchronizer=self._binance_time_synchronizer,
domain=self._domain,
auth=self._auth)
self._rest_assistant = None
self._order_book_tracker = OrderBookTracker(
data_source=BinanceAPIOrderBookDataSource(
trading_pairs=trading_pairs,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler),
trading_pairs=trading_pairs,
domain=self._domain)
self._user_stream_tracker = UserStreamTracker(
data_source=BinanceAPIUserStreamDataSource(
auth=self._auth,
domain=self._domain,
throttler=self._throttler,
api_factory=self._api_factory,
time_synchronizer=self._binance_time_synchronizer))
self._ev_loop = asyncio.get_event_loop()
self._poll_notifier = asyncio.Event()
self._last_timestamp = 0
self._order_not_found_records = {} # Dict[client_order_id:str, count:int]
self._trading_rules = {} # Dict[trading_pair:str, TradingRule]
self._trade_fees = {} # Dict[trading_pair:str, (maker_fee_percent:Decimal, taken_fee_percent:Decimal)]
self._last_update_trade_fees_timestamp = 0
self._status_polling_task = None
self._user_stream_tracker_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._last_poll_timestamp = 0
self._last_trades_poll_binance_timestamp = 0
self._order_tracker: ClientOrderTracker = ClientOrderTracker(connector=self)
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.auth = BinanceAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
client_config_map = ClientConfigAdapter(ClientConfigMap())
self.connector = BinanceExchange(
client_config_map=client_config_map,
binance_api_key="",
binance_api_secret="",
trading_pairs=[],
trading_required=False,
domain=self.domain)
self.connector._web_assistants_factory._auth = self.auth
self.data_source = BinanceAPIUserStreamDataSource(
auth=self.auth,
trading_pairs=[self.trading_pair],
connector=self.connector,
api_factory=self.connector._web_assistants_factory,
domain=self.domain
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
self.connector._set_trading_pair_symbol_map(bidict({self.ex_trading_pair: self.trading_pair}))
class BinanceUserStreamDataSourceUnitTests(unittest.TestCase):
# the level is required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "COINALPHA"
cls.quote_asset = "HBOT"
cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
cls.ex_trading_pair = cls.base_asset + cls.quote_asset
cls.domain = "com"
cls.listen_key = "TEST_LISTEN_KEY"
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.data_source = BinanceAPIUserStreamDataSource(
auth=BinanceAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider),
domain=self.domain,
throttler=self.throttler,
time_synchronizer=self.time_synchronizer,
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
def tearDown(self) -> None:
self.listening_task and self.listening_task.cancel()
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def _is_logged(self, log_level: str, message: str) -> bool:
return any(record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def _raise_exception(self, exception_class):
raise exception_class
def _create_exception_and_unlock_test_with_event(self, exception):
self.resume_test_event.set()
raise exception
def _create_return_value_and_unlock_test_with_event(self, value):
self.resume_test_event.set()
return value
def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1):
ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout))
return ret
def _error_response(self) -> Dict[str, Any]:
resp = {
"code": "ERROR CODE",
"msg": "ERROR MESSAGE"
}
return resp
def _user_update_event(self):
# Balance Update
resp = {
"e": "balanceUpdate",
"E": 1573200697110,
"a": "BTC",
"d": "100.00000000",
"T": 1573200697068
}
return json.dumps(resp)
def _successfully_subscribed_event(self):
resp = {
"result": None,
"id": 1
}
return resp
@aioresponses()
def test_get_listen_key_log_exception(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_api.post(regex_url, status=400, body=json.dumps(self._error_response()))
with self.assertRaises(IOError):
self.async_run_with_timeout(self.data_source._get_listen_key())
@aioresponses()
def test_get_listen_key_successful(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_response = {
"listenKey": self.listen_key
}
mock_api.post(regex_url, body=json.dumps(mock_response))
result: str = self.async_run_with_timeout(self.data_source._get_listen_key())
self.assertEqual(self.listen_key, result)
@aioresponses()
def test_ping_listen_key_log_exception(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_api.put(regex_url, status=400, body=json.dumps(self._error_response()))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(self.data_source._ping_listen_key())
self.assertTrue(self._is_logged("WARNING", f"Failed to refresh the listen key {self.listen_key}: {self._error_response()}"))
self.assertFalse(result)
@aioresponses()
def test_ping_listen_key_successful(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_api.put(regex_url, body=json.dumps({}))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(self.data_source._ping_listen_key())
self.assertTrue(result)
@patch("hummingbot.connector.exchange.binance.binance_api_user_stream_data_source.BinanceAPIUserStreamDataSource"
"._ping_listen_key",
new_callable=AsyncMock)
def test_manage_listen_key_task_loop_keep_alive_failed(self, mock_ping_listen_key):
mock_ping_listen_key.side_effect = (lambda *args, **kwargs:
self._create_return_value_and_unlock_test_with_event(False))
self.data_source._current_listen_key = self.listen_key
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(self._is_logged("ERROR", "Error occurred renewing listen key ..."))
self.assertIsNone(self.data_source._current_listen_key)
self.assertFalse(self.data_source._listen_key_initialized_event.is_set())
@patch("hummingbot.connector.exchange.binance.binance_api_user_stream_data_source.BinanceAPIUserStreamDataSource."
"_ping_listen_key",
new_callable=AsyncMock)
def test_manage_listen_key_task_loop_keep_alive_successful(self, mock_ping_listen_key):
mock_ping_listen_key.side_effect = (lambda *args, **kwargs:
self._create_return_value_and_unlock_test_with_event(True))
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._current_listen_key = self.listen_key
self.data_source._listen_key_initialized_event.set()
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(self._is_logged("INFO", f"Refreshed listen key {self.listen_key}."))
self.assertGreater(self.data_source._last_listen_key_ping_ts, 0)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_get_listen_key_successful_with_user_update_event(self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_response = {
"listenKey": self.listen_key
}
mock_api.post(regex_url, body=json.dumps(mock_response))
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(mock_ws.return_value, self._user_update_event())
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue)
)
msg = self.async_run_with_timeout(msg_queue.get())
self.assertEqual(json.loads(self._user_update_event()), msg)
mock_ws.return_value.ping.assert_called()
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_does_not_queue_empty_payload(self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_response = {
"listenKey": self.listen_key
}
mock_api.post(regex_url, body=json.dumps(mock_response))
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(mock_ws.return_value, "")
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue)
)
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(mock_ws.return_value)
self.assertEqual(0, msg_queue.qsize())
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_connection_failed(self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_response = {
"listenKey": self.listen_key
}
mock_api.post(regex_url, body=json.dumps(mock_response))
mock_ws.side_effect = lambda *arg, **kwars: self._create_exception_and_unlock_test_with_event(
Exception("TEST ERROR."))
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue)
)
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged("ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds..."))
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_iter_message_throws_exception(self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.BINANCE_USER_STREAM_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))
mock_response = {
"listenKey": self.listen_key
}
mock_api.post(regex_url, body=json.dumps(mock_response))
msg_queue: asyncio.Queue = asyncio.Queue()
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
mock_ws.return_value.receive.side_effect = (lambda *args, **kwargs:
self._create_exception_and_unlock_test_with_event(
Exception("TEST ERROR")))
mock_ws.close.return_value = None
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue)
)
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds..."))
class BinanceUserStreamDataSourceUnitTests(unittest.TestCase):
# the level is required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "COINALPHA"
cls.quote_asset = "HBOT"
cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
cls.ex_trading_pair = cls.base_asset + cls.quote_asset
cls.domain = "com"
cls.listen_key = "TEST_LISTEN_KEY"
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.binance_client = MockBinanceClient(api_key="TEST_API_KEY")
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.data_source = BinanceAPIUserStreamDataSource(
binance_client=self.binance_client,
domain=self.domain,
throttler=self.throttler)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
def tearDown(self) -> None:
self.listening_task and self.listening_task.cancel()
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def _is_logged(self, log_level: str, message: str) -> bool:
return any(
record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def _raise_exception(self, exception_class):
raise exception_class
def _error_response(self) -> Dict[str, Any]:
resp = {"code": "ERROR CODE", "msg": "ERROR MESSAGE"}
return resp
def _user_update_event(self):
# Balance Update
resp = {
"e": "balanceUpdate",
"E": 1573200697110,
"a": "BTC",
"d": "100.00000000",
"T": 1573200697068
}
return ujson.dumps(resp)
def test_last_recv_time(self):
# Initial last_recv_time
self.assertEqual(0, self.data_source.last_recv_time)
def test_get_throttler_instance(self):
self.assertIsInstance(self.data_source._get_throttler_instance(),
AsyncThrottler)
@patch("aiohttp.ClientSession.post")
def test_get_listen_key_log_exception(self, mock_api):
self.mocking_assistant.configure_http_request_mock(mock_api)
self.mocking_assistant.add_http_response(mock_api, 400,
self._error_response())
with self.assertRaises(IOError):
self.ev_loop.run_until_complete(self.data_source.get_listen_key())
@patch("aiohttp.ClientSession.post")
def test_get_listen_key_successful(self, mock_api):
self.mocking_assistant.configure_http_request_mock(mock_api)
self.mocking_assistant.add_http_response(
mock_api, 200, {"listenKey": self.listen_key})
result: str = self.ev_loop.run_until_complete(
self.data_source.get_listen_key())
self.assertEqual(self.listen_key, result)
@patch("aiohttp.ClientSession.put")
def test_ping_listen_key_log_exception(self, mock_api):
self.mocking_assistant.configure_http_request_mock(mock_api)
self.mocking_assistant.add_http_response(mock_api, 400,
self._error_response())
result: bool = self.ev_loop.run_until_complete(
self.data_source.ping_listen_key(listen_key=self.listen_key))
self.assertTrue(
self._is_logged(
"WARNING",
f"Failed to refresh the listen key {self.listen_key}: {self._error_response()}"
))
self.assertFalse(result)
@patch("aiohttp.ClientSession.put")
def test_ping_listen_key_successful(self, mock_api):
self.mocking_assistant.configure_http_request_mock(mock_api)
self.mocking_assistant.add_http_response(mock_api, 200, {})
result: bool = self.ev_loop.run_until_complete(
self.data_source.ping_listen_key(listen_key=self.listen_key))
self.assertTrue(result)
@patch("aiohttp.ClientSession.post")
@patch("websockets.connect", new_callable=AsyncMock)
@patch(
"hummingbot.core.data_type.user_stream_tracker_data_source.UserStreamTrackerDataSource.wait_til_next_tick",
new_callable=AsyncMock)
def test_listen_for_user_stream_no_listen_key(self, mock_next_tick,
mock_ws, mock_post):
# mock_next_tick.return_value = None
self.mocking_assistant.configure_http_request_mock(mock_post)
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_text_message(
mock_ws.return_value, self._user_update_event())
# Add REST API response for get_listen_key()
self.mocking_assistant.add_http_response(
mock_post, 200, {"listenKey": self.listen_key})
# Add REST API response for _ping_listen_key()
# self.mocking_assistant.add_http_response(mock_post, 200, {})
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(self.ev_loop, msg_queue))
msg = self.ev_loop.run_until_complete(msg_queue.get())
self.assertTrue(msg, self._user_update_event)