def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.auth = KucoinAuth(self.api_key,
self.api_passphrase,
self.api_secret_key,
time_provider=self.mock_time_provider)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.api_factory = web_utils.build_api_factory(
throttler=self.throttler,
time_synchronizer=self.time_synchronizer,
auth=self.auth)
self.data_source = KucoinAPIUserStreamDataSource(
throttler=self.throttler, api_factory=self.api_factory)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.auth = GateIoAuth(api_key=self.api_key,
secret_key=self.api_secret_key,
time_provider=self.mock_time_provider)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
client_config_map = ClientConfigAdapter(ClientConfigMap())
self.connector = GateIoExchange(client_config_map=client_config_map,
gate_io_api_key="",
gate_io_secret_key="",
trading_pairs=[],
trading_required=False)
self.connector._web_assistants_factory._auth = self.auth
self.data_source = GateIoAPIUserStreamDataSource(
self.auth,
trading_pairs=[self.trading_pair],
connector=self.connector,
api_factory=self.connector._web_assistants_factory)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.connector._set_trading_pair_symbol_map(
bidict({self.ex_trading_pair: self.trading_pair}))
def test_time_calculated_with_mean_of_all_offsets(self, _,
seconds_counter_mock):
first_time = 1640000003.0
second_time = 16400000010.0
third_time = 1640000016.0
seconds_counter_mock.side_effect = [2, 4, 6, 10, 11, 13, 25]
time_provider = TimeSynchronizer()
for time in [first_time, second_time, third_time]:
self.async_run_with_timeout(
time_provider.update_server_time_offset_with_time_provider(
time_provider=self.configurable_timestamp_provider(time *
1e3)))
synchronized_time = time_provider.time()
first_expected_offset = first_time - (4 + 2) / 2
second_expected_offset = second_time - (10 + 6) / 2
third_expected_offset = third_time - (13 + 11) / 2
expected_offsets = [
first_expected_offset, second_expected_offset,
third_expected_offset
]
seconds_difference_when_calculating_current_time = 25
self.assertEqual(
statistics.median(expected_offsets) +
seconds_difference_when_calculating_current_time,
synchronized_time)
def test_time_with_registered_offsets_returns_local_time(
self, time_mock, seconds_counter_mock):
now = 1640000000.0
time_mock.side_effect = [now]
seconds_counter_mock.side_effect = [2, 3]
time_provider = TimeSynchronizer()
synchronized_time = time_provider.time()
self.assertEqual(now + (2 - 3), synchronized_time)
def __init__(self,
binance_api_key: str,
binance_api_secret: str,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
):
self._domain = domain
self._binance_time_synchronizer = TimeSynchronizer()
super().__init__()
self._trading_required = trading_required
self._auth = BinanceAuth(
api_key=binance_api_key,
secret_key=binance_api_secret,
time_provider=self._binance_time_synchronizer)
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._api_factory = web_utils.build_api_factory(
throttler=self._throttler,
time_synchronizer=self._binance_time_synchronizer,
domain=self._domain,
auth=self._auth)
self._rest_assistant = None
self._order_book_tracker = OrderBookTracker(
data_source=BinanceAPIOrderBookDataSource(
trading_pairs=trading_pairs,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler),
trading_pairs=trading_pairs,
domain=self._domain)
self._user_stream_tracker = UserStreamTracker(
data_source=BinanceAPIUserStreamDataSource(
auth=self._auth,
domain=self._domain,
throttler=self._throttler,
api_factory=self._api_factory,
time_synchronizer=self._binance_time_synchronizer))
self._ev_loop = asyncio.get_event_loop()
self._poll_notifier = asyncio.Event()
self._last_timestamp = 0
self._order_not_found_records = {} # Dict[client_order_id:str, count:int]
self._trading_rules = {} # Dict[trading_pair:str, TradingRule]
self._trade_fees = {} # Dict[trading_pair:str, (maker_fee_percent:Decimal, taken_fee_percent:Decimal)]
self._last_update_trade_fees_timestamp = 0
self._status_polling_task = None
self._user_stream_tracker_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._last_poll_timestamp = 0
self._last_trades_poll_binance_timestamp = 0
self._order_tracker: ClientOrderTracker = ClientOrderTracker(connector=self)
def test_time_with_one_registered_offset(self, _, seconds_counter_mock):
now = 1640000020.0
seconds_counter_mock.side_effect = [10, 30, 31]
time_provider = TimeSynchronizer()
self.async_run_with_timeout(
time_provider.update_server_time_offset_with_time_provider(
time_provider=self.configurable_timestamp_provider(now * 1e3)))
synchronized_time = time_provider.time()
seconds_difference_getting_time = 30 - 10
seconds_difference_when_calculating_current_time = 31
self.assertEqual(
now - seconds_difference_getting_time +
seconds_difference_when_calculating_current_time,
synchronized_time)
def setUpClass(cls):
cls.domain = "com"
cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop()
api_key = conf.latoken_api_key
secret_key = conf.latoken_secret_key
cls.trading_pair = "BTC-USDT"
ts = TimeSynchronizer()
auth = LatokenAuth(api_key, secret_key, ts)
cls.api_factory = LatokenWebAssistantsFactory(auth=auth)
async def api_request(path: str,
api_factory: Optional[WebAssistantsFactory] = None,
throttler: Optional[AsyncThrottler] = None,
time_synchronizer: Optional[TimeSynchronizer] = None,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
method: RESTMethod = RESTMethod.GET,
is_auth_required: bool = False,
return_err: bool = False,
limit_id: Optional[str] = None,
timeout: Optional[float] = None,
headers: Dict[str, Any] = {}):
throttler = throttler or create_throttler()
time_synchronizer = time_synchronizer or TimeSynchronizer()
# If api_factory is not provided a default one is created
# The default instance has no authentication capabilities and all authenticated requests will fail
api_factory = api_factory or build_api_factory(
throttler=throttler,
time_synchronizer=time_synchronizer,
domain=domain,
)
rest_assistant = await api_factory.get_rest_assistant()
local_headers = {"Content-Type": "application/x-www-form-urlencoded"}
local_headers.update(headers)
url = rest_url(path, domain=domain)
request = RESTRequest(method=method,
url=url,
params=params,
data=data,
headers=local_headers,
is_auth_required=is_auth_required,
throttler_limit_id=limit_id if limit_id else path)
async with throttler.execute_task(limit_id=limit_id if limit_id else path):
response = await rest_assistant.call(request=request, timeout=timeout)
if response.status != 200:
if return_err:
error_response = await response.json()
return error_response
else:
error_response = await response.text()
if error_response is not None and "ret_code" in error_response and "ret_msg" in error_response:
raise IOError(
f"The request to Bybit failed. Error: {error_response}. Request: {request}"
)
else:
raise IOError(
f"Error executing request {method.name} {path}. "
f"HTTP status is {response.status}. "
f"Error: {error_response}")
return await response.json()
def setUpClass(cls):
cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop()
cls.trading_pairs = ["BTC-USD"]
cls.user_stream_tracker: CoinflexUserStreamTracker = CoinflexUserStreamTracker(
domain=os.getenv("COINFLEX_DOMAIN", "live"),
auth=CoinflexAuth(api_key=cls.api_key,
secret_key=cls.api_secret,
time_provider=TimeSynchronizer()))
cls.user_stream_tracker_task: asyncio.Task = safe_ensure_future(
cls.user_stream_tracker.start())
def test_build_api_factory(self):
api_factory = web_utils.build_api_factory(
time_synchronizer=TimeSynchronizer(),
time_provider=lambda: None,
)
self.assertIsInstance(api_factory, WebAssistantsFactory)
self.assertIsNone(api_factory._auth)
self.assertTrue(2, len(api_factory._rest_pre_processors))
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.async_task = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self.time_synchronnizer = TimeSynchronizer()
self.time_synchronnizer.add_time_offset_ms_sample(1000)
self.ob_data_source = BybitAPIOrderBookDataSource(
trading_pairs=[self.trading_pair],
throttler=self.throttler,
time_synchronizer=self.time_synchronnizer)
self.ob_data_source.logger().setLevel(1)
self.ob_data_source.logger().addHandler(self)
BybitAPIOrderBookDataSource._trading_pair_symbol_map = {
self.domain: bidict({self.ex_trading_pair: self.trading_pair})
}
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task = None
self.async_tasks: List[asyncio.Task] = []
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.data_source = BinancePerpetualAPIOrderBookDataSource(
time_synchronizer=self.time_synchronizer,
trading_pairs=[self.trading_pair],
domain=self.domain,
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.mocking_assistant = NetworkMockingAssistant()
self.resume_test_event = asyncio.Event()
BinancePerpetualAPIOrderBookDataSource._trading_pair_symbol_map = {
self.domain: bidict({self.ex_trading_pair: self.trading_pair})
}
def __init__(
self,
client_config_map: "ClientConfigAdapter",
bitmex_api_key: str = None,
bitmex_api_secret: str = None,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain: str = CONSTANTS.DOMAIN,
):
self._bitmex_time_synchronizer = TimeSynchronizer()
self._auth: BitmexAuth = BitmexAuth(api_key=bitmex_api_key,
api_secret=bitmex_api_secret)
self._trading_pairs = trading_pairs
self._trading_required = trading_required
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._domain = domain
self._api_factory = web_utils.build_api_factory(auth=self._auth)
self._rest_assistant: Optional[RESTAssistant] = None
self._ws_assistant: Optional[WSAssistant] = None
ExchangeBase.__init__(self, client_config_map=client_config_map)
self._user_stream_tracker = BitmexUserStreamTracker(
auth=self._auth,
domain=self._domain,
throttler=self._throttler,
api_factory=self._api_factory,
time_synchronizer=self._bitmex_time_synchronizer)
self._order_book_tracker = BitmexOrderBookTracker(
trading_pairs=trading_pairs,
domain=self._domain,
throttler=self._throttler,
api_factory=self._api_factory)
self._ev_loop = asyncio.get_event_loop()
self._poll_notifier = asyncio.Event()
self._order_not_found_records = defaultdict(int)
self._last_timestamp = 0
self._trading_rules = {}
self._in_flight_orders = {}
self._status_polling_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._user_stream_tracker_task = None
self._last_poll_timestamp = 0
self._client_order_tracker: ClientOrderTracker = ClientOrderTracker(
connector=self)
self._trading_pair_to_multipliers = {}
self._trading_pair_price_estimate_for_quantize = {}
self._token_multiplier = {}
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task = None
self.mocking_assistant = NetworkMockingAssistant()
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(1000)
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.data_source = BinanceAPIOrderBookDataSource(
trading_pairs=[self.trading_pair],
throttler=self.throttler,
domain=self.domain,
time_synchronizer=self.time_synchronizer)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
BinanceAPIOrderBookDataSource._trading_pair_symbol_map = {
"com":
bidict({f"{self.base_asset}{self.quote_asset}": self.trading_pair})
}
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.auth = BinanceAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
client_config_map = ClientConfigAdapter(ClientConfigMap())
self.connector = BinanceExchange(
client_config_map=client_config_map,
binance_api_key="",
binance_api_secret="",
trading_pairs=[],
trading_required=False,
domain=self.domain)
self.connector._web_assistants_factory._auth = self.auth
self.data_source = BinanceAPIUserStreamDataSource(
auth=self.auth,
trading_pairs=[self.trading_pair],
connector=self.connector,
api_factory=self.connector._web_assistants_factory,
domain=self.domain
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
self.connector._set_trading_pair_symbol_map(bidict({self.ex_trading_pair: self.trading_pair}))
def __init__(self,
kucoin_api_key: str,
kucoin_passphrase: str,
kucoin_secret_key: str,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain: str = CONSTANTS.DEFAULT_DOMAIN):
self._domain = domain
self._time_synchronizer = TimeSynchronizer()
super().__init__()
self._auth = KucoinAuth(
api_key=kucoin_api_key,
passphrase=kucoin_passphrase,
secret_key=kucoin_secret_key,
time_provider=self._time_synchronizer)
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._api_factory = web_utils.build_api_factory(
throttler=self._throttler,
time_synchronizer=self._time_synchronizer,
domain=self._domain,
auth=self._auth)
self._last_poll_timestamp = 0
self._last_timestamp = 0
self._trading_pairs = trading_pairs
self._order_book_tracker = OrderBookTracker(
data_source=KucoinAPIOrderBookDataSource(
trading_pairs=trading_pairs,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler,
time_synchronizer=self._time_synchronizer),
trading_pairs=trading_pairs,
domain=self._domain)
self._user_stream_tracker = UserStreamTracker(
data_source=KucoinAPIUserStreamDataSource(
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler))
self._poll_notifier = asyncio.Event()
self._status_polling_task = None
self._user_stream_tracker_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._trading_fees_polling_task = None
self._trading_required = trading_required
self._trading_rules = {}
self._trading_fees = {}
self._order_tracker: ClientOrderTracker = ClientOrderTracker(connector=self)
async def api_request(path: str,
api_factory: Optional[WebAssistantsFactory] = None,
throttler: Optional[AsyncThrottler] = None,
time_synchronizer: Optional[TimeSynchronizer] = None,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
method: RESTMethod = RESTMethod.GET,
is_auth_required: bool = False,
return_err: bool = False,
limit_id: Optional[str] = None,
timeout: Optional[float] = None,
headers=None) -> Union[str, Dict[str, Any]]:
if headers is None:
headers = {}
throttler = throttler or create_throttler()
time_synchronizer = time_synchronizer or TimeSynchronizer()
# If api_factory is not provided a default one is created
# The default instance has no authentication capabilities and all authenticated requests will fail
api_factory = api_factory or build_api_factory(
throttler=throttler,
time_synchronizer=time_synchronizer,
domain=domain,
)
rest_assistant = await api_factory.get_rest_assistant()
local_headers = {"Content-Type": "application/json"}
local_headers.update(headers)
url = private_rest_url(
path, domain=domain) if is_auth_required else public_rest_url(
path, domain=domain)
# top_level_function_name = cast(FrameType, cast(FrameType, inspect.currentframe()).f_back).f_code.co_name
# print(f"top_level_function_name={top_level_function_name} limit_id={limit_id} url={url}")
request = RESTRequest(method=method,
url=url,
params=params,
data=data,
headers=local_headers,
is_auth_required=is_auth_required,
throttler_limit_id=limit_id if limit_id else path)
async with throttler.execute_task(limit_id=limit_id if limit_id else path):
response = await rest_assistant.call(request=request, timeout=timeout)
if response.status != 200 and not return_err:
raise IOError(f"Error for Response: {response}.")
return await response.json()
def __init__(
self,
client_config_map: "ClientConfigAdapter",
bybit_api_key: str,
bybit_api_secret: str,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
):
self._domain = domain
self._time_synchronizer = TimeSynchronizer()
super().__init__(client_config_map)
self._trading_required = trading_required
self._auth = BybitAuth(api_key=bybit_api_key,
secret_key=bybit_api_secret,
time_provider=self._time_synchronizer)
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._api_factory = web_utils.build_api_factory(
throttler=self._throttler,
time_synchronizer=self._time_synchronizer,
domain=self._domain,
auth=self._auth)
self._rest_assistant = None
self._order_book_tracker = OrderBookTracker(
data_source=BybitAPIOrderBookDataSource(
trading_pairs=trading_pairs,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler),
trading_pairs=trading_pairs,
domain=self._domain)
self._user_stream_tracker = UserStreamTracker(
data_source=BybitAPIUserStreamDataSource(
auth=self._auth,
domain=self._domain,
throttler=self._throttler,
api_factory=self._api_factory,
time_synchronizer=self._time_synchronizer))
self._poll_notifier = asyncio.Event()
self._last_timestamp = 0
self._trading_rules = {}
self._trade_fees = {}
self._status_polling_task = None
self._user_stream_tracker_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._last_poll_timestamp = 0
self._order_tracker: ClientOrderTracker = ClientOrderTracker(
connector=self)
async def api_request(path: str,
api_factory: Optional[WebAssistantsFactory] = None,
throttler: Optional[AsyncThrottler] = None,
time_synchronizer: Optional[TimeSynchronizer] = None,
domain: str = CONSTANTS.DOMAIN,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
method: RESTMethod = RESTMethod.GET,
is_auth_required: bool = False,
return_err: bool = False,
api_version: str = CONSTANTS.API_VERSION,
limit_id: Optional[str] = None,
timeout: Optional[float] = None):
throttler = throttler or create_throttler()
time_synchronizer = time_synchronizer or TimeSynchronizer()
# If api_factory is not provided a default one is created
# The default instance has no authentication capabilities and all authenticated requests will fail
api_factory = api_factory or build_api_factory(
throttler=throttler,
time_synchronizer=time_synchronizer,
domain=domain,
)
rest_assistant = await api_factory.get_rest_assistant()
async with throttler.execute_task(limit_id=limit_id if limit_id else path):
url = rest_url(path, domain, api_version)
request = RESTRequest(
method=method,
url=url,
params=params,
data=data,
is_auth_required=is_auth_required,
throttler_limit_id=limit_id if limit_id else path
)
response = await rest_assistant.call(request=request, timeout=timeout)
if response.status != 200:
if return_err:
error_response = await response.json()
return error_response
else:
error_response = await response.text()
raise IOError(f"Error executing request {method.name} {path}. "
f"HTTP status is {response.status}. "
f"Error: {error_response}")
return await response.json()
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.data_source = BinanceAPIUserStreamDataSource(
auth=BinanceAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider),
domain=self.domain,
throttler=self.throttler,
time_synchronizer=self.time_synchronizer,
)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
def setUpClass(cls):
cls.ev_loop: asyncio.BaseEventLoop = asyncio.get_event_loop()
ts = TimeSynchronizer()
auth = LatokenAuth(conf.latoken_api_key, conf.latoken_secret_key, ts)
cls.trading_pair = ["ETH-USDT"]
throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
api_factory = LatokenWebAssistantsFactory(auth=auth)
data_source = LatokenAPIUserStreamDataSource(auth=auth,
domain=domain,
api_factory=api_factory,
throttler=throttler)
cls.user_stream_tracker: LatokenUserStreamTracker = LatokenUserStreamTracker(
auth=auth, data_source=data_source, domain=domain)
cls.user_stream_tracker_task: asyncio.Task = asyncio.ensure_future(
cls.user_stream_tracker.start())
def build_api_factory(
throttler: Optional[AsyncThrottler] = None,
time_synchronizer: Optional[TimeSynchronizer] = None,
time_provider: Optional[Callable] = None,
auth: Optional[AuthBase] = None, ) -> WebAssistantsFactory:
throttler = throttler or create_throttler()
time_synchronizer = time_synchronizer or TimeSynchronizer()
time_provider = time_provider or (lambda: get_current_server_time(throttler=throttler))
api_factory = WebAssistantsFactory(
throttler=throttler,
auth=auth,
rest_pre_processors=[
TimeSynchronizerRESTPreProcessor(synchronizer=time_synchronizer, time_provider=time_provider),
])
return api_factory
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.emulated_time = 1640001112.223
self.auth = BinancePerpetualAuth(api_key=self.api_key,
api_secret=self.secret_key,
time_provider=self)
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.data_source = BinancePerpetualUserStreamDataSource(
auth=self.auth,
domain=self.domain,
throttler=self.throttler,
time_synchronizer=self.time_synchronizer)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.mock_done_event = asyncio.Event()
self.resume_test_event = asyncio.Event()
def build_api_factory(
throttler: Optional[AsyncThrottler] = None,
time_synchronizer: Optional[TimeSynchronizer] = None,
domain: str = CONSTANTS.DEFAULT_DOMAIN,
time_provider: Optional[Callable] = None,
auth: Optional[AuthBase] = None,
) -> WebAssistantsFactory:
time_synchronizer = time_synchronizer or TimeSynchronizer()
time_provider = time_provider or (lambda: get_current_server_time(
throttler=throttler,
domain=domain,
))
api_factory = WebAssistantsFactory(auth=auth,
rest_pre_processors=[
TimeSynchronizerRESTPreProcessor(
synchronizer=time_synchronizer,
time_provider=time_provider),
])
return api_factory
def __init__(self, client_config_map: "ClientConfigAdapter"):
super().__init__(client_config_map)
self._last_poll_timestamp = 0
self._last_timestamp = 0
self._trading_rules = {}
self._trading_fees = {}
self._status_polling_task = None
self._user_stream_tracker_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._trading_fees_polling_task = None
self._time_synchronizer = TimeSynchronizer()
self._throttler = AsyncThrottler(self.rate_limits_rules)
self._poll_notifier = asyncio.Event()
# init Auth and Api factory
self._auth: AuthBase = self.authenticator
self._web_assistants_factory: WebAssistantsFactory = self._create_web_assistants_factory(
)
# init OrderBook Data Source and Tracker
self._orderbook_ds: OrderBookTrackerDataSource = self._create_order_book_data_source(
)
self._set_order_book_tracker(
OrderBookTracker(data_source=self._orderbook_ds,
trading_pairs=self.trading_pairs,
domain=self.domain))
# init UserStream Data Source and Tracker
self._userstream_ds = self._create_user_stream_data_source()
self._user_stream_tracker = UserStreamTracker(
data_source=self._userstream_ds)
self._order_tracker: ClientOrderTracker = ClientOrderTracker(
connector=self)
class BinanceExchange(ExchangeBase):
SHORT_POLL_INTERVAL = 5.0
UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0
LONG_POLL_INTERVAL = 120.0
MAX_ORDER_UPDATE_RETRIEVAL_RETRIES_WITH_FAILURES = 3
def __init__(self,
binance_api_key: str,
binance_api_secret: str,
trading_pairs: Optional[List[str]] = None,
trading_required: bool = True,
domain="com"):
self._domain = domain
self._binance_time_synchronizer = TimeSynchronizer()
super().__init__()
self._trading_required = trading_required
self._auth = BinanceAuth(api_key=binance_api_key,
secret_key=binance_api_secret,
time_provider=self._binance_time_synchronizer)
self._api_factory = WebAssistantsFactory(auth=self._auth)
self._rest_assistant = None
self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self._order_book_tracker = BinanceOrderBookTracker(
trading_pairs=trading_pairs,
domain=domain,
api_factory=self._api_factory,
throttler=self._throttler)
self._user_stream_tracker = BinanceUserStreamTracker(
auth=self._auth, domain=domain, throttler=self._throttler)
self._ev_loop = asyncio.get_event_loop()
self._poll_notifier = asyncio.Event()
self._last_timestamp = 0
self._order_not_found_records = {
} # Dict[client_order_id:str, count:int]
self._trading_rules = {} # Dict[trading_pair:str, TradingRule]
self._trade_fees = {
} # Dict[trading_pair:str, (maker_fee_percent:Decimal, taken_fee_percent:Decimal)]
self._last_update_trade_fees_timestamp = 0
self._status_polling_task = None
self._user_stream_event_listener_task = None
self._trading_rules_polling_task = None
self._last_poll_timestamp = 0
self._last_trades_poll_binance_timestamp = 0
self._order_tracker: ClientOrderTracker = ClientOrderTracker(
connector=self)
@classmethod
def logger(cls) -> HummingbotLogger:
global s_logger
if s_logger is None:
s_logger = logging.getLogger(__name__)
return s_logger
@property
def name(self) -> str:
if self._domain == "com":
return "binance"
else:
return f"binance_{self._domain}"
@property
def order_books(self) -> Dict[str, OrderBook]:
return self._order_book_tracker.order_books
@property
def trading_rules(self) -> Dict[str, TradingRule]:
return self._trading_rules
@property
def in_flight_orders(self) -> Dict[str, InFlightOrder]:
return self._order_tracker.active_orders
@property
def limit_orders(self) -> List[LimitOrder]:
return [
in_flight_order.to_limit_order()
for in_flight_order in self.in_flight_orders.values()
]
@property
def tracking_states(self) -> Dict[str, any]:
"""
Returns a dictionary associating current active orders client id to their JSON representation
"""
return {
key: value.to_json()
for key, value in self.in_flight_orders.items()
}
@property
def order_book_tracker(self) -> BinanceOrderBookTracker:
return self._order_book_tracker
@property
def user_stream_tracker(self) -> BinanceUserStreamTracker:
return self._user_stream_tracker
@property
def status_dict(self) -> Dict[str, bool]:
"""
Returns a dictionary with the values of all the conditions that determine if the connector is ready to operate.
The key of each entry is the condition name, and the value is True if condition is ready, False otherwise.
"""
return {
"symbols_mapping_initialized":
BinanceAPIOrderBookDataSource.trading_pair_symbol_map_ready(
domain=self._domain),
"order_books_initialized":
self._order_book_tracker.ready,
"account_balance":
len(self._account_balances) > 0
if self._trading_required else True,
"trading_rule_initialized":
len(self._trading_rules) > 0,
}
@property
def ready(self) -> bool:
"""
Returns True if the connector is ready to operate (all connections established with the exchange). If it is
not ready it returns False.
"""
return all(self.status_dict.values())
@staticmethod
def binance_order_type(order_type: OrderType) -> str:
return order_type.name.upper()
@staticmethod
def to_hb_order_type(binance_type: str) -> OrderType:
return OrderType[binance_type]
def supported_order_types(self):
return [OrderType.LIMIT, OrderType.LIMIT_MAKER]
async def start_network(self):
"""
Start all required tasks to update the status of the connector. Those tasks include:
- The order book tracker
- The polling loop to update the trading rules
- The polling loop to update order status and balance status using REST API (backup for main update process)
- The background task to process the events received through the user stream tracker (websocket connection)
"""
self._order_book_tracker.start()
self._trading_rules_polling_task = safe_ensure_future(
self._trading_rules_polling_loop())
if self._trading_required:
self._status_polling_task = safe_ensure_future(
self._status_polling_loop())
self._user_stream_tracker_task = safe_ensure_future(
self._user_stream_tracker.start())
self._user_stream_event_listener_task = safe_ensure_future(
self._user_stream_event_listener())
async def stop_network(self):
"""
This function is executed when the connector is stopped. It perform a general cleanup and stops all background
tasks that require the connection with the exchange to work.
"""
# Reset timestamps and _poll_notifier for status_polling_loop
self._last_poll_timestamp = 0
self._last_timestamp = 0
self._poll_notifier = asyncio.Event()
self._order_book_tracker.stop()
if self._status_polling_task is not None:
self._status_polling_task.cancel()
if self._user_stream_tracker_task is not None:
self._user_stream_tracker_task.cancel()
if self._user_stream_event_listener_task is not None:
self._user_stream_event_listener_task.cancel()
if self._trading_rules_polling_task is not None:
self._trading_rules_polling_task.cancel()
self._status_polling_task = self._user_stream_tracker_task = self._user_stream_event_listener_task = None
async def check_network(self) -> NetworkStatus:
"""
Checks connectivity with the exchange using the API
"""
try:
await self._api_request(
method=RESTMethod.GET,
path_url=CONSTANTS.PING_PATH_URL,
)
except asyncio.CancelledError:
raise
except Exception:
return NetworkStatus.NOT_CONNECTED
return NetworkStatus.CONNECTED
def restore_tracking_states(self, saved_states: Dict[str, any]):
"""
Restore in-flight orders from saved tracking states, this is st the connector can pick up on where it left off
when it disconnects.
:param saved_states: The saved tracking_states.
"""
self._order_tracker.restore_tracking_states(
tracking_states=saved_states)
def tick(self, timestamp: float):
"""
Includes the logic that has to be processed every time a new tick happens in the bot. Particularly it enables
the execution of the status update polling loop using an event.
"""
now = time.time()
poll_interval = (self.SHORT_POLL_INTERVAL if
now - self.user_stream_tracker.last_recv_time > 60.0
else self.LONG_POLL_INTERVAL)
last_tick = int(self._last_timestamp / poll_interval)
current_tick = int(timestamp / poll_interval)
if current_tick > last_tick:
if not self._poll_notifier.is_set():
self._poll_notifier.set()
self._last_timestamp = timestamp
def get_order_book(self, trading_pair: str) -> OrderBook:
"""
Returns the current order book for a particular market
:param trading_pair: the pair of tokens for which the order book should be retrieved
"""
if trading_pair not in self._order_book_tracker.order_books:
raise ValueError(f"No order book exists for '{trading_pair}'.")
return self._order_book_tracker.order_books[trading_pair]
def start_tracking_order(self, order_id: str,
exchange_order_id: Optional[str],
trading_pair: str, trade_type: TradeType,
price: Decimal, amount: Decimal,
order_type: OrderType):
"""
Starts tracking an order by adding it to the order tracker.
:param order_id: the order identifier
:param exchange_order_id: the identifier for the order in the exchange
:param trading_pair: the token pair for the operation
:param trade_type: the type of order (buy or sell)
:param price: the price for the order
:param amount: the amount for the order
:order type: type of execution for the order (MARKET, LIMIT, LIMIT_MAKER)
"""
self._order_tracker.start_tracking_order(
InFlightOrder(
client_order_id=order_id,
exchange_order_id=exchange_order_id,
trading_pair=trading_pair,
order_type=order_type,
trade_type=trade_type,
amount=amount,
price=price,
))
def stop_tracking_order(self, order_id: str):
"""
Stops tracking an order
:param order_id: The id of the order that will not be tracked any more
"""
self._order_tracker.stop_tracking_order(client_order_id=order_id)
def get_order_price_quantum(self, trading_pair: str,
price: Decimal) -> Decimal:
"""
Used by quantize_order_price() in _create_order()
Returns a price step, a minimum price increment for a given trading pair.
:param trading_pair: the trading pair to check for market conditions
:param price: the starting point price
"""
trading_rule = self._trading_rules[trading_pair]
return trading_rule.min_price_increment
def get_order_size_quantum(self, trading_pair: str,
order_size: Decimal) -> Decimal:
"""
Used by quantize_order_price() in _create_order()
Returns an order amount step, a minimum amount increment for a given trading pair.
:param trading_pair: the trading pair to check for market conditions
:param order_size: the starting point order price
"""
trading_rule = self._trading_rules[trading_pair]
return trading_rule.min_base_amount_increment
def quantize_order_amount(self,
trading_pair: str,
amount: Decimal,
price: Decimal = s_decimal_0) -> Decimal:
"""
Applies the trading rules to calculate the correct order amount for the market
:param trading_pair: the token pair for which the order will be created
:param amount: the intended amount for the order
:param price: the intended price for the order
:return: the quantized order amount after applying the trading rules
"""
trading_rule = self._trading_rules[trading_pair]
quantized_amount: Decimal = super().quantize_order_amount(
trading_pair, amount)
# Check against min_order_size and min_notional_size. If not passing either check, return 0.
if quantized_amount < trading_rule.min_order_size:
return s_decimal_0
if price == s_decimal_0:
current_price: Decimal = self.get_price(trading_pair, False)
notional_size = current_price * quantized_amount
else:
notional_size = price * quantized_amount
# Add 1% as a safety factor in case the prices changed while making the order.
if notional_size < trading_rule.min_notional_size * Decimal("1.01"):
return s_decimal_0
return quantized_amount
def get_fee(self,
base_currency: str,
quote_currency: str,
order_type: OrderType,
order_side: TradeType,
amount: Decimal,
price: Decimal = s_decimal_NaN,
is_maker: Optional[bool] = None) -> TradeFeeBase:
"""
Calculates the estimated fee an order would pay based on the connector configuration
:param base_currency: the order base currency
:param quote_currency: the order quote currency
:param order_type: the type of order (MARKET, LIMIT, LIMIT_MAKER)
:param order_side: if the order is for buying or selling
:param amount: the order amount
:param price: the order price
:return: the estimated fee for the order
"""
"""
To get trading fee, this function is simplified by using fee override configuration. Most parameters to this
function are ignore except order_type. Use OrderType.LIMIT_MAKER to specify you want trading fee for
maker order.
"""
is_maker = order_type is OrderType.LIMIT_MAKER
return DeductedFromReturnsTradeFee(
percent=self.estimate_fee_pct(is_maker))
def buy(self,
trading_pair: str,
amount: Decimal,
order_type: OrderType = OrderType.LIMIT,
price: Decimal = s_decimal_NaN,
**kwargs) -> str:
"""
Creates a promise to create a buy order using the parameters.
:param trading_pair: the token pair to operate with
:param amount: the order amount
:param order_type: the type of order to create (MARKET, LIMIT, LIMIT_MAKER)
:param price: the order price
:return: the id assigned by the connector to the order (the client id)
"""
new_order_id = binance_utils.get_new_client_order_id(
is_buy=True, trading_pair=trading_pair)
safe_ensure_future(
self._create_order(TradeType.BUY, new_order_id, trading_pair,
amount, order_type, price))
return new_order_id
def sell(self,
trading_pair: str,
amount: Decimal,
order_type: OrderType = OrderType.MARKET,
price: Decimal = s_decimal_NaN,
**kwargs) -> str:
"""
Creates a promise to create a sell order using the parameters.
:param trading_pair: the token pair to operate with
:param amount: the order amount
:param order_type: the type of order to create (MARKET, LIMIT, LIMIT_MAKER)
:param price: the order price
:return: the id assigned by the connector to the order (the client id)
"""
order_id = binance_utils.get_new_client_order_id(
is_buy=False, trading_pair=trading_pair)
safe_ensure_future(
self._create_order(TradeType.SELL, order_id, trading_pair, amount,
order_type, price))
return order_id
def cancel(self, trading_pair: str, order_id: str):
"""
Creates a promise to cancel an order in the exchange
:param trading_pair: the trading pair the order to cancel operates with
:param order_id: the client id of the order to cancel
:return: the client id of the order to cancel
"""
safe_ensure_future(self._execute_cancel(trading_pair, order_id))
return order_id
async def cancel_all(self,
timeout_seconds: float) -> List[CancellationResult]:
"""
Cancels all currently active orders. The cancellations are performed in parallel tasks.
:param timeout_seconds: the maximum time (in seconds) the cancel logic should run
:return: a list of CancellationResult instances, one for each of the orders to be cancelled
"""
incomplete_orders = [
o for o in self.in_flight_orders.values() if not o.is_done
]
tasks = [
self._execute_cancel(o.trading_pair, o.client_order_id)
for o in incomplete_orders
]
order_id_set = set([o.client_order_id for o in incomplete_orders])
successful_cancellations = []
try:
async with timeout(timeout_seconds):
cancellation_results = await safe_gather(
*tasks, return_exceptions=True)
for cr in cancellation_results:
if isinstance(cr, Exception):
continue
if isinstance(cr, dict) and "origClientOrderId" in cr:
client_order_id = cr.get("origClientOrderId")
order_id_set.remove(client_order_id)
successful_cancellations.append(
CancellationResult(client_order_id, True))
except Exception:
self.logger().network(
"Unexpected error cancelling orders.",
exc_info=True,
app_warning_msg=
"Failed to cancel order with Binance. Check API key and network connection."
)
failed_cancellations = [
CancellationResult(oid, False) for oid in order_id_set
]
return successful_cancellations + failed_cancellations
async def _create_order(self,
trade_type: TradeType,
order_id: str,
trading_pair: str,
amount: Decimal,
order_type: OrderType,
price: Optional[Decimal] = Decimal("NaN")):
"""
Creates a an order in the exchange using the parameters to configure it
:param trade_type: the side of the order (BUY of SELL)
:param order_id: the id that should be assigned to the order (the client id)
:param trading_pair: the token pair to operate with
:param amount: the order amount
:param order_type: the type of order to create (MARKET, LIMIT, LIMIT_MAKER)
:param price: the order price
"""
trading_rule: TradingRule = self._trading_rules[trading_pair]
price = self.quantize_order_price(trading_pair, price)
quantize_amount_price = Decimal("0") if price.is_nan() else price
amount = self.quantize_order_amount(trading_pair=trading_pair,
amount=amount,
price=quantize_amount_price)
self.start_tracking_order(order_id=order_id,
exchange_order_id=None,
trading_pair=trading_pair,
trade_type=trade_type,
price=price,
amount=amount,
order_type=order_type)
if amount < trading_rule.min_order_size:
self.logger().warning(
f"{trade_type.name.title()} order amount {amount} is lower than the minimum order"
f" size {trading_rule.min_order_size}. The order will not be created."
)
order_update: OrderUpdate = OrderUpdate(
client_order_id=order_id,
trading_pair=trading_pair,
update_timestamp=int(self.current_timestamp * 1e3),
new_state=OrderState.FAILED,
)
self._order_tracker.process_order_update(order_update)
return
order_result = None
amount_str = f"{amount:f}"
price_str = f"{price:f}"
type_str = BinanceExchange.binance_order_type(order_type)
side_str = CONSTANTS.SIDE_BUY if trade_type is TradeType.BUY else CONSTANTS.SIDE_SELL
symbol = await BinanceAPIOrderBookDataSource.exchange_symbol_associated_to_pair(
trading_pair=trading_pair,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler)
api_params = {
"symbol": symbol,
"side": side_str,
"quantity": amount_str,
"type": type_str,
"newClientOrderId": order_id,
"price": price_str
}
if order_type == OrderType.LIMIT:
api_params["timeInForce"] = CONSTANTS.TIME_IN_FORCE_GTC
try:
order_result = await self._api_request(
method=RESTMethod.POST,
path_url=CONSTANTS.ORDER_PATH_URL,
data=api_params,
is_auth_required=True)
exchange_order_id = str(order_result["orderId"])
order_update: OrderUpdate = OrderUpdate(
client_order_id=order_id,
exchange_order_id=exchange_order_id,
trading_pair=trading_pair,
update_timestamp=int(order_result["transactTime"]),
new_state=OrderState.OPEN,
)
self._order_tracker.process_order_update(order_update)
except asyncio.CancelledError:
raise
except Exception as e:
self.logger().network(
f"Error submitting {side_str} {type_str} order to Binance for "
f"{amount} {trading_pair} "
f"{price}.",
exc_info=True,
app_warning_msg=str(e))
order_update: OrderUpdate = OrderUpdate(
client_order_id=order_id,
trading_pair=trading_pair,
update_timestamp=int(self.current_timestamp * 1e3),
new_state=OrderState.FAILED,
)
self._order_tracker.process_order_update(order_update)
async def _execute_cancel(self, trading_pair: str, order_id: str):
"""
Requests the exchange to cancel an active order
:param trading_pair: the trading pair the order to cancel operates with
:param order_id: the client id of the order to cancel
"""
tracked_order = self._order_tracker.fetch_tracked_order(order_id)
if tracked_order is not None:
try:
symbol = await BinanceAPIOrderBookDataSource.exchange_symbol_associated_to_pair(
trading_pair=trading_pair,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler)
api_params = {
"symbol": symbol,
"origClientOrderId": order_id,
}
cancel_result = await self._api_request(
method=RESTMethod.DELETE,
path_url=CONSTANTS.ORDER_PATH_URL,
params=api_params,
is_auth_required=True)
if cancel_result.get("status") == "CANCELED":
order_update: OrderUpdate = OrderUpdate(
client_order_id=order_id,
trading_pair=tracked_order.trading_pair,
update_timestamp=int(self.current_timestamp * 1e3),
new_state=OrderState.CANCELLED,
)
self._order_tracker.process_order_update(order_update)
return cancel_result
except asyncio.CancelledError:
raise
except Exception:
self.logger().exception(
f"There was a an error when requesting cancellation of order {order_id}"
)
raise
async def _status_polling_loop(self):
"""
Performs all required operation to keep the connector updated and synchronized with the exchange.
It contains the backup logic to update status using API requests in case the main update source (the user stream
data source websocket) fails.
It also updates the time synchronizer. This is necessary because Binance require the time of the client to be
the same as the time in the exchange.
Executes when the _poll_notifier event is enabled by the `tick` function.
"""
while True:
try:
await self._poll_notifier.wait()
await self._update_time_synchronizer()
await safe_gather(
self._update_balances(),
self._update_order_fills_from_trades(),
)
await self._update_order_status()
self._last_poll_timestamp = self.current_timestamp
except asyncio.CancelledError:
raise
except Exception:
self.logger().network(
"Unexpected error while fetching account updates.",
exc_info=True,
app_warning_msg=
"Could not fetch account updates from Binance. "
"Check API key and network connection.")
await asyncio.sleep(0.5)
finally:
self._poll_notifier = asyncio.Event()
async def _trading_rules_polling_loop(self):
"""
Updates the trading rules by requesting the latest definitions from the exchange.
Executes regularly every 30 minutes
"""
while True:
try:
await safe_gather(self._update_trading_rules(), )
await asyncio.sleep(30 * 60)
except asyncio.CancelledError:
raise
except Exception:
self.logger().network(
"Unexpected error while fetching trading rules.",
exc_info=True,
app_warning_msg=
"Could not fetch new trading rules from Binance. "
"Check network connection.")
await asyncio.sleep(0.5)
async def _update_trading_rules(self):
exchange_info = await self._api_request(
method=RESTMethod.GET, path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL)
trading_rules_list = await self._format_trading_rules(exchange_info)
self._trading_rules.clear()
for trading_rule in trading_rules_list:
self._trading_rules[trading_rule.trading_pair] = trading_rule
async def _format_trading_rules(
self, exchange_info_dict: Dict[str, Any]) -> List[TradingRule]:
"""
Example:
{
"symbol": "ETHBTC",
"baseAssetPrecision": 8,
"quotePrecision": 8,
"orderTypes": ["LIMIT", "MARKET"],
"filters": [
{
"filterType": "PRICE_FILTER",
"minPrice": "0.00000100",
"maxPrice": "100000.00000000",
"tickSize": "0.00000100"
}, {
"filterType": "LOT_SIZE",
"minQty": "0.00100000",
"maxQty": "100000.00000000",
"stepSize": "0.00100000"
}, {
"filterType": "MIN_NOTIONAL",
"minNotional": "0.00100000"
}
]
}
"""
trading_pair_rules = exchange_info_dict.get("symbols", [])
retval = []
for rule in filter(binance_utils.is_exchange_information_valid,
trading_pair_rules):
try:
trading_pair = await BinanceAPIOrderBookDataSource.trading_pair_associated_to_exchange_symbol(
symbol=rule.get("symbol"),
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler)
filters = rule.get("filters")
price_filter = [
f for f in filters if f.get("filterType") == "PRICE_FILTER"
][0]
lot_size_filter = [
f for f in filters if f.get("filterType") == "LOT_SIZE"
][0]
min_notional_filter = [
f for f in filters if f.get("filterType") == "MIN_NOTIONAL"
][0]
min_order_size = Decimal(lot_size_filter.get("minQty"))
tick_size = price_filter.get("tickSize")
step_size = Decimal(lot_size_filter.get("stepSize"))
min_notional = Decimal(min_notional_filter.get("minNotional"))
retval.append(
TradingRule(trading_pair,
min_order_size=min_order_size,
min_price_increment=Decimal(tick_size),
min_base_amount_increment=Decimal(step_size),
min_notional_size=Decimal(min_notional)))
except Exception:
self.logger().exception(
f"Error parsing the trading pair rule {rule}. Skipping.")
return retval
async def _user_stream_event_listener(self):
"""
This functions runs in background continuously processing the events received from the exchange by the user
stream data source. It keeps reading events from the queue until the task is interrupted.
The events received are balance updates, order updates and trade events.
"""
async for event_message in self._iter_user_event_queue():
try:
event_type = event_message.get("e")
# Refer to https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md
# As per the order update section in Binance the ID of the order being cancelled is under the "C" key
if event_type == "executionReport":
execution_type = event_message.get("x")
if execution_type != "CANCELED":
client_order_id = event_message.get("c")
else:
client_order_id = event_message.get("C")
if execution_type == "TRADE":
tracked_order = self._order_tracker.fetch_order(
client_order_id=client_order_id)
if tracked_order is not None:
trade_update = TradeUpdate(
trade_id=str(event_message["t"]),
client_order_id=client_order_id,
exchange_order_id=str(event_message["i"]),
trading_pair=tracked_order.trading_pair,
fee_asset=event_message["N"],
fee_paid=Decimal(event_message["n"]),
fill_base_amount=Decimal(event_message["l"]),
fill_quote_amount=Decimal(event_message["l"]) *
Decimal(event_message["L"]),
fill_price=Decimal(event_message["L"]),
fill_timestamp=int(event_message["T"]),
)
self._order_tracker.process_trade_update(
trade_update)
tracked_order = self.in_flight_orders.get(client_order_id)
if tracked_order is not None:
order_update = OrderUpdate(
trading_pair=tracked_order.trading_pair,
update_timestamp=int(event_message["E"]),
new_state=CONSTANTS.ORDER_STATE[
event_message["X"]],
client_order_id=client_order_id,
exchange_order_id=str(event_message["i"]),
)
self._order_tracker.process_order_update(
order_update=order_update)
elif event_type == "outboundAccountPosition":
balances = event_message["B"]
for balance_entry in balances:
asset_name = balance_entry["a"]
free_balance = Decimal(balance_entry["f"])
total_balance = Decimal(balance_entry["f"]) + Decimal(
balance_entry["l"])
self._account_available_balances[
asset_name] = free_balance
self._account_balances[asset_name] = total_balance
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error in user stream listener loop.",
exc_info=True)
await asyncio.sleep(5.0)
async def _update_order_fills_from_trades(self):
"""
This is intended to be a backup measure to get filled events with trade ID for orders,
in case Binance's user stream events are not working.
NOTE: It is not required to copy this functionality in other connectors.
This is separated from _update_order_status which only updates the order status without producing filled
events, since Binance's get order endpoint does not return trade IDs.
The minimum poll interval for order status is 10 seconds.
"""
small_interval_last_tick = self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL
small_interval_current_tick = self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL
long_interval_last_tick = self._last_poll_timestamp / self.LONG_POLL_INTERVAL
long_interval_current_tick = self.current_timestamp / self.LONG_POLL_INTERVAL
if (long_interval_current_tick > long_interval_last_tick or
(self.in_flight_orders
and small_interval_current_tick > small_interval_last_tick)):
query_time = int(self._last_trades_poll_binance_timestamp * 1e3)
self._last_trades_poll_binance_timestamp = self._binance_time_synchronizer.time(
)
order_by_exchange_id_map = {}
for order in self._order_tracker.all_orders.values():
order_by_exchange_id_map[order.exchange_order_id] = order
tasks = []
trading_pairs = self._order_book_tracker._trading_pairs
for trading_pair in trading_pairs:
params = {
"symbol":
await BinanceAPIOrderBookDataSource.
exchange_symbol_associated_to_pair(
trading_pair=trading_pair,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler)
}
if self._last_poll_timestamp > 0:
params["startTime"] = query_time
tasks.append(
self._api_request(method=RESTMethod.GET,
path_url=CONSTANTS.MY_TRADES_PATH_URL,
params=params,
is_auth_required=True))
self.logger().debug(
f"Polling for order fills of {len(tasks)} trading pairs.")
results = await safe_gather(*tasks, return_exceptions=True)
for trades, trading_pair in zip(results, trading_pairs):
if isinstance(trades, Exception):
self.logger().network(
f"Error fetching trades update for the order {trading_pair}: {trades}.",
app_warning_msg=
f"Failed to fetch trade update for {trading_pair}.")
continue
for trade in trades:
exchange_order_id = str(trade["orderId"])
if exchange_order_id in order_by_exchange_id_map:
# This is a fill for a tracked order
tracked_order = order_by_exchange_id_map[
exchange_order_id]
trade_update = TradeUpdate(
trade_id=str(trade["id"]),
client_order_id=tracked_order.client_order_id,
exchange_order_id=exchange_order_id,
trading_pair=trading_pair,
fee_asset=trade["commissionAsset"],
fee_paid=Decimal(trade["commission"]),
fill_base_amount=Decimal(trade["qty"]),
fill_quote_amount=Decimal(trade["quoteQty"]),
fill_price=Decimal(trade["price"]),
fill_timestamp=int(trade["time"]),
)
self._order_tracker.process_trade_update(trade_update)
elif self.is_confirmed_new_order_filled_event(
str(trade["id"]), exchange_order_id, trading_pair):
# This is a fill of an order registered in the DB but not tracked any more
self._current_trade_fills.add(
TradeFillOrderDetails(market=self.display_name,
exchange_trade_id=str(
trade["id"]),
symbol=trading_pair))
self.trigger_event(
MarketEvent.OrderFilled,
OrderFilledEvent(
timestamp=float(trade["time"]) * 1e-3,
order_id=self._exchange_order_ids.get(
str(trade["orderId"]), None),
trading_pair=trading_pair,
trade_type=TradeType.BUY
if trade["isBuyer"] else TradeType.SELL,
order_type=OrderType.LIMIT_MAKER
if trade["isMaker"] else OrderType.LIMIT,
price=Decimal(trade["price"]),
amount=Decimal(trade["qty"]),
trade_fee=DeductedFromReturnsTradeFee(
flat_fees=[
TokenAmount(
trade["commissionAsset"],
Decimal(trade["commission"]))
]),
exchange_trade_id=str(trade["id"])))
self.logger().info(
f"Recreating missing trade in TradeFill: {trade}")
async def _update_order_status(self):
# This is intended to be a backup measure to close straggler orders, in case Binance's user stream events
# are not working.
# The minimum poll interval for order status is 10 seconds.
last_tick = self._last_poll_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL
current_tick = self.current_timestamp / self.UPDATE_ORDER_STATUS_MIN_INTERVAL
tracked_orders: List[InFlightOrder] = list(
self.in_flight_orders.values())
if current_tick > last_tick and len(tracked_orders) > 0:
tasks = [
self._api_request(method=RESTMethod.GET,
path_url=CONSTANTS.ORDER_PATH_URL,
params={
"symbol":
await BinanceAPIOrderBookDataSource.
exchange_symbol_associated_to_pair(
trading_pair=o.trading_pair,
domain=self._domain,
api_factory=self._api_factory,
throttler=self._throttler),
"origClientOrderId":
o.client_order_id
},
is_auth_required=True)
for o in tracked_orders
]
self.logger().debug(
f"Polling for order status updates of {len(tasks)} orders.")
results = await safe_gather(*tasks, return_exceptions=True)
for order_update, tracked_order in zip(results, tracked_orders):
client_order_id = tracked_order.client_order_id
# If the order has already been cancelled or has failed do nothing
if client_order_id not in self.in_flight_orders:
continue
if isinstance(order_update, Exception):
self.logger().network(
f"Error fetching status update for the order {client_order_id}: {order_update}.",
app_warning_msg=
f"Failed to fetch status update for the order {client_order_id}."
)
self._order_not_found_records[client_order_id] = (
self._order_not_found_records.get(client_order_id, 0) +
1)
if (self._order_not_found_records[client_order_id] >= self.
MAX_ORDER_UPDATE_RETRIEVAL_RETRIES_WITH_FAILURES):
# Wait until the order not found error have repeated a few times before actually treating
# it as failed. See: https://github.com/CoinAlpha/hummingbot/issues/601
order_update: OrderUpdate = OrderUpdate(
client_order_id=client_order_id,
trading_pair=tracked_order.trading_pair,
update_timestamp=int(self.current_timestamp * 1e3),
new_state=OrderState.FAILED,
)
self._order_tracker.process_order_update(order_update)
else:
# Update order execution status
new_state = CONSTANTS.ORDER_STATE[order_update["status"]]
update = OrderUpdate(
client_order_id=client_order_id,
exchange_order_id=str(order_update["orderId"]),
trading_pair=tracked_order.trading_pair,
update_timestamp=int(order_update["updateTime"]),
new_state=new_state,
)
self._order_tracker.process_order_update(update)
async def _iter_user_event_queue(self) -> AsyncIterable[Dict[str, any]]:
while True:
try:
yield await self._user_stream_tracker.user_stream.get()
except asyncio.CancelledError:
raise
except Exception:
self.logger().network(
"Unknown error. Retrying after 1 seconds.",
exc_info=True,
app_warning_msg=
"Could not fetch user events from Binance. Check API key and network connection."
)
await asyncio.sleep(1.0)
async def _update_balances(self):
local_asset_names = set(self._account_balances.keys())
remote_asset_names = set()
try:
account_info = await self._api_request(
method=RESTMethod.GET,
path_url=CONSTANTS.ACCOUNTS_PATH_URL,
is_auth_required=True)
balances = account_info["balances"]
for balance_entry in balances:
asset_name = balance_entry["asset"]
free_balance = Decimal(balance_entry["free"])
total_balance = Decimal(balance_entry["free"]) + Decimal(
balance_entry["locked"])
self._account_available_balances[asset_name] = free_balance
self._account_balances[asset_name] = total_balance
remote_asset_names.add(asset_name)
asset_names_to_remove = local_asset_names.difference(
remote_asset_names)
for asset_name in asset_names_to_remove:
del self._account_available_balances[asset_name]
del self._account_balances[asset_name]
except IOError:
self.logger().exception(
"Error getting account balances from server")
async def _update_time_synchronizer(self):
try:
await self._binance_time_synchronizer.update_server_time_offset_with_time_provider(
time_provider=self._get_current_server_time())
except asyncio.CancelledError:
raise
except Exception:
self.logger().exception(
"Error requesting time from Binance server")
raise
async def _api_request(self,
method: RESTMethod,
path_url: str,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
is_auth_required: bool = False) -> Dict[str, Any]:
headers = {
"Content-Type":
"application/json" if method == RESTMethod.POST else
"application/x-www-form-urlencoded"
}
client = await self._get_rest_assistant()
if is_auth_required:
url = binance_utils.private_rest_url(path_url, domain=self._domain)
else:
url = binance_utils.public_rest_url(path_url, domain=self._domain)
request = RESTRequest(method=method,
url=url,
data=data,
params=params,
headers=headers,
is_auth_required=is_auth_required)
async with self._throttler.execute_task(limit_id=path_url):
response = await client.call(request)
if response.status != 200:
data = await response.text()
raise IOError(
f"Error fetching data from {url}. HTTP status is {response.status} ({data})."
)
try:
parsed_response = await response.json()
except Exception:
raise IOError(f"Error parsing data from {response}.")
if "code" in parsed_response and "msg" in parsed_response:
raise IOError(
f"The request to Binance failed. Error: {parsed_response}. Request: {request}"
)
return parsed_response
async def _get_current_server_time(self):
response = await self._api_request(
method=RESTMethod.GET,
path_url=CONSTANTS.SERVER_TIME_PATH_URL,
)
return response["serverTime"]
async def _get_rest_assistant(self) -> RESTAssistant:
if self._rest_assistant is None:
self._rest_assistant = await self._api_factory.get_rest_assistant()
return self._rest_assistant
class LatokenUserStreamDataSourceUnitTests(unittest.TestCase):
# the level is required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "d8ae67f2-f954-4014-98c8-64b1ac334c64"
cls.quote_asset = "0c3a106d-bde3-4c13-a26e-3fd2394529e5"
cls.trading_pair = "ETH-USDT"
cls.trading_pairs = [cls.trading_pair]
cls.ex_trading_pair = cls.base_asset + '/' + cls.quote_asset
cls.domain = "com"
cls.listen_key = 'ffffffff-ffff-ffff-ffff-ffffffffff'
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.mock_time_provider = MagicMock()
self.mock_time_provider.time.return_value = 1000
self.auth = LatokenAuth(api_key="TEST_API_KEY",
secret_key="TEST_SECRET",
time_provider=self.mock_time_provider)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
client_config_map = ClientConfigAdapter(ClientConfigMap())
self.connector = LatokenExchange(client_config_map=client_config_map,
latoken_api_key="",
latoken_api_secret="",
trading_pairs=[],
trading_required=False,
domain=self.domain)
self.connector._web_assistants_factory._auth = self.auth
self.data_source = LatokenAPIUserStreamDataSource(
auth=self.auth,
trading_pairs=[self.trading_pair],
connector=self.connector,
api_factory=self.connector._web_assistants_factory,
domain=self.domain)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
self.connector._set_trading_pair_symbol_map(
bidict({self.ex_trading_pair: self.trading_pair}))
def tearDown(self) -> None:
self.listening_task and self.listening_task.cancel()
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def _is_logged(self, log_level: str, message: str) -> bool:
return any(
record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def _raise_exception(self, exception_class):
raise exception_class
def _create_exception_and_unlock_test_with_event(self, exception):
self.resume_test_event.set()
raise exception
def _create_return_value_and_unlock_test_with_event(self, value):
self.resume_test_event.set()
return value
def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1):
ret = self.ev_loop.run_until_complete(
asyncio.wait_for(coroutine, timeout))
return ret
def _error_response(self) -> Dict[str, Any]:
resp = {"code": "ERROR CODE", "msg": "ERROR MESSAGE"}
return resp
def _user_update_event(self):
# Balance Update, so not the initial balance
return b'MESSAGE\ndestination:/user/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/v1/account\nmessage-id:9e8188c8-682c-41cd-9a14-722bf6dfd99e\ncontent-length:346\nsubscription:2\n\n{"payload":[{"id":"44d36460-46dc-4828-a17c-63b1a047b054","status":"ACCOUNT_STATUS_ACTIVE","type":"ACCOUNT_TYPE_SPOT","timestamp":1650120265819,"currency":"620f2019-33c0-423b-8a9d-cde4d7f8ef7f","available":"34.001000000000000000","blocked":"0.999000000000000000","user":"******"}],"nonce":1,"timestamp":1650120265830}\x00'
def _successfully_subscribed_event(self):
return b'CONNECTED\nserver:vertx-stomp/3.9.6\nheart-beat:1000,1000\nsession:37a8e962-7fa7-4eab-b163-146eeafdef63\nversion:1.1\n\n\x00 '
@aioresponses()
def test_get_listen_key_log_exception(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url,
status=400,
body=json.dumps(self._error_response()))
with self.assertRaises(IOError):
self.async_run_with_timeout(self.data_source._get_listen_key())
@aioresponses()
def test_get_listen_key_successful(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
result: str = self.async_run_with_timeout(
self.data_source._get_listen_key())
self.assertEqual(self.listen_key, result)
@aioresponses()
def test_ping_listen_key_log_exception(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url,
status=400,
body=json.dumps(self._error_response()))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(
self.data_source._ping_listen_key())
self.assertTrue(
self._is_logged(
"WARNING",
f"Failed to refresh the listen key {self.listen_key}: {self._error_response()}"
))
self.assertFalse(result)
@aioresponses()
def test_ping_listen_key_successful(self, mock_api):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(
self.data_source._ping_listen_key())
self.assertTrue(result)
@patch(
"hummingbot.connector.exchange.latoken.latoken_api_user_stream_data_source.LatokenAPIUserStreamDataSource"
"._ping_listen_key",
new_callable=AsyncMock)
def test_manage_listen_key_task_loop_keep_alive_failed(
self, mock_ping_listen_key):
mock_ping_listen_key.side_effect = (
lambda *args, **kwargs: self.
_create_return_value_and_unlock_test_with_event(False))
self.data_source._current_listen_key = self.listen_key
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(
self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged("ERROR", "Error occurred renewing listen key ..."))
self.assertIsNone(self.data_source._current_listen_key)
self.assertFalse(
self.data_source._listen_key_initialized_event.is_set())
@patch(
"hummingbot.connector.exchange.latoken.latoken_api_user_stream_data_source.LatokenAPIUserStreamDataSource."
"_ping_listen_key",
new_callable=AsyncMock)
def test_manage_listen_key_task_loop_keep_alive_successful(
self, mock_ping_listen_key):
mock_ping_listen_key.side_effect = (
lambda *args, **kwargs: self.
_create_return_value_and_unlock_test_with_event(True))
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._current_listen_key = self.listen_key
self.data_source._listen_key_initialized_event.set()
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(
self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged("INFO",
f"Refreshed listen key {self.listen_key}."))
self.assertGreater(self.data_source._last_listen_key_ping_ts, 0)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_get_listen_key_successful_with_user_update_event(
self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(
mock_ws.return_value,
self._successfully_subscribed_event(),
message_type=aiohttp.WSMsgType.BINARY)
self.mocking_assistant.add_websocket_aiohttp_message(
mock_ws.return_value,
self._user_update_event(),
message_type=aiohttp.WSMsgType.BINARY)
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
msg = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(msg, self._user_update_event)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_does_not_queue_empty_payload(
self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(
mock_ws.return_value, "")
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(
mock_ws.return_value)
self.assertEqual(0, msg_queue.qsize())
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_connection_failed(self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
mock_ws.side_effect = lambda *arg, **kwars: self._create_exception_and_unlock_test_with_event(
Exception("TEST ERROR."))
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds..."
))
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_iter_message_throws_exception(
self, mock_api, mock_ws):
url = web_utils.private_rest_url(path_url=CONSTANTS.USER_ID_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
'id': 'ffffffff-ffff-ffff-ffff-ffffffffff',
'status': 'ACTIVE',
'role': 'INVESTOR',
'email': '*****@*****.**',
'phone': '',
'authorities': [],
'forceChangePassword': None,
'authType': 'API_KEY',
'socials': []
}
mock_api.get(regex_url, body=json.dumps(mock_response))
msg_queue: asyncio.Queue = asyncio.Queue()
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
mock_ws.return_value.receive.side_effect = (
lambda *args, **kwargs: self.
_create_exception_and_unlock_test_with_event(
Exception("TEST ERROR")))
mock_ws.close.return_value = None
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds..."
))
class TestKucoinAPIOrderBookDataSource(unittest.TestCase):
# logging.Level required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "COINALPHA"
cls.quote_asset = "HBOT"
cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
cls.ws_endpoint = "ws://someEndpoint"
cls.api_key = "someKey"
cls.api_passphrase = "somePassPhrase"
cls.api_secret_key = "someSecretKey"
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.async_task = None
self.mocking_assistant = NetworkMockingAssistant()
self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
self.time_synchronnizer = TimeSynchronizer()
self.time_synchronnizer.add_time_offset_ms_sample(1000)
self.ob_data_source = KucoinAPIOrderBookDataSource(
trading_pairs=[self.trading_pair],
throttler=self.throttler,
time_synchronizer=self.time_synchronnizer)
self.ob_data_source.logger().setLevel(1)
self.ob_data_source.logger().addHandler(self)
KucoinAPIOrderBookDataSource._trading_pair_symbol_map = {
CONSTANTS.DEFAULT_DOMAIN:
bidict({self.trading_pair: self.trading_pair})
}
def tearDown(self) -> None:
self.async_task and self.async_task.cancel()
KucoinAPIOrderBookDataSource._trading_pair_symbol_map = {}
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def _is_logged(self, log_level: str, message: str) -> bool:
return any(
record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1):
ret = self.ev_loop.run_until_complete(
asyncio.wait_for(coroutine, timeout))
return ret
@staticmethod
def get_snapshot_mock() -> Dict:
snapshot = {
"code": "200000",
"data": {
"time": 1630556205455,
"sequence": "1630556205456",
"bids": [["0.3003", "4146.5645"]],
"asks": [["0.3004", "1553.6412"]]
}
}
return snapshot
@aioresponses()
def test_get_last_traded_prices(self, mock_api):
KucoinAPIOrderBookDataSource._trading_pair_symbol_map[
CONSTANTS.DEFAULT_DOMAIN]["TKN1-TKN2"] = "TKN1-TKN2"
url1 = web_utils.rest_url(
path_url=CONSTANTS.TICKER_PRICE_CHANGE_PATH_URL,
domain=CONSTANTS.DEFAULT_DOMAIN)
url1 = f"{url1}?symbol={self.trading_pair}"
regex_url = re.compile(f"^{url1}".replace(".",
r"\.").replace("?", r"\?"))
resp = {
"code": "200000",
"data": {
"sequence": "1550467636704",
"bestAsk": "0.03715004",
"size": "0.17",
"price": "100",
"bestBidSize": "3.803",
"bestBid": "0.03710768",
"bestAskSize": "1.788",
"time": 1550653727731
}
}
mock_api.get(regex_url, body=json.dumps(resp))
url2 = web_utils.rest_url(
path_url=CONSTANTS.TICKER_PRICE_CHANGE_PATH_URL,
domain=CONSTANTS.DEFAULT_DOMAIN)
url2 = f"{url2}?symbol=TKN1-TKN2"
regex_url = re.compile(f"^{url2}".replace(".",
r"\.").replace("?", r"\?"))
resp = {
"code": "200000",
"data": {
"sequence": "1550467636704",
"bestAsk": "0.03715004",
"size": "0.17",
"price": "200",
"bestBidSize": "3.803",
"bestBid": "0.03710768",
"bestAskSize": "1.788",
"time": 1550653727731
}
}
mock_api.get(regex_url, body=json.dumps(resp))
ret = self.async_run_with_timeout(
coroutine=KucoinAPIOrderBookDataSource.get_last_traded_prices(
[self.trading_pair, "TKN1-TKN2"]))
ticker_requests = [(key, value)
for key, value in mock_api.requests.items()
if key[1].human_repr().startswith(url1)
or key[1].human_repr().startswith(url2)]
request_params = ticker_requests[0][1][0].kwargs["params"]
self.assertEqual(f"{self.base_asset}-{self.quote_asset}",
request_params["symbol"])
request_params = ticker_requests[1][1][0].kwargs["params"]
self.assertEqual("TKN1-TKN2", request_params["symbol"])
self.assertEqual(ret[self.trading_pair], 100)
self.assertEqual(ret["TKN1-TKN2"], 200)
@aioresponses()
def test_fetch_trading_pairs(self, mock_api):
KucoinAPIOrderBookDataSource._trading_pair_symbol_map = {}
url = web_utils.rest_url(path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL)
resp = {
"data": [{
"symbol": self.trading_pair,
"name": self.trading_pair,
"baseCurrency": self.base_asset,
"quoteCurrency": self.quote_asset,
"enableTrading": True,
}, {
"symbol": "SOME-PAIR",
"name": "SOME-PAIR",
"baseCurrency": "SOME",
"quoteCurrency": "PAIR",
"enableTrading": False,
}]
}
mock_api.get(url, body=json.dumps(resp))
ret = self.async_run_with_timeout(
coroutine=KucoinAPIOrderBookDataSource.fetch_trading_pairs(
throttler=self.throttler,
time_synchronizer=self.time_synchronnizer,
))
self.assertEqual(1, len(ret))
self.assertEqual(self.trading_pair, ret[0])
@aioresponses()
def test_fetch_trading_pairs_exception_raised(self, mock_api):
KucoinAPIOrderBookDataSource._trading_pair_symbol_map = {}
url = web_utils.rest_url(path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=Exception)
result: Dict[str] = self.async_run_with_timeout(
self.ob_data_source.fetch_trading_pairs())
self.assertEqual(0, len(result))
@aioresponses()
def test_get_snapshot_raises(self, mock_api):
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, status=500)
with self.assertRaises(IOError):
self.async_run_with_timeout(
coroutine=self.ob_data_source.get_snapshot(self.trading_pair))
@aioresponses()
def test_get_snapshot(self, mock_api):
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
resp = self.get_snapshot_mock()
mock_api.get(regex_url, body=json.dumps(resp))
ret = self.async_run_with_timeout(
coroutine=self.ob_data_source.get_snapshot(self.trading_pair))
self.assertEqual(ret, resp) # shallow comparison ok
@aioresponses()
def test_get_new_order_book(self, mock_api):
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
resp = self.get_snapshot_mock()
mock_api.get(regex_url, body=json.dumps(resp))
ret = self.async_run_with_timeout(
coroutine=self.ob_data_source.get_new_order_book(
self.trading_pair))
bid_entries = list(ret.bid_entries())
ask_entries = list(ret.ask_entries())
self.assertEqual(1, len(bid_entries))
self.assertEqual(0.3003, bid_entries[0].price)
self.assertEqual(4146.5645, bid_entries[0].amount)
self.assertEqual(int(resp["data"]["sequence"]),
bid_entries[0].update_id)
self.assertEqual(1, len(ask_entries))
self.assertEqual(0.3004, ask_entries[0].price)
self.assertEqual(1553.6412, ask_entries[0].amount)
self.assertEqual(int(resp["data"]["sequence"]),
ask_entries[0].update_id)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
@patch(
"hummingbot.connector.exchange.kucoin.kucoin_web_utils.next_message_id"
)
def test_listen_for_subscriptions_subscribes_to_trades_and_order_diffs(
self, mock_api, id_mock, ws_connect_mock):
id_mock.side_effect = [1, 2]
url = web_utils.rest_url(path_url=CONSTANTS.PUBLIC_WS_DATA_PATH_URL)
resp = {
"code": "200000",
"data": {
"instanceServers": [{
"endpoint": "wss://test.url/endpoint",
"protocol": "websocket",
"encrypt": True,
"pingInterval": 50000,
"pingTimeout": 10000
}],
"token":
"testToken"
}
}
mock_api.post(url, body=json.dumps(resp))
ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock(
)
result_subscribe_trades = {"type": "ack", "id": 1}
result_subscribe_diffs = {"type": "ack", "id": 2}
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_trades))
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_diffs))
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_subscriptions())
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(
ws_connect_mock.return_value)
sent_subscription_messages = self.mocking_assistant.json_messages_sent_through_websocket(
websocket_mock=ws_connect_mock.return_value)
self.assertEqual(2, len(sent_subscription_messages))
expected_trade_subscription = {
"id": 1,
"type": "subscribe",
"topic": f"/market/match:{self.trading_pair}",
"privateChannel": False,
"response": False
}
self.assertEqual(expected_trade_subscription,
sent_subscription_messages[0])
expected_diff_subscription = {
"id": 2,
"type": "subscribe",
"topic": f"/market/level2:{self.trading_pair}",
"privateChannel": False,
"response": False
}
self.assertEqual(expected_diff_subscription,
sent_subscription_messages[1])
self.assertTrue(
self._is_logged(
"INFO",
"Subscribed to public order book and trade channels..."))
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
@patch(
"hummingbot.connector.exchange.kucoin.kucoin_web_utils.next_message_id"
)
@patch(
"hummingbot.connector.exchange.kucoin.kucoin_api_order_book_data_source.KucoinAPIOrderBookDataSource._time"
)
def test_listen_for_subscriptions_sends_ping_message_before_ping_interval_finishes(
self, mock_api, time_mock, id_mock, ws_connect_mock):
id_mock.side_effect = [1, 2, 3, 4]
time_mock.side_effect = [
1000, 1100, 1101, 1102
] # Simulate first ping interval is already due
url = web_utils.rest_url(path_url=CONSTANTS.PUBLIC_WS_DATA_PATH_URL)
resp = {
"code": "200000",
"data": {
"instanceServers": [{
"endpoint": "wss://test.url/endpoint",
"protocol": "websocket",
"encrypt": True,
"pingInterval": 20000,
"pingTimeout": 10000
}],
"token":
"testToken"
}
}
mock_api.post(url, body=json.dumps(resp))
ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock(
)
result_subscribe_trades = {"type": "ack", "id": 1}
result_subscribe_diffs = {"type": "ack", "id": 2}
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_trades))
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_diffs))
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_subscriptions())
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(
ws_connect_mock.return_value)
sent_messages = self.mocking_assistant.json_messages_sent_through_websocket(
websocket_mock=ws_connect_mock.return_value)
expected_ping_message = {
"id": 3,
"type": "ping",
}
self.assertEqual(expected_ping_message, sent_messages[-1])
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
@patch(
"hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep"
)
def test_listen_for_subscriptions_raises_cancel_exception(
self, mock_api, _, ws_connect_mock):
url = web_utils.rest_url(path_url=CONSTANTS.PUBLIC_WS_DATA_PATH_URL)
resp = {
"code": "200000",
"data": {
"instanceServers": [{
"endpoint": "wss://test.url/endpoint",
"protocol": "websocket",
"encrypt": True,
"pingInterval": 50000,
"pingTimeout": 10000
}],
"token":
"testToken"
}
}
mock_api.post(url, body=json.dumps(resp))
ws_connect_mock.side_effect = asyncio.CancelledError
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_subscriptions())
self.async_run_with_timeout(self.listening_task)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
@patch(
"hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep"
)
def test_listen_for_subscriptions_logs_exception_details(
self, mock_api, sleep_mock, ws_connect_mock):
url = web_utils.rest_url(path_url=CONSTANTS.PUBLIC_WS_DATA_PATH_URL)
resp = {
"code": "200000",
"data": {
"instanceServers": [{
"endpoint": "wss://test.url/endpoint",
"protocol": "websocket",
"encrypt": True,
"pingInterval": 50000,
"pingTimeout": 10000
}],
"token":
"testToken"
}
}
mock_api.post(url, body=json.dumps(resp))
sleep_mock.side_effect = asyncio.CancelledError
ws_connect_mock.side_effect = Exception("TEST ERROR.")
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_subscriptions())
self.async_run_with_timeout(self.listening_task)
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error occurred when listening to order book streams. Retrying in 5 seconds..."
))
def test_listen_for_trades_cancelled_when_listening(self):
mock_queue = MagicMock()
mock_queue.get.side_effect = asyncio.CancelledError()
self.ob_data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_trades(self.ev_loop, msg_queue))
self.async_run_with_timeout(self.listening_task)
def test_listen_for_trades_logs_exception(self):
incomplete_resp = {
"type": "message",
"topic": f"/market/match:{self.trading_pair}",
}
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
incomplete_resp, asyncio.CancelledError()
]
self.ob_data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_trades(self.ev_loop, msg_queue))
try:
self.async_run_with_timeout(self.listening_task)
except asyncio.CancelledError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error when processing public trade updates from exchange"
))
def test_listen_for_trades_successful(self):
mock_queue = AsyncMock()
trade_event = {
"type": "message",
"topic": f"/market/match:{self.trading_pair}",
"subject": "trade.l3match",
"data": {
"sequence": "1545896669145",
"type": "match",
"symbol": self.trading_pair,
"side": "buy",
"price": "0.08200000000000000000",
"size": "0.01022222000000000000",
"tradeId": "5c24c5da03aa673885cd67aa",
"takerOrderId": "5c24c5d903aa6772d55b371e",
"makerOrderId": "5c2187d003aa677bd09d5c93",
"time": "1545913818099033203"
}
}
mock_queue.get.side_effect = [trade_event, asyncio.CancelledError()]
self.ob_data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
try:
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_trades(self.ev_loop, msg_queue))
except asyncio.CancelledError:
pass
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(trade_event["data"]["tradeId"], msg.trade_id)
def test_listen_for_order_book_diffs_cancelled(self):
mock_queue = AsyncMock()
mock_queue.get.side_effect = asyncio.CancelledError()
self.ob_data_source._message_queue[
CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
self.async_run_with_timeout(self.listening_task)
def test_listen_for_order_book_diffs_logs_exception(self):
incomplete_resp = {
"type": "message",
"topic": f"/market/level2:{self.trading_pair}",
}
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
incomplete_resp, asyncio.CancelledError()
]
self.ob_data_source._message_queue[
CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
try:
self.async_run_with_timeout(self.listening_task)
except asyncio.CancelledError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error when processing public order book updates from exchange"
))
def test_listen_for_order_book_diffs_successful(self):
mock_queue = AsyncMock()
diff_event = {
"type": "message",
"topic": "/market/level2:BTC-USDT",
"subject": "trade.l2update",
"data": {
"sequenceStart": 1545896669105,
"sequenceEnd": 1545896669106,
"symbol": f"{self.trading_pair}",
"changes": {
"asks": [["6", "1", "1545896669105"]],
"bids": [["4", "1", "1545896669106"]]
}
}
}
mock_queue.get.side_effect = [diff_event, asyncio.CancelledError()]
self.ob_data_source._message_queue[
CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
try:
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
except asyncio.CancelledError:
pass
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(diff_event["data"]["sequenceEnd"], msg.update_id)
@aioresponses()
def test_listen_for_order_book_snapshots_cancelled_when_fetching_snapshot(
self, mock_api):
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=asyncio.CancelledError)
with self.assertRaises(asyncio.CancelledError):
self.async_run_with_timeout(
self.ob_data_source.listen_for_order_book_snapshots(
self.ev_loop, asyncio.Queue()))
@aioresponses()
@patch(
"hummingbot.connector.exchange.kucoin.kucoin_api_order_book_data_source"
".KucoinAPIOrderBookDataSource._sleep")
def test_listen_for_order_book_snapshots_log_exception(
self, mock_api, sleep_mock):
msg_queue: asyncio.Queue = asyncio.Queue()
sleep_mock.side_effect = asyncio.CancelledError
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=Exception)
try:
self.async_run_with_timeout(
self.ob_data_source.listen_for_order_book_snapshots(
self.ev_loop, msg_queue))
except asyncio.CancelledError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
f"Unexpected error fetching order book snapshot for {self.trading_pair}."
))
@aioresponses()
def test_listen_for_order_book_snapshots_successful(
self,
mock_api,
):
msg_queue: asyncio.Queue = asyncio.Queue()
url = web_utils.rest_url(path_url=CONSTANTS.SNAPSHOT_NO_AUTH_PATH_URL)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
snapshot_data = {
"code": "200000",
"data": {
"sequence": "3262786978",
"time": 1550653727731,
"bids": [["6500.12", "0.45054140"], ["6500.11", "0.45054140"]],
"asks": [["6500.16", "0.57753524"], ["6500.15", "0.57753524"]]
}
}
mock_api.get(regex_url, body=json.dumps(snapshot_data))
self.listening_task = self.ev_loop.create_task(
self.ob_data_source.listen_for_order_book_snapshots(
self.ev_loop, msg_queue))
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertEqual(int(snapshot_data["data"]["sequence"]), msg.update_id)
class BinancePerpetualUserStreamDataSourceUnitTests(unittest.TestCase):
# the level is required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "COINALPHA"
cls.quote_asset = "HBOT"
cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
cls.ex_trading_pair = cls.base_asset + cls.quote_asset
cls.domain = CONSTANTS.TESTNET_DOMAIN
cls.api_key = "TEST_API_KEY"
cls.secret_key = "TEST_SECRET_KEY"
cls.listen_key = "TEST_LISTEN_KEY"
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task: Optional[asyncio.Task] = None
self.mocking_assistant = NetworkMockingAssistant()
self.emulated_time = 1640001112.223
self.auth = BinancePerpetualAuth(api_key=self.api_key,
api_secret=self.secret_key,
time_provider=self)
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(0)
self.data_source = BinancePerpetualUserStreamDataSource(
auth=self.auth,
domain=self.domain,
throttler=self.throttler,
time_synchronizer=self.time_synchronizer)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.mock_done_event = asyncio.Event()
self.resume_test_event = asyncio.Event()
def tearDown(self) -> None:
self.listening_task and self.listening_task.cancel()
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1):
ret = self.ev_loop.run_until_complete(
asyncio.wait_for(coroutine, timeout))
return ret
def _is_logged(self, log_level: str, message: str) -> bool:
return any(
record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def _raise_exception(self, exception_class):
raise exception_class
def _mock_responses_done_callback(self, *_, **__):
self.mock_done_event.set()
def _create_exception_and_unlock_test_with_event(self, exception):
self.resume_test_event.set()
raise exception
def _successful_get_listen_key_response(self) -> str:
resp = {"listenKey": self.listen_key}
return ujson.dumps(resp)
def _error_response(self) -> Dict[str, Any]:
resp = {"code": "ERROR CODE", "msg": "ERROR MESSAGE"}
return resp
def _simulate_user_update_event(self):
# Order Trade Update
resp = {
"e": "ORDER_TRADE_UPDATE",
"E": 1591274595442,
"T": 1591274595453,
"i": "SfsR",
"o": {
"s": "BTCUSD_200925",
"c": "TEST",
"S": "SELL",
"o": "TRAILING_STOP_MARKET",
"f": "GTC",
"q": "2",
"p": "0",
"ap": "0",
"sp": "9103.1",
"x": "NEW",
"X": "NEW",
"i": 8888888,
"l": "0",
"z": "0",
"L": "0",
"ma": "BTC",
"N": "BTC",
"n": "0",
"T": 1591274595442,
"t": 0,
"rp": "0",
"b": "0",
"a": "0",
"m": False,
"R": False,
"wt": "CONTRACT_PRICE",
"ot": "TRAILING_STOP_MARKET",
"ps": "LONG",
"cp": False,
"AP": "9476.8",
"cr": "5.0",
"pP": False,
},
}
return ujson.dumps(resp)
def time(self):
# Implemented to emulate a TimeSynchronizer
return self.emulated_time
def test_last_recv_time(self):
# Initial last_recv_time
self.assertEqual(0, self.data_source.last_recv_time)
@aioresponses()
def test_get_listen_key_exception_raised(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
status=400,
body=ujson.dumps(self._error_response))
with self.assertRaises(IOError):
self.async_run_with_timeout(self.data_source.get_listen_key())
@aioresponses()
def test_get_listen_key_successful(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
body=self._successful_get_listen_key_response())
result: str = self.async_run_with_timeout(
self.data_source.get_listen_key())
self.assertEqual(self.listen_key, result)
@aioresponses()
def test_ping_listen_key_failed_log_warning(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.put(regex_url,
status=400,
body=ujson.dumps(self._error_response()))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(
self.data_source.ping_listen_key())
self.assertTrue(
self._is_logged(
"WARNING",
f"Failed to refresh the listen key {self.listen_key}: {self._error_response()}"
))
self.assertFalse(result)
@aioresponses()
def test_ping_listen_key_successful(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.put(regex_url, body=ujson.dumps({}))
self.data_source._current_listen_key = self.listen_key
result: bool = self.async_run_with_timeout(
self.data_source.ping_listen_key())
self.assertTrue(result)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_create_websocket_connection_log_exception(self, mock_api,
mock_ws):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
body=self._successful_get_listen_key_response())
mock_ws.side_effect = Exception("TEST ERROR.")
msg_queue = asyncio.Queue()
try:
self.async_run_with_timeout(
self.data_source.listen_for_user_stream(msg_queue))
except asyncio.exceptions.TimeoutError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds... Error: TEST ERROR.",
))
@aioresponses()
def test_manage_listen_key_task_loop_keep_alive_failed(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.put(regex_url,
status=400,
body=ujson.dumps(self._error_response()),
callback=self._mock_responses_done_callback)
self.data_source._current_listen_key = self.listen_key
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(
self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.mock_done_event.wait())
self.assertTrue(
self._is_logged("ERROR", "Error occurred renewing listen key... "))
self.assertIsNone(self.data_source._current_listen_key)
self.assertFalse(
self.data_source._listen_key_initialized_event.is_set())
@aioresponses()
def test_manage_listen_key_task_loop_keep_alive_successful(self, mock_api):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.put(regex_url,
body=ujson.dumps({}),
callback=self._mock_responses_done_callback)
self.data_source._current_listen_key = self.listen_key
# Simulate LISTEN_KEY_KEEP_ALIVE_INTERVAL reached
self.data_source._last_listen_key_ping_ts = 0
self.listening_task = self.ev_loop.create_task(
self.data_source._manage_listen_key_task_loop())
self.async_run_with_timeout(self.mock_done_event.wait())
self.assertTrue(
self._is_logged("INFO",
f"Refreshed listen key {self.listen_key}."))
self.assertGreater(self.data_source._last_listen_key_ping_ts, 0)
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_create_websocket_connection_failed(
self, mock_api, mock_ws):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
body=self._successful_get_listen_key_response())
mock_ws.side_effect = Exception("TEST ERROR.")
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
try:
self.async_run_with_timeout(msg_queue.get())
except asyncio.exceptions.TimeoutError:
pass
self.assertTrue(
self._is_logged(
"INFO", f"Successfully obtained listen key {self.listen_key}"))
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds... Error: TEST ERROR.",
))
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
@patch(
"hummingbot.core.data_type.user_stream_tracker_data_source.UserStreamTrackerDataSource._sleep"
)
def test_listen_for_user_stream_iter_message_throws_exception(
self, mock_api, _, mock_ws):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {"listenKey": self.listen_key}
mock_api.post(regex_url, body=ujson.dumps(mock_response))
msg_queue: asyncio.Queue = asyncio.Queue()
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
mock_ws.return_value.receive.side_effect = Exception("TEST ERROR")
mock_ws.return_value.closed = False
mock_ws.return_value.close.side_effect = Exception
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
try:
self.async_run_with_timeout(msg_queue.get())
except Exception:
pass
self.assertTrue(
self._is_logged(
"INFO", f"Successfully obtained listen key {self.listen_key}"))
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error while listening to user stream. Retrying after 5 seconds... Error: TEST ERROR",
))
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_successful(self, mock_api, mock_ws):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
body=self._successful_get_listen_key_response())
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(
mock_ws.return_value, self._simulate_user_update_event())
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
msg = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(msg, self._simulate_user_update_event)
mock_ws.return_value.ping.assert_called()
@aioresponses()
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_user_stream_does_not_queue_empty_payload(
self, mock_api, mock_ws):
url = web_utils.rest_url(
path_url=CONSTANTS.BINANCE_USER_STREAM_ENDPOINT,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.post(regex_url,
body=self._successful_get_listen_key_response())
mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
self.mocking_assistant.add_websocket_aiohttp_message(
mock_ws.return_value, "")
msg_queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_user_stream(msg_queue))
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(
mock_ws.return_value)
self.assertEqual(0, msg_queue.qsize())
class BinanceAPIOrderBookDataSourceUnitTests(unittest.TestCase):
# logging.Level required to receive logs from the data source logger
level = 0
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.ev_loop = asyncio.get_event_loop()
cls.base_asset = "COINALPHA"
cls.quote_asset = "HBOT"
cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
cls.ex_trading_pair = cls.base_asset + cls.quote_asset
cls.domain = "com"
def setUp(self) -> None:
super().setUp()
self.log_records = []
self.listening_task = None
self.mocking_assistant = NetworkMockingAssistant()
self.time_synchronizer = TimeSynchronizer()
self.time_synchronizer.add_time_offset_ms_sample(1000)
self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS)
self.data_source = BinanceAPIOrderBookDataSource(
trading_pairs=[self.trading_pair],
throttler=self.throttler,
domain=self.domain,
time_synchronizer=self.time_synchronizer)
self.data_source.logger().setLevel(1)
self.data_source.logger().addHandler(self)
self.resume_test_event = asyncio.Event()
BinanceAPIOrderBookDataSource._trading_pair_symbol_map = {
"com":
bidict({f"{self.base_asset}{self.quote_asset}": self.trading_pair})
}
def tearDown(self) -> None:
self.listening_task and self.listening_task.cancel()
BinanceAPIOrderBookDataSource._trading_pair_symbol_map = {}
super().tearDown()
def handle(self, record):
self.log_records.append(record)
def _is_logged(self, log_level: str, message: str) -> bool:
return any(
record.levelname == log_level and record.getMessage() == message
for record in self.log_records)
def _create_exception_and_unlock_test_with_event(self, exception):
self.resume_test_event.set()
raise exception
def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1):
ret = self.ev_loop.run_until_complete(
asyncio.wait_for(coroutine, timeout))
return ret
def _successfully_subscribed_event(self):
resp = {"result": None, "id": 1}
return resp
def _trade_update_event(self):
resp = {
"e": "trade",
"E": 123456789,
"s": self.ex_trading_pair,
"t": 12345,
"p": "0.001",
"q": "100",
"b": 88,
"a": 50,
"T": 123456785,
"m": True,
"M": True
}
return resp
def _order_diff_event(self):
resp = {
"e": "depthUpdate",
"E": 123456789,
"s": self.ex_trading_pair,
"U": 157,
"u": 160,
"b": [["0.0024", "10"]],
"a": [["0.0026", "100"]]
}
return resp
def _snapshot_response(self):
resp = {
"lastUpdateId": 1027024,
"bids": [["4.00000000", "431.00000000"]],
"asks": [["4.00000200", "12.00000000"]]
}
return resp
@aioresponses()
def test_get_last_trade_prices(self, mock_api):
url = web_utils.public_rest_url(
path_url=CONSTANTS.TICKER_PRICE_CHANGE_PATH_URL,
domain=self.domain)
url = f"{url}?symbol={self.base_asset}{self.quote_asset}"
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response = {
"symbol": "BNBBTC",
"priceChange": "-94.99999800",
"priceChangePercent": "-95.960",
"weightedAvgPrice": "0.29628482",
"prevClosePrice": "0.10002000",
"lastPrice": "100.0",
"lastQty": "200.00000000",
"bidPrice": "4.00000000",
"bidQty": "100.00000000",
"askPrice": "4.00000200",
"askQty": "100.00000000",
"openPrice": "99.00000000",
"highPrice": "100.00000000",
"lowPrice": "0.10000000",
"volume": "8913.30000000",
"quoteVolume": "15.30000000",
"openTime": 1499783499040,
"closeTime": 1499869899040,
"firstId": 28385,
"lastId": 28460,
"count": 76,
}
mock_api.get(regex_url, body=json.dumps(mock_response))
result: Dict[str, float] = self.async_run_with_timeout(
self.data_source.get_last_traded_prices(
trading_pairs=[self.trading_pair],
throttler=self.throttler,
time_synchronizer=self.time_synchronizer))
self.assertEqual(1, len(result))
self.assertEqual(100, result[self.trading_pair])
@aioresponses()
def test_get_all_mid_prices(self, mock_api):
url = web_utils.public_rest_url(CONSTANTS.SERVER_TIME_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
response = {"serverTime": 1640000003000}
mock_api.get(regex_url, body=json.dumps(response))
url = web_utils.public_rest_url(
path_url=CONSTANTS.TICKER_PRICE_CHANGE_PATH_URL,
domain=self.domain)
mock_response: List[Dict[str, Any]] = [
{
# Truncated Response
"symbol": self.ex_trading_pair,
"bidPrice": "99",
"askPrice": "101",
},
{
# Truncated Response for unrecognized pair
"symbol": "BCCBTC",
"bidPrice": "99",
"askPrice": "101",
}
]
mock_api.get(url, body=json.dumps(mock_response))
result: Dict[str, float] = self.async_run_with_timeout(
self.data_source.get_all_mid_prices())
self.assertEqual(1, len(result))
self.assertEqual(100, result[self.trading_pair])
@aioresponses()
def test_fetch_trading_pairs(self, mock_api):
BinanceAPIOrderBookDataSource._trading_pair_symbol_map = {}
url = web_utils.public_rest_url(
path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL, domain=self.domain)
mock_response: Dict[str, Any] = {
"timezone":
"UTC",
"serverTime":
1639598493658,
"rateLimits": [],
"exchangeFilters": [],
"symbols": [
{
"symbol":
"ETHBTC",
"status":
"TRADING",
"baseAsset":
"ETH",
"baseAssetPrecision":
8,
"quoteAsset":
"BTC",
"quotePrecision":
8,
"quoteAssetPrecision":
8,
"baseCommissionPrecision":
8,
"quoteCommissionPrecision":
8,
"orderTypes": [
"LIMIT", "LIMIT_MAKER", "MARKET", "STOP_LOSS_LIMIT",
"TAKE_PROFIT_LIMIT"
],
"icebergAllowed":
True,
"ocoAllowed":
True,
"quoteOrderQtyMarketAllowed":
True,
"isSpotTradingAllowed":
True,
"isMarginTradingAllowed":
True,
"filters": [],
"permissions": ["SPOT", "MARGIN"]
},
{
"symbol":
"LTCBTC",
"status":
"TRADING",
"baseAsset":
"LTC",
"baseAssetPrecision":
8,
"quoteAsset":
"BTC",
"quotePrecision":
8,
"quoteAssetPrecision":
8,
"baseCommissionPrecision":
8,
"quoteCommissionPrecision":
8,
"orderTypes": [
"LIMIT", "LIMIT_MAKER", "MARKET", "STOP_LOSS_LIMIT",
"TAKE_PROFIT_LIMIT"
],
"icebergAllowed":
True,
"ocoAllowed":
True,
"quoteOrderQtyMarketAllowed":
True,
"isSpotTradingAllowed":
True,
"isMarginTradingAllowed":
True,
"filters": [],
"permissions": ["SPOT", "MARGIN"]
},
{
"symbol":
"BNBBTC",
"status":
"TRADING",
"baseAsset":
"BNB",
"baseAssetPrecision":
8,
"quoteAsset":
"BTC",
"quotePrecision":
8,
"quoteAssetPrecision":
8,
"baseCommissionPrecision":
8,
"quoteCommissionPrecision":
8,
"orderTypes": [
"LIMIT", "LIMIT_MAKER", "MARKET", "STOP_LOSS_LIMIT",
"TAKE_PROFIT_LIMIT"
],
"icebergAllowed":
True,
"ocoAllowed":
True,
"quoteOrderQtyMarketAllowed":
True,
"isSpotTradingAllowed":
True,
"isMarginTradingAllowed":
True,
"filters": [],
"permissions": ["MARGIN"]
},
]
}
mock_api.get(url, body=json.dumps(mock_response))
result: Dict[str] = self.async_run_with_timeout(
self.data_source.fetch_trading_pairs(
time_synchronizer=self.time_synchronizer))
self.assertEqual(2, len(result))
self.assertIn("ETH-BTC", result)
self.assertIn("LTC-BTC", result)
self.assertNotIn("BNB-BTC", result)
@aioresponses()
def test_fetch_trading_pairs_exception_raised(self, mock_api):
BinanceAPIOrderBookDataSource._trading_pair_symbol_map = {}
url = web_utils.public_rest_url(
path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL, domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=Exception)
result: Dict[str] = self.async_run_with_timeout(
self.data_source.fetch_trading_pairs(
time_synchronizer=self.time_synchronizer))
self.assertEqual(0, len(result))
@aioresponses()
def test_get_snapshot_successful(self, mock_api):
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, body=json.dumps(self._snapshot_response()))
result: Dict[str, Any] = self.async_run_with_timeout(
self.data_source.get_snapshot(self.trading_pair))
self.assertEqual(self._snapshot_response(), result)
@aioresponses()
def test_get_snapshot_catch_exception(self, mock_api):
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, status=400)
with self.assertRaises(IOError):
self.async_run_with_timeout(
self.data_source.get_snapshot(self.trading_pair))
@aioresponses()
def test_get_new_order_book(self, mock_api):
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_response: Dict[str, Any] = {
"lastUpdateId": 1,
"bids": [["4.00000000", "431.00000000"]],
"asks": [["4.00000200", "12.00000000"]]
}
mock_api.get(regex_url, body=json.dumps(mock_response))
result: OrderBook = self.async_run_with_timeout(
self.data_source.get_new_order_book(self.trading_pair))
self.assertEqual(1, result.snapshot_uid)
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_subscriptions_subscribes_to_trades_and_order_diffs(
self, ws_connect_mock):
ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock(
)
result_subscribe_trades = {"result": None, "id": 1}
result_subscribe_diffs = {"result": None, "id": 2}
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_trades))
self.mocking_assistant.add_websocket_aiohttp_message(
websocket_mock=ws_connect_mock.return_value,
message=json.dumps(result_subscribe_diffs))
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_subscriptions())
self.mocking_assistant.run_until_all_aiohttp_messages_delivered(
ws_connect_mock.return_value)
sent_subscription_messages = self.mocking_assistant.json_messages_sent_through_websocket(
websocket_mock=ws_connect_mock.return_value)
self.assertEqual(2, len(sent_subscription_messages))
expected_trade_subscription = {
"method": "SUBSCRIBE",
"params": [f"{self.ex_trading_pair.lower()}@trade"],
"id": 1
}
self.assertEqual(expected_trade_subscription,
sent_subscription_messages[0])
expected_diff_subscription = {
"method": "SUBSCRIBE",
"params": [f"{self.ex_trading_pair.lower()}@depth@100ms"],
"id": 2
}
self.assertEqual(expected_diff_subscription,
sent_subscription_messages[1])
self.assertTrue(
self._is_logged(
"INFO",
"Subscribed to public order book and trade channels..."))
@patch(
"hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep"
)
@patch("aiohttp.ClientSession.ws_connect")
def test_listen_for_subscriptions_raises_cancel_exception(
self, mock_ws, _: AsyncMock):
mock_ws.side_effect = asyncio.CancelledError
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_subscriptions())
self.async_run_with_timeout(self.listening_task)
@patch(
"hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep"
)
@patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
def test_listen_for_subscriptions_logs_exception_details(
self, mock_ws, sleep_mock):
mock_ws.side_effect = Exception("TEST ERROR.")
sleep_mock.side_effect = lambda _: self._create_exception_and_unlock_test_with_event(
asyncio.CancelledError())
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_subscriptions())
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error occurred when listening to order book streams. Retrying in 5 seconds..."
))
def test_subscribe_channels_raises_cancel_exception(self):
mock_ws = MagicMock()
mock_ws.send.side_effect = asyncio.CancelledError
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.data_source._subscribe_channels(mock_ws))
self.async_run_with_timeout(self.listening_task)
def test_subscribe_channels_raises_exception_and_logs_error(self):
mock_ws = MagicMock()
mock_ws.send.side_effect = Exception("Test Error")
with self.assertRaises(Exception):
self.listening_task = self.ev_loop.create_task(
self.data_source._subscribe_channels(mock_ws))
self.async_run_with_timeout(self.listening_task)
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error occurred subscribing to order book trading and delta streams..."
))
def test_listen_for_trades_cancelled_when_listening(self):
mock_queue = MagicMock()
mock_queue.get.side_effect = asyncio.CancelledError()
self.data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_trades(self.ev_loop, msg_queue))
self.async_run_with_timeout(self.listening_task)
def test_listen_for_trades_logs_exception(self):
incomplete_resp = {
"m": 1,
"i": 2,
}
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
incomplete_resp, asyncio.CancelledError()
]
self.data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_trades(self.ev_loop, msg_queue))
try:
self.async_run_with_timeout(self.listening_task)
except asyncio.CancelledError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error when processing public trade updates from exchange"
))
def test_listen_for_trades_successful(self):
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
self._trade_update_event(),
asyncio.CancelledError()
]
self.data_source._message_queue[
CONSTANTS.TRADE_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
try:
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_trades(self.ev_loop, msg_queue))
except asyncio.CancelledError:
pass
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(12345, msg.trade_id)
def test_listen_for_order_book_diffs_cancelled(self):
mock_queue = AsyncMock()
mock_queue.get.side_effect = asyncio.CancelledError()
self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
with self.assertRaises(asyncio.CancelledError):
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
self.async_run_with_timeout(self.listening_task)
def test_listen_for_order_book_diffs_logs_exception(self):
incomplete_resp = {
"m": 1,
"i": 2,
}
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
incomplete_resp, asyncio.CancelledError()
]
self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
try:
self.async_run_with_timeout(self.listening_task)
except asyncio.CancelledError:
pass
self.assertTrue(
self._is_logged(
"ERROR",
"Unexpected error when processing public order book updates from exchange"
))
def test_listen_for_order_book_diffs_successful(self):
mock_queue = AsyncMock()
mock_queue.get.side_effect = [
self._order_diff_event(),
asyncio.CancelledError()
]
self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue
msg_queue: asyncio.Queue = asyncio.Queue()
try:
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_order_book_diffs(
self.ev_loop, msg_queue))
except asyncio.CancelledError:
pass
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertTrue(12345, msg.update_id)
@aioresponses()
def test_listen_for_order_book_snapshots_cancelled_when_fetching_snapshot(
self, mock_api):
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=asyncio.CancelledError)
with self.assertRaises(asyncio.CancelledError):
self.async_run_with_timeout(
self.data_source.listen_for_order_book_snapshots(
self.ev_loop, asyncio.Queue()))
@aioresponses()
@patch(
"hummingbot.connector.exchange.binance.binance_api_order_book_data_source"
".BinanceAPIOrderBookDataSource._sleep")
def test_listen_for_order_book_snapshots_log_exception(
self, mock_api, sleep_mock):
msg_queue: asyncio.Queue = asyncio.Queue()
sleep_mock.side_effect = lambda _: self._create_exception_and_unlock_test_with_event(
asyncio.CancelledError())
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, exception=Exception)
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_order_book_snapshots(
self.ev_loop, msg_queue))
self.async_run_with_timeout(self.resume_test_event.wait())
self.assertTrue(
self._is_logged(
"ERROR",
f"Unexpected error fetching order book snapshot for {self.trading_pair}."
))
@aioresponses()
def test_listen_for_order_book_snapshots_successful(
self,
mock_api,
):
msg_queue: asyncio.Queue = asyncio.Queue()
url = web_utils.public_rest_url(path_url=CONSTANTS.SNAPSHOT_PATH_URL,
domain=self.domain)
regex_url = re.compile(f"^{url}".replace(".",
r"\.").replace("?", r"\?"))
mock_api.get(regex_url, body=json.dumps(self._snapshot_response()))
self.listening_task = self.ev_loop.create_task(
self.data_source.listen_for_order_book_snapshots(
self.ev_loop, msg_queue))
msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get())
self.assertEqual(1027024, msg.update_id)
