def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: # This does not check for any limits. Can there be any limits like with trades # in the deposit/withdrawal binance api? Can't see anything in the docs: # https://github.com/binance-exchange/binance-official-api-docs/blob/master/wapi-api.md#deposit-history-user_data # # Note that all timestamps should be in milliseconds, so we multiply by 1k options = { 'timestamp': ts_now_in_ms(), 'startTime': start_ts * 1000, 'endTime': end_ts * 1000, } result = self.api_query_dict('depositHistory.html', options=options) raw_data = result.get('depositList', []) options['timestamp'] = ts_now_in_ms() result = self.api_query_dict('withdrawHistory.html', options=options) raw_data.extend(result.get('withdrawList', [])) log.debug('binance deposit/withdrawal history result', results_num=len(raw_data)) movements = [] for raw_movement in raw_data: movement = self._deserialize_asset_movement(raw_movement) if movement: movements.append(movement) return movements
def _api_query( self, endpoint: Literal['balance', 'user_transactions'], method: Literal['post'] = 'post', options: Optional[Dict[str, Any]] = None, ) -> Response: """Request a Bistamp API v2 endpoint (from `endpoint`). """ call_options = options.copy() if options else {} data = call_options or None request_url = f'{self.base_uri}/v2/{endpoint}/' query_params = '' nonce = str(uuid.uuid4()) timestamp = str(ts_now_in_ms()) payload_string = urlencode(call_options) content_type = '' if payload_string == '' else 'application/x-www-form-urlencoded' message = ( 'BITSTAMP ' f'{self.api_key}' f'{method.upper()}' f'{request_url.replace("https://", "")}' f'{query_params}' f'{content_type}' f'{nonce}' f'{timestamp}' 'v2' f'{payload_string}' ) signature = hmac.new( self.secret, msg=message.encode('utf-8'), digestmod=hashlib.sha256, ).hexdigest() self.session.headers.update({ 'X-Auth-Signature': signature, 'X-Auth-Nonce': nonce, 'X-Auth-Timestamp': timestamp, }) if content_type: self.session.headers.update({'Content-Type': content_type}) else: self.session.headers.pop('Content-Type', None) log.debug('Bitstamp API request', request_url=request_url) try: response = self.session.request( method=method, url=request_url, data=data, ) except requests.exceptions.RequestException as e: raise RemoteError( f'Bitstamp {method} request at {request_url} connection error: {str(e)}.', ) from e return response
def _api_query_dict_within_time_delta( self, start_ts: Timestamp, end_ts: Timestamp, time_delta: Timestamp, api_type: BINANCE_API_TYPE, method: Literal['depositHistory.html', 'withdrawHistory.html'], ) -> List[Dict[str, Any]]: """Request via `api_query_dict()` from `start_ts` `end_ts` using a time delta (offset) less than `time_delta`. Be aware of: - If `start_ts` equals zero, the Binance launch timestamp is used (from BINANCE_LAUNCH_TS). This value is not stored in the `used_query_ranges` table, but 0. - Timestamps are converted to milliseconds. """ if method == 'depositHistory.html': query_schema = 'depositList' elif method == 'withdrawHistory.html': query_schema = 'withdrawList' else: raise AssertionError( f'Unexpected {self.name} method case: {method}.') results: List[Dict[str, Any]] = [] # Create required time references in milliseconds start_ts = Timestamp(start_ts * 1000) end_ts = Timestamp(end_ts * 1000) offset = time_delta * 1000 - 1 # less than time_delta if start_ts == Timestamp(0): from_ts = BINANCE_LAUNCH_TS * 1000 else: from_ts = start_ts to_ts = ( from_ts + offset # Case request with offset if end_ts - from_ts > offset else end_ts # Case request without offset (1 request) ) while True: options = { 'timestamp': ts_now_in_ms(), 'startTime': from_ts, 'endTime': to_ts, } result = self.api_query_dict(api_type, method, options=options) results.extend(result.get(query_schema, [])) # Case stop requesting if to_ts >= end_ts: break from_ts = to_ts + 1 to_ts = min(to_ts + offset, end_ts) return results
def api_query( # noqa: F811 self, method: str, options: Optional[Dict[str, Any]] = None, ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ Queries Bittrex with given method and options """ if not options: options = {} nonce = str(ts_now_in_ms()) method_type = 'public' if method in BITTREX_MARKET_METHODS: method_type = 'market' elif method in BITTREX_ACCOUNT_METHODS: method_type = 'account' request_url = self.uri + method_type + '/' + method + '?' if method_type != 'public': request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&' request_url += urlencode(options) signature = hmac.new( self.secret, request_url.encode(), hashlib.sha512, ).hexdigest() self.session.headers.update({'apisign': signature}) log.debug('Bittrex API query', request_url=request_url) try: response = self.session.get(request_url) except requests.exceptions.ConnectionError as e: raise RemoteError(f'Bittrex API request failed due to {str(e)}') if response.status_code != 200: raise RemoteError( f'Bittrex query responded with error status code: {response.status_code}' f' and text: {response.text}', ) try: json_ret = rlk_jsonloads_dict(response.text) except JSONDecodeError: raise RemoteError( f'Bittrex returned invalid JSON response: {response.text}') if json_ret['success'] is not True: raise RemoteError(json_ret['message']) result = json_ret['result'] assert isinstance(result, dict) or isinstance(result, list) return result
def _query_continuously( self, method: Literal['get', 'post'], endpoint: str, options: Optional[Dict[str, Any]] = None, ) -> requests.Response: """Queries endpoint until anything but 429 is returned May raise: - RemoteError if something is wrong connecting to the exchange """ v_endpoint = f'/v1/{endpoint}' url = f'{self.base_uri}{v_endpoint}' retries_left = QUERY_RETRY_TIMES while retries_left > 0: if endpoint in ('mytrades', 'balances', 'transfers', 'roles', 'balances/earn'): # private endpoints timestamp = str(ts_now_in_ms()) payload = {'request': v_endpoint, 'nonce': timestamp} if options is not None: payload.update(options) encoded_payload = json.dumps(payload).encode() b64 = b64encode(encoded_payload) signature = hmac.new(self.secret, b64, hashlib.sha384).hexdigest() self.session.headers.update({ 'X-GEMINI-PAYLOAD': b64.decode(), 'X-GEMINI-SIGNATURE': signature, }) try: response = self.session.request( method=method, url=url, timeout=GLOBAL_REQUESTS_TIMEOUT, ) except requests.exceptions.RequestException as e: raise RemoteError( f'Gemini {method} query at {url} connection error: {str(e)}', ) from e if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: # Backoff a bit by sleeping. Sleep more, the more retries have been made gevent.sleep(QUERY_RETRY_TIMES / retries_left) retries_left -= 1 else: # get out of the retry loop, we did not get 429 complaint break return response
def first_connection(self) -> None: if self.first_connection_made: return # If it's the first time, populate the binance pair trade symbols # We know exchangeInfo returns a dict exchange_data = self.api_query_dict('exchangeInfo') self._symbols_to_pair = create_binance_symbols_to_pair(exchange_data) server_time = self.api_query_dict('time') self.offset_ms = server_time['serverTime'] - ts_now_in_ms() self.first_connection_made = True
def _single_query(self, command: str, req: Dict[str, Any]) -> Optional[requests.Response]: """A single api query for poloniex Returns the response if all went well or None if a recoverable poloniex error occured such as a 504. Can raise: - RemoteError if there is a problem with the response - ConnectionError if there is a problem connecting to poloniex. """ if command in ('returnTicker', 'returnCurrencies'): log.debug(f'Querying poloniex for {command}') response = self.session.get(self.public_uri + command, timeout=DEFAULT_TIMEOUT_TUPLE) else: req['command'] = command with self.nonce_lock: # Protect this region with a lock since poloniex will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail req['nonce'] = ts_now_in_ms() post_data = str.encode(urlencode(req)) sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest() self.session.headers.update({'Sign': sign}) response = self.session.post( 'https://poloniex.com/tradingApi', req, timeout=DEFAULT_TIMEOUT_TUPLE, ) if response.status_code == 504: # backoff and repeat return None if response.status_code != 200: raise RemoteError( f'Poloniex query responded with error status code: {response.status_code}' f' and text: {response.text}', ) # else all is good return response
def _single_api_query( self, request_url: str, options: Dict[str, Any], method: Literal['get', 'put', 'delete'], public_endpoint: bool = False, ) -> requests.Response: payload = '' if method == 'get' else json.dumps(options) if not public_endpoint: api_content_hash = hashlib.sha512(payload.encode()).hexdigest() api_timestamp = str(ts_now_in_ms()) presign_str = api_timestamp + request_url + method.upper( ) + api_content_hash signature = hmac.new( self.secret, presign_str.encode(), hashlib.sha512, ).hexdigest() self.session.headers.update({ 'Api-Key': self.api_key, 'Api-Timestamp': api_timestamp, 'Api-Content-Hash': api_content_hash, 'Api-Signature': signature, }) else: self.session.headers.pop('Api-Key') log.debug('Bittrex v3 API query', request_url=request_url) try: response = self.session.request( method=method, url=request_url, json=options if method != 'get' else None, ) except requests.exceptions.RequestException as e: raise RemoteError( f'Bittrex API request failed due to {str(e)}') from e return response
def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]: if not options: options = {} backoff = self.initial_backoff while True: with self.nonce_lock: # Protect this region with a lock since binance will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS: api_version = 3 # Recommended recvWindows is 5000 but we get timeouts with it options['recvWindow'] = 10000 options['timestamp'] = str(ts_now_in_ms() + self.offset_ms) signature = hmac.new( self.secret, urlencode(options).encode('utf-8'), hashlib.sha256, ).hexdigest() options['signature'] = signature elif method in V1_ENDPOINTS: api_version = 1 else: raise ValueError( 'Unexpected binance api method {}'.format(method)) apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/' request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?' request_url += urlencode(options) log.debug('Binance API request', request_url=request_url) try: response = self.session.get(request_url) except requests.exceptions.ConnectionError as e: raise RemoteError( f'Binance API request failed due to {str(e)}') limit_ban = response.status_code == 429 and backoff > self.backoff_limit if limit_ban or response.status_code not in (200, 429): code = 'no code found' msg = 'no message found' try: result = rlk_jsonloads(response.text) if isinstance(result, dict): code = result.get('code', code) msg = result.get('msg', msg) except JSONDecodeError: pass raise RemoteError( 'Binance API request {} for {} failed with HTTP status ' 'code: {}, error code: {} and error message: {}'.format( response.url, method, response.status_code, code, msg, )) elif response.status_code == 429: if backoff > self.backoff_limit: break # Binance has limits and if we hit them we should backoff # https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md#limits log.debug('Got 429 from Binance. Backing off', seconds=backoff) gevent.sleep(backoff) backoff = backoff * 2 continue else: # success break try: json_ret = rlk_jsonloads(response.text) except JSONDecodeError: raise RemoteError( f'Binance returned invalid JSON response: {response.text}') return json_ret
def api_query( self, api_type: BINANCE_API_TYPE, method: str, options: Optional[Dict] = None, ) -> Union[List, Dict]: """Performs a binance api query May raise: - RemoteError - BinancePermissionError """ call_options = options.copy() if options else {} while True: with self.nonce_lock: # Protect this region with a lock since binance will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail if 'signature' in call_options: del call_options['signature'] is_v3_api_method = api_type == 'api' and method in V3_METHODS is_new_futures_api = api_type in ('fapi', 'dapi') call_needs_signature = ( (api_type == 'fapi' and method in FAPI_METHODS) or (api_type == 'dapi' and method in FAPI_METHODS) or # same as fapi (api_type == 'sapi' and method in SAPI_METHODS) or (api_type == 'wapi' and method in WAPI_METHODS) or is_v3_api_method) if call_needs_signature: if api_type in ('sapi', 'dapi'): api_version = 1 elif api_type == 'fapi': api_version = 2 elif api_type == 'wapi' or is_v3_api_method: api_version = 3 else: raise AssertionError( f'Should never get to signed binance api call for ' f'api_type: {api_type} and method {method}', ) # Recommended recvWindows is 5000 but we get timeouts with it call_options['recvWindow'] = 10000 call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms) signature = hmac.new( self.secret, urlencode(call_options).encode('utf-8'), hashlib.sha256, ).hexdigest() call_options['signature'] = signature elif api_type == 'api' and method in V1_METHODS: api_version = 1 else: raise AssertionError( f'Unexpected {self.name} API method {method}') api_subdomain = api_type if is_new_futures_api else 'api' request_url = ( f'https://{api_subdomain}.{self.uri}{api_type}/v{str(api_version)}/{method}?' ) request_url += urlencode(call_options) log.debug(f'{self.name} API request', request_url=request_url) try: response = self.session.get(request_url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'{self.name} API request failed due to {str(e)}', ) from e if response.status_code not in (200, 418, 429): code = 'no code found' msg = 'no message found' try: result = json.loads(response.text) if isinstance(result, dict): code = result.get('code', code) msg = result.get('msg', msg) except JSONDecodeError: pass exception_class: Union[Type[RemoteError], Type[BinancePermissionError]] if response.status_code == 401 and code == REJECTED_MBX_KEY: # Either API key permission error or if futures/dapi then not enables yet exception_class = BinancePermissionError else: exception_class = RemoteError raise exception_class( '{} API request {} for {} failed with HTTP status ' 'code: {}, error code: {} and error message: {}'.format( self.name, response.url, method, response.status_code, code, msg, )) if response.status_code in (418, 429): # Binance has limits and if we hit them we should backoff. # A Retry-After header is sent with a 418 or 429 responses and # will give the number of seconds required to wait, in the case # of a 429, to prevent a ban, or, in the case of a 418, until # the ban is over. # https://binance-docs.github.io/apidocs/spot/en/#limits retry_after = int(response.headers.get('retry-after', '0')) # Spoiler. They actually seem to always return 0 here. So we don't # wait at all. Won't be much of an improvement but force 1 sec wait if 0 returns retry_after = max( 1, retry_after ) # wait at least 1 sec even if api says otherwise log.debug( f'Got status code {response.status_code} from {self.name}. Backing off', seconds=retry_after, ) if retry_after > RETRY_AFTER_LIMIT: raise RemoteError( '{} API request {} for {} failed with HTTP status ' 'code: {} due to a too long retry after value ({} > {})' .format( self.name, response.url, method, response.status_code, retry_after, RETRY_AFTER_LIMIT, )) gevent.sleep(retry_after) continue # else success break try: json_ret = json.loads(response.text) except JSONDecodeError as e: raise RemoteError( f'{self.name} returned invalid JSON response: {response.text}', ) from e return json_ret
def _api_query( self, endpoint: Literal[ 'configs_list_currency', 'configs_list_pair_exchange', 'configs_map_currency_symbol', 'movements', 'trades', 'wallets', ], options: Optional[Dict[str, Any]] = None, ) -> Response: """Request a Bitfinex API v2 endpoint (from `endpoint`). """ call_options = options.copy() if options else {} for header in ('Content-Type', 'bfx-nonce', 'bfx-signature'): self.session.headers.pop(header, None) if endpoint == 'configs_list_currency': method = 'get' api_path = 'v2/conf/pub:list:currency' request_url = f'{self.base_uri}/{api_path}' elif endpoint == 'configs_list_pair_exchange': method = 'get' api_path = 'v2/conf/pub:list:pair:exchange' request_url = f'{self.base_uri}/{api_path}' elif endpoint == 'configs_map_currency_symbol': method = 'get' api_path = 'v2/conf/pub:map:currency:sym' request_url = f'{self.base_uri}/{api_path}' elif endpoint == 'movements': method = 'post' api_path = 'v2/auth/r/movements/hist' request_url = f'{self.base_uri}/{api_path}?{urlencode(call_options)}' elif endpoint == 'trades': method = 'post' api_path = 'v2/auth/r/trades/hist' request_url = f'{self.base_uri}/{api_path}?{urlencode(call_options)}' elif endpoint == 'wallets': method = 'post' api_path = 'v2/auth/r/wallets' request_url = f'{self.base_uri}/{api_path}' else: raise AssertionError(f'Unexpected {self.name} endpoint type: {endpoint}') with self.nonce_lock: # Protect this region with a lock since Bitfinex will reject # non-increasing nonces for authenticated endpoints if endpoint in ('movements', 'trades', 'wallets'): nonce = str(ts_now_in_ms()) message = f'/api/{api_path}{nonce}' signature = hmac.new( self.secret, msg=message.encode('utf-8'), digestmod=hashlib.sha384, ).hexdigest() self.session.headers.update({ 'Content-Type': 'application/json', 'bfx-nonce': nonce, 'bfx-signature': signature, }) log.debug(f'{self.name} API request', request_url=request_url) try: response = self.session.request( method=method, url=request_url, ) except requests.exceptions.RequestException as e: raise RemoteError( f'{self.name} {method} request at {request_url} connection error: {str(e)}.', ) from e return response
def test_api_query_retry_on_status_code_429(function_scope_binance): """Test when Binance API returns 429 and the request is retried, the signature is not polluted by any attribute from the previous call. It also tests getting the `retry-after` seconds to backoff from the response header. NB: basically remove `call_options['signature']`. """ binance = function_scope_binance offset_ms = 1000 call_options = { 'fromId': 0, 'limit': 1000, 'symbol': 'BUSDUSDT', 'recvWindow': 10000, 'timestamp': str(ts_now_in_ms() + offset_ms), } signature = hmac.new( binance.secret, urlencode(call_options).encode('utf-8'), hashlib.sha256, ).hexdigest() call_options['signature'] = signature base_url = 'https://api.binance.com/api/v3/myTrades?' exp_request_url = base_url + urlencode(call_options) # NB: all calls must have the same signature (time frozen) expected_calls = [ call(exp_request_url, timeout=DEFAULT_TIMEOUT_TUPLE), call(exp_request_url, timeout=DEFAULT_TIMEOUT_TUPLE), call(exp_request_url, timeout=DEFAULT_TIMEOUT_TUPLE), ] def get_mocked_response(): responses = [ MockResponse(429, '[]', headers={'retry-after': '1'}), MockResponse(418, '[]', headers={'retry-after': '5'}), MockResponse(418, '[]', headers={'retry-after': str(RETRY_AFTER_LIMIT + 1)}), ] for response in responses: yield response def mock_response(url, timeout): # pylint: disable=unused-argument return next(get_response) get_response = get_mocked_response() offset_ms_patch = patch.object(binance, 'offset_ms', new=1000) binance_patch = patch.object(binance.session, 'get', side_effect=mock_response) with ExitStack() as stack: stack.enter_context(offset_ms_patch) binance_mock_get = stack.enter_context(binance_patch) with pytest.raises(RemoteError) as e: binance.api_query( api_type='api', method='myTrades', options={ 'fromId': 0, 'limit': 1000, 'symbol': 'BUSDUSDT', }, ) assert 'myTrades failed with HTTP status code: 418' in str(e.value) assert binance_mock_get.call_args_list == expected_calls
else: api_path = 'api/v1/withdrawals' elif case == KucoinCase.OLD_TRADES: assert isinstance(options, dict) api_path = 'api/v1/hist-orders' elif case == KucoinCase.TRADES: assert isinstance(options, dict) api_path = 'api/v1/fills' else: raise AssertionError(f'Unexpected case: {case}') retries_left = API_REQUEST_RETRY_TIMES retries_after_seconds = API_REQUEST_RETRIES_AFTER_SECONDS while retries_left >= 0: timestamp = str(ts_now_in_ms()) method = 'GET' request_url = f'{self.base_uri}/{api_path}' message = f'{timestamp}{method}/{api_path}' if case in PAGINATED_CASES: if call_options != {}: urlencoded_options = urlencode(call_options) request_url = f'{request_url}?{urlencoded_options}' message = f'{message}?{urlencoded_options}' signature = base64.b64encode( hmac.new( self.secret, msg=message.encode('utf-8'), digestmod=hashlib.sha256, ).digest(), ).decode('utf-8')
def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]: call_options = options.copy() if options else {} while True: with self.nonce_lock: # Protect this region with a lock since binance will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail if 'signature' in call_options: del call_options['signature'] if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS or method in SAPI_ENDPOINTS: if method in SAPI_ENDPOINTS: api_version = 1 else: api_version = 3 # Recommended recvWindows is 5000 but we get timeouts with it call_options['recvWindow'] = 10000 call_options['timestamp'] = str(ts_now_in_ms() + self.offset_ms) signature = hmac.new( self.secret, urlencode(call_options).encode('utf-8'), hashlib.sha256, ).hexdigest() call_options['signature'] = signature elif method in V1_ENDPOINTS: api_version = 1 else: raise ValueError( 'Unexpected binance api method {}'.format(method)) if method in WAPI_ENDPOINTS: apistr = 'wapi/' elif method in SAPI_ENDPOINTS: apistr = 'sapi/' else: apistr = 'api/' request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?' request_url += urlencode(call_options) log.debug('Binance API request', request_url=request_url) try: response = self.session.get(request_url) except requests.exceptions.RequestException as e: raise RemoteError( f'Binance API request failed due to {str(e)}') from e if response.status_code not in (200, 418, 429): code = 'no code found' msg = 'no message found' try: result = rlk_jsonloads(response.text) if isinstance(result, dict): code = result.get('code', code) msg = result.get('msg', msg) except JSONDecodeError: pass raise RemoteError( 'Binance API request {} for {} failed with HTTP status ' 'code: {}, error code: {} and error message: {}'.format( response.url, method, response.status_code, code, msg, )) if response.status_code in (418, 429): # Binance has limits and if we hit them we should backoff. # A Retry-After header is sent with a 418 or 429 responses and # will give the number of seconds required to wait, in the case # of a 429, to prevent a ban, or, in the case of a 418, until # the ban is over. # https://binance-docs.github.io/apidocs/spot/en/#limits retry_after = int(response.headers.get('retry-after', '0')) log.debug( f'Got status code {response.status_code} from Binance. Backing off', seconds=retry_after, ) if retry_after > RETRY_AFTER_LIMIT: raise RemoteError( 'Binance API request {} for {} failed with HTTP status ' 'code: {} due to a too long retry after value (> {})'. format( response.url, method, response.status_code, RETRY_AFTER_LIMIT, )) gevent.sleep(retry_after) continue else: # success break try: json_ret = rlk_jsonloads(response.text) except JSONDecodeError as e: raise RemoteError( f'Binance returned invalid JSON response: {response.text}' ) from e return json_ret