def test_bitstamp_exchange_assets_are_known(mock_bitstamp): request_url = f'{mock_bitstamp.base_uri}/v2/trading-pairs-info' try: response = requests.get(request_url) except requests.exceptions.RequestException as e: raise RemoteError( f'Bitstamp get request at {request_url} connection error: {str(e)}.', ) from e if response.status_code != 200: raise RemoteError( f'Bitstamp query responded with error status code: {response.status_code} ' f'and text: {response.text}', ) try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError as e: raise RemoteError(f'Bitstamp returned invalid JSON response: {response.text}') from e # Extract the unique symbols from the exchange pairs pairs = [raw_result.get('name') for raw_result in response_list] symbols = set() for pair in pairs: symbols.update(set(pair.split('/'))) for symbol in symbols: try: asset_from_bitstamp(symbol) except UnknownAsset as e: test_warnings.warn(UserWarning( f'Found unknown asset {e.asset_name} in {mock_bitstamp.name}. ' f'Support for it has to be added', ))
def _get_tom_pool_fee_rewards_from_api(self) -> FeeRewards: """Do a GET request to the Tom pool fee rewards API. """ fee_rewards: FeeRewards = [] try: response = self.session.get(TOM_POOL_FEE_REWARDS_API_URL) except requests.exceptions.RequestException as e: msg = ( f'AdEx get request at {TOM_POOL_FEE_REWARDS_API_URL} connection error: {str(e)}.' ) self.msg_aggregator.add_error( f'Got remote error while querying AdEx fee rewards API: {msg}', ) return fee_rewards if response.status_code != HTTPStatus.OK: msg = ( f'AdEx fee rewards API query responded with error status code: ' f'{response.status_code} and text: {response.text}.') self.msg_aggregator.add_error( f'Got remote error while querying AdEx fee rewards API: {msg}', ) return fee_rewards try: fee_rewards = rlk_jsonloads_list(response.text) except JSONDecodeError: msg = f'AdEx fee rewards API returned invalid JSON response: {response.text}.' self.msg_aggregator.add_error( f'Got remote error while querying AdEx fee rewards API: {msg}', ) return fee_rewards return fee_rewards
def _public_api_query( self, endpoint: str, ) -> List[Any]: """Performs a Gemini API Query for a public endpoint You can optionally provide extra arguments to the endpoint via the options argument. Raises RemoteError if something went wrong with connecting or reading from the exchange """ response = self._query_continuously(method='get', endpoint=endpoint) if response.status_code != HTTPStatus.OK: raise RemoteError( f'Gemini query at {response.url} responded with error ' f'status code: {response.status_code} and text: {response.text}', ) try: json_ret = rlk_jsonloads_list(response.text) except JSONDecodeError: raise RemoteError( f'Gemini query at {response.url} ' f'returned invalid JSON response: {response.text}', ) return json_ret
def _query_exchange_pairs(self) -> ExchangePairsResponse: """Query and return the list of the exchange (trades) pairs in `<ExchangePairsResponse>.pairs`. Otherwise populate <ExchangePairsResponse> with data that each endpoint can process as an unsuccessful request. """ was_successful = True pairs = [] response = self._api_query('configs_list_pair_exchange') if response.status_code != HTTPStatus.OK: was_successful = False log.error( f'{self.name} exchange pairs list query failed. Check further logs' ) else: try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError: was_successful = False log.error( f'{self.name} exchange pairs list returned invalid JSON response. ' f'Check further logs', ) else: pairs = [ pair for pair in response_list[0] if not pair.startswith(BITFINEX_EXCHANGE_TEST_ASSETS) and not pair.endswith(BITFINEX_EXCHANGE_TEST_ASSETS) ] return ExchangePairsResponse( success=was_successful, response=response, pairs=pairs, )
def _query_currencies(self) -> CurrenciesResponse: """Query and return the list of all the currencies supported in `<CurrenciesResponse>.currencies`. Otherwise populate <CurrenciesResponse> with data that each endpoint can process as an unsuccessful request. """ was_successful = True currencies = [] response = self._api_query('configs_list_currency') if response.status_code != HTTPStatus.OK: was_successful = False log.error( f'{self.name} currencies list query failed. Check further logs' ) else: try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError: was_successful = False log.error( f'{self.name} currencies list returned invalid JSON response. ' f'Check further logs', ) else: currencies = [ currency for currency in response_list[0] if currency not in set(BITFINEX_EXCHANGE_TEST_ASSETS) ] return CurrenciesResponse( success=was_successful, response=response, currencies=currencies, )
def api_query( # noqa: F811 self, endpoint: str, method: Literal['get', 'put', 'delete'] = 'get', options: Optional[Dict[str, Any]] = None, ) -> List[Dict[str, Any]]: """ Queries Bittrex api v3 for given endpoint, method and options """ given_options = options.copy() if options else {} backoff = self.initial_backoff request_url = self.uri + endpoint if given_options: # iso8601 dates need special handling in bittrex since they can't parse them urlencoded # https://github.com/Bittrex/bittrex.github.io/issues/72#issuecomment-498335240 start_date = given_options.pop('startDate', None) end_date = given_options.pop('endDate', None) request_url += '?' + urlencode(given_options) if start_date is not None: request_url += f'&startDate={start_date}' if end_date is not None: request_url += f'&endDate={end_date}' while True: response = self._single_api_query( request_url=request_url, options=given_options, method=method, public_endpoint=endpoint in BITTREX_V3_PUBLIC_ENDPOINTS, ) should_backoff = (response.status_code == HTTPStatus.TOO_MANY_REQUESTS and backoff < self.backoff_limit) if should_backoff: log.debug('Got 429 from Bittrex. Backing off', seconds=backoff) gevent.sleep(backoff) backoff = backoff * 2 continue # else we got a result break if response.status_code != HTTPStatus.OK: self._check_for_system_clock_not_synced_error(response) raise RemoteError( f'Bittrex query responded with error status code: {response.status_code}' f' and text: {response.text}', ) try: result = rlk_jsonloads_list(response.text) except JSONDecodeError as e: raise RemoteError( f'Bittrex returned invalid JSON response: {response.text}' ) from e return result
def _api_query(self, command: str, req: Optional[Dict] = None) -> Union[Dict, List]: if req is None: req = {} if command == 'returnTicker' or command == 'returnCurrencies': log.debug(f'Querying poloniex for {command}') ret = self.session.get(self.public_uri + command) return rlk_jsonloads(ret.text) req['command'] = command with self.lock: # Protect this region with a lock since poloniex will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail req['nonce'] = int(time.time() * 1000) post_data = str.encode(urlencode(req)) sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest() self.session.headers.update({'Sign': sign}) log.debug( 'Poloniex private API query', command=command, post_data=req, ) ret = self.session.post('https://poloniex.com/tradingApi', req) if ret.status_code != 200: raise RemoteError( f'Poloniex query responded with error status code: {ret.status_code}' f' and text: {ret.text}', ) try: if command == 'returnLendingHistory': return rlk_jsonloads_list(ret.text) else: # For some reason poloniex can also return [] for an empty trades result if ret.text == '[]': return {} else: result = rlk_jsonloads_dict(ret.text) return _post_process(result) except JSONDecodeError: raise RemoteError( f'Poloniex returned invalid JSON response: {ret.text}')
def _query_currency_map(self) -> CurrencyMapResponse: """Query the list that maps standard currency symbols with the version of the Bitfinex API. If the request is successful and the list format as well, return it as dict in `<CurrencyMapResponse>.currency_map`. Otherwise populate <CurrencyMapResponse> with data that each endpoint can process as an unsuccessful request. API result format is: [[[<bitfinex_symbol>, <symbol>], ...]] May raise IndexError if the list is empty. """ was_successful = True currency_map = {} response = self._api_query('configs_map_currency_symbol') if response.status_code != HTTPStatus.OK: was_successful = False log.error( f'{self.name} currency map query failed. Check further logs') else: try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError: was_successful = False log.error( f'{self.name} currency map returned invalid JSON response. Check further logs', ) else: currency_map = { bfx_symbol: symbol for bfx_symbol, symbol in response_list[0] if bfx_symbol not in set(BITFINEX_EXCHANGE_TEST_ASSETS) } currency_map.update(BITFINEX_TO_WORLD) return CurrencyMapResponse( success=was_successful, response=response, currency_map=currency_map, )
def _api_query(self, command: str, req: Optional[Dict] = None) -> Union[Dict, List]: """An api query to poloniex. May make multiple requests Can raise: - RemoteError if there is a problem reaching poloniex or with the returned response """ if req is None: req = {} log.debug( 'Poloniex API query', command=command, post_data=req, ) tries = QUERY_RETRY_TIMES while tries >= 0: try: response = self._single_query(command, req) except requests.exceptions.ConnectionError as e: raise RemoteError(f'Poloniex API request failed due to {str(e)}') if response is None: if tries >= 1: backoff_seconds = 20 / tries log.debug( f'Got a recoverable poloniex error. ' f'Backing off for {backoff_seconds}', ) gevent.sleep(backoff_seconds) tries -= 1 continue else: break if response is None: raise RemoteError( f'Got a recoverable poloniex error and did not manage to get a ' f'request through even after {QUERY_RETRY_TIMES} ' f'incremental backoff retries', ) result: Union[Dict, List] try: if command == 'returnLendingHistory': result = rlk_jsonloads_list(response.text) else: # For some reason poloniex can also return [] for an empty trades result if response.text == '[]': result = {} else: result = rlk_jsonloads_dict(response.text) result = _post_process(result) except JSONDecodeError: raise RemoteError(f'Poloniex returned invalid JSON response: {response.text}') if isinstance(result, dict) and 'error' in result: raise RemoteError( 'Poloniex query for "{}" returned error: {}'.format( command, result['error'], )) return result
def get_coins_list(self) -> List[Dict[str, Any]]: response_data = self._query('coins') return rlk_jsonloads_list(response_data)
call_options = options.copy() limit = options.get('limit', API_MAX_LIMIT) results: Union[List[Trade], List[AssetMovement]] = [] # type: ignore while True: response = self._api_query( endpoint=endpoint, method='post', options=call_options, ) if response.status_code != HTTPStatus.OK: return self._process_unsuccessful_response( response=response, case=response_case, ) try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError: msg = f'Bitstamp returned invalid JSON response: {response.text}.' log.error(msg) self.msg_aggregator.add_error( f'Got remote error while querying Bistamp trades: {msg}', ) no_results: Union[List[Trade], List[AssetMovement]] = [] # type: ignore return no_results has_results = False is_result_timesamp_gt_end_ts = False result: Union[Trade, AssetMovement] for raw_result in response_list: if raw_result['type'] not in raw_result_type_filter: continue
def query_balances(self) -> ExchangeQueryBalances: """Return the account exchange balances on Bitfinex The wallets endpoint returns a list where each item is a currency wallet. Each currency wallet has type (i.e. exchange, margin, funding), currency, balance, etc. Currencies (tickers) are in Bitfinex format and must be standardized. Endpoint documentation: https://docs.bitfinex.com/reference#rest-auth-wallets """ self.first_connection() response = self._api_query('wallets') if response.status_code != HTTPStatus.OK: result, msg = self._process_unsuccessful_response( response=response, case='balances', ) return result, msg try: response_list = rlk_jsonloads_list(response.text) except JSONDecodeError as e: msg = f'{self.name} returned invalid JSON response: {response.text}.' log.error(msg) raise RemoteError(msg) from e # Wallet items indices currency_index = 1 balance_index = 2 assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) for wallet in response_list: if len(wallet) < API_WALLET_MIN_RESULT_LENGTH or wallet[ balance_index] <= 0: log.error( f'Error processing a {self.name} balance result. ' f'Found less items than expected', wallet=wallet, ) self.msg_aggregator.add_error( f'Failed to deserialize a {self.name} balance result. ' f'Check logs for details. Ignoring it.', ) continue try: asset = asset_from_bitfinex( bitfinex_name=wallet[currency_index], currency_map=self.currency_map, ) except (UnknownAsset, UnsupportedAsset) as e: asset_tag = 'unknown' if isinstance( e, UnknownAsset) else 'unsupported' self.msg_aggregator.add_warning( f'Found {asset_tag} {self.name} asset {e.asset_name} due to: {str(e)}. ' f'Ignoring its balance query.', ) continue try: usd_price = Inquirer().find_usd_price(asset=asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing {self.name} balance result due to inability to ' f'query USD price: {str(e)}. Skipping balance result.', ) continue amount = FVal(wallet[balance_index]) assets_balance[asset] += Balance( amount=amount, usd_value=amount * usd_price, ) return dict(assets_balance), ''
def _api_query( self, endpoint: str, request_method: Literal['GET', 'POST'] = 'GET', options: Optional[Dict[str, Any]] = None, query_options: Optional[Dict[str, Any]] = None, ) -> Tuple[List[Any], Optional[str]]: """Performs a coinbase PRO API Query for endpoint You can optionally provide extra arguments to the endpoint via the options argument. Returns a tuple of the result and optional pagination cursor. Raises RemoteError if something went wrong with connecting or reading from the exchange Raises CoinbaseProPermissionError if the API Key does not have sufficient permissions for the endpoint """ request_url = f'/{endpoint}' timestamp = str(int(time.time())) if options: stringified_options = json.dumps(options, separators=(',', ':')) else: stringified_options = '' options = {} if query_options: request_url += '?' + urlencode(query_options) message = timestamp + request_method + request_url + stringified_options log.debug( 'Coinbase Pro API query', request_method=request_method, request_url=request_url, options=options, ) if 'products' not in endpoint: try: signature = hmac.new( b64decode(self.secret), message.encode(), hashlib.sha256, ).digest() except binascii.Error as e: raise RemoteError('Provided API Secret is invalid') from e self.session.headers.update({ 'CB-ACCESS-SIGN': b64encode(signature).decode('utf-8'), 'CB-ACCESS-TIMESTAMP': timestamp, }) retries_left = QUERY_RETRY_TIMES while retries_left > 0: full_url = self.base_uri + request_url try: response = self.session.request( request_method.lower(), full_url, data=stringified_options, ) except requests.exceptions.RequestException as e: raise RemoteError( f'Coinbase Pro {request_method} query at ' f'{full_url} connection error: {str(e)}', ) from e if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: # Backoff a bit by sleeping. Sleep more, the more retries have been made gevent.sleep(QUERY_RETRY_TIMES / retries_left) retries_left -= 1 else: # get out of the retry loop, we did not get 429 complaint break json_ret: Union[List[Any], Dict[str, Any]] if response.status_code == HTTPStatus.BAD_REQUEST: json_ret = rlk_jsonloads_dict(response.text) if json_ret['message'] == 'invalid signature': raise CoinbaseProPermissionError( f'While doing {request_method} at {endpoint} endpoint the API secret ' f'created an invalid signature.', ) # else do nothing and a generic remote error will be thrown below elif response.status_code == HTTPStatus.FORBIDDEN: raise CoinbaseProPermissionError( f'API key does not have permission for {endpoint}', ) if response.status_code != HTTPStatus.OK: raise RemoteError( f'Coinbase Pro {request_method} query at {full_url} responded with error ' f'status code: {response.status_code} and text: {response.text}', ) try: json_ret = rlk_jsonloads_list(response.text) except JSONDecodeError as e: raise RemoteError( f'Coinbase Pro {request_method} query at {full_url} ' f'returned invalid JSON response: {response.text}', ) from e return json_ret, response.headers.get('cb-after', None)