def get_poap_airdrop_data(name: str, data_dir: Path) -> Dict[str, Any]: airdrops_dir = data_dir / 'airdrops_poap' airdrops_dir.mkdir(parents=True, exist_ok=True) filename = airdrops_dir / f'{name}.json' if not filename.is_file(): # if not cached, get it from the gist try: request = requests.get(url=POAP_AIRDROPS[name][0], timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'POAP airdrops Gist request failed due to {str(e)}') from e try: json_data = jsonloads_dict(request.content.decode('utf-8')) except JSONDecodeError as e: raise RemoteError( f'POAP airdrops Gist contains an invalid JSON {str(e)}') from e with open(filename, 'w') as outfile: outfile.write(rlk_jsondumps(json_data)) infile = open(filename, 'r') data_dict = jsonloads_dict(infile.read()) return data_dict
def api_query(self, method: str, req: Optional[dict] = None) -> dict: # Pretty ugly ... mock a kraken remote eror if self.remote_errors: raise RemoteError('Kraken remote error') if self.use_original_kraken: return super().api_query(method, req) if method == 'Balance': if self.random_balance_data: return generate_random_kraken_balance_response() # else return self.balance_data_return if method == 'TradesHistory': assert req, 'Should have given arguments for kraken TradesHistory endpoint call' if self.random_trade_data: return generate_random_kraken_trades_data( start=req['start'], end=req['end'], tradeable_pairs=list(self.tradeable_pairs.keys()), ) # else return jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE) if method == 'Ledgers': assert req, 'Should have given arguments for kraken Ledgers endpoint call' ledger_type = req['type'] if self.random_ledgers_data: return generate_random_kraken_ledger_data( start=req['start'], end=req['end'], ledger_type=ledger_type, ) # else use specific data if ledger_type in ('deposit', 'withdrawal'): data = json.loads( KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type == 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, ) new_data: Dict[str, Any] = {'ledger': {}} for key, val in data['ledger'].items(): try: ts = int(val['time']) except ValueError: ts = req[ 'start'] # can happen for tests of invalid data -- let it through if ts < req['start'] or ts > req['end']: continue new_data['ledger'][key] = val new_data['count'] = len(new_data['ledger']) response = json.dumps(new_data) else: raise AssertionError( 'Unknown ledger type at kraken ledgers mock query') return jsonloads_dict(response) # else return super().api_query(method, req)
def get_cryptocyrrency_map(self) -> List[Dict[str, Any]]: # TODO: Both here and in cryptocompare the cache funcionality is the same # Extract the caching part into its own function somehow and abstract it # away invalidate_cache = True coinlist_cache_path = os.path.join(self.data_directory, 'cmc_coinlist.json') if os.path.isfile(coinlist_cache_path): log.info('Found coinmarketcap coinlist cache', path=coinlist_cache_path) with open(coinlist_cache_path, 'r') as f: try: file_data = jsonloads_dict(f.read()) now = ts_now() invalidate_cache = False # If we got a cache and it's over a month old then requery coinmarketcap if file_data['time'] < now and now - file_data['time'] > 2629800: log.info('Coinmarketcap coinlist cache is now invalidated') invalidate_cache = True except JSONDecodeError: invalidate_cache = True if invalidate_cache: data = self._get_cryptocyrrency_map() # Also save the cache with open(coinlist_cache_path, 'w') as f: now = ts_now() log.info('Writing coinmarketcap coinlist cache', timestamp=now) write_data = {'time': now, 'data': data} f.write(rlk_jsondumps(write_data)) else: # in any case take the data data = file_data['data'] return data
def query_ipstack() -> Optional[GeolocationData]: """ 10,000 requests per month tied to the API Key https://ipstack.com/ """ try: response = requests.get( 'http://api.ipstack.com/check?access_key=affd920d6e1008a614900dbc31d52fa6', timeout=LOCATION_DATA_QUERY_TIMEOUT, ) except requests.exceptions.RequestException: return None if response.status_code != HTTPStatus.OK: return None try: json_ret = jsonloads_dict(response.text) except JSONDecodeError: return None return GeolocationData( country_code=json_ret.get('country_code', 'unknown'), city=json_ret.get('city', 'unknown'), )
def query_ipinfo() -> Optional[GeolocationData]: """ 50,000 requests per month tied to the API Key https://ipinfo.io/developers """ try: response = requests.get( 'https://ipinfo.io/json?token=16ab40aad9bd5b', timeout=LOCATION_DATA_QUERY_TIMEOUT, ) except requests.exceptions.RequestException: return None if response.status_code != HTTPStatus.OK: return None try: json_ret = jsonloads_dict(response.text) except JSONDecodeError: return None return GeolocationData( country_code=json_ret.get('country', 'unknown'), city=json_ret.get('city', 'unknown'), )
def _request_chain_metadata(self) -> Dict[str, Any]: """Subscan API metadata documentation: https://docs.api.subscan.io/#metadata """ response = self._request_explorer_api(endpoint='metadata') if response.status_code != HTTPStatus.OK: message = ( f'{self.chain} chain metadata request was not successful. ' f'Response status code: {response.status_code}. ' f'Response text: {response.text}.' ) log.error(message) raise RemoteError(message) try: result = jsonloads_dict(response.text) except JSONDecodeError as e: message = ( f'{self.chain} chain metadata request returned invalid JSON ' f'response: {response.text}.' ) log.error(message) raise RemoteError(message) from e log.debug(f'{self.chain} subscan API metadata', result=result) return result
def test_kucoin_exchange_assets_are_known(mock_kucoin): request_url = f'{mock_kucoin.base_uri}/api/v1/currencies' try: response = requests.get(request_url) except requests.exceptions.RequestException as e: raise RemoteError( f'Kucoin get request at {request_url} connection error: {str(e)}.', ) from e if response.status_code != HTTPStatus.OK: raise RemoteError( f'Kucoin query responded with error status code: {response.status_code} ' f'and text: {response.text}', ) try: response_dict = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Kucoin returned invalid JSON response: {response.text}') from e # Extract the unique symbols from the exchange pairs unsupported_assets = set(UNSUPPORTED_KUCOIN_ASSETS) common_items = unsupported_assets.intersection( set(WORLD_TO_KUCOIN.values())) assert not common_items, f'Kucoin assets {common_items} should not be unsupported' for entry in response_dict['data']: symbol = entry['currency'] try: asset_from_kucoin(symbol) except UnsupportedAsset: assert symbol in unsupported_assets except UnknownAsset as e: test_warnings.warn( UserWarning( f'Found unknown asset {e.asset_name} in kucoin. ' f'Support for it has to be added', ))
def _get_cryptocyrrency_map(self) -> List[Dict[str, Any]]: start = 1 limit = 5000 result: List[Dict[str, Any]] = [] while True: response_data = jsonloads_dict( self._query(f'v1/cryptocurrency/map?start={start}&limit={limit}'), ) result.extend(response_data['data']) if len(response_data['data']) != limit: break return result
def _process_dict_response(response: requests.Response) -> Dict: """Processess a dict response returned from the Rotkehlchen server and returns the result for success or raises RemoteError if an error happened""" if response.status_code not in HANDLABLE_STATUS_CODES: raise RemoteError( f'Unexpected status response({response.status_code}) from ' 'rotki server', ) result_dict = jsonloads_dict(response.text) if 'error' in result_dict: raise RemoteError(result_dict['error']) return result_dict
def _check_and_get_response(response: Response, method: str) -> Union[str, Dict]: """Checks the kraken response and if it's succesfull returns the result. If there is recoverable error a string is returned explaining the error May raise: - RemoteError if there is an unrecoverable/unexpected remote error """ if response.status_code in (520, 525, 504): log.debug(f'Kraken returned status code {response.status_code}') return 'Usual kraken 5xx shenanigans' if response.status_code != 200: raise RemoteError( 'Kraken API request {} for {} failed with HTTP status ' 'code: {}'.format( response.url, method, response.status_code, )) try: decoded_json = jsonloads_dict(response.text) except json.decoder.JSONDecodeError as e: raise RemoteError(f'Invalid JSON in Kraken response. {e}') from e error = decoded_json.get('error', None) if error: if isinstance(error, list) and len(error) != 0: error = error[0] if 'Rate limit exceeded' in error: log.debug(f'Kraken: Got rate limit exceeded error: {error}') return 'Rate limited exceeded' # else raise RemoteError(error) result = decoded_json.get('result', None) if result is None: if method == 'Balance': return {} raise RemoteError(f'Missing result in kraken response for {method}') return result
def _single_grant_api_query(self, query_str: str) -> Dict[str, Any]: backoff = 1 backoff_limit = 33 while backoff < backoff_limit: log.debug(f'Querying gitcoin: {query_str}') try: response = self.session.get(query_str, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: if 'Max retries exceeded with url' in str(e): log.debug( f'Got max retries exceeded from gitcoin. Will ' f'backoff for {backoff} seconds.', ) gevent.sleep(backoff) backoff = backoff * 2 if backoff >= backoff_limit: raise RemoteError( 'Getting gitcoin error even ' 'after we incrementally backed off', ) from e continue raise RemoteError( f'Gitcoin API request failed due to {str(e)}') from e if response.status_code != 200: raise RemoteError( f'Gitcoin API request {response.url} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Gitcoin API request {response.url} returned invalid ' f'JSON response: {response.text}', ) from e if 'error' in json_ret: raise RemoteError( f'Gitcoin API request {response.url} returned an error: {json_ret["error"]}', ) break # success return json_ret
def query_balances(self) -> ExchangeQueryBalances: """Return the account balances May raise RemoteError """ accounts_response = self._api_query(KucoinCase.BALANCES) if accounts_response.status_code != HTTPStatus.OK: result, msg = self._process_unsuccessful_response( response=accounts_response, case=KucoinCase.BALANCES, ) return result, msg try: response_dict = jsonloads_dict(accounts_response.text) except JSONDecodeError as e: msg = f'Kucoin balances returned an invalid JSON response: {accounts_response.text}.' log.error(msg) raise RemoteError(msg) from e account_balances = self._deserialize_accounts_balances(response_dict=response_dict) return account_balances, ''
def _query(self, path: str) -> Dict: """ May raise: - RemoteError if there is a problem querying Github """ try: response = requests.get(f'{self.prefix}{path}') except requests.exceptions.RequestException as e: raise RemoteError(f'Failed to query Github: {str(e)}') from e if response.status_code != 200: raise RemoteError( f'Github API request {response.url} for {path} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Github returned invalid JSON response: {response.text}' ) from e return json_ret
def query_ipwhoisio() -> Optional[GeolocationData]: """10,000 requests per month per IP https://ipwhois.io/documentation """ try: response = requests.get( 'http://free.ipwhois.io/json/', timeout=LOCATION_DATA_QUERY_TIMEOUT, ) except requests.exceptions.RequestException: return None if response.status_code != HTTPStatus.OK: return None try: json_ret = jsonloads_dict(response.text) except JSONDecodeError: return None return GeolocationData( country_code=json_ret.get('country_code', 'unknown'), city=json_ret.get('city', 'unknown'), )
def _api_query( self, endpoint: str, options: Optional[Dict[str, Any]] = None, ) -> Tuple[Union[List[Any], Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: # noqa: E501 """Performs a bitpanda API Query for endpoint You can optionally provide extra arguments to the endpoint via the options argument. Returns a tuple of: - The result data - Optional meta dict containing total_count, page and page_size - Optional links dict containing next, last and self links Raises RemoteError if something went wrong with connecting or reading from the exchange """ request_url = f'{self.uri}/{endpoint}' retries_left = QUERY_RETRY_TIMES if options is not None: request_url += '?' + urlencode(options) while retries_left > 0: log.debug( 'Bitpanda API query', request_url=request_url, options=options, ) try: response = self.session.get(request_url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'Bitpanda API request failed due to {str(e)}') from e if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: backoff_in_seconds = int(20 / retries_left) retries_left -= 1 log.debug( f'Got a 429 from Bitpanda query of {request_url}. Will backoff ' f'for {backoff_in_seconds} seconds. {retries_left} retries left', ) gevent.sleep(backoff_in_seconds) continue if response.status_code != HTTPStatus.OK: raise RemoteError( f'Bitpanda API request failed with response: {response.text} ' f'and status code: {response.status_code}', ) # we got it, so break break else: # retries left are zero raise RemoteError( f'Ran out of retries for Bitpanda query of {request_url}') try: decoded_json = jsonloads_dict(response.text) except json.decoder.JSONDecodeError as e: raise RemoteError( f'Invalid JSON {response.text} in Bitpanda response. {e}' ) from e if 'data' not in decoded_json: raise RemoteError( f'Invalid JSON {response.text} in Bitpanda response. Expected "data" key', ) log.debug(f'Got Bitpanda response: {decoded_json}') return decoded_json['data'], decoded_json.get( 'meta'), decoded_json.get('links')
def _api_query( self, endpoint: str, options: Optional[Dict[str, Any]] = None, pagination_next_uri: str = None, ignore_pagination: bool = False, ) -> List[Any]: """Performs a coinbase API Query for endpoint You can optionally provide extra arguments to the endpoint via the options argument. If this is an ongoing paginating call then provide pagination_next_uri. If you want just the first results then set ignore_pagination to True. """ request_verb = "GET" if pagination_next_uri: request_url = pagination_next_uri else: request_url = f'/{self.apiversion}/{endpoint}' if options: request_url += urlencode(options) timestamp = str(int(time.time())) message = timestamp + request_verb + request_url signature = hmac.new( self.secret, message.encode(), hashlib.sha256, ).hexdigest() log.debug('Coinbase API query', request_url=request_url) self.session.headers.update({ 'CB-ACCESS-SIGN': signature, 'CB-ACCESS-TIMESTAMP': timestamp, # This is needed to guarantee the up to the given date # API version response. 'CB-VERSION': '2019-08-25', }) full_url = self.base_uri + request_url try: response = self.session.get(full_url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'Coinbase API request failed due to {str(e)}') from e if response.status_code == 403: raise CoinbasePermissionError( f'API key does not have permission for {endpoint}') if response.status_code != 200: raise RemoteError( f'Coinbase query {full_url} responded with error status code: ' f'{response.status_code} and text: {response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Coinbase returned invalid JSON response: {response.text}' ) from e if 'data' not in json_ret: raise RemoteError( f'Coinbase json response does not contain data: {response.text}' ) final_data = json_ret['data'] # If we got pagination recursively gather all the subsequent queries if 'pagination' in json_ret and not ignore_pagination: if 'next_uri' not in json_ret['pagination']: raise RemoteError( 'Coinbase json response contained no "next_uri" key') next_uri = json_ret['pagination']['next_uri'] if not next_uri: # As per the docs: https://developers.coinbase.com/api/v2?python#pagination # once we get an empty next_uri we are done return final_data additional_data = self._api_query( endpoint=endpoint, options=options, pagination_next_uri=next_uri, ) final_data.extend(additional_data) return final_data
def _api_query( self, endpoint: str, options: Optional[Dict[str, Any]] = None, ignore_pagination: bool = False, ) -> List[Any]: """Performs a coinbase API Query for endpoint You can optionally provide extra arguments to the endpoint via the options argument. If you want just the first results then set ignore_pagination to True. """ all_items: List[Any] = [] request_verb = "GET" # initialize next_uri before loop next_uri = f'/{self.apiversion}/{endpoint}' if options: next_uri += urlencode(options) while True: timestamp = str(int(time.time())) message = timestamp + request_verb + next_uri signature = hmac.new( self.secret, message.encode(), hashlib.sha256, ).hexdigest() log.debug('Coinbase API query', request_url=next_uri) self.session.headers.update({ 'CB-ACCESS-SIGN': signature, 'CB-ACCESS-TIMESTAMP': timestamp, # This is needed to guarantee the up to the given date # API version response. 'CB-VERSION': '2019-08-25', }) full_url = self.base_uri + next_uri try: response = self.session.get(full_url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'Coinbase API request failed due to {str(e)}') from e if response.status_code == 403: raise CoinbasePermissionError( f'API key does not have permission for {endpoint}') if response.status_code != 200: raise RemoteError( f'Coinbase query {full_url} responded with error status code: ' f'{response.status_code} and text: {response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Coinbase returned invalid JSON response: {response.text}', ) from e if 'data' not in json_ret: raise RemoteError( f'Coinbase json response does not contain data: {response.text}' ) # `data` attr is a list in itself all_items.extend(json_ret['data']) if ignore_pagination or 'pagination' not in json_ret: # break out of the loop, no need to handle pagination break if 'next_uri' not in json_ret['pagination']: raise RemoteError( 'Coinbase json response contained no "next_uri" key') # otherwise, let the loop run to gather subsequent queries # this next_uri will be used in next iteration next_uri = json_ret['pagination']['next_uri'] if not next_uri: # As per the docs: https://developers.coinbase.com/api/v2?python#pagination # once we get an empty next_uri we are done break return all_items
def get_coin_by_id(self, coinpaprika_id: str) -> Dict[str, Any]: response_data = self._query(f'coins/{coinpaprika_id}') return jsonloads_dict(response_data)
case=case, time=min(current_query_ts + time_step, end_ts), ) response = self._api_query( case=case, options=call_options, ) if response.status_code != HTTPStatus.OK: return self._process_unsuccessful_response( response=response, case=case, ) try: response_dict = jsonloads_dict(response.text) except JSONDecodeError as e: msg = f'Kucoin {case} returned an invalid JSON response: {response.text}.' log.error(msg) self.msg_aggregator.add_error( f'Got remote error while querying kucoin {case}: {msg}', ) raise RemoteError(msg) from e try: response_data = response_dict['data'] total_page = response_data['totalPage'] current_page = response_data['currentPage'] raw_results = response_data['items'] except KeyError as e: msg = f'Kucoin {case} JSON response is missing key: {str(e)}' log.error(msg, response_dict)
def _make_request( self, endpoint: str, start_time: Optional[Timestamp] = None, end_time: Optional[Timestamp] = None, limit: int = PAGINATION_LIMIT, ) -> Union[List[Dict[str, Any]], Dict[str, List[Any]]]: """Performs an FTX API Query for endpoint adding the needed information to authenticate user and handling errors. This function can raise: - RemoteError """ request_verb = "GET" backoff = INITIAL_BACKOFF_TIME # Use a while loop to retry request if rate limit is reached while True: request_url = '/api/' + endpoint options = {'limit': limit} if start_time is not None: options['start_time'] = start_time if end_time is not None: options['end_time'] = end_time if len(options) != 0: request_url += '?' + urlencode(options) timestamp = int(time.time() * 1000) signature_payload = f'{timestamp}{request_verb}{request_url}'.encode( ) signature = hmac.new(self.secret, signature_payload, 'sha256').hexdigest() log.debug('FTX API query', request_url=request_url) self.session.headers.update({ 'FTX-SIGN': signature, 'FTX-TS': str(timestamp), }) full_url = self.base_uri + request_url try: response = self.session.get(full_url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'FTX API request {full_url} failed due to {str(e)}' ) from e if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: if backoff < BACKOFF_LIMIT: log.debug( f'FTX rate limit exceeded on request {request_url}. Backing off', seconds=backoff, ) gevent.sleep(backoff) backoff = backoff * 2 continue # We got a result here break if response.status_code != HTTPStatus.OK: raise RemoteError( f'FTX query {full_url} responded with error status code: ' f'{response.status_code} and text: {response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'FTX returned invalid JSON response: {response.text}') from e if 'result' not in json_ret: raise RemoteError( f'FTX json response does not contain data: {response.text}') return json_ret['result']
def _query( self, module: str, action: str, options: Optional[Dict[str, Any]] = None, timeout: Optional[Tuple[int, int]] = None, ) -> Union[List[Dict[str, Any]], str, List[EthereumTransaction], Dict[ str, Any]]: """Queries etherscan May raise: - RemoteError if there are any problems with reaching Etherscan or if an unexpected response is returned """ query_str = f'https://api.etherscan.io/api?module={module}&action={action}' if options: for name, value in options.items(): query_str += f'&{name}={value}' api_key = self._get_api_key() if api_key is None: if not self.warning_given: self.msg_aggregator.add_warning( 'You do not have an Etherscan API key configured. rotki ' 'etherscan queries will still work but will be very slow. ' 'If you are not using your own ethereum node, it is recommended ' 'to go to https://etherscan.io/register, create an API ' 'key and then input it in the external service credentials setting of rotki', ) self.warning_given = True else: query_str += f'&apikey={api_key}' backoff = 1 backoff_limit = 33 while backoff < backoff_limit: log.debug(f'Querying etherscan: {query_str}') try: response = self.session.get( query_str, timeout=timeout if timeout else DEFAULT_TIMEOUT_TUPLE) # noqa: E501 except requests.exceptions.RequestException as e: if 'Max retries exceeded with url' in str(e): log.debug( f'Got max retries exceeded from etherscan. Will ' f'backoff for {backoff} seconds.', ) gevent.sleep(backoff) backoff = backoff * 2 if backoff >= backoff_limit: raise RemoteError( 'Getting Etherscan max connections error even ' 'after we incrementally backed off', ) from e continue raise RemoteError( f'Etherscan API request failed due to {str(e)}') from e if response.status_code != 200: raise RemoteError( f'Etherscan API request {response.url} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Etherscan API request {response.url} returned invalid ' f'JSON response: {response.text}', ) from e try: result = json_ret.get('result', None) if result is None: raise RemoteError( f'Unexpected format of Etherscan response for request {response.url}. ' f'Missing a result in response. Response was: {response.text}', ) # sucessful proxy calls do not include a status status = int(json_ret.get('status', 1)) if status != 1: if status == 0 and 'rate limit reached' in result: log.debug( f'Got response: {response.text} from etherscan. Will ' f'backoff for {backoff} seconds.', ) gevent.sleep(backoff) # Continue increasing backoff until limit is reached. # If limit is reached then keep sleeping with the limit. # Etherscan will let the query go through eventually if backoff * 2 < backoff_limit: backoff = backoff * 2 continue transaction_endpoint_and_none_found = ( status == 0 and json_ret['message'] == 'No transactions found' and action in ('txlist', 'txlistinternal', 'tokentx')) logs_endpoint_and_none_found = (status == 0 and json_ret['message'] == 'No records found' and 'getLogs' in action) if transaction_endpoint_and_none_found or logs_endpoint_and_none_found: # Can't realize that result is always a list here so we ignore mypy warning return [] # type: ignore # else raise RemoteError( f'Etherscan returned error response: {json_ret}') except KeyError as e: raise RemoteError( f'Unexpected format of Etherscan response for request {response.url}. ' f'Missing key entry for {str(e)}. Response was: {response.text}', ) from e # success, break out of the loop and return result return result return result
def _query( self, module: Literal['validator'], endpoint: Optional[Literal['balancehistory', 'performance', 'eth1', 'deposits']], encoded_args: str, ) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """ May raise: - RemoteError due to problems querying beaconcha.in API """ if endpoint is None: # for now only validator data query_str = f'{self.url}{module}/{encoded_args}' elif endpoint == 'eth1': query_str = f'{self.url}{module}/{endpoint}/{encoded_args}' else: query_str = f'{self.url}{module}/{encoded_args}/{endpoint}' api_key = self._get_api_key() if api_key is not None: query_str += f'?apikey={api_key}' times = QUERY_RETRY_TIMES backoff_in_seconds = 10 log.debug(f'Querying beaconcha.in API for {query_str}') while True: try: response = self.session.get(query_str, timeout=BEACONCHAIN_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'Querying {query_str} failed due to {str(e)}') from e if response.status_code == 429: minute_rate_limit = response.headers.get( 'x-ratelimit-limit-minute', 'unknown') user_minute_rate_limit = response.headers.get( 'x-ratelimit-remaining-minute', 'unknown') # noqa: E501 daily_rate_limit = response.headers.get( 'x-ratelimit-limit-day', 'unknown') user_daily_rate_limit = response.headers.get( 'x-ratelimit-remaining-day', 'unknown') # noqa: E501 month_rate_limit = response.headers.get( 'x-ratelimit-limit-month', 'unknown') user_month_rate_limit = response.headers.get( 'x-ratelimit-remaining-month', 'unknown') # noqa: E501 if times == 0: msg = ( f'Beaconchain API request {response.url} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text} after 5 retries') log.debug( f'{msg} minute limit: {user_minute_rate_limit}/{minute_rate_limit}, ' f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, ' f'monthly limit: {user_month_rate_limit}/{month_rate_limit}', ) raise RemoteError(msg) retry_after = response.headers.get('retry-after', None) if retry_after: retry_after_secs = int(retry_after) if retry_after_secs > MAX_WAIT_SECS: msg = ( f'Beaconchain API request {response.url} got rate limited. Would ' f'need to wait for {retry_after} seconds which is more than the ' f'wait limit of {MAX_WAIT_SECS} seconds. Bailing out.' ) log.debug( f'{msg} minute limit: {user_minute_rate_limit}/{minute_rate_limit}, ' f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, ' f'monthly limit: {user_month_rate_limit}/{month_rate_limit}', ) raise RemoteError(msg) # else sleep_seconds = retry_after_secs else: # Rate limited. Try incremental backoff since retry-after header is missing sleep_seconds = backoff_in_seconds * (QUERY_RETRY_TIMES - times + 1) times -= 1 log.debug( f'Beaconchain API request {response.url} got rate limited. Sleeping ' f'for {sleep_seconds}. We have {times} tries left.' f'minute limit: {user_minute_rate_limit}/{minute_rate_limit}, ' f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, ' f'monthly limit: {user_month_rate_limit}/{month_rate_limit}', ) gevent.sleep(sleep_seconds) continue # else break if response.status_code != 200: raise RemoteError( f'Beaconchain API request {response.url} failed ' f'with HTTP status code {response.status_code} and text ' f'{response.text}', ) try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Beaconchain API returned invalid JSON response: {response.text}', ) from e if json_ret.get('status') != 'OK': raise RemoteError( f'Beaconchain API returned non-OK status. Response: {json_ret}' ) if 'data' not in json_ret: raise RemoteError( f'Beaconchain API did not contain a data key. Response: {json_ret}' ) return json_ret['data']
def all_coins(self) -> Dict[str, Any]: """ Gets the mapping of all the cryptocompare coins May raise: - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ # Get coin list of cryptocompare invalidate_cache = True coinlist_cache_path = os.path.join(self.data_directory, 'cryptocompare_coinlist.json') if os.path.isfile(coinlist_cache_path): log.info('Found cryptocompare coinlist cache', path=coinlist_cache_path) with open(coinlist_cache_path, 'r') as f: try: data = jsonloads_dict(f.read()) now = ts_now() invalidate_cache = False # If we got a cache and its' over a month old then requery cryptocompare if data['time'] < now and now - data['time'] > 2629800: log.info('Cryptocompare coinlist cache is now invalidated') invalidate_cache = True data = data['data'] except JSONDecodeError: invalidate_cache = True if invalidate_cache: data = self._api_query('all/coinlist') # Also save the cache with open(coinlist_cache_path, 'w') as f: now = ts_now() log.info('Writing coinlist cache', timestamp=now) write_data = {'time': now, 'data': data} f.write(rlk_jsondumps(write_data)) else: # in any case take the data data = data['data'] # As described in the docs # https://min-api.cryptocompare.com/documentation?key=Other&cat=allCoinsWithContentEndpoint # This is not the entire list of assets in the system, so I am manually adding # here assets I am aware of that they already have historical data for in thei # cryptocompare system data['DAO'] = object() data['USDT'] = object() data['VEN'] = object() data['AIR*'] = object() # This is Aircoin # This is SpendCoin (https://coinmarketcap.com/currencies/spendcoin/) data['SPND'] = object() # This is eBitcoinCash (https://coinmarketcap.com/currencies/ebitcoin-cash/) data['EBCH'] = object() # This is Educare (https://coinmarketcap.com/currencies/educare/) data['EKT'] = object() # This is Knoxstertoken (https://coinmarketcap.com/currencies/knoxstertoken/) data['FKX'] = object() # This is FNKOS (https://coinmarketcap.com/currencies/fnkos/) data['FNKOS'] = object() # This is FansTime (https://coinmarketcap.com/currencies/fanstime/) data['FTI'] = object() # This is Gene Source Code Chain # (https://coinmarketcap.com/currencies/gene-source-code-chain/) data['GENE*'] = object() # This is GazeCoin (https://coinmarketcap.com/currencies/gazecoin/) data['GZE'] = object() # This is probaly HarmonyCoin (https://coinmarketcap.com/currencies/harmonycoin-hmc/) data['HMC*'] = object() # This is IoTChain (https://coinmarketcap.com/currencies/iot-chain/) data['ITC'] = object() # This is Luna Coin (https://coinmarketcap.com/currencies/luna-coin/) data['LUNA'] = object # This is MFTU (https://coinmarketcap.com/currencies/mainstream-for-the-underground/) data['MFTU'] = object() # This is Nexxus (https://coinmarketcap.com/currencies/nexxus/) data['NXX'] = object() # This is Owndata (https://coinmarketcap.com/currencies/owndata/) data['OWN'] = object() # This is PiplCoin (https://coinmarketcap.com/currencies/piplcoin/) data['PIPL'] = object() # This is PKG Token (https://coinmarketcap.com/currencies/pkg-token/) data['PKG'] = object() # This is Quibitica https://coinmarketcap.com/currencies/qubitica/ data['QBIT'] = object() # This is DPRating https://coinmarketcap.com/currencies/dprating/ data['RATING'] = object() # This is RocketPool https://coinmarketcap.com/currencies/rocket-pool/ data['RPL'] = object() # This is SpeedMiningService (https://coinmarketcap.com/currencies/speed-mining-service/) data['SMS'] = object() # This is SmartShare (https://coinmarketcap.com/currencies/smartshare/) data['SSP'] = object() # This is ThoreCoin (https://coinmarketcap.com/currencies/thorecoin/) data['THR'] = object() # This is Transcodium (https://coinmarketcap.com/currencies/transcodium/) data['TNS'] = object() return data
def _api_query(self, path: str) -> Dict[str, Any]: """Queries cryptocompare - May raise RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ querystr = f'https://min-api.cryptocompare.com/data/{path}' log.debug('Querying cryptocompare', url=querystr) api_key = self._get_api_key() if api_key: querystr += '?' if '?' not in querystr else '&' querystr += f'api_key={api_key}' tries = CRYPTOCOMPARE_QUERY_RETRY_TIMES while tries >= 0: try: response = self.session.get(querystr, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError(f'Cryptocompare API request failed due to {str(e)}') from e try: json_ret = jsonloads_dict(response.text) except JSONDecodeError as e: raise RemoteError( f'Cryptocompare returned invalid JSON response: {response.text}', ) from e try: # backoff and retry 3 times = 1 + 1.5 + 3 = at most 5.5 secs # Failing is also fine, since all calls have secondary data sources # for example coingecko if json_ret.get('Message', None) == RATE_LIMIT_MSG: self.last_rate_limit = ts_now() if tries >= 1: backoff_seconds = 3 / tries log.debug( f'Got rate limited by cryptocompare. ' f'Backing off for {backoff_seconds}', ) gevent.sleep(backoff_seconds) tries -= 1 continue # else log.debug( f'Got rate limited by cryptocompare and did not manage to get a ' f'request through even after {CRYPTOCOMPARE_QUERY_RETRY_TIMES} ' f'incremental backoff retries', ) if json_ret.get('Response', 'Success') != 'Success': error_message = f'Failed to query cryptocompare for: "{querystr}"' if 'Message' in json_ret: error_message += f'. Error: {json_ret["Message"]}' log.warning( 'Cryptocompare query failure', url=querystr, error=error_message, status_code=response.status_code, ) raise RemoteError(error_message) return json_ret['Data'] if 'Data' in json_ret else json_ret except KeyError as e: raise RemoteError( f'Unexpected format of Cryptocompare json_response. ' f'Missing key entry for {str(e)}', ) from e raise AssertionError('We should never get here')
def api_query(self, method: str, req: Optional[dict] = None) -> dict: # Pretty ugly ... mock a kraken remote eror if self.remote_errors: raise RemoteError('Kraken remote error') if self.use_original_kraken: return super().api_query(method, req) if method == 'Balance': if self.random_balance_data: return generate_random_kraken_balance_response() # else return self.balance_data_return if method == 'TradesHistory': assert req, 'Should have given arguments for kraken TradesHistory endpoint call' if self.random_trade_data: return generate_random_kraken_trades_data( start=req['start'], end=req['end'], tradeable_pairs=list(self.tradeable_pairs.keys()), ) # else return jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE) if method == 'AssetPairs': dir_path = Path(__file__).resolve().parent.parent filepath = dir_path / 'data' / 'assets_kraken.json' with open(filepath) as f: return jsonloads_dict(f.read())['result'] if method == 'Ledgers': if req is None: req = {} ledger_type: str = req.get('type', '') if self.random_ledgers_data: assert req is not None return generate_random_kraken_ledger_data( start=req.get('start', 0), end=req.get('end', ts_now), ledger_type=ledger_type, ) # else use specific data if ledger_type in ('deposit', 'withdrawal'): data = json.loads( KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type == 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, ) else: data = json.loads(KRAKEN_GENERAL_LEDGER_RESPONSE) new_data: Dict[str, Any] = {'ledger': {}} for key, val in data['ledger'].items(): try: ts = int(val['time']) except ValueError: # can happen for tests of invalid data -- let it through ts = req.get('start', 0) if ts < req.get('start', 0) or ts > req.get('end', ts_now): continue new_data['ledger'][key] = val new_data['count'] = len(new_data['ledger']) response = json.dumps(new_data) return jsonloads_dict(response) # else return super().api_query(method, req)
def test_jsonloads_dict(): result = jsonloads_dict('{"foo": 1, "boo": "value"}') assert result == {'foo': 1, 'boo': 'value'} with pytest.raises(JSONDecodeError) as e: jsonloads_dict('["foo", "boo", 3]') assert 'Returned json is not a dict' in str(e.value)
def _api_query(self, command: str, req: Optional[Dict] = None) -> Union[Dict, List]: """An api query to poloniex. May make multiple requests Can raise: - RemoteError if there is a problem reaching poloniex or with the returned response """ if req is None: req = {} log.debug( 'Poloniex API query', command=command, post_data=req, ) tries = QUERY_RETRY_TIMES while tries >= 0: try: response = self._single_query(command, req) except requests.exceptions.RequestException as e: raise RemoteError( f'Poloniex API request failed due to {str(e)}') from e if response is None: if tries >= 1: backoff_seconds = 20 / tries log.debug( f'Got a recoverable poloniex error. ' f'Backing off for {backoff_seconds}', ) gevent.sleep(backoff_seconds) tries -= 1 continue else: break if response is None: raise RemoteError( f'Got a recoverable poloniex error and did not manage to get a ' f'request through even after {QUERY_RETRY_TIMES} ' f'incremental backoff retries', ) result: Union[Dict, List] try: if command == 'returnLendingHistory': result = jsonloads_list(response.text) else: # For some reason poloniex can also return [] for an empty trades result if response.text == '[]': result = {} else: result = jsonloads_dict(response.text) result = _post_process(result) except JSONDecodeError as e: raise RemoteError( f'Poloniex returned invalid JSON response: {response.text}' ) from e if isinstance(result, dict) and 'error' in result: raise RemoteError( 'Poloniex query for "{}" returned error: {}'.format( command, result['error'], )) return result
def _api_query( self, endpoint: str, request_method: Literal['GET', 'POST'] = 'GET', options: Optional[Dict[str, Any]] = None, query_options: Optional[Dict[str, Any]] = None, ) -> Tuple[List[Any], Optional[str]]: """Performs a coinbase PRO API Query for endpoint You can optionally provide extra arguments to the endpoint via the options argument. Returns a tuple of the result and optional pagination cursor. Raises RemoteError if something went wrong with connecting or reading from the exchange Raises CoinbaseProPermissionError if the API Key does not have sufficient permissions for the endpoint """ request_url = f'/{endpoint}' timestamp = str(int(time.time())) if options: stringified_options = json.dumps(options, separators=(',', ':')) else: stringified_options = '' options = {} if query_options: request_url += '?' + urlencode(query_options) message = timestamp + request_method + request_url + stringified_options if 'products' not in endpoint: try: signature = hmac.new( b64decode(self.secret), message.encode(), hashlib.sha256, ).digest() except binascii.Error as e: raise RemoteError('Provided API Secret is invalid') from e self.session.headers.update({ 'CB-ACCESS-SIGN': b64encode(signature).decode('utf-8'), 'CB-ACCESS-TIMESTAMP': timestamp, }) retries_left = QUERY_RETRY_TIMES while retries_left > 0: log.debug( 'Coinbase Pro API query', request_method=request_method, request_url=request_url, options=options, ) full_url = self.base_uri + request_url try: response = self.session.request( request_method.lower(), full_url, data=stringified_options, timeout=DEFAULT_TIMEOUT_TUPLE, ) except requests.exceptions.RequestException as e: raise RemoteError( f'Coinbase Pro {request_method} query at ' f'{full_url} connection error: {str(e)}', ) from e if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: # Backoff a bit by sleeping. Sleep more, the more retries have been made backoff_secs = QUERY_RETRY_TIMES / retries_left log.debug( f'Backing off coinbase pro api query for {backoff_secs} secs' ) gevent.sleep(backoff_secs) retries_left -= 1 else: # get out of the retry loop, we did not get 429 complaint break json_ret: Union[List[Any], Dict[str, Any]] if response.status_code == HTTPStatus.BAD_REQUEST: json_ret = jsonloads_dict(response.text) if json_ret['message'] == 'invalid signature': raise CoinbaseProPermissionError( f'While doing {request_method} at {endpoint} endpoint the API secret ' f'created an invalid signature.', ) # else do nothing and a generic remote error will be thrown below elif response.status_code == HTTPStatus.FORBIDDEN: raise CoinbaseProPermissionError( f'API key does not have permission for {endpoint}', ) if response.status_code != HTTPStatus.OK: raise RemoteError( f'Coinbase Pro {request_method} query at {full_url} responded with error ' f'status code: {response.status_code} and text: {response.text}', ) try: json_ret = jsonloads_list(response.text) except JSONDecodeError as e: raise RemoteError( f'Coinbase Pro {request_method} query at {full_url} ' f'returned invalid JSON response: {response.text}', ) from e return json_ret, response.headers.get('cb-after', None)
def query_balances(self) -> ExchangeQueryBalances: """Return the account balances on Bistamp The balance endpoint returns a dict where the keys (str) are related to assets and the values (str) amounts. The keys that end with `_balance` contain the exact amount of an asset the account is holding (available amount + orders amount, per asset). """ response = self._api_query('balance') if response.status_code != HTTPStatus.OK: result, msg = self._process_unsuccessful_response( response=response, case='balances', ) return result, msg try: response_dict = jsonloads_dict(response.text) except JSONDecodeError as e: msg = f'Bitstamp returned invalid JSON response: {response.text}.' log.error(msg) raise RemoteError(msg) from e assets_balance: Dict[Asset, Balance] = {} for entry, amount in response_dict.items(): if not entry.endswith('_balance'): continue symbol = entry.split('_')[0] # If no `_`, defaults to entry try: amount = deserialize_asset_amount(amount) if amount == ZERO: continue asset = asset_from_bitstamp(symbol) except DeserializationError as e: log.error( 'Error processing a Bitstamp balance.', entry=entry, error=str(e), ) self.msg_aggregator.add_error( 'Failed to deserialize a Bitstamp balance. ' 'Check logs for details. Ignoring it.', ) continue except (UnknownAsset, UnsupportedAsset) as e: log.error(str(e)) asset_tag = 'unknown' if isinstance( e, UnknownAsset) else 'unsupported' self.msg_aggregator.add_warning( f'Found {asset_tag} Bistamp asset {e.asset_name}. Ignoring its balance query.', ) continue try: usd_price = Inquirer().find_usd_price(asset=asset) except RemoteError as e: log.error(str(e)) self.msg_aggregator.add_error( f'Error processing Bitstamp balance result due to inability to ' f'query USD price: {str(e)}. Skipping balance entry.', ) continue assets_balance[asset] = Balance( amount=amount, usd_value=amount * usd_price, ) return assets_balance, ''