def get_poap_airdrop_data(name: str, data_dir: Path) -> Dict[str, Any]:
airdrops_dir = data_dir / 'airdrops_poap'
airdrops_dir.mkdir(parents=True, exist_ok=True)
filename = airdrops_dir / f'{name}.json'
if not filename.is_file():
# if not cached, get it from the gist
try:
request = requests.get(url=POAP_AIRDROPS[name][0],
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'POAP airdrops Gist request failed due to {str(e)}') from e
try:
json_data = jsonloads_dict(request.content.decode('utf-8'))
except JSONDecodeError as e:
raise RemoteError(
f'POAP airdrops Gist contains an invalid JSON {str(e)}') from e
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(json_data))
infile = open(filename, 'r')
data_dict = jsonloads_dict(infile.read())
return data_dict
def api_query(self, method: str, req: Optional[dict] = None) -> dict:
# Pretty ugly ... mock a kraken remote eror
if self.remote_errors:
raise RemoteError('Kraken remote error')
if self.use_original_kraken:
return super().api_query(method, req)
if method == 'Balance':
if self.random_balance_data:
return generate_random_kraken_balance_response()
# else
return self.balance_data_return
if method == 'TradesHistory':
assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
if self.random_trade_data:
return generate_random_kraken_trades_data(
start=req['start'],
end=req['end'],
tradeable_pairs=list(self.tradeable_pairs.keys()),
)
# else
return jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
if method == 'Ledgers':
assert req, 'Should have given arguments for kraken Ledgers endpoint call'
ledger_type = req['type']
if self.random_ledgers_data:
return generate_random_kraken_ledger_data(
start=req['start'],
end=req['end'],
ledger_type=ledger_type,
)
# else use specific data
if ledger_type in ('deposit', 'withdrawal'):
data = json.loads(
KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type
== 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, )
new_data: Dict[str, Any] = {'ledger': {}}
for key, val in data['ledger'].items():
try:
ts = int(val['time'])
except ValueError:
ts = req[
'start'] # can happen for tests of invalid data -- let it through
if ts < req['start'] or ts > req['end']:
continue
new_data['ledger'][key] = val
new_data['count'] = len(new_data['ledger'])
response = json.dumps(new_data)
else:
raise AssertionError(
'Unknown ledger type at kraken ledgers mock query')
return jsonloads_dict(response)
# else
return super().api_query(method, req)
def get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
# TODO: Both here and in cryptocompare the cache funcionality is the same
# Extract the caching part into its own function somehow and abstract it
# away
invalidate_cache = True
coinlist_cache_path = os.path.join(self.data_directory, 'cmc_coinlist.json')
if os.path.isfile(coinlist_cache_path):
log.info('Found coinmarketcap coinlist cache', path=coinlist_cache_path)
with open(coinlist_cache_path, 'r') as f:
try:
file_data = jsonloads_dict(f.read())
now = ts_now()
invalidate_cache = False
# If we got a cache and it's over a month old then requery coinmarketcap
if file_data['time'] < now and now - file_data['time'] > 2629800:
log.info('Coinmarketcap coinlist cache is now invalidated')
invalidate_cache = True
except JSONDecodeError:
invalidate_cache = True
if invalidate_cache:
data = self._get_cryptocyrrency_map()
# Also save the cache
with open(coinlist_cache_path, 'w') as f:
now = ts_now()
log.info('Writing coinmarketcap coinlist cache', timestamp=now)
write_data = {'time': now, 'data': data}
f.write(rlk_jsondumps(write_data))
else:
# in any case take the data
data = file_data['data']
return data
def query_ipstack() -> Optional[GeolocationData]:
"""
10,000 requests per month tied to the API Key
https://ipstack.com/
"""
try:
response = requests.get(
'http://api.ipstack.com/check?access_key=affd920d6e1008a614900dbc31d52fa6',
timeout=LOCATION_DATA_QUERY_TIMEOUT,
)
except requests.exceptions.RequestException:
return None
if response.status_code != HTTPStatus.OK:
return None
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError:
return None
return GeolocationData(
country_code=json_ret.get('country_code', 'unknown'),
city=json_ret.get('city', 'unknown'),
)
def query_ipinfo() -> Optional[GeolocationData]:
"""
50,000 requests per month tied to the API Key
https://ipinfo.io/developers
"""
try:
response = requests.get(
'https://ipinfo.io/json?token=16ab40aad9bd5b',
timeout=LOCATION_DATA_QUERY_TIMEOUT,
)
except requests.exceptions.RequestException:
return None
if response.status_code != HTTPStatus.OK:
return None
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError:
return None
return GeolocationData(
country_code=json_ret.get('country', 'unknown'),
city=json_ret.get('city', 'unknown'),
)
def _request_chain_metadata(self) -> Dict[str, Any]:
"""Subscan API metadata documentation:
https://docs.api.subscan.io/#metadata
"""
response = self._request_explorer_api(endpoint='metadata')
if response.status_code != HTTPStatus.OK:
message = (
f'{self.chain} chain metadata request was not successful. '
f'Response status code: {response.status_code}. '
f'Response text: {response.text}.'
)
log.error(message)
raise RemoteError(message)
try:
result = jsonloads_dict(response.text)
except JSONDecodeError as e:
message = (
f'{self.chain} chain metadata request returned invalid JSON '
f'response: {response.text}.'
)
log.error(message)
raise RemoteError(message) from e
log.debug(f'{self.chain} subscan API metadata', result=result)
return result
def test_kucoin_exchange_assets_are_known(mock_kucoin):
request_url = f'{mock_kucoin.base_uri}/api/v1/currencies'
try:
response = requests.get(request_url)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Kucoin get request at {request_url} connection error: {str(e)}.',
) from e
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Kucoin query responded with error status code: {response.status_code} '
f'and text: {response.text}', )
try:
response_dict = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Kucoin returned invalid JSON response: {response.text}') from e
# Extract the unique symbols from the exchange pairs
unsupported_assets = set(UNSUPPORTED_KUCOIN_ASSETS)
common_items = unsupported_assets.intersection(
set(WORLD_TO_KUCOIN.values()))
assert not common_items, f'Kucoin assets {common_items} should not be unsupported'
for entry in response_dict['data']:
symbol = entry['currency']
try:
asset_from_kucoin(symbol)
except UnsupportedAsset:
assert symbol in unsupported_assets
except UnknownAsset as e:
test_warnings.warn(
UserWarning(
f'Found unknown asset {e.asset_name} in kucoin. '
f'Support for it has to be added', ))
def _get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
start = 1
limit = 5000
result: List[Dict[str, Any]] = []
while True:
response_data = jsonloads_dict(
self._query(f'v1/cryptocurrency/map?start={start}&limit={limit}'),
)
result.extend(response_data['data'])
if len(response_data['data']) != limit:
break
return result
def _process_dict_response(response: requests.Response) -> Dict:
"""Processess a dict response returned from the Rotkehlchen server and returns
the result for success or raises RemoteError if an error happened"""
if response.status_code not in HANDLABLE_STATUS_CODES:
raise RemoteError(
f'Unexpected status response({response.status_code}) from '
'rotki server', )
result_dict = jsonloads_dict(response.text)
if 'error' in result_dict:
raise RemoteError(result_dict['error'])
return result_dict
def _check_and_get_response(response: Response,
method: str) -> Union[str, Dict]:
"""Checks the kraken response and if it's succesfull returns the result.
If there is recoverable error a string is returned explaining the error
May raise:
- RemoteError if there is an unrecoverable/unexpected remote error
"""
if response.status_code in (520, 525, 504):
log.debug(f'Kraken returned status code {response.status_code}')
return 'Usual kraken 5xx shenanigans'
if response.status_code != 200:
raise RemoteError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
try:
decoded_json = jsonloads_dict(response.text)
except json.decoder.JSONDecodeError as e:
raise RemoteError(f'Invalid JSON in Kraken response. {e}') from e
error = decoded_json.get('error', None)
if error:
if isinstance(error, list) and len(error) != 0:
error = error[0]
if 'Rate limit exceeded' in error:
log.debug(f'Kraken: Got rate limit exceeded error: {error}')
return 'Rate limited exceeded'
# else
raise RemoteError(error)
result = decoded_json.get('result', None)
if result is None:
if method == 'Balance':
return {}
raise RemoteError(f'Missing result in kraken response for {method}')
return result
def _single_grant_api_query(self, query_str: str) -> Dict[str, Any]:
backoff = 1
backoff_limit = 33
while backoff < backoff_limit:
log.debug(f'Querying gitcoin: {query_str}')
try:
response = self.session.get(query_str,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
if 'Max retries exceeded with url' in str(e):
log.debug(
f'Got max retries exceeded from gitcoin. Will '
f'backoff for {backoff} seconds.', )
gevent.sleep(backoff)
backoff = backoff * 2
if backoff >= backoff_limit:
raise RemoteError(
'Getting gitcoin error even '
'after we incrementally backed off', ) from e
continue
raise RemoteError(
f'Gitcoin API request failed due to {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'Gitcoin API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Gitcoin API request {response.url} returned invalid '
f'JSON response: {response.text}', ) from e
if 'error' in json_ret:
raise RemoteError(
f'Gitcoin API request {response.url} returned an error: {json_ret["error"]}',
)
break # success
return json_ret
def query_balances(self) -> ExchangeQueryBalances:
"""Return the account balances
May raise RemoteError
"""
accounts_response = self._api_query(KucoinCase.BALANCES)
if accounts_response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=accounts_response,
case=KucoinCase.BALANCES,
)
return result, msg
try:
response_dict = jsonloads_dict(accounts_response.text)
except JSONDecodeError as e:
msg = f'Kucoin balances returned an invalid JSON response: {accounts_response.text}.'
log.error(msg)
raise RemoteError(msg) from e
account_balances = self._deserialize_accounts_balances(response_dict=response_dict)
return account_balances, ''
def _query(self, path: str) -> Dict:
"""
May raise:
- RemoteError if there is a problem querying Github
"""
try:
response = requests.get(f'{self.prefix}{path}')
except requests.exceptions.RequestException as e:
raise RemoteError(f'Failed to query Github: {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'Github API request {response.url} for {path} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Github returned invalid JSON response: {response.text}'
) from e
return json_ret
def query_ipwhoisio() -> Optional[GeolocationData]:
"""10,000 requests per month per IP
https://ipwhois.io/documentation
"""
try:
response = requests.get(
'http://free.ipwhois.io/json/',
timeout=LOCATION_DATA_QUERY_TIMEOUT,
)
except requests.exceptions.RequestException:
return None
if response.status_code != HTTPStatus.OK:
return None
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError:
return None
return GeolocationData(
country_code=json_ret.get('country_code', 'unknown'),
city=json_ret.get('city', 'unknown'),
)
def _api_query(
self,
endpoint: str,
options: Optional[Dict[str, Any]] = None,
) -> Tuple[Union[List[Any], Dict[str, Any]], Optional[Dict[str, Any]],
Optional[Dict[str, Any]]]: # noqa: E501
"""Performs a bitpanda API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Returns a tuple of:
- The result data
- Optional meta dict containing total_count, page and page_size
- Optional links dict containing next, last and self links
Raises RemoteError if something went wrong with connecting or reading from the exchange
"""
request_url = f'{self.uri}/{endpoint}'
retries_left = QUERY_RETRY_TIMES
if options is not None:
request_url += '?' + urlencode(options)
while retries_left > 0:
log.debug(
'Bitpanda API query',
request_url=request_url,
options=options,
)
try:
response = self.session.get(request_url,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bitpanda API request failed due to {str(e)}') from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
backoff_in_seconds = int(20 / retries_left)
retries_left -= 1
log.debug(
f'Got a 429 from Bitpanda query of {request_url}. Will backoff '
f'for {backoff_in_seconds} seconds. {retries_left} retries left',
)
gevent.sleep(backoff_in_seconds)
continue
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Bitpanda API request failed with response: {response.text} '
f'and status code: {response.status_code}', )
# we got it, so break
break
else: # retries left are zero
raise RemoteError(
f'Ran out of retries for Bitpanda query of {request_url}')
try:
decoded_json = jsonloads_dict(response.text)
except json.decoder.JSONDecodeError as e:
raise RemoteError(
f'Invalid JSON {response.text} in Bitpanda response. {e}'
) from e
if 'data' not in decoded_json:
raise RemoteError(
f'Invalid JSON {response.text} in Bitpanda response. Expected "data" key',
)
log.debug(f'Got Bitpanda response: {decoded_json}')
return decoded_json['data'], decoded_json.get(
'meta'), decoded_json.get('links')
def _api_query(
self,
endpoint: str,
options: Optional[Dict[str, Any]] = None,
pagination_next_uri: str = None,
ignore_pagination: bool = False,
) -> List[Any]:
"""Performs a coinbase API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
If this is an ongoing paginating call then provide pagination_next_uri.
If you want just the first results then set ignore_pagination to True.
"""
request_verb = "GET"
if pagination_next_uri:
request_url = pagination_next_uri
else:
request_url = f'/{self.apiversion}/{endpoint}'
if options:
request_url += urlencode(options)
timestamp = str(int(time.time()))
message = timestamp + request_verb + request_url
signature = hmac.new(
self.secret,
message.encode(),
hashlib.sha256,
).hexdigest()
log.debug('Coinbase API query', request_url=request_url)
self.session.headers.update({
'CB-ACCESS-SIGN': signature,
'CB-ACCESS-TIMESTAMP': timestamp,
# This is needed to guarantee the up to the given date
# API version response.
'CB-VERSION': '2019-08-25',
})
full_url = self.base_uri + request_url
try:
response = self.session.get(full_url,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Coinbase API request failed due to {str(e)}') from e
if response.status_code == 403:
raise CoinbasePermissionError(
f'API key does not have permission for {endpoint}')
if response.status_code != 200:
raise RemoteError(
f'Coinbase query {full_url} responded with error status code: '
f'{response.status_code} and text: {response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Coinbase returned invalid JSON response: {response.text}'
) from e
if 'data' not in json_ret:
raise RemoteError(
f'Coinbase json response does not contain data: {response.text}'
)
final_data = json_ret['data']
# If we got pagination recursively gather all the subsequent queries
if 'pagination' in json_ret and not ignore_pagination:
if 'next_uri' not in json_ret['pagination']:
raise RemoteError(
'Coinbase json response contained no "next_uri" key')
next_uri = json_ret['pagination']['next_uri']
if not next_uri:
# As per the docs: https://developers.coinbase.com/api/v2?python#pagination
# once we get an empty next_uri we are done
return final_data
additional_data = self._api_query(
endpoint=endpoint,
options=options,
pagination_next_uri=next_uri,
)
final_data.extend(additional_data)
return final_data
def _api_query(
self,
endpoint: str,
options: Optional[Dict[str, Any]] = None,
ignore_pagination: bool = False,
) -> List[Any]:
"""Performs a coinbase API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
If you want just the first results then set ignore_pagination to True.
"""
all_items: List[Any] = []
request_verb = "GET"
# initialize next_uri before loop
next_uri = f'/{self.apiversion}/{endpoint}'
if options:
next_uri += urlencode(options)
while True:
timestamp = str(int(time.time()))
message = timestamp + request_verb + next_uri
signature = hmac.new(
self.secret,
message.encode(),
hashlib.sha256,
).hexdigest()
log.debug('Coinbase API query', request_url=next_uri)
self.session.headers.update({
'CB-ACCESS-SIGN': signature,
'CB-ACCESS-TIMESTAMP': timestamp,
# This is needed to guarantee the up to the given date
# API version response.
'CB-VERSION': '2019-08-25',
})
full_url = self.base_uri + next_uri
try:
response = self.session.get(full_url,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Coinbase API request failed due to {str(e)}') from e
if response.status_code == 403:
raise CoinbasePermissionError(
f'API key does not have permission for {endpoint}')
if response.status_code != 200:
raise RemoteError(
f'Coinbase query {full_url} responded with error status code: '
f'{response.status_code} and text: {response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Coinbase returned invalid JSON response: {response.text}',
) from e
if 'data' not in json_ret:
raise RemoteError(
f'Coinbase json response does not contain data: {response.text}'
)
# `data` attr is a list in itself
all_items.extend(json_ret['data'])
if ignore_pagination or 'pagination' not in json_ret:
# break out of the loop, no need to handle pagination
break
if 'next_uri' not in json_ret['pagination']:
raise RemoteError(
'Coinbase json response contained no "next_uri" key')
# otherwise, let the loop run to gather subsequent queries
# this next_uri will be used in next iteration
next_uri = json_ret['pagination']['next_uri']
if not next_uri:
# As per the docs: https://developers.coinbase.com/api/v2?python#pagination
# once we get an empty next_uri we are done
break
return all_items
def get_coin_by_id(self, coinpaprika_id: str) -> Dict[str, Any]:
response_data = self._query(f'coins/{coinpaprika_id}')
return jsonloads_dict(response_data)
case=case,
time=min(current_query_ts + time_step, end_ts),
)
response = self._api_query(
case=case,
options=call_options,
)
if response.status_code != HTTPStatus.OK:
return self._process_unsuccessful_response(
response=response,
case=case,
)
try:
response_dict = jsonloads_dict(response.text)
except JSONDecodeError as e:
msg = f'Kucoin {case} returned an invalid JSON response: {response.text}.'
log.error(msg)
self.msg_aggregator.add_error(
f'Got remote error while querying kucoin {case}: {msg}', )
raise RemoteError(msg) from e
try:
response_data = response_dict['data']
total_page = response_data['totalPage']
current_page = response_data['currentPage']
raw_results = response_data['items']
except KeyError as e:
msg = f'Kucoin {case} JSON response is missing key: {str(e)}'
log.error(msg, response_dict)
def _make_request(
self,
endpoint: str,
start_time: Optional[Timestamp] = None,
end_time: Optional[Timestamp] = None,
limit: int = PAGINATION_LIMIT,
) -> Union[List[Dict[str, Any]], Dict[str, List[Any]]]:
"""Performs an FTX API Query for endpoint adding the needed information to
authenticate user and handling errors.
This function can raise:
- RemoteError
"""
request_verb = "GET"
backoff = INITIAL_BACKOFF_TIME
# Use a while loop to retry request if rate limit is reached
while True:
request_url = '/api/' + endpoint
options = {'limit': limit}
if start_time is not None:
options['start_time'] = start_time
if end_time is not None:
options['end_time'] = end_time
if len(options) != 0:
request_url += '?' + urlencode(options)
timestamp = int(time.time() * 1000)
signature_payload = f'{timestamp}{request_verb}{request_url}'.encode(
)
signature = hmac.new(self.secret, signature_payload,
'sha256').hexdigest()
log.debug('FTX API query', request_url=request_url)
self.session.headers.update({
'FTX-SIGN': signature,
'FTX-TS': str(timestamp),
})
full_url = self.base_uri + request_url
try:
response = self.session.get(full_url,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'FTX API request {full_url} failed due to {str(e)}'
) from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
if backoff < BACKOFF_LIMIT:
log.debug(
f'FTX rate limit exceeded on request {request_url}. Backing off',
seconds=backoff,
)
gevent.sleep(backoff)
backoff = backoff * 2
continue
# We got a result here
break
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'FTX query {full_url} responded with error status code: '
f'{response.status_code} and text: {response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'FTX returned invalid JSON response: {response.text}') from e
if 'result' not in json_ret:
raise RemoteError(
f'FTX json response does not contain data: {response.text}')
return json_ret['result']
def _query(
self,
module: str,
action: str,
options: Optional[Dict[str, Any]] = None,
timeout: Optional[Tuple[int, int]] = None,
) -> Union[List[Dict[str, Any]], str, List[EthereumTransaction], Dict[
str, Any]]:
"""Queries etherscan
May raise:
- RemoteError if there are any problems with reaching Etherscan or if
an unexpected response is returned
"""
query_str = f'https://api.etherscan.io/api?module={module}&action={action}'
if options:
for name, value in options.items():
query_str += f'&{name}={value}'
api_key = self._get_api_key()
if api_key is None:
if not self.warning_given:
self.msg_aggregator.add_warning(
'You do not have an Etherscan API key configured. rotki '
'etherscan queries will still work but will be very slow. '
'If you are not using your own ethereum node, it is recommended '
'to go to https://etherscan.io/register, create an API '
'key and then input it in the external service credentials setting of rotki',
)
self.warning_given = True
else:
query_str += f'&apikey={api_key}'
backoff = 1
backoff_limit = 33
while backoff < backoff_limit:
log.debug(f'Querying etherscan: {query_str}')
try:
response = self.session.get(
query_str,
timeout=timeout
if timeout else DEFAULT_TIMEOUT_TUPLE) # noqa: E501
except requests.exceptions.RequestException as e:
if 'Max retries exceeded with url' in str(e):
log.debug(
f'Got max retries exceeded from etherscan. Will '
f'backoff for {backoff} seconds.', )
gevent.sleep(backoff)
backoff = backoff * 2
if backoff >= backoff_limit:
raise RemoteError(
'Getting Etherscan max connections error even '
'after we incrementally backed off', ) from e
continue
raise RemoteError(
f'Etherscan API request failed due to {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'Etherscan API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Etherscan API request {response.url} returned invalid '
f'JSON response: {response.text}', ) from e
try:
result = json_ret.get('result', None)
if result is None:
raise RemoteError(
f'Unexpected format of Etherscan response for request {response.url}. '
f'Missing a result in response. Response was: {response.text}',
)
# sucessful proxy calls do not include a status
status = int(json_ret.get('status', 1))
if status != 1:
if status == 0 and 'rate limit reached' in result:
log.debug(
f'Got response: {response.text} from etherscan. Will '
f'backoff for {backoff} seconds.', )
gevent.sleep(backoff)
# Continue increasing backoff until limit is reached.
# If limit is reached then keep sleeping with the limit.
# Etherscan will let the query go through eventually
if backoff * 2 < backoff_limit:
backoff = backoff * 2
continue
transaction_endpoint_and_none_found = (
status == 0
and json_ret['message'] == 'No transactions found'
and action in ('txlist', 'txlistinternal', 'tokentx'))
logs_endpoint_and_none_found = (status == 0
and json_ret['message']
== 'No records found'
and 'getLogs' in action)
if transaction_endpoint_and_none_found or logs_endpoint_and_none_found:
# Can't realize that result is always a list here so we ignore mypy warning
return [] # type: ignore
# else
raise RemoteError(
f'Etherscan returned error response: {json_ret}')
except KeyError as e:
raise RemoteError(
f'Unexpected format of Etherscan response for request {response.url}. '
f'Missing key entry for {str(e)}. Response was: {response.text}',
) from e
# success, break out of the loop and return result
return result
return result
def _query(
self,
module: Literal['validator'],
endpoint: Optional[Literal['balancehistory', 'performance', 'eth1',
'deposits']],
encoded_args: str,
) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
"""
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
if endpoint is None: # for now only validator data
query_str = f'{self.url}{module}/{encoded_args}'
elif endpoint == 'eth1':
query_str = f'{self.url}{module}/{endpoint}/{encoded_args}'
else:
query_str = f'{self.url}{module}/{encoded_args}/{endpoint}'
api_key = self._get_api_key()
if api_key is not None:
query_str += f'?apikey={api_key}'
times = QUERY_RETRY_TIMES
backoff_in_seconds = 10
log.debug(f'Querying beaconcha.in API for {query_str}')
while True:
try:
response = self.session.get(query_str,
timeout=BEACONCHAIN_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Querying {query_str} failed due to {str(e)}') from e
if response.status_code == 429:
minute_rate_limit = response.headers.get(
'x-ratelimit-limit-minute', 'unknown')
user_minute_rate_limit = response.headers.get(
'x-ratelimit-remaining-minute', 'unknown') # noqa: E501
daily_rate_limit = response.headers.get(
'x-ratelimit-limit-day', 'unknown')
user_daily_rate_limit = response.headers.get(
'x-ratelimit-remaining-day', 'unknown') # noqa: E501
month_rate_limit = response.headers.get(
'x-ratelimit-limit-month', 'unknown')
user_month_rate_limit = response.headers.get(
'x-ratelimit-remaining-month', 'unknown') # noqa: E501
if times == 0:
msg = (
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text} after 5 retries')
log.debug(
f'{msg} minute limit: {user_minute_rate_limit}/{minute_rate_limit}, '
f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, '
f'monthly limit: {user_month_rate_limit}/{month_rate_limit}',
)
raise RemoteError(msg)
retry_after = response.headers.get('retry-after', None)
if retry_after:
retry_after_secs = int(retry_after)
if retry_after_secs > MAX_WAIT_SECS:
msg = (
f'Beaconchain API request {response.url} got rate limited. Would '
f'need to wait for {retry_after} seconds which is more than the '
f'wait limit of {MAX_WAIT_SECS} seconds. Bailing out.'
)
log.debug(
f'{msg} minute limit: {user_minute_rate_limit}/{minute_rate_limit}, '
f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, '
f'monthly limit: {user_month_rate_limit}/{month_rate_limit}',
)
raise RemoteError(msg)
# else
sleep_seconds = retry_after_secs
else:
# Rate limited. Try incremental backoff since retry-after header is missing
sleep_seconds = backoff_in_seconds * (QUERY_RETRY_TIMES -
times + 1)
times -= 1
log.debug(
f'Beaconchain API request {response.url} got rate limited. Sleeping '
f'for {sleep_seconds}. We have {times} tries left.'
f'minute limit: {user_minute_rate_limit}/{minute_rate_limit}, '
f'daily limit: {user_daily_rate_limit}/{daily_rate_limit}, '
f'monthly limit: {user_month_rate_limit}/{month_rate_limit}',
)
gevent.sleep(sleep_seconds)
continue
# else
break
if response.status_code != 200:
raise RemoteError(
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Beaconchain API returned invalid JSON response: {response.text}',
) from e
if json_ret.get('status') != 'OK':
raise RemoteError(
f'Beaconchain API returned non-OK status. Response: {json_ret}'
)
if 'data' not in json_ret:
raise RemoteError(
f'Beaconchain API did not contain a data key. Response: {json_ret}'
)
return json_ret['data']
def all_coins(self) -> Dict[str, Any]:
"""
Gets the mapping of all the cryptocompare coins
May raise:
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
# Get coin list of cryptocompare
invalidate_cache = True
coinlist_cache_path = os.path.join(self.data_directory, 'cryptocompare_coinlist.json')
if os.path.isfile(coinlist_cache_path):
log.info('Found cryptocompare coinlist cache', path=coinlist_cache_path)
with open(coinlist_cache_path, 'r') as f:
try:
data = jsonloads_dict(f.read())
now = ts_now()
invalidate_cache = False
# If we got a cache and its' over a month old then requery cryptocompare
if data['time'] < now and now - data['time'] > 2629800:
log.info('Cryptocompare coinlist cache is now invalidated')
invalidate_cache = True
data = data['data']
except JSONDecodeError:
invalidate_cache = True
if invalidate_cache:
data = self._api_query('all/coinlist')
# Also save the cache
with open(coinlist_cache_path, 'w') as f:
now = ts_now()
log.info('Writing coinlist cache', timestamp=now)
write_data = {'time': now, 'data': data}
f.write(rlk_jsondumps(write_data))
else:
# in any case take the data
data = data['data']
# As described in the docs
# https://min-api.cryptocompare.com/documentation?key=Other&cat=allCoinsWithContentEndpoint
# This is not the entire list of assets in the system, so I am manually adding
# here assets I am aware of that they already have historical data for in thei
# cryptocompare system
data['DAO'] = object()
data['USDT'] = object()
data['VEN'] = object()
data['AIR*'] = object() # This is Aircoin
# This is SpendCoin (https://coinmarketcap.com/currencies/spendcoin/)
data['SPND'] = object()
# This is eBitcoinCash (https://coinmarketcap.com/currencies/ebitcoin-cash/)
data['EBCH'] = object()
# This is Educare (https://coinmarketcap.com/currencies/educare/)
data['EKT'] = object()
# This is Knoxstertoken (https://coinmarketcap.com/currencies/knoxstertoken/)
data['FKX'] = object()
# This is FNKOS (https://coinmarketcap.com/currencies/fnkos/)
data['FNKOS'] = object()
# This is FansTime (https://coinmarketcap.com/currencies/fanstime/)
data['FTI'] = object()
# This is Gene Source Code Chain
# (https://coinmarketcap.com/currencies/gene-source-code-chain/)
data['GENE*'] = object()
# This is GazeCoin (https://coinmarketcap.com/currencies/gazecoin/)
data['GZE'] = object()
# This is probaly HarmonyCoin (https://coinmarketcap.com/currencies/harmonycoin-hmc/)
data['HMC*'] = object()
# This is IoTChain (https://coinmarketcap.com/currencies/iot-chain/)
data['ITC'] = object()
# This is Luna Coin (https://coinmarketcap.com/currencies/luna-coin/)
data['LUNA'] = object
# This is MFTU (https://coinmarketcap.com/currencies/mainstream-for-the-underground/)
data['MFTU'] = object()
# This is Nexxus (https://coinmarketcap.com/currencies/nexxus/)
data['NXX'] = object()
# This is Owndata (https://coinmarketcap.com/currencies/owndata/)
data['OWN'] = object()
# This is PiplCoin (https://coinmarketcap.com/currencies/piplcoin/)
data['PIPL'] = object()
# This is PKG Token (https://coinmarketcap.com/currencies/pkg-token/)
data['PKG'] = object()
# This is Quibitica https://coinmarketcap.com/currencies/qubitica/
data['QBIT'] = object()
# This is DPRating https://coinmarketcap.com/currencies/dprating/
data['RATING'] = object()
# This is RocketPool https://coinmarketcap.com/currencies/rocket-pool/
data['RPL'] = object()
# This is SpeedMiningService (https://coinmarketcap.com/currencies/speed-mining-service/)
data['SMS'] = object()
# This is SmartShare (https://coinmarketcap.com/currencies/smartshare/)
data['SSP'] = object()
# This is ThoreCoin (https://coinmarketcap.com/currencies/thorecoin/)
data['THR'] = object()
# This is Transcodium (https://coinmarketcap.com/currencies/transcodium/)
data['TNS'] = object()
return data
def _api_query(self, path: str) -> Dict[str, Any]:
"""Queries cryptocompare
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
querystr = f'https://min-api.cryptocompare.com/data/{path}'
log.debug('Querying cryptocompare', url=querystr)
api_key = self._get_api_key()
if api_key:
querystr += '?' if '?' not in querystr else '&'
querystr += f'api_key={api_key}'
tries = CRYPTOCOMPARE_QUERY_RETRY_TIMES
while tries >= 0:
try:
response = self.session.get(querystr, timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(f'Cryptocompare API request failed due to {str(e)}') from e
try:
json_ret = jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Cryptocompare returned invalid JSON response: {response.text}',
) from e
try:
# backoff and retry 3 times = 1 + 1.5 + 3 = at most 5.5 secs
# Failing is also fine, since all calls have secondary data sources
# for example coingecko
if json_ret.get('Message', None) == RATE_LIMIT_MSG:
self.last_rate_limit = ts_now()
if tries >= 1:
backoff_seconds = 3 / tries
log.debug(
f'Got rate limited by cryptocompare. '
f'Backing off for {backoff_seconds}',
)
gevent.sleep(backoff_seconds)
tries -= 1
continue
# else
log.debug(
f'Got rate limited by cryptocompare and did not manage to get a '
f'request through even after {CRYPTOCOMPARE_QUERY_RETRY_TIMES} '
f'incremental backoff retries',
)
if json_ret.get('Response', 'Success') != 'Success':
error_message = f'Failed to query cryptocompare for: "{querystr}"'
if 'Message' in json_ret:
error_message += f'. Error: {json_ret["Message"]}'
log.warning(
'Cryptocompare query failure',
url=querystr,
error=error_message,
status_code=response.status_code,
)
raise RemoteError(error_message)
return json_ret['Data'] if 'Data' in json_ret else json_ret
except KeyError as e:
raise RemoteError(
f'Unexpected format of Cryptocompare json_response. '
f'Missing key entry for {str(e)}',
) from e
raise AssertionError('We should never get here')
def api_query(self, method: str, req: Optional[dict] = None) -> dict:
# Pretty ugly ... mock a kraken remote eror
if self.remote_errors:
raise RemoteError('Kraken remote error')
if self.use_original_kraken:
return super().api_query(method, req)
if method == 'Balance':
if self.random_balance_data:
return generate_random_kraken_balance_response()
# else
return self.balance_data_return
if method == 'TradesHistory':
assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
if self.random_trade_data:
return generate_random_kraken_trades_data(
start=req['start'],
end=req['end'],
tradeable_pairs=list(self.tradeable_pairs.keys()),
)
# else
return jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
if method == 'AssetPairs':
dir_path = Path(__file__).resolve().parent.parent
filepath = dir_path / 'data' / 'assets_kraken.json'
with open(filepath) as f:
return jsonloads_dict(f.read())['result']
if method == 'Ledgers':
if req is None:
req = {}
ledger_type: str = req.get('type', '')
if self.random_ledgers_data:
assert req is not None
return generate_random_kraken_ledger_data(
start=req.get('start', 0),
end=req.get('end', ts_now),
ledger_type=ledger_type,
)
# else use specific data
if ledger_type in ('deposit', 'withdrawal'):
data = json.loads(
KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type
== 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, )
else:
data = json.loads(KRAKEN_GENERAL_LEDGER_RESPONSE)
new_data: Dict[str, Any] = {'ledger': {}}
for key, val in data['ledger'].items():
try:
ts = int(val['time'])
except ValueError:
# can happen for tests of invalid data -- let it through
ts = req.get('start', 0)
if ts < req.get('start', 0) or ts > req.get('end', ts_now):
continue
new_data['ledger'][key] = val
new_data['count'] = len(new_data['ledger'])
response = json.dumps(new_data)
return jsonloads_dict(response)
# else
return super().api_query(method, req)
def test_jsonloads_dict():
result = jsonloads_dict('{"foo": 1, "boo": "value"}')
assert result == {'foo': 1, 'boo': 'value'}
with pytest.raises(JSONDecodeError) as e:
jsonloads_dict('["foo", "boo", 3]')
assert 'Returned json is not a dict' in str(e.value)
def _api_query(self,
command: str,
req: Optional[Dict] = None) -> Union[Dict, List]:
"""An api query to poloniex. May make multiple requests
Can raise:
- RemoteError if there is a problem reaching poloniex or with the returned response
"""
if req is None:
req = {}
log.debug(
'Poloniex API query',
command=command,
post_data=req,
)
tries = QUERY_RETRY_TIMES
while tries >= 0:
try:
response = self._single_query(command, req)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Poloniex API request failed due to {str(e)}') from e
if response is None:
if tries >= 1:
backoff_seconds = 20 / tries
log.debug(
f'Got a recoverable poloniex error. '
f'Backing off for {backoff_seconds}', )
gevent.sleep(backoff_seconds)
tries -= 1
continue
else:
break
if response is None:
raise RemoteError(
f'Got a recoverable poloniex error and did not manage to get a '
f'request through even after {QUERY_RETRY_TIMES} '
f'incremental backoff retries', )
result: Union[Dict, List]
try:
if command == 'returnLendingHistory':
result = jsonloads_list(response.text)
else:
# For some reason poloniex can also return [] for an empty trades result
if response.text == '[]':
result = {}
else:
result = jsonloads_dict(response.text)
result = _post_process(result)
except JSONDecodeError as e:
raise RemoteError(
f'Poloniex returned invalid JSON response: {response.text}'
) from e
if isinstance(result, dict) and 'error' in result:
raise RemoteError(
'Poloniex query for "{}" returned error: {}'.format(
command,
result['error'],
))
return result
def _api_query(
self,
endpoint: str,
request_method: Literal['GET', 'POST'] = 'GET',
options: Optional[Dict[str, Any]] = None,
query_options: Optional[Dict[str, Any]] = None,
) -> Tuple[List[Any], Optional[str]]:
"""Performs a coinbase PRO API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Returns a tuple of the result and optional pagination cursor.
Raises RemoteError if something went wrong with connecting or reading from the exchange
Raises CoinbaseProPermissionError if the API Key does not have sufficient
permissions for the endpoint
"""
request_url = f'/{endpoint}'
timestamp = str(int(time.time()))
if options:
stringified_options = json.dumps(options, separators=(',', ':'))
else:
stringified_options = ''
options = {}
if query_options:
request_url += '?' + urlencode(query_options)
message = timestamp + request_method + request_url + stringified_options
if 'products' not in endpoint:
try:
signature = hmac.new(
b64decode(self.secret),
message.encode(),
hashlib.sha256,
).digest()
except binascii.Error as e:
raise RemoteError('Provided API Secret is invalid') from e
self.session.headers.update({
'CB-ACCESS-SIGN':
b64encode(signature).decode('utf-8'),
'CB-ACCESS-TIMESTAMP':
timestamp,
})
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
log.debug(
'Coinbase Pro API query',
request_method=request_method,
request_url=request_url,
options=options,
)
full_url = self.base_uri + request_url
try:
response = self.session.request(
request_method.lower(),
full_url,
data=stringified_options,
timeout=DEFAULT_TIMEOUT_TUPLE,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at '
f'{full_url} connection error: {str(e)}', ) from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
# Backoff a bit by sleeping. Sleep more, the more retries have been made
backoff_secs = QUERY_RETRY_TIMES / retries_left
log.debug(
f'Backing off coinbase pro api query for {backoff_secs} secs'
)
gevent.sleep(backoff_secs)
retries_left -= 1
else:
# get out of the retry loop, we did not get 429 complaint
break
json_ret: Union[List[Any], Dict[str, Any]]
if response.status_code == HTTPStatus.BAD_REQUEST:
json_ret = jsonloads_dict(response.text)
if json_ret['message'] == 'invalid signature':
raise CoinbaseProPermissionError(
f'While doing {request_method} at {endpoint} endpoint the API secret '
f'created an invalid signature.', )
# else do nothing and a generic remote error will be thrown below
elif response.status_code == HTTPStatus.FORBIDDEN:
raise CoinbaseProPermissionError(
f'API key does not have permission for {endpoint}', )
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
try:
json_ret = jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} '
f'returned invalid JSON response: {response.text}', ) from e
return json_ret, response.headers.get('cb-after', None)
def query_balances(self) -> ExchangeQueryBalances:
"""Return the account balances on Bistamp
The balance endpoint returns a dict where the keys (str) are related to
assets and the values (str) amounts. The keys that end with `_balance`
contain the exact amount of an asset the account is holding (available
amount + orders amount, per asset).
"""
response = self._api_query('balance')
if response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=response,
case='balances',
)
return result, msg
try:
response_dict = jsonloads_dict(response.text)
except JSONDecodeError as e:
msg = f'Bitstamp returned invalid JSON response: {response.text}.'
log.error(msg)
raise RemoteError(msg) from e
assets_balance: Dict[Asset, Balance] = {}
for entry, amount in response_dict.items():
if not entry.endswith('_balance'):
continue
symbol = entry.split('_')[0] # If no `_`, defaults to entry
try:
amount = deserialize_asset_amount(amount)
if amount == ZERO:
continue
asset = asset_from_bitstamp(symbol)
except DeserializationError as e:
log.error(
'Error processing a Bitstamp balance.',
entry=entry,
error=str(e),
)
self.msg_aggregator.add_error(
'Failed to deserialize a Bitstamp balance. '
'Check logs for details. Ignoring it.', )
continue
except (UnknownAsset, UnsupportedAsset) as e:
log.error(str(e))
asset_tag = 'unknown' if isinstance(
e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} Bistamp asset {e.asset_name}. Ignoring its balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
log.error(str(e))
self.msg_aggregator.add_error(
f'Error processing Bitstamp balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry.', )
continue
assets_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
return assets_balance, ''
