def _private_api_query(
self,
endpoint: str,
options: Optional[Dict[str, Any]] = None,
) -> Union[Dict[str, Any], List[Any]]:
"""Performs a Gemini API Query for a private endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Raises RemoteError if something went wrong with connecting or reading from the exchange
Raises GeminiPermissionError if the API Key does not have sufficient
permissions for the endpoint
"""
response = self._query_continuously(method='post', endpoint=endpoint, options=options)
json_ret: Union[List[Any], Dict[str, Any]]
if response.status_code == HTTPStatus.FORBIDDEN:
raise GeminiPermissionError(
f'API key does not have permission for {endpoint}',
)
if response.status_code == HTTPStatus.BAD_REQUEST:
if 'InvalidSignature' in response.text:
raise GeminiPermissionError('Invalid API Key or API secret')
# else let it be handled by the generic non-200 code error below
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Gemini query at {response.url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
deserialization_fn: Union[Callable[[str], Dict[str, Any]], Callable[[str], List[Any]]]
deserialization_fn = jsonloads_dict if endpoint == 'roles' else jsonloads_list
try:
json_ret = deserialization_fn(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Gemini query at {response.url} '
f'returned invalid JSON response: {response.text}',
) from e
return json_ret
def _check_and_get_response(response: Response, method: str) -> Union[str, Dict]:
"""Checks the kraken response and if it's succesfull returns the result.
If there is recoverable error a string is returned explaining the error
May raise:
- RemoteError if there is an unrecoverable/unexpected remote error
"""
if response.status_code in (520, 525, 504):
log.debug(f'Kraken returned status code {response.status_code}')
return 'Usual kraken 5xx shenanigans'
elif response.status_code != 200:
raise RemoteError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
try:
decoded_json = rlk_jsonloads_dict(response.text)
except json.decoder.JSONDecodeError as e:
raise RemoteError(f'Invalid JSON in Kraken response. {e}')
try:
if decoded_json['error']:
if isinstance(decoded_json['error'], list):
error = decoded_json['error'][0]
else:
error = decoded_json['error']
if 'Rate limit exceeded' in error:
log.debug(f'Kraken: Got rate limit exceeded error: {error}')
return 'Rate limited exceeded'
else:
raise RemoteError(error)
result = decoded_json['result']
except KeyError as e:
raise RemoteError(f'Unexpected format of Kraken response. Missing key: {e}')
return result
def _parse_int(line: str, entry: str) -> int:
try:
if line == '-':
result = 0
else:
result = int(line)
except ValueError as e:
raise RemoteError(
f'Could not parse {line} as an integer for {entry}') from e
return result
def __init__(self, url: str) -> None:
"""
- May raise requests.RequestException if there is a problem connecting to the subgraph"""
transport = RequestsHTTPTransport(url=url)
try:
self.client = Client(transport=transport,
fetch_schema_from_transport=False)
except (requests.exceptions.RequestException) as e:
raise RemoteError(
f'Failed to connect to the graph at {url} due to {str(e)}'
) from e
def request_get(uri, timeout=ALL_REMOTES_TIMEOUT):
response = requests.get(uri)
if response.status_code != 200:
raise RemoteError('Get {} returned status code {}'.format(uri, response.status_code))
try:
result = rlk_jsonloads(response.text)
except json.decoder.JSONDecodeError:
raise ValueError('{} returned malformed json'.format(uri))
return result
def api_query(
self,
method: str,
options: Optional[Dict] = None,
) -> Union[List, Dict]:
"""
Queries Bittrex with given method and options
"""
if not options:
options = {}
nonce = str(int(time.time() * 1000))
method_type = 'public'
if method in BITTREX_MARKET_METHODS:
method_type = 'market'
elif method in BITTREX_ACCOUNT_METHODS:
method_type = 'account'
request_url = self.uri + method_type + '/' + method + '?'
if method_type != 'public':
request_url += 'apikey=' + self.api_key.decode() + "&nonce=" + nonce + '&'
request_url += urlencode(options)
signature = hmac.new(
self.secret,
request_url.encode(),
hashlib.sha512,
).hexdigest()
self.session.headers.update({'apisign': signature})
log.debug('Bittrex API query', request_url=request_url)
response = self.session.get(request_url)
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError('Bittrex returned invalid JSON response')
if json_ret['success'] is not True:
raise RemoteError(json_ret['message'])
return json_ret['result']
def _api_query(self, command: str, req: Optional[Dict] = None) -> Union[Dict, List]:
if req is None:
req = {}
if command == 'returnTicker' or command == 'returnCurrencies':
log.debug(f'Querying poloniex for {command}')
ret = self.session.get(self.public_uri + command)
return rlk_jsonloads(ret.text)
req['command'] = command
with self.lock:
# Protect this region with a lock since poloniex will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
req['nonce'] = int(time.time() * 1000)
post_data = str.encode(urlencode(req))
sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
self.session.headers.update({'Sign': sign})
log.debug(
'Poloniex private API query',
command=command,
post_data=req,
)
ret = self.session.post('https://poloniex.com/tradingApi', req)
if ret.status_code != 200:
raise RemoteError(
f'Poloniex query responded with error status code: {ret.status_code}'
f' and text: {ret.text}',
)
try:
if command == 'returnLendingHistory':
return rlk_jsonloads_list(ret.text)
else:
result = rlk_jsonloads_dict(ret.text)
return _post_process(result)
except JSONDecodeError:
raise RemoteError(f'Poloniex returned invalid JSON response: {ret.text}')
def _get_account_balance(
self,
account: SubstrateAddress,
node_interface: SubstrateInterface,
) -> FVal:
"""Given an account get its amount of chain native token.
More information about an account balance in the Substrate AccountData
documentation.
"""
log.debug(
f'{self.chain} querying {self.chain_properties.token.identifier} balance',
url=node_interface.url,
account=account,
)
try:
with gevent.Timeout(KUSAMA_NODE_CONNECTION_TIMEOUT):
result = node_interface.query(
module='System',
storage_function='Account',
params=[account],
)
except (
requests.exceptions.RequestException,
SubstrateRequestException,
ValueError,
WebSocketException,
gevent.Timeout,
) as e:
msg = str(e)
if isinstance(e, gevent.Timeout):
msg = f'a timeout of {msg}'
message = (
f'{self.chain} failed to request {self.chain_properties.token.identifier} account '
f'balance at endpoint {node_interface.url} due to: {msg}'
)
log.error(message, account=account)
raise RemoteError(message) from e
log.debug(
f'{self.chain} account balance',
account=account,
result=result,
)
balance = ZERO
if result is not None:
account_data = result.value['data']
balance = (
FVal(account_data['free'] + account_data['reserved']) /
FVal('10') ** self.chain_properties.token_decimals
)
return balance
def _api_query(
self,
endpoint: Literal['balance', 'user_transactions'],
method: Literal['post'] = 'post',
options: Optional[Dict[str, Any]] = None,
) -> Response:
"""Request a Bistamp API v2 endpoint (from `endpoint`).
"""
call_options = options.copy() if options else {}
data = call_options or None
request_url = f'{self.base_uri}/v2/{endpoint}/'
query_params = ''
nonce = str(uuid.uuid4())
timestamp = str(ts_now_in_ms())
payload_string = urlencode(call_options)
content_type = '' if payload_string == '' else 'application/x-www-form-urlencoded'
message = ('BITSTAMP '
f'{self.api_key}'
f'{method.upper()}'
f'{request_url.replace("https://", "")}'
f'{query_params}'
f'{content_type}'
f'{nonce}'
f'{timestamp}'
'v2'
f'{payload_string}')
signature = hmac.new(
self.secret,
msg=message.encode('utf-8'),
digestmod=hashlib.sha256,
).hexdigest()
self.session.headers.update({
'X-Auth-Signature': signature,
'X-Auth-Nonce': nonce,
'X-Auth-Timestamp': timestamp,
})
if content_type:
self.session.headers.update({'Content-Type': content_type})
log.debug('Bitstamp API request', request_url=request_url)
try:
response = self.session.request(
method=method,
url=request_url,
data=data,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bitstamp {method} request at {request_url} connection error: {str(e)}.',
) from e
return response
def query_balances(self, **kwargs: Any) -> ExchangeQueryBalances:
assets_balance: Dict[Asset, Balance] = {}
try:
resp_info = self._api_query('get', 'user/balance')
except RemoteError as e:
msg = (
'ICONOMI API request failed. Could not reach ICONOMI due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
if resp_info['currency'] != 'USD':
raise RemoteError('Iconomi API did not return values in USD')
for balance_info in resp_info['assetList']:
ticker = balance_info['ticker']
try:
asset = iconomi_asset(ticker)
# There seems to be a bug in the ICONOMI API regarding balance_info['value'].
# The value is supposed to be in USD, but is actually returned
# in EUR. So let's use the Inquirer for now.
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing ICONOMI balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
amount = FVal(balance_info['balance'])
assets_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'unknown' if isinstance(e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} ICONOMI asset {ticker}. '
f' Ignoring its balance query.',
)
for balance_info in resp_info['daaList']:
ticker = balance_info['ticker']
self.msg_aggregator.add_warning(
f'Found unsupported ICONOMI strategy {ticker}. '
f' Ignoring its balance query.',
)
return assets_balance, ''
def query(
self,
querystr: str,
param_types: Optional[Dict[str, Any]] = None,
param_values: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Queries The Graph for a particular query
May raise:
- RemoteError: If there is a problem querying the subgraph and there
are no retries left.
"""
prefix = ''
if param_types is not None:
prefix = 'query '
prefix += json.dumps(param_types).replace('"', '').replace(
'{', '(').replace('}', ')')
prefix += '{'
querystr = prefix + querystr
log.debug(f'Querying The Graph for {querystr}')
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
try:
result = self.client.execute(gql(querystr),
variable_values=param_values)
# need to catch Exception here due to stupidity of gql library
except (requests.exceptions.RequestException, Exception) as e: # pylint: disable=broad-except # noqa: E501
# NB: the lack of a good API error handling by The Graph combined
# with gql v2 raising bare exceptions doesn't allow us to act
# better on failed requests. Currently all trigger the retry logic.
# TODO: upgrade to gql v3 and amend this code on any improvement
# The Graph does on its API error handling.
exc_msg = str(e)
retries_left -= 1
base_msg = f'The Graph query to {querystr} failed due to {exc_msg}'
if retries_left:
sleep_seconds = RETRY_BACKOFF_FACTOR * pow(
2, QUERY_RETRY_TIMES - retries_left)
retry_msg = (
f'Retrying query after {sleep_seconds} seconds. '
f'Retries left: {retries_left}.')
log.error(f'{base_msg}. {retry_msg}')
gevent.sleep(sleep_seconds)
else:
raise RemoteError(f'{base_msg}. No retries left.') from e
else:
break
log.debug('Got result from The Graph query')
return result
def test_query_balances_skips_inquirer_error(mock_bitstamp):
"""Test an entry that can't get its USD price because of a remote error is
skipped
"""
inquirer = MagicMock()
inquirer.find_usd_price.side_effect = RemoteError('test')
def mock_api_query_response(endpoint): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, '{"link_balance": "1.00000000"}')
with patch('rotkehlchen.exchanges.bitstamp.Inquirer', return_value=inquirer):
with patch.object(mock_bitstamp, '_api_query', side_effect=mock_api_query_response):
assert mock_bitstamp.query_balances() == ({}, '')
def get_poap_airdrop_data(name: str, data_dir: Path) -> Dict[str, Any]:
airdrops_dir = data_dir / 'airdrops_poap'
airdrops_dir.mkdir(parents=True, exist_ok=True)
filename = airdrops_dir / f'{name}.json'
if not filename.is_file():
# if not cached, get it from the gist
try:
request = requests.get(POAP_AIRDROPS[name][0])
except requests.exceptions.RequestException as e:
raise RemoteError(f'POAP airdrops Gist request failed due to {str(e)}') from e
try:
json_data = rlk_jsonloads_dict(request.content.decode('utf-8'))
except JSONDecodeError as e:
raise RemoteError(f'POAP airdrops Gist contains an invalid JSON {str(e)}') from e
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(json_data))
infile = open(filename, 'r')
data_dict = rlk_jsonloads_dict(infile.read())
return data_dict
def _check_for_system_clock_not_synced_error(response: Response) -> None:
if response.status_code == HTTPStatus.UNAUTHORIZED:
try:
result = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Bittrex returned invalid JSON response: {response.text}')
if result.get('code', None) == 'INVALID_TIMESTAMP':
raise SystemClockNotSyncedError(
current_time=str(datetime.now()),
remote_server='Bittrex',
)
return None
def _query(self, path: str) -> Dict:
"""
May raise:
- RemoteError if there is a problem querying Github
"""
try:
response = requests.get(f'{self.prefix}{path}')
except requests.exceptions.RequestException as e:
raise RemoteError(f'Failed to query Github: {str(e)}')
if response.status_code != 200:
raise RemoteError(
f'Github API request {response.url} for {path} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}',
)
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Github returned invalid JSON response: {response.text}')
return json_ret
def _query_continuously(
self,
method: Literal['get', 'post'],
endpoint: str,
options: Optional[Dict[str, Any]] = None,
) -> requests.Response:
"""Queries endpoint until anything but 429 is returned
May raise:
- RemoteError if something is wrong connecting to the exchange
"""
v_endpoint = f'/v1/{endpoint}'
url = f'{self.base_uri}{v_endpoint}'
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
if endpoint in ('mytrades', 'balances', 'transfers', 'roles'):
# private endpoints
timestamp = str(ts_now_in_ms())
payload = {'request': v_endpoint, 'nonce': timestamp}
if options is not None:
payload.update(options)
encoded_payload = json.dumps(payload).encode()
b64 = b64encode(encoded_payload)
signature = hmac.new(self.secret, b64,
hashlib.sha384).hexdigest()
self.session.headers.update({
'X-GEMINI-PAYLOAD': b64.decode(),
'X-GEMINI-SIGNATURE': signature,
})
try:
response = self.session.request(
method=method,
url=url,
timeout=GLOBAL_REQUESTS_TIMEOUT,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Gemini {method} query at {url} connection error: {str(e)}',
) from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
# Backoff a bit by sleeping. Sleep more, the more retries have been made
gevent.sleep(QUERY_RETRY_TIMES / retries_left)
retries_left -= 1
else:
# get out of the retry loop, we did not get 429 complaint
break
return response
def _query_lending_balances(
self,
balances: DefaultDict[Asset, Balance],
) -> DefaultDict[Asset, Balance]:
data = self.api_query_dict('sapi', 'lending/union/account')
positions = data.get('positionAmountVos', None)
if positions is None:
raise RemoteError(
f'Could not find key positionAmountVos in lending account data '
f'{data} returned by {self.name}.', )
for entry in positions:
try:
amount = FVal(entry['amount'])
if amount == ZERO:
continue
asset = asset_from_binance(entry['asset'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported {self.name} asset {e.asset_name}. '
f'Ignoring its lending balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown {self.name} asset {e.asset_name}. '
f'Ignoring its lending balance query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error at deserializing {self.name} asset. {msg}. '
f'Ignoring its lending balance query.', )
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return balances
def get_balance_history(
self, validator_indices: List[int]) -> Dict[int, ValidatorBalance]:
"""Get the balance history of all the validators given from the indices list
https://beaconcha.in/api/v1/docs/index.html#/Validator/get_api_v1_validator__indexOrPubkey__balancehistory
Queries in chunks of 100 due to api limitations.
NOTICE: Do not use yet. The results seem incosistent. The list can accept
up to 100 validators, but the balance history is for the last 100 epochs
of each validator, limited to 100 results. So it's not really useful.
Their devs said they will have a look as this may not be desired behaviour.
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
chunks = list(get_chunks(validator_indices, n=100))
data = []
for chunk in chunks:
result = self._query(
module='validator',
endpoint='balancehistory',
encoded_args=','.join(str(x) for x in chunk),
)
if isinstance(result, list):
data.extend(result)
else:
data.append(result)
# We are only interested in last epoch, so get its value
balances: Dict[int, ValidatorBalance] = {}
try:
for entry in data:
index = entry['validatorindex']
epoch = entry['epoch']
if index in balances and balances[index].epoch >= epoch:
continue
balances[index] = ValidatorBalance(
epoch=epoch,
balance=entry['balance'],
effective_balance=entry['effectivebalance'],
)
except KeyError as e:
raise RemoteError(
f'Beaconchai.in balance response processing error. Missing key entry {str(e)}',
) from e
return balances
def _request_chain_metadata(self) -> Dict[str, Any]:
"""Subscan API metadata documentation:
https://docs.api.subscan.io/#metadata
"""
response = self._request_explorer_api(endpoint='metadata')
if response.status_code != HTTPStatus.OK:
message = (
f'{self.chain} chain metadata request was not successful. '
f'Response status code: {response.status_code}. '
f'Response text: {response.text}.')
log.error(message)
raise RemoteError(message)
try:
result = rlk_jsonloads_dict(response.text)
except JSONDecodeError as e:
message = (
f'{self.chain} chain metadata request returned invalid JSON '
f'response: {response.text}.')
log.error(message)
raise RemoteError(message) from e
log.debug(f'{self.chain} subscan API metadata', result=result)
return result
def get_account_balance(self, account: ChecksumEthAddress) -> FVal:
"""Gets the balance of the given account in WEI
May raise:
- RemoteError due to self._query(). Also if the returned result can't be parsed as a number
"""
result = self._query(module='account', action='balance', options={'address': account})
try:
amount = FVal(result)
except ValueError:
raise RemoteError(
f'Etherscan returned non-numeric result for account balance {result}',
)
return amount
def _decode_response_json(response: requests.Response) -> Any:
"""Decodes a python requests response to json and returns it.
May raise:
- RemoteError if the response does not contain valid json
"""
try:
json_response = response.json()
except ValueError as e:
raise RemoteError(
f'Could not decode json from {response.text} to {response.request.method} '
f'query {response.url}', ) from e
return json_response
def api_query(self, method: str, req: Optional[dict] = None) -> dict:
tries = KRAKEN_QUERY_TRIES
query_method = (self._query_public if method in KRAKEN_PUBLIC_METHODS
else self._query_private)
while tries > 0:
if self.call_counter + MAX_CALL_COUNTER_INCREASE > self.call_limit:
# If we are close to the limit, check how much our call counter reduced
# https://www.kraken.com/features/api#api-call-rate-limit
secs_since_last_call = ts_now() - self.last_query_ts
self.call_counter = max(
0,
self.call_counter -
int(secs_since_last_call / self.reduction_every_secs),
)
# If still at limit, sleep for an amount big enough for smallest tier reduction
if self.call_counter + MAX_CALL_COUNTER_INCREASE > self.call_limit:
backoff_in_seconds = self.reduction_every_secs * 2
log.debug(
f'Doing a Kraken API call would now exceed our call counter limit. '
f'Backing off for {backoff_in_seconds} seconds',
call_counter=self.call_counter,
)
tries -= 1
gevent.sleep(backoff_in_seconds)
continue
log.debug(
'Kraken API query',
method=method,
data=req,
call_counter=self.call_counter,
)
result = query_method(method, req)
if isinstance(result, str):
# Got a recoverable error
backoff_in_seconds = int(KRAKEN_BACKOFF_DIVIDEND / tries)
log.debug(
f'Got recoverable error {result} in a Kraken query of {method}. Will backoff '
f'for {backoff_in_seconds} seconds', )
tries -= 1
gevent.sleep(backoff_in_seconds)
continue
# else success
return result
raise RemoteError(
f'After {KRAKEN_QUERY_TRIES} kraken queries for {method} could still not be completed',
)
def _request_explorer_api(self, endpoint: Literal['metadata']) -> Response:
if endpoint == 'metadata':
url = f'{self.chain.chain_explorer_api()}/scan/metadata'
else:
raise AssertionError(f'Unexpected {self.chain} endpoint type: {endpoint}')
log.debug(f'{self.chain} subscan API request', request_url=url)
try:
response = requests.post(url=url)
except requests.exceptions.RequestException as e:
message = f'{self.chain} failed to post request at {url}. Connection error: {str(e)}.'
log.error(message)
raise RemoteError(message) from e
return response
def _query(self, path: str) -> str:
backoff = INITIAL_BACKOFF
while True:
response = self.session.get(f'{self.prefix}{path}')
if response.status_code == 429 and backoff < self.backoff_limit:
gevent.sleep(backoff)
backoff *= 2
continue
elif response.status_code != 200:
raise RemoteError(
f'Coinpaprika API request {response.url} for {path} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}', )
return response.text
def _query(
self,
module: Literal['validator'],
endpoint: Literal['balancehistory', 'performance', 'eth1'],
encoded_args: str,
) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
"""
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
if endpoint == 'eth1':
query_str = f'{self.url}{module}/{endpoint}/{encoded_args}'
else:
query_str = f'{self.url}{module}/{encoded_args}/{endpoint}'
times = QUERY_RETRY_TIMES
backoff_in_seconds = 10
while True:
try:
response = self.session.get(query_str)
except requests.exceptions.RequestException as e:
raise RemoteError(f'Querying {query_str} failed due to {str(e)}')
if response.status_code == 429:
if times == 0:
raise RemoteError(
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text} after 5 retries',
)
# We got rate limited. Let's try incremental backoff
gevent.sleep(backoff_in_seconds * (QUERY_RETRY_TIMES - times + 1))
continue
else:
break
if response.status_code != 200:
raise RemoteError(
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}',
)
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Beaconchain API returned invalid JSON response: {response.text}')
if json_ret.get('status') != 'OK':
raise RemoteError(f'Beaconchain API returned non-OK status. Response: {json_ret}')
if 'data' not in json_ret:
raise RemoteError(f'Beaconchain API did not contain a data key. Response: {json_ret}')
return json_ret['data']
def _check_node_synchronization(self, node_interface: SubstrateInterface) -> BlockNumber:
"""Check the node synchronization comparing the last block obtained via
the node interface against the last block obtained via Subscan API.
Return the last block obtained via the node interface.
May raise:
- RemoteError: the last block/chain metadata requests fail or
there is an error deserializing the chain metadata.
"""
# Last block via node interface
last_block = self._get_last_block(node_interface=node_interface)
# Last block via Subscan API
try:
chain_metadata = self._request_chain_metadata()
except RemoteError:
self.msg_aggregator.add_warning(
f'Unable to verify that {self.chain} node at endpoint {node_interface.url} '
f'is synced with the chain. Balances and other queries may be incorrect.',
)
return last_block
# Check node synchronization
try:
metadata_last_block = BlockNumber(
deserialize_int_from_str(
symbol=chain_metadata['data']['blockNum'],
location='subscan api',
),
)
except (KeyError, DeserializationError) as e:
message = f'{self.chain} failed to deserialize the chain metadata response: {str(e)}.'
log.error(message, chain_metadata=chain_metadata)
raise RemoteError(message) from e
log.debug(
f'{self.chain} subscan API metadata last block',
metadata_last_block=metadata_last_block,
)
if metadata_last_block - last_block > self.chain.blocks_threshold():
self.msg_aggregator.add_warning(
f'Found that {self.chain} node at endpoint {node_interface.url} '
f'is not synced with the chain. Node last block is {last_block}, '
f'expected last block is {metadata_last_block}. '
f'Balances and other queries may be incorrect.',
)
return last_block
def _get_chain_id(self, node_interface: SubstrateInterface) -> SubstrateChainId:
"""Return the chain identifier.
"""
log.debug(f'{self.chain} querying chain ID', url=node_interface.url)
try:
chain_id = node_interface.chain
except (requests.exceptions.RequestException, SubstrateRequestException) as e:
message = (
f'{self.chain} failed to request chain ID '
f'at endpoint: {node_interface.url} due to: {str(e)}.'
)
log.error(message)
raise RemoteError(message) from e
log.debug(f'{self.chain} chain ID', chain_id=chain_id)
return SubstrateChainId(chain_id)
def _check_chain_id(self, node_interface: SubstrateInterface) -> None:
"""Validate a node connects to the expected chain.
May raise:
- RemoteError: the chain ID request fails, or the chain ID is not the
expected one.
"""
# Check connection and chain ID
chain_id = self._get_chain_id(node_interface=node_interface)
if chain_id != str(self.chain):
message = (
f'{self.chain} found unexpected chain {chain_id} when attempted '
f'to connect to node at endpoint: {node_interface.url}, ')
log.error(message)
raise RemoteError(message)
def wait_until_a_node_is_available(
substrate_manager: SubstrateManager,
seconds: int,
) -> None:
"""Temporarily suspends the caller execution until a node is available or
this function timeouts.
"""
try:
with gevent.Timeout(seconds):
while len(substrate_manager.available_nodes_call_order) == 0:
gevent.sleep(0.1)
except gevent.Timeout as e:
chain = substrate_manager.chain
raise RemoteError(
f"{chain} manager does not have nodes availables after waiting "
f"{seconds} seconds. {chain} balances won't be queried.", ) from e
def api_query(self,
method: str,
options: Optional[Dict] = None) -> Union[List, Dict]:
if not options:
options = {}
with self.lock:
# Protect this region with a lock since binance will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
if method in V3_ENDPOINTS:
api_version = 3
# Recommended recvWindows is 5000 but we get timeouts with it
options['recvWindow'] = 10000
options['timestamp'] = str(int(time.time() * 1000))
signature = hmac.new(self.secret,
urlencode(options).encode('utf-8'),
hashlib.sha256).hexdigest()
options['signature'] = signature
elif method in V1_ENDPOINTS:
api_version = 1
else:
raise ValueError(
'Unexpected binance api method {}'.format(method))
request_url = self.uri + 'v' + str(
api_version) + '/' + method + '?'
request_url += urlencode(options)
log.debug('Binance API request', request_url=request_url)
response = self.session.get(request_url)
if response.status_code != 200:
result = rlk_jsonloads(response.text)
raise RemoteError(
'Binance API request {} for {} failed with HTTP status '
'code: {}, error code: {} and error message: {}'.format(
response.url,
method,
response.status_code,
result['code'],
result['msg'],
))
json_ret = rlk_jsonloads(response.text)
return json_ret
