def _single_query(self, command: str,
req: Dict[str, Any]) -> Optional[requests.Response]:
"""A single api query for poloniex
Returns the response if all went well or None if a recoverable poloniex
error occured such as a 504.
Can raise:
- RemoteError if there is a problem with the response
- ConnectionError if there is a problem connecting to poloniex.
"""
if command in ('returnTicker', 'returnCurrencies'):
log.debug(f'Querying poloniex for {command}')
response = self.session.get(self.public_uri + command)
else:
req['command'] = command
with self.nonce_lock:
# Protect this region with a lock since poloniex will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
req['nonce'] = ts_now_in_ms()
post_data = str.encode(urlencode(req))
sign = hmac.new(self.secret, post_data,
hashlib.sha512).hexdigest()
self.session.headers.update({'Sign': sign})
response = self.session.post('https://poloniex.com/tradingApi',
req)
if response.status_code == 504:
# backoff and repeat
return None
if response.status_code != 200:
raise RemoteError(
f'Poloniex query responded with error status code: {response.status_code}'
f' and text: {response.text}', )
# else all is good
return response
def _single_api_query(
self,
request_url: str,
options: Dict[str, Any],
method: Literal['get', 'put', 'delete'],
public_endpoint: bool = False,
) -> requests.Response:
payload = '' if method == 'get' else json.dumps(options)
if not public_endpoint:
api_content_hash = hashlib.sha512(payload.encode()).hexdigest()
api_timestamp = str(ts_now_in_ms())
presign_str = api_timestamp + request_url + method.upper(
) + api_content_hash
signature = hmac.new(
self.secret,
presign_str.encode(),
hashlib.sha512,
).hexdigest()
self.session.headers.update({
'Api-Key': self.api_key,
'Api-Timestamp': api_timestamp,
'Api-Content-Hash': api_content_hash,
'Api-Signature': signature,
})
else:
self.session.headers.pop('Api-Key')
log.debug('Bittrex v3 API query', request_url=request_url)
try:
response = self.session.request(
method=method,
url=request_url,
json=options if method != 'get' else None,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bittrex API request failed due to {str(e)}') from e
return response
def request_get(uri: str,
timeout: int = ALL_REMOTES_TIMEOUT) -> Union[Dict, List]:
# TODO make this a bit more smart. Perhaps conditional on the type of request.
# Not all requests would need repeated attempts
response = retry_calls(
5,
'',
uri,
requests.get,
uri,
)
if response.status_code != 200:
raise RemoteError('Get {} returned status code {}'.format(
uri, response.status_code))
try:
result = rlk_jsonloads(response.text)
except json.decoder.JSONDecodeError:
raise ValueError('{} returned malformed json'.format(uri))
return result
def _get_vaults_of_address(
self,
user_address: ChecksumEthAddress,
proxy_address: ChecksumEthAddress,
) -> List[MakerdaoVault]:
"""Gets the vaults of a single address
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result.
- BlockchainQueryError if an ethereum node is used and the contract call
queries fail for some reason
"""
result = MAKERDAO_GET_CDPS.call(
ethereum=self.ethereum,
method_name='getCdpsAsc',
arguments=[MAKERDAO_CDP_MANAGER.address, proxy_address],
)
vaults = []
for idx, identifier in enumerate(result[0]):
try:
urn = deserialize_ethereum_address(result[1][idx])
except DeserializationError as e:
raise RemoteError(
f'Failed to deserialize address {result[1][idx]} '
f'when processing vaults of {user_address}',
) from e
vault = self._query_vault_data(
identifier=identifier,
owner=user_address,
urn=urn,
ilk=result[2][idx],
)
if vault:
vaults.append(vault)
self.vault_mappings[user_address].append(vault)
return vaults
def _query_public(self,
method: str,
req: Optional[dict] = None) -> Union[Dict, str]:
"""API queries that do not require a valid key/secret pair.
Arguments:
method -- API method name (string, no default)
req -- additional API request parameters (default: {})
"""
if req is None:
req = {}
urlpath = f'{KRAKEN_BASE_URL}/{KRAKEN_API_VERSION}/public/{method}'
try:
response = self.session.post(urlpath,
data=req,
timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Kraken API request failed due to {str(e)}') from e
self._manage_call_counter(method)
return _check_and_get_response(response, method)
def query_statistics_renderer(self) -> str:
"""Queries for the source of the statistics_renderer from the server
Raises RemoteError if there are problems reaching the server or if
there is an error returned by the server
"""
signature, data = self.sign('statistics_renderer')
self.session.headers.update({ # type: ignore
'API-SIGN': base64.b64encode(signature.digest()),
})
try:
response = self.session.get(
self.uri + 'statistics_renderer',
data=data,
timeout=ROTKEHLCHEN_SERVER_TIMEOUT,
)
except requests.ConnectionError:
raise RemoteError('Could not connect to rotkehlchen server')
result = _process_dict_response(response)
return result['data']
def retry_calls(
times: int,
location: str,
method: str,
function: Callable[..., Any],
*args: Any,
) -> Any:
tries = times
while True:
try:
result = function(*args)
return result
except (requests.exceptions.ConnectionError,
RecoverableRequestError) as e:
if isinstance(e, RecoverableRequestError):
time.sleep(5)
tries -= 1
if tries == 0:
raise RemoteError(
"{} query for {} failed after {} tries. Reason: {}".format(
location, method, times, e))
def _get_account_proxy(
self, address: ChecksumEthAddress) -> Optional[ChecksumEthAddress]:
"""Checks if a DSR proxy exists for the given address and returns it if it does
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result. Also this error can be raised
if there is a problem deserializing the result address.
- BlockchainQueryError if an ethereum node is used and the contract call
queries fail for some reason
"""
result = MAKERDAO_PROXY_REGISTRY.call(self.ethereum,
'proxies',
arguments=[address])
if int(result, 16) != 0:
try:
return deserialize_ethereum_address(result)
except DeserializationError as e:
msg = f'Failed to deserialize {result} DSR proxy for address {address}'
log.error(msg)
raise RemoteError(msg) from e
return None
def query_cryptocompare_for_fiat_price(asset: Asset) -> Price:
log.debug('Get usd price from cryptocompare', asset=asset)
cc_asset_str = asset.to_cryptocompare()
resp = retry_calls(
times=5,
location='find_usd_price',
handle_429=False,
backoff_in_seconds=0,
method_name='requests.get',
function=requests.get,
# function's arguments
url=(
u'https://min-api.cryptocompare.com/data/price?'
'fsym={}&tsyms=USD'.format(cc_asset_str)
),
)
if resp.status_code != 200:
raise RemoteError('Cant reach cryptocompare to get USD value of {}'.format(asset))
resp = rlk_jsonloads_dict(resp.text)
# If there is an error in the response skip this token
if 'USD' not in resp:
error_message = ''
if resp.get('Response', None) == 'Error':
error_message = resp.get('Message', '')
log.error(
'Cryptocompare usd price query failed',
asset=asset,
error=error_message,
)
return Price(ZERO)
price = Price(FVal(resp['USD']))
log.debug('Got usd price from cryptocompare', asset=asset, price=price)
return price
def get_performance(self, validator_indices: List[int]) -> Dict[int, ValidatorPerformance]:
"""Get the performance of all the validators given from the indices list
Queries in chunks of 100 due to api limitations
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
chunks = list(get_chunks(validator_indices, n=100))
data = []
for chunk in chunks:
result = self._query(
module='validator',
endpoint='performance',
encoded_args=','.join(str(x) for x in chunk),
)
if isinstance(result, list):
data.extend(result)
else:
data.append(result)
try:
performance = {}
for entry in data:
index = entry['validatorindex']
performance[index] = ValidatorPerformance(
balance=entry['balance'],
performance_1d=entry['performance1d'],
performance_1w=entry['performance7d'],
performance_1m=entry['performance31d'],
performance_1y=entry['performance365d'],
)
except KeyError as e:
raise RemoteError(
f'Beaconchai.in performance response processing error. Missing key entry {str(e)}',
)
return performance
def upload_data(
self,
data_blob: B64EncodedBytes,
our_hash: str,
last_modify_ts: Timestamp,
compression_type: Literal['zlib'],
) -> Dict:
"""Uploads data to the server and returns the response dict
Raises RemoteError if there are problems reaching the server or if
there is an error returned by the server
"""
signature, data = self.sign(
'save_data',
data_blob=data_blob,
original_hash=our_hash,
last_modify_ts=last_modify_ts,
index=0,
length=len(data_blob),
compression=compression_type,
)
self.session.headers.update({
'API-SIGN':
base64.b64encode(signature.digest()), # type: ignore
})
try:
response = self.session.put(
self.uri + 'save_data',
data=data,
timeout=ROTKEHLCHEN_SERVER_TIMEOUT * 10,
)
except requests.exceptions.RequestException as e:
msg = f'Could not connect to rotki server due to {str(e)}'
logger.error(msg)
raise RemoteError(msg) from e
return _process_dict_response(response)
def _get_new_staking_events_graph(
self,
addresses: List[ChecksumEthAddress],
identity_address_map: Dict[ChecksumAddress, ChecksumAddress],
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> List[Union[Bond, Unbond, UnbondRequest, ChannelWithdraw]]:
"""Returns events of the addresses within the time range and inserts/updates
the used query range of the addresses as well.
May raise:
- RemoteError: when there is a problem either querying the subgraph or
deserializing the events.
"""
all_events: List[Union[Bond, Unbond, UnbondRequest,
ChannelWithdraw]] = []
for event_type_ in AdexEventType:
try:
# TODO: fix. type -> overload does not work well with enum in this case
events = self._get_staking_events_graph( # type: ignore
addresses=addresses,
identity_address_map=identity_address_map,
event_type=event_type_,
from_timestamp=from_timestamp,
to_timestamp=to_timestamp,
)
except DeserializationError as e:
raise RemoteError(e) from e
all_events.extend(events)
for address in addresses:
self.database.update_used_query_range(
name=f'{ADEX_EVENTS_PREFIX}_{address}',
start_ts=from_timestamp,
end_ts=to_timestamp,
)
return all_events
def _query_private(self, method: str, req: Optional[dict] = None) -> Union[Dict, str]:
"""API queries that require a valid key/secret pair.
Arguments:
method -- API method name (string, no default)
req -- additional API request parameters (default: {})
"""
if req is None:
req = {}
urlpath = '/' + KRAKEN_API_VERSION + '/private/' + method
with self.nonce_lock:
# Protect this section, or else, non increasing nonces will be rejected
req['nonce'] = int(1000 * time.time())
post_data = urlencode(req)
# any unicode strings must be turned to bytes
hashable = (str(req['nonce']) + post_data).encode()
message = urlpath.encode() + hashlib.sha256(hashable).digest()
signature = hmac.new(
base64.b64decode(self.secret),
message,
hashlib.sha512,
)
self.session.headers.update({
'API-Sign': base64.b64encode(signature.digest()), # type: ignore
})
try:
response = self.session.post(
KRAKEN_BASE_URL + urlpath,
data=post_data.encode(),
)
except requests.exceptions.ConnectionError as e:
raise RemoteError(f'Kraken API request failed due to {str(e)}')
self._manage_call_counter(method)
return _check_and_get_response(response, method)
def query_balances(self) -> ExchangeQueryBalances:
"""Return the account balances
May raise RemoteError
"""
accounts_response = self._api_query(KucoinCase.BALANCES)
if accounts_response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=accounts_response,
case=KucoinCase.BALANCES,
)
return result, msg
try:
response_dict = jsonloads_dict(accounts_response.text)
except JSONDecodeError as e:
msg = f'Kucoin balances returned an invalid JSON response: {accounts_response.text}.'
log.error(msg)
raise RemoteError(msg) from e
account_balances = self._deserialize_accounts_balances(
response_dict=response_dict)
return account_balances, ''
def _get_blocknumber_by_time_from_subgraph(self, ts: Timestamp) -> int:
"""Queries Ethereum Blocks Subgraph for closest block at or before given timestamp"""
response = self.blocks_subgraph.query(
f"""
{{
blocks(
first: 1, orderBy: timestamp, orderDirection: desc,
where: {{timestamp_lte: "{ts}"}}
) {{
id
number
timestamp
}}
}}
""", )
try:
result = int(response['blocks'][0]['number'])
except (IndexError, KeyError) as e:
raise RemoteError(
f'Got unexpected ethereum blocks subgraph response: {response}',
) from e
else:
return result
def get_blocknumber_by_time(self, ts: Timestamp) -> int:
"""Performs the etherscan api call to get the blocknumber by a specific timestamp
May raise:
- RemoteError if there are any problems with reaching Etherscan or if
an unexpected response is returned
"""
if ts < 1438269989:
return 0 # etherscan does not handle timestamps close and before genesis well
options = {'timestamp': ts, 'closest': 'before'}
result = self._query(
module='block',
action='getblocknobytime',
options=options,
)
if not isinstance(result, int):
# At this point the blocknumber string returned by etherscan should be an int
raise RemoteError(
f'Got unexpected etherscan response: {result} to getblocknobytime call',
)
return result
def find_usd_price(
self,
asset: typing.Asset,
asset_btc_price: Optional[FVal] = None,
) -> FVal:
if self.kraken and self.kraken.first_connection_made and asset_btc_price is not None:
return self.query_kraken_for_price(asset, asset_btc_price)
# Adjust some ETH tokens to how cryptocompare knows them
if asset == S_RDN:
# remove this if cryptocompare changes the symbol
asset = cast(typing.EthToken, 'RDN*')
if asset == S_DATACOIN:
asset = cast(typing.NonEthTokenBlockchainAsset, 'DATA')
resp = retry_calls(
5, 'find_usd_price', 'requests.get', requests.get,
u'https://min-api.cryptocompare.com/data/price?'
'fsym={}&tsyms=USD'.format(asset))
if resp.status_code != 200:
raise RemoteError(
'Cant reach cryptocompare to get USD value of {}'.format(
asset))
resp = rlk_jsonloads(resp.text)
# If there is an error in the response skip this token
if 'USD' not in resp:
if resp['Response'] == 'Error':
print(
'Could not query USD price for {}. Error: "{}"'.format(
asset, resp['Message']), )
else:
print('Could not query USD price for {}'.format(asset))
return FVal(0)
return FVal(resp['USD'])
def watcher_query(
self,
method: Literal['GET', 'PUT', 'PATCH', 'DELETE'],
data: Optional[Dict[str, Any]],
) -> Any:
if data is None:
data = {}
signature, _ = self.sign('watchers', **data)
self.session.headers.update({
'API-SIGN': base64.b64encode(signature.digest()), # type: ignore
})
try:
response = self.session.request(
method=method,
url=self.uri + 'watchers',
json=data,
timeout=ROTKEHLCHEN_SERVER_TIMEOUT,
)
except requests.exceptions.ConnectionError:
raise RemoteError('Could not connect to rotki server')
return _decode_premium_json(response)
def pull_data(self) -> Dict:
"""Pulls data from the server and returns the response dict
Returns None if there is no DB saved in the server.
Raises RemoteError if there are problems reaching the server or if
there is an error returned by the server
"""
signature, data = self.sign('get_saved_data')
self.session.headers.update({
'API-SIGN':
base64.b64encode(signature.digest()), # type: ignore
})
try:
response = self.session.get(
self.uri + 'get_saved_data',
data=data,
timeout=ROTKEHLCHEN_SERVER_TIMEOUT,
)
except requests.exceptions.ConnectionError:
raise RemoteError('Could not connect to rotki server')
return _process_dict_response(response)
def query(self, method: Callable, call_order: Sequence[NodeName], **kwargs: Any) -> Any:
"""Queries ethereum related data by performing the provided method to all given nodes
The first node in the call order that gets a succcesful response returns.
If none get a result then a remote error is raised
"""
for node in call_order:
web3 = self.web3_mapping.get(node, None)
if web3 is None and node != NodeName.ETHERSCAN:
continue
try:
result = method(web3, **kwargs)
except (RemoteError, BlockchainQueryError):
# Catch all possible errors here and just try next node call
continue
return result
# no node in the call order list was succesfully queried
raise RemoteError(
f'Failed to query {str(callable)} after trying the following '
f'nodes: {[str(x) for x in call_order]}',
)
def get_transaction_receipt(
self,
tx_hash: str,
) -> Dict[str, Any]:
tx_receipt = self.covalent.get_transaction_receipt(tx_hash)
if tx_receipt is None:
tx_receipt = self.w3.eth.get_transaction(tx_hash).__dict__ # type: ignore
return tx_receipt
try:
# Turn hex numbers to int
tx_receipt.pop('from_address_label', None)
tx_receipt.pop('to_address_label', None)
block_number = tx_receipt['block_height']
tx_receipt['blockNumber'] = tx_receipt.pop('block_height', None)
tx_receipt['cumulativeGasUsed'] = tx_receipt.pop('gas_spent', None)
tx_receipt['gasUsed'] = tx_receipt['cumulativeGasUsed']
successful = tx_receipt.pop('successful', None)
tx_receipt['status'] = 1 if successful else 0
tx_receipt['transactionIndex'] = 0
txhash = tx_receipt.pop('tx_hash')
tx_receipt['hash'] = txhash
# TODO input and nonce is decoded in Covalent api, encoded in future
tx_receipt['input'] = '0x'
tx_receipt['nonce'] = 0
for index, receipt_log in enumerate(tx_receipt['log_events']):
receipt_log['blockNumber'] = block_number
receipt_log['logIndex'] = receipt_log.pop('log_offset', None)
receipt_log['transactionIndex'] = 0
tx_receipt['log_events'][index] = receipt_log
except (DeserializationError, ValueError) as e:
raise RemoteError(
f'Couldnt deserialize transaction receipt '
f'data from covalent {tx_receipt} due to {str(e)}',
) from e
return tx_receipt
def query(self, method: Callable, call_order: Sequence[NodeName],
**kwargs: Any) -> Any:
"""Queries ethereum related data by performing the provided method to all given nodes
The first node in the call order that gets a succcesful response returns.
If none get a result then a remote error is raised
"""
for node in call_order:
web3 = self.web3_mapping.get(node, None)
if web3 is None and node != NodeName.ETHERSCAN:
continue
try:
result = method(web3, **kwargs)
except (
RemoteError,
requests.exceptions.RequestException,
BlockchainQueryError,
TransactionNotFound,
BlockNotFound,
KeyError, # saw this happen inside web3.py if resulting json contains unexpected key. Probably fixed as written below, but no risking it. # noqa: E501
BadResponseFormat, # should replace the above KeyError after https://github.com/ethereum/web3.py/pull/2188 # noqa: E501
) as e:
log.warning(
f'Failed to query {node} for {str(method)} due to {str(e)}'
)
# Catch all possible errors here and just try next node call
continue
return result
# no node in the call order list was succesfully queried
raise RemoteError(
f'Failed to query {str(method)} after trying the following '
f'nodes: {[str(x) for x in call_order]}. Check logs for details.',
)
def _generate_reports(
self,
start_ts: Timestamp,
end_ts: Timestamp,
report_type: Literal['fills', 'account'],
tempdir: str,
) -> List[str]:
"""
Generates all the reports to get historical data from coinbase.
https://docs.pro.coinbase.com/#reports
There are 2 type of reports.
1. Fill reports which are per product id (market)
2. Account reports which are per account id
The fill reports have the following data format:
portfolio,trade id,product,side,created at,size,size unit,price,fee,
total,price/fee/total unit
The account reports have the following data format:
portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id
Returns a list of filepaths where the reports were written.
- Raises the same exceptions as _api_query()
- Can raise KeyError if the API does not return the expected response format.
"""
start_date = timestamp_to_iso8601(start_ts)
end_date = timestamp_to_iso8601(end_ts)
if report_type == 'fills':
account_or_product_ids = self._get_products_ids()
identifier_key = 'product_id'
else:
account_or_product_ids = self._get_account_ids()
identifier_key = 'account_id'
report_ids = []
options = {
'type': report_type,
'start_date': start_date,
'end_date': end_date,
'format': 'csv',
# The only way to disable emailing the report link is to give an invalid link
'email': '*****@*****.**',
}
for identifier in account_or_product_ids:
options[identifier_key] = identifier
post_result = self._api_query('reports',
request_method='POST',
options=options)
report_ids.append(post_result['id'])
# At this point all reports must have been queued for creation at the server
# Now wait until they are ready and pull them one by one
report_paths = []
last_change_ts = ts_now()
while True:
finished_ids_indices = []
for idx, report_id in enumerate(report_ids):
get_result = self._api_query(f'reports/{report_id}',
request_method='GET')
# Have to add assert here for mypy since the endpoint string is
# a variable string and can't be overloaded and type checked
assert isinstance(get_result, dict)
if get_result['status'] != 'ready':
continue
# a report is ready here so let's reset the timer
last_change_ts = ts_now()
file_url = get_result['file_url']
response = requests.get(file_url)
length = len(response.content)
# empty fill reports have length of 95, empty account reports 85
# So we assume a report of more than 100 chars has data.
if length > 100:
log.debug(
f'Got a populated report for id: {report_id}. Writing it to disk'
)
filepath = os.path.join(tempdir, f'report_{report_id}.csv')
with open(filepath, 'wb') as f:
f.write(response.content)
report_paths.append(filepath)
else:
log.debug(
f'Got report for id: {report_id} with length {length}. Skipping it'
)
finished_ids_indices.append(idx)
if ts_now() - last_change_ts > SECS_TO_WAIT_FOR_REPORT:
raise RemoteError(
f'There has been no response from CoinbasePro reports for over '
f' {MINS_TO_WAIT_FOR_REPORT} minutes. Bailing out.', )
# Delete the report ids that have been downloaded. Note: reverse order
# so that we don't mess up the indices
for idx in reversed(finished_ids_indices):
del report_ids[idx]
# When there is no more ids to query break out of the loop
if len(report_ids) == 0:
break
return report_paths
def _get_logs(
self,
web3: Optional[Web3],
contract_address: ChecksumEthAddress,
abi: List,
event_name: str,
argument_filters: Dict[str, Any],
from_block: int,
to_block: Union[int, Literal['latest']] = 'latest',
) -> List[Dict[str, Any]]:
"""Queries logs of an ethereum contract
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result
"""
event_abi = find_matching_event_abi(abi=abi, event_name=event_name)
_, filter_args = construct_event_filter_params(
event_abi=event_abi,
abi_codec=Web3().codec,
contract_address=contract_address,
argument_filters=argument_filters,
fromBlock=from_block,
toBlock=to_block,
)
if event_abi['anonymous']:
# web3.py does not handle the anonymous events correctly and adds the first topic
filter_args['topics'] = filter_args['topics'][1:]
events: List[Dict[str, Any]] = []
start_block = from_block
if web3 is not None:
until_block = web3.eth.blockNumber if to_block == 'latest' else to_block
while start_block <= until_block:
filter_args['fromBlock'] = start_block
end_block = min(start_block + 250000, until_block)
filter_args['toBlock'] = end_block
log.debug(
'Querying node for contract event',
contract_address=contract_address,
event_name=event_name,
argument_filters=argument_filters,
from_block=filter_args['fromBlock'],
to_block=filter_args['toBlock'],
)
# WTF: for some reason the first time we get in here the loop resets
# to the start without querying eth_getLogs and ends up with double logging
new_events_web3 = cast(List[Dict[str, Any]],
web3.eth.getLogs(filter_args))
# Turn all HexBytes into hex strings
for e_idx, event in enumerate(new_events_web3):
new_events_web3[e_idx]['blockHash'] = event[
'blockHash'].hex()
new_topics = []
for topic in (event['topics']):
new_topics.append(topic.hex())
new_events_web3[e_idx]['topics'] = new_topics
new_events_web3[e_idx]['transactionHash'] = event[
'transactionHash'].hex()
start_block = end_block + 1
events.extend(new_events_web3)
else: # etherscan
until_block = (self.etherscan.get_latest_block_number()
if to_block == 'latest' else to_block)
while start_block <= until_block:
end_block = min(start_block + 300000, until_block)
new_events = self.etherscan.get_logs(
contract_address=contract_address,
topics=filter_args['topics'], # type: ignore
from_block=start_block,
to_block=end_block,
)
# Turn all Hex ints to ints
for e_idx, event in enumerate(new_events):
try:
new_events[e_idx]['address'] = to_checksum_address(
event['address'])
new_events[e_idx][
'blockNumber'] = deserialize_int_from_hex(
symbol=event['blockNumber'],
location='etherscan log query',
)
new_events[e_idx][
'timeStamp'] = deserialize_int_from_hex(
symbol=event['timeStamp'],
location='etherscan log query',
)
new_events[e_idx][
'gasPrice'] = deserialize_int_from_hex(
symbol=event['gasPrice'],
location='etherscan log query',
)
new_events[e_idx][
'gasUsed'] = deserialize_int_from_hex(
symbol=event['gasUsed'],
location='etherscan log query',
)
new_events[e_idx][
'logIndex'] = deserialize_int_from_hex(
symbol=event['logIndex'],
location='etherscan log query',
)
new_events[e_idx][
'transactionIndex'] = deserialize_int_from_hex(
symbol=event['transactionIndex'],
location='etherscan log query',
)
except DeserializationError as e:
raise RemoteError(
'Couldnt decode an etherscan event due to {str(e)}}',
) from e
start_block = end_block + 1
events.extend(new_events)
return events
def _api_query( # noqa: F811
self,
endpoint: str,
request_method: Literal['GET', 'POST'] = 'GET',
options: Optional[Dict[str, Any]] = None,
) -> Union[List[Any], Dict[str, Any]]:
"""Performs a coinbase PRO API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Raises RemoteError if something went wrong with connecting or reading from the exchange
Raises CoinbaseProPermissionError if the API Key does not have sufficient
permissions for the endpoint
"""
request_url = f'/{endpoint}'
timestamp = str(int(time.time()))
if options:
stringified_options = json.dumps(options, separators=(',', ':'))
else:
stringified_options = ''
options = {}
message = timestamp + request_method + request_url + stringified_options
log.debug(
'Coinbase Pro API query',
request_method=request_method,
request_url=request_url,
options=options,
)
if 'products' not in endpoint:
try:
signature = hmac.new(
b64decode(self.secret),
message.encode(),
hashlib.sha256,
).digest()
except binascii.Error:
raise RemoteError('Provided API Secret is invalid')
self.session.headers.update({
'CB-ACCESS-SIGN':
b64encode(signature).decode('utf-8'),
'CB-ACCESS-TIMESTAMP':
timestamp,
})
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
full_url = self.base_uri + request_url
try:
response = self.session.request(
request_method.lower(),
full_url,
data=stringified_options,
)
except requests.exceptions.ConnectionError as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at '
f'{full_url} connection error: {str(e)}', )
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
# Backoff a bit by sleeping. Sleep more, the more retries have been made
gevent.sleep(QUERY_RETRY_TIMES / retries_left)
retries_left -= 1
else:
# get out of the retry loop, we did not get 429 complaint
break
json_ret: Union[List[Any], Dict[str, Any]]
if response.status_code == HTTPStatus.BAD_REQUEST:
json_ret = rlk_jsonloads_dict(response.text)
if json_ret['message'] == 'invalid signature':
raise CoinbaseProPermissionError(
f'While doing {request_method} at {endpoint} endpoint the API secret '
f'created an invalid signature.', )
# else do nothing and a generic remote error will be thrown below
elif response.status_code == HTTPStatus.FORBIDDEN:
raise CoinbaseProPermissionError(
f'API key does not have permission for {endpoint}', )
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
loading_function: Union[Callable[[str], Dict[str, Any]],
Callable[[str], List[Any]]]
if any(x in endpoint for x in ('accounts', 'products')):
loading_function = rlk_jsonloads_list
else:
loading_function = rlk_jsonloads_dict
try:
json_ret = loading_function(response.text)
except JSONDecodeError:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} '
f'returned invalid JSON response: {response.text}', )
return json_ret
def _get_logs(
self,
web3: Optional[Web3],
contract_address: ChecksumEthAddress,
abi: List,
event_name: str,
argument_filters: Dict[str, Any],
from_block: int,
to_block: Union[int, Literal['latest']] = 'latest',
) -> List[Dict[str, Any]]:
"""Queries logs of an ethereum contract
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result
"""
event_abi = find_matching_event_abi(abi=abi, event_name=event_name)
_, filter_args = construct_event_filter_params(
event_abi=event_abi,
abi_codec=Web3().codec,
contract_address=contract_address,
argument_filters=argument_filters,
fromBlock=from_block,
toBlock=to_block,
)
if event_abi['anonymous']:
# web3.py does not handle the anonymous events correctly and adds the first topic
filter_args['topics'] = filter_args['topics'][1:]
events: List[Dict[str, Any]] = []
start_block = from_block
if web3 is not None:
events = _query_web3_get_logs(
web3=web3,
filter_args=filter_args,
from_block=from_block,
to_block=to_block,
contract_address=contract_address,
event_name=event_name,
argument_filters=argument_filters,
)
else: # etherscan
until_block = (self.etherscan.get_latest_block_number()
if to_block == 'latest' else to_block)
blocks_step = 300000
while start_block <= until_block:
while True: # loop to continuously reduce block range if need b
end_block = min(start_block + blocks_step, until_block)
try:
new_events = self.etherscan.get_logs(
contract_address=contract_address,
topics=filter_args['topics'], # type: ignore
from_block=start_block,
to_block=end_block,
)
except RemoteError as e:
if 'Please select a smaller result dataset' in str(e):
blocks_step = blocks_step // 2
if blocks_step < 100:
raise # stop trying
# else try with the smaller step
continue
# else some other error
raise
break # we must have a result
# Turn all Hex ints to ints
for e_idx, event in enumerate(new_events):
try:
block_number = deserialize_int_from_hex(
symbol=event['blockNumber'],
location='etherscan log query',
)
log_index = deserialize_int_from_hex(
symbol=event['logIndex'],
location='etherscan log query',
)
# Try to see if the event is a duplicate that got returned
# in the previous iteration
for previous_event in reversed(events):
if previous_event['blockNumber'] < block_number:
break
same_event = (previous_event['logIndex']
== log_index
and previous_event['transactionHash']
== event['transactionHash'])
if same_event:
events.pop()
new_events[e_idx][
'address'] = deserialize_ethereum_address(
event['address'], )
new_events[e_idx]['blockNumber'] = block_number
new_events[e_idx][
'timeStamp'] = deserialize_int_from_hex(
symbol=event['timeStamp'],
location='etherscan log query',
)
new_events[e_idx][
'gasPrice'] = deserialize_int_from_hex(
symbol=event['gasPrice'],
location='etherscan log query',
)
new_events[e_idx][
'gasUsed'] = deserialize_int_from_hex(
symbol=event['gasUsed'],
location='etherscan log query',
)
new_events[e_idx]['logIndex'] = log_index
new_events[e_idx][
'transactionIndex'] = deserialize_int_from_hex(
symbol=event['transactionIndex'],
location='etherscan log query',
)
except DeserializationError as e:
raise RemoteError(
'Couldnt decode an etherscan event due to {str(e)}}',
) from e
# etherscan will only return 1000 events in one go. If more than 1000
# are returned such as when no filter args are provided then continue
# the query from the last block
if len(new_events) == 1000:
start_block = new_events[-1]['blockNumber']
else:
start_block = end_block + 1
events.extend(new_events)
return events
def _api_query(
self,
verb: str,
path: str,
options: Optional[Dict] = None,
) -> Union[List, Dict]:
"""
Queries Bitmex with the given verb for the given path and options
"""
assert verb in ('get', 'post', 'push'), (
'Given verb {} is not a valid HTTP verb'.format(verb))
# 20 seconds expiration
expires = int(time.time()) + 20
request_path_no_args = '/api/v1/' + path
data = ''
if not options:
options = {}
request_path = request_path_no_args
else:
request_path = request_path_no_args + '?' + urlencode(options)
if path in BITMEX_PRIVATE_ENDPOINTS:
self._generate_signature(
verb=verb,
path=request_path_no_args,
expires=expires,
data=data,
)
self.session.headers.update({
'api-expires': str(expires),
})
if data != '':
self.session.headers.update({
'Content-Type': 'application/json',
'Content-Length': str(len(data)),
})
request_url = self.uri + request_path
response = getattr(self.session, verb)(request_url, data=data)
if response.status_code not in (200, 401):
raise RemoteError(
'Bitmex api request for {} failed with HTTP status code {}'.
format(
response.url,
response.status_code,
))
try:
json_ret = rlk_jsonloads(response.text)
except JSONDecodeError:
raise RemoteError('Bitmex returned invalid JSON response')
if 'error' in json_ret:
raise RemoteError(json_ret['error']['message'])
return json_ret
def get_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> List[PriceHistoryEntry]:
"""
Get historical price data from cryptocompare
Returns a sorted list of price entries.
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
cache_key = PairCacheKey(from_asset.identifier + '_' +
to_asset.identifier)
got_cached_value = self._got_cached_price(cache_key, timestamp)
if got_cached_value:
return self.price_history[cache_key].data
now_ts = ts_now()
cryptocompare_hourquerylimit = 2000
calculated_history: List[Dict[str, Any]] = []
if historical_data_start <= timestamp:
end_date = historical_data_start
else:
end_date = timestamp
while True:
pr_end_date = end_date
end_date = Timestamp(end_date +
(cryptocompare_hourquerylimit) * 3600)
log.debug(
'Querying cryptocompare for hourly historical price',
from_asset=from_asset,
to_asset=to_asset,
cryptocompare_hourquerylimit=cryptocompare_hourquerylimit,
end_date=end_date,
)
resp = self.query_endpoint_histohour(
from_asset=from_asset,
to_asset=to_asset,
limit=2000,
to_timestamp=end_date,
)
if pr_end_date != resp['TimeFrom']:
# If we get more than we needed, since we are close to the now_ts
# then skip all the already included entries
diff = pr_end_date - resp['TimeFrom']
# If the start date has less than 3600 secs difference from previous
# end date then do nothing. If it has more skip all already included entries
if diff >= 3600:
if resp['Data'][diff // 3600]['time'] != pr_end_date:
raise RemoteError(
'Unexpected fata format in cryptocompare query_endpoint_histohour. '
'Expected to find the previous date timestamp during '
'cryptocompare historical data fetching', )
# just add only the part from the previous timestamp and on
resp['Data'] = resp['Data'][diff // 3600:]
# The end dates of a cryptocompare query do not match. The end date
# can have up to 3600 secs different to the requested one since this is
# hourly historical data but no more.
end_dates_dont_match = (end_date < now_ts
and resp['TimeTo'] != end_date)
if end_dates_dont_match:
if resp['TimeTo'] - end_date >= 3600:
raise RemoteError(
'Unexpected fata format in cryptocompare query_endpoint_histohour. '
'End dates do not match.', )
else:
# but if it's just a drift within an hour just update the end_date so that
# it can be picked up by the next iterations in the loop
end_date = resp['TimeTo']
# If last time slot and first new are the same, skip the first new slot
last_entry_equal_to_first = (len(calculated_history) != 0
and calculated_history[-1]['time']
== resp['Data'][0]['time'])
if last_entry_equal_to_first:
resp['Data'] = resp['Data'][1:]
calculated_history += resp['Data']
if end_date >= now_ts:
break
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# and now since we actually queried the data let's also cache them
filename = self.data_directory / ('price_history_' + cache_key +
'.json')
log.info(
'Updating price history cache',
filename=filename,
from_asset=from_asset,
to_asset=to_asset,
)
write_history_data_in_file(
data=calculated_history,
filepath=filename,
start_ts=historical_data_start,
end_ts=now_ts,
)
# Finally save the objects in memory and return them
data_including_time = {
'data': calculated_history,
'start_time': historical_data_start,
'end_time': end_date,
}
self.price_history_file[cache_key] = filename
self.price_history[cache_key] = _dict_history_to_data(
data_including_time)
return self.price_history[cache_key].data
def _api_query(self, path: str) -> Dict[str, Any]:
"""Queries cryptocompare
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
querystr = f'https://min-api.cryptocompare.com/data/{path}'
log.debug('Querying cryptocompare', url=querystr)
api_key = self._get_api_key()
if api_key:
querystr += f'&api_key={api_key}'
tries = CRYPTOCOMPARE_QUERY_RETRY_TIMES
while tries >= 0:
try:
response = self.session.get(querystr)
except requests.exceptions.ConnectionError as e:
raise RemoteError(
f'Cryptocompare API request failed due to {str(e)}')
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(
f'Cryptocompare returned invalid JSON response: {response.text}'
)
try:
if json_ret.get('Message', None) == RATE_LIMIT_MSG:
if tries >= 1:
backoff_seconds = 20 / tries
log.debug(
f'Got rate limited by cryptocompare. '
f'Backing off for {backoff_seconds}', )
gevent.sleep(backoff_seconds)
tries -= 1
continue
else:
log.debug(
f'Got rate limited by cryptocompare and did not manage to get a '
f'request through even after {CRYPTOCOMPARE_QUERY_RETRY_TIMES} '
f'incremental backoff retries', )
if json_ret.get('Response', 'Success') != 'Success':
error_message = f'Failed to query cryptocompare for: "{querystr}"'
if 'Message' in json_ret:
error_message += f'. Error: {json_ret["Message"]}'
log.error(
'Cryptocompare query failure',
url=querystr,
error=error_message,
status_code=response.status_code,
)
raise RemoteError(error_message)
return json_ret['Data'] if 'Data' in json_ret else json_ret
except KeyError as e:
raise RemoteError(
f'Unexpected format of Cryptocompare json_response. '
f'Missing key entry for {str(e)}', )
raise AssertionError('We should never get here')
def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]:
if not options:
options = {}
backoff = self.initial_backoff
while True:
with self.lock:
# Protect this region with a lock since binance will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
api_version = 3
# Recommended recvWindows is 5000 but we get timeouts with it
options['recvWindow'] = 10000
options['timestamp'] = str(int(time.time() * 1000))
signature = hmac.new(
self.secret,
urlencode(options).encode('utf-8'),
hashlib.sha256,
).hexdigest()
options['signature'] = signature
elif method in V1_ENDPOINTS:
api_version = 1
else:
raise ValueError('Unexpected binance api method {}'.format(method))
apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
request_url += urlencode(options)
log.debug('Binance API request', request_url=request_url)
response = self.session.get(request_url)
limit_ban = response.status_code == 429 and backoff > self.backoff_limit
if limit_ban or response.status_code not in (200, 429):
code = 'no code found'
msg = 'no message found'
try:
result = rlk_jsonloads(response.text)
if isinstance(result, dict):
code = result.get('code', code)
msg = result.get('msg', msg)
except JSONDecodeError:
pass
raise RemoteError(
'Binance API request {} for {} failed with HTTP status '
'code: {}, error code: {} and error message: {}'.format(
response.url,
method,
response.status_code,
code,
msg,
))
elif response.status_code == 429:
if backoff > self.backoff_limit:
break
# Binance has limits and if we hit them we should backoff
# https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md#limits
log.debug('Got 429 from Binance. Backing off', seconds=backoff)
gevent.sleep(backoff)
backoff = backoff * 2
continue
else:
# success
break
try:
json_ret = rlk_jsonloads(response.text)
except JSONDecodeError:
raise RemoteError(f'Binance returned invalid JSON response: {response.text}')
return json_ret
