def get_bitcoin_addresses_balances(
accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
"""Queries blockchain.info or blockstream for the balances of accounts
May raise:
- RemotError if there is a problem querying blockchain.info or blockstream
"""
source = 'blockchain.info'
balances: Dict[BTCAddress, FVal] = {}
try:
if _have_bc1_accounts(accounts):
# if 1 account is bech32 we have to query blockstream. blockchaininfo won't work
source = 'blockstream'
balances = {}
for account in accounts:
url = f'https://blockstream.info/api/address/{account}'
response_data = request_get_dict(url=url,
handle_429=True,
backoff_in_seconds=4)
stats = response_data['chain_stats']
balance = int(stats['funded_txo_sum']) - int(
stats['spent_txo_sum'])
balances[account] = satoshis_to_btc(balance)
else:
params = '|'.join(accounts)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
for idx, entry in enumerate(btc_resp['addresses']):
balances[accounts[idx]] = satoshis_to_btc(
FVal(entry['final_balance']))
except (
requests.exceptions.ConnectionError,
UnableToDecryptRemoteData,
requests.exceptions.Timeout,
) as e:
raise RemoteError(
f'bitcoin external API request for balances failed due to {str(e)}'
) from e # noqa: E501
except KeyError as e:
raise RemoteError(
f'Malformed response when querying bitcoin blockchain via {source}.'
f'Did not find key {e}', ) from e
return balances
def _api_query(self, path: str) -> Dict[str, Any]:
querystr = f'{self.prefix}{path}'
log.debug('Querying cryptocompare', url=querystr)
resp = request_get_dict(querystr)
# These endpoints are wrapped in a response object
wrapped_response = 'all/coinlist' in path or 'histohour' in path
log.debug(f'Wrapped response: {wrapped_response}')
if wrapped_response:
if 'Response' not in resp or resp['Response'] != 'Success':
error_message = 'Failed to query cryptocompare for: "{}"'.format(
querystr)
if 'Message' in resp:
error_message += ". Error: {}".format(resp['Message'])
log.error('Cryptocompare query failure',
url=querystr,
error=error_message)
raise ValueError(error_message)
# for histohour we want all the data, including the wrapper to get TimeFrom
# and TimeTo
if 'histohour' not in path:
return resp['Data']
# else not a wrapped response
return resp
def _query_exchanges_rateapi(base: Asset, quote: Asset) -> Optional[Price]:
assert base.is_fiat(), 'fiat currency should have been provided'
assert quote.is_fiat(), 'fiat currency should have been provided'
log.debug(
'Querying api.exchangeratesapi.io fiat pair',
base_currency=base.identifier,
quote_currency=quote.identifier,
)
querystr = (
f'https://api.exchangeratesapi.io/latest?base={base.identifier}&symbols={quote.identifier}'
)
try:
resp = request_get_dict(querystr)
return Price(FVal(resp['rates'][quote.identifier]))
except (
RemoteError,
KeyError,
requests.exceptions.TooManyRedirects,
UnableToDecryptRemoteData,
):
log.error(
'Querying api.exchangeratesapi.io for fiat pair failed',
base_currency=base.identifier,
quote_currency=quote.identifier,
)
return None
def get_eth_balance(self, account: typing.EthAddress) -> FVal:
if not self.connected:
log.debug(
'Querying etherscan for account balance',
sensitive_log=True,
eth_address=account,
)
eth_resp = request_get_dict(
'https://api.etherscan.io/api?module=account&action=balance&address=%s'
% account,
)
if eth_resp['status'] != 1:
raise ValueError('Failed to query etherscan for accounts balance')
amount = FVal(eth_resp['result'])
log.debug(
'Etherscan account balance result',
sensitive_log=True,
eth_address=account,
wei_amount=amount,
)
return from_wei(amount)
else:
wei_amount = self.web3.eth.getBalance(account) # pylint: disable=no-member
log.debug(
'Ethereum node account balance result',
sensitive_log=True,
eth_address=account,
wei_amount=wei_amount,
)
return from_wei(wei_amount)
def query_eth_highest_block(self) -> Optional[int]:
""" Attempts to query an external service for the block height
Returns the highest blockNumber"""
url = 'https://api.blockcypher.com/v1/eth/main'
log.debug('Querying blockcypher for ETH highest block', url=url)
eth_resp: Optional[Dict[str, str]]
try:
eth_resp = request_get_dict(url)
except (RemoteError, UnableToDecryptRemoteData):
eth_resp = None
block_number: Optional[int]
if eth_resp and 'height' in eth_resp:
block_number = int(eth_resp['height'])
log.debug('ETH highest block result', block=block_number)
else:
try:
block_number = self.etherscan.get_latest_block_number()
log.debug('ETH highest block result', block=block_number)
except RemoteError:
block_number = None
return block_number
def get_multieth_balance(
self,
accounts: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, FVal]:
"""Returns a dict with keys being accounts and balances in ETH"""
balances = {}
if not self.connected:
if len(accounts) > 20:
new_accounts = [
accounts[x:x + 2] for x in range(0, len(accounts), 2)
]
else:
new_accounts = [accounts]
for account_slice in new_accounts:
log.debug(
'Querying etherscan for multiple accounts balance',
sensitive_log=True,
eth_accounts=account_slice,
)
eth_resp = request_get_dict(
'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s'
% ','.join(account_slice), )
if eth_resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for accounts balance')
eth_accounts = eth_resp['result']
for account_entry in eth_accounts:
amount = FVal(account_entry['balance'])
# Etherscan does not return accounts checksummed so make sure they
# are converted properly here
checksum_account = to_checksum_address(
account_entry['account'])
balances[checksum_account] = from_wei(amount)
log.debug(
'Etherscan account balance result',
sensitive_log=True,
eth_address=account_entry['account'],
wei_amount=amount,
)
else:
for account in accounts:
amount = FVal(self.web3.eth.getBalance(account)) # pylint: disable=no-member
log.debug(
'Ethereum node balance result',
sensitive_log=True,
eth_address=account,
wei_amount=amount,
)
balances[account] = from_wei(amount)
return balances
def get_token_contract_creation_time(token_address: str) -> Timestamp:
resp = request_get_dict(
f'http://api.etherscan.io/api?module=account&action=txlist&address='
f'{token_address}&startblock=0&endblock=999999999&sort=asc',
)
if resp['status'] != 1:
raise ValueError('Failed to query etherscan for token {token_address} creation')
tx_list = resp['result']
if len(tx_list) == 0:
raise ValueError('Etherscan query of {token_address} transactions returned empty list')
return Timestamp(tx_list[0]['timeStamp'].to_int(exact=True))
def query_eth_highest_block() -> Optional[int]:
""" Attempts to query blockcypher for the block height
Returns the highest blockNumber"""
url = 'https://api.blockcypher.com/v1/eth/main'
log.debug('Querying ETH highest block', url=url)
eth_resp = request_get_dict(url)
if 'height' not in eth_resp:
return None
block_number = int(eth_resp['height'])
log.debug('ETH highest block result', block=block_number)
return block_number
def _check_blockchaininfo_for_transactions(
accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
have_transactions = {}
params = '|'.join(accounts)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.infoq/
backoff_in_seconds=15,
)
for idx, entry in enumerate(btc_resp['addresses']):
balance = satoshis_to_btc(entry['final_balance'])
have_transactions[accounts[idx]] = (entry['n_tx'] != 0, balance)
return have_transactions
def _check_blockstream_for_transactions(
accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
"""May raise connection errors or KeyError"""
have_transactions = {}
for account in accounts:
url = f'https://blockstream.info/api/address/{account}'
response_data = request_get_dict(url=url,
handle_429=True,
backoff_in_seconds=4)
stats = response_data['chain_stats']
balance = satoshis_to_btc(
int(stats['funded_txo_sum']) - int(stats['spent_txo_sum']))
have_txs = stats['tx_count'] != 0
have_transactions[account] = (have_txs, balance)
return have_transactions
def _api_query(self, path: str, only_data: bool = True) -> Dict[str, Any]:
querystr = f'{self.prefix}{path}'
log.debug('Querying cryptocompare', url=querystr)
resp = request_get_dict(querystr)
if 'Response' not in resp or resp['Response'] != 'Success':
error_message = 'Failed to query cryptocompare for: "{}"'.format(querystr)
if 'Message' in resp:
error_message += ". Error: {}".format(resp['Message'])
log.error('Cryptocompare query failure', url=querystr, error=error_message)
raise ValueError(error_message)
if only_data:
return resp['Data']
return resp
def _query_exchanges_rateapi(base: FiatAsset, quote: FiatAsset) -> Optional[Price]:
log.debug(
'Querying api.exchangeratesapi.io fiat pair',
base_currency=base,
quote_currency=quote,
)
querystr = f'https://api.exchangeratesapi.io/latest?base={base}&symbols={quote}'
try:
resp = request_get_dict(querystr)
return Price(FVal(resp['rates'][quote]))
except (ValueError, RemoteError, KeyError, requests.exceptions.TooManyRedirects):
log.error(
'Querying api.exchangeratesapi.io for fiat pair failed',
base_currency=base,
quote_currency=quote,
)
return None
def _query_currency_converterapi(base: FiatAsset,
quote: FiatAsset) -> Optional[Price]:
log.debug(
'Query free.currencyconverterapi.com fiat pair',
base_currency=base,
quote_currency=quote,
)
pair = f'{base}_{quote}'
querystr = (f'https://free.currencyconverterapi.com/api/v6/convert?'
f'q={pair}&apiKey={CURRENCYCONVERTER_API_KEY}')
try:
resp = request_get_dict(querystr)
return Price(FVal(resp['results'][pair]['val']))
except (ValueError, RemoteError, KeyError):
log.error(
'Querying free.currencyconverterapi.com fiat pair failed',
base_currency=base,
quote_currency=quote,
)
return None
def _query_currency_converterapi(base: Asset, quote: Asset) -> Optional[Price]:
assert base.is_fiat(), 'fiat currency should have been provided'
assert quote.is_fiat(), 'fiat currency should have been provided'
log.debug(
'Query free.currencyconverterapi.com fiat pair',
base_currency=base.identifier,
quote_currency=quote.identifier,
)
pair = f'{base.identifier}_{quote.identifier}'
querystr = (f'https://free.currencyconverterapi.com/api/v6/convert?'
f'q={pair}&apiKey={CURRENCYCONVERTER_API_KEY}')
try:
resp = request_get_dict(querystr)
return Price(FVal(resp['results'][pair]['val']))
except (ValueError, RemoteError, KeyError, UnableToDecryptRemoteData):
log.error(
'Querying free.currencyconverterapi.com fiat pair failed',
base_currency=base.identifier,
quote_currency=quote.identifier,
)
return None
def query_ethereum_txlist(
address: EthAddress,
msg_aggregator: MessagesAggregator,
internal: bool,
from_block: Optional[int] = None,
to_block: Optional[int] = None,
) -> List[EthereumTransaction]:
"""Query ethereum tx list"""
log.debug(
'Querying etherscan for tx list',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
)
result = list()
if internal:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address))
else:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address))
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = request_get_dict(reqstring)
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
log.error(
'Querying etherscan for tx list failed',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
error=resp['message'],
)
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp), )
log.debug('Etherscan tx list query result',
results_num=len(resp['result']))
for v in resp['result']:
try:
tx = deserialize_transaction_from_etherscan(data=v,
internal=internal)
except DeserializationError as e:
msg_aggregator.add_warning(f'{str(e)}. Skipping transaction')
continue
result.append(tx)
return result
def get_multitoken_balance(
self,
token: EthereumToken,
accounts: List[typing.EthAddress],
) -> Dict[typing.EthAddress, FVal]:
"""Return a dictionary with keys being accounts and value balances of token
Balance value is normalized through the token decimals.
"""
balances = {}
if self.connected:
token_contract = self.web3.eth.contract( # pylint: disable=no-member
address=token.ethereum_address,
abi=self.token_abi,
)
for account in accounts:
log.debug(
'Ethereum node query for token balance',
sensitive_log=True,
eth_address=account,
token_address=token.ethereum_address,
token_symbol=token.decimals,
)
token_amount = FVal(token_contract.functions.balanceOf(account).call())
if token_amount != 0:
balances[account] = token_amount / (FVal(10) ** FVal(token.decimals))
log.debug(
'Ethereum node result for token balance',
sensitive_log=True,
eth_address=account,
token_address=token.ethereum_address,
token_symbol=token.symbol,
amount=token_amount,
)
else:
for account in accounts:
log.debug(
'Querying Etherscan for token balance',
sensitive_log=True,
eth_address=account,
token_address=token.ethereum_address,
token_symbol=token.symbol,
)
resp = request_get_dict(
'https://api.etherscan.io/api?module=account&action='
'tokenbalance&contractaddress={}&address={}'.format(
token.ethereum_address,
account,
))
if resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for {} token balance of {}'.format(
token.symbol,
account,
))
token_amount = FVal(resp['result'])
if token_amount != 0:
balances[account] = token_amount / (FVal(10) ** FVal(token.decimals))
log.debug(
'Etherscan result for token balance',
sensitive_log=True,
eth_address=account,
token_address=token.ethereum_address,
token_symbol=token.symbol,
amount=token_amount,
)
return balances
def query_ethereum_txlist(
address: EthAddress,
internal: bool,
from_block: Optional[int] = None,
to_block: Optional[int] = None,
) -> List[EthereumTransaction]:
log.debug(
'Querying etherscan for tx list',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
)
result = list()
if internal:
reqstring = (
'https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address)
)
else:
reqstring = (
'https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address)
)
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = request_get_dict(reqstring)
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
log.error(
'Querying etherscan for tx list failed',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
error=resp['message'],
)
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp),
)
log.debug('Etherscan tx list query result', results_num=len(resp['result']))
for v in resp['result']:
# internal tx list contains no gasprice
gas_price = FVal(-1) if internal else FVal(v['gasPrice'])
result.append(EthereumTransaction(
timestamp=Timestamp(convert_to_int(v['timeStamp'])),
block_number=convert_to_int(v['blockNumber']),
hash=v['hash'],
from_address=v['from'],
to_address=v['to'],
value=FVal(v['value']),
gas=FVal(v['gas']),
gas_price=gas_price,
gas_used=FVal(v['gasUsed']),
))
return result
def query_btc_accounts_balances(
accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
"""Queries blockchain.info for the balance of account
May raise:
- RemotError if there is a problem querying blockchain.info or blockcypher
"""
source = 'blockchain.info'
balances = {}
try:
if any(account.lower()[0:3] == 'bc1' for account in accounts):
# if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work
source = 'blockcypher.com'
# the bech32 accounts have to be given lowercase to the
# blockcypher query. No idea why.
new_accounts = []
for x in accounts:
lowered = x.lower()
if lowered[0:3] == 'bc1':
new_accounts.append(lowered)
else:
new_accounts.append(x)
# blockcypher's batching takes up as many api queries as the batch,
# and the api rate limit is 3 requests per second. So we should make
# sure each batch is of max size 3
# https://www.blockcypher.com/dev/bitcoin/#batching
batches = [
new_accounts[x:x + 3]
for x in range(0, len(new_accounts), 3)
]
total_idx = 0
for batch in batches:
params = ';'.join(batch)
url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
response_data = request_get(url=url,
handle_429=True,
backoff_in_seconds=4)
if isinstance(response_data, dict):
# If only one account was requested put it in a list so the
# rest of the code works
response_data = [response_data]
for idx, entry in enumerate(response_data):
# we don't use the returned address as it may be lowercased
balances[accounts[total_idx + idx]] = satoshis_to_btc(
FVal(entry['final_balance']), )
total_idx += len(batch)
else:
params = '|'.join(accounts)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
for idx, entry in enumerate(btc_resp['addresses']):
balances[accounts[idx]] = satoshis_to_btc(
FVal(entry['final_balance']))
except (requests.exceptions.ConnectionError,
UnableToDecryptRemoteData) as e:
raise RemoteError(
f'bitcoin external API request failed due to {str(e)}')
except KeyError as e:
raise RemoteError(
f'Malformed response when querying bitcoin blockchain via {source}.'
f'Did not find key {e}', )
return balances
