def get_bitcoin_addresses_balances( accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]: """Queries blockchain.info or blockstream for the balances of accounts May raise: - RemotError if there is a problem querying blockchain.info or blockstream """ source = 'blockchain.info' balances: Dict[BTCAddress, FVal] = {} try: if _have_bc1_accounts(accounts): # if 1 account is bech32 we have to query blockstream. blockchaininfo won't work source = 'blockstream' balances = {} for account in accounts: url = f'https://blockstream.info/api/address/{account}' response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4) stats = response_data['chain_stats'] balance = int(stats['funded_txo_sum']) - int( stats['spent_txo_sum']) balances[account] = satoshis_to_btc(balance) else: params = '|'.join(accounts) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) for idx, entry in enumerate(btc_resp['addresses']): balances[accounts[idx]] = satoshis_to_btc( FVal(entry['final_balance'])) except ( requests.exceptions.ConnectionError, UnableToDecryptRemoteData, requests.exceptions.Timeout, ) as e: raise RemoteError( f'bitcoin external API request for balances failed due to {str(e)}' ) from e # noqa: E501 except KeyError as e: raise RemoteError( f'Malformed response when querying bitcoin blockchain via {source}.' f'Did not find key {e}', ) from e return balances
def _api_query(self, path: str) -> Dict[str, Any]: querystr = f'{self.prefix}{path}' log.debug('Querying cryptocompare', url=querystr) resp = request_get_dict(querystr) # These endpoints are wrapped in a response object wrapped_response = 'all/coinlist' in path or 'histohour' in path log.debug(f'Wrapped response: {wrapped_response}') if wrapped_response: if 'Response' not in resp or resp['Response'] != 'Success': error_message = 'Failed to query cryptocompare for: "{}"'.format( querystr) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) log.error('Cryptocompare query failure', url=querystr, error=error_message) raise ValueError(error_message) # for histohour we want all the data, including the wrapper to get TimeFrom # and TimeTo if 'histohour' not in path: return resp['Data'] # else not a wrapped response return resp
def _query_exchanges_rateapi(base: Asset, quote: Asset) -> Optional[Price]: assert base.is_fiat(), 'fiat currency should have been provided' assert quote.is_fiat(), 'fiat currency should have been provided' log.debug( 'Querying api.exchangeratesapi.io fiat pair', base_currency=base.identifier, quote_currency=quote.identifier, ) querystr = ( f'https://api.exchangeratesapi.io/latest?base={base.identifier}&symbols={quote.identifier}' ) try: resp = request_get_dict(querystr) return Price(FVal(resp['rates'][quote.identifier])) except ( RemoteError, KeyError, requests.exceptions.TooManyRedirects, UnableToDecryptRemoteData, ): log.error( 'Querying api.exchangeratesapi.io for fiat pair failed', base_currency=base.identifier, quote_currency=quote.identifier, ) return None
def get_eth_balance(self, account: typing.EthAddress) -> FVal: if not self.connected: log.debug( 'Querying etherscan for account balance', sensitive_log=True, eth_address=account, ) eth_resp = request_get_dict( 'https://api.etherscan.io/api?module=account&action=balance&address=%s' % account, ) if eth_resp['status'] != 1: raise ValueError('Failed to query etherscan for accounts balance') amount = FVal(eth_resp['result']) log.debug( 'Etherscan account balance result', sensitive_log=True, eth_address=account, wei_amount=amount, ) return from_wei(amount) else: wei_amount = self.web3.eth.getBalance(account) # pylint: disable=no-member log.debug( 'Ethereum node account balance result', sensitive_log=True, eth_address=account, wei_amount=wei_amount, ) return from_wei(wei_amount)
def query_eth_highest_block(self) -> Optional[int]: """ Attempts to query an external service for the block height Returns the highest blockNumber""" url = 'https://api.blockcypher.com/v1/eth/main' log.debug('Querying blockcypher for ETH highest block', url=url) eth_resp: Optional[Dict[str, str]] try: eth_resp = request_get_dict(url) except (RemoteError, UnableToDecryptRemoteData): eth_resp = None block_number: Optional[int] if eth_resp and 'height' in eth_resp: block_number = int(eth_resp['height']) log.debug('ETH highest block result', block=block_number) else: try: block_number = self.etherscan.get_latest_block_number() log.debug('ETH highest block result', block=block_number) except RemoteError: block_number = None return block_number
def get_multieth_balance( self, accounts: List[ChecksumEthAddress], ) -> Dict[ChecksumEthAddress, FVal]: """Returns a dict with keys being accounts and balances in ETH""" balances = {} if not self.connected: if len(accounts) > 20: new_accounts = [ accounts[x:x + 2] for x in range(0, len(accounts), 2) ] else: new_accounts = [accounts] for account_slice in new_accounts: log.debug( 'Querying etherscan for multiple accounts balance', sensitive_log=True, eth_accounts=account_slice, ) eth_resp = request_get_dict( 'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s' % ','.join(account_slice), ) if eth_resp['status'] != 1: raise ValueError( 'Failed to query etherscan for accounts balance') eth_accounts = eth_resp['result'] for account_entry in eth_accounts: amount = FVal(account_entry['balance']) # Etherscan does not return accounts checksummed so make sure they # are converted properly here checksum_account = to_checksum_address( account_entry['account']) balances[checksum_account] = from_wei(amount) log.debug( 'Etherscan account balance result', sensitive_log=True, eth_address=account_entry['account'], wei_amount=amount, ) else: for account in accounts: amount = FVal(self.web3.eth.getBalance(account)) # pylint: disable=no-member log.debug( 'Ethereum node balance result', sensitive_log=True, eth_address=account, wei_amount=amount, ) balances[account] = from_wei(amount) return balances
def get_token_contract_creation_time(token_address: str) -> Timestamp: resp = request_get_dict( f'http://api.etherscan.io/api?module=account&action=txlist&address=' f'{token_address}&startblock=0&endblock=999999999&sort=asc', ) if resp['status'] != 1: raise ValueError('Failed to query etherscan for token {token_address} creation') tx_list = resp['result'] if len(tx_list) == 0: raise ValueError('Etherscan query of {token_address} transactions returned empty list') return Timestamp(tx_list[0]['timeStamp'].to_int(exact=True))
def query_eth_highest_block() -> Optional[int]: """ Attempts to query blockcypher for the block height Returns the highest blockNumber""" url = 'https://api.blockcypher.com/v1/eth/main' log.debug('Querying ETH highest block', url=url) eth_resp = request_get_dict(url) if 'height' not in eth_resp: return None block_number = int(eth_resp['height']) log.debug('ETH highest block result', block=block_number) return block_number
def _check_blockchaininfo_for_transactions( accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]: have_transactions = {} params = '|'.join(accounts) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.infoq/ backoff_in_seconds=15, ) for idx, entry in enumerate(btc_resp['addresses']): balance = satoshis_to_btc(entry['final_balance']) have_transactions[accounts[idx]] = (entry['n_tx'] != 0, balance) return have_transactions
def _check_blockstream_for_transactions( accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]: """May raise connection errors or KeyError""" have_transactions = {} for account in accounts: url = f'https://blockstream.info/api/address/{account}' response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4) stats = response_data['chain_stats'] balance = satoshis_to_btc( int(stats['funded_txo_sum']) - int(stats['spent_txo_sum'])) have_txs = stats['tx_count'] != 0 have_transactions[account] = (have_txs, balance) return have_transactions
def _api_query(self, path: str, only_data: bool = True) -> Dict[str, Any]: querystr = f'{self.prefix}{path}' log.debug('Querying cryptocompare', url=querystr) resp = request_get_dict(querystr) if 'Response' not in resp or resp['Response'] != 'Success': error_message = 'Failed to query cryptocompare for: "{}"'.format(querystr) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) log.error('Cryptocompare query failure', url=querystr, error=error_message) raise ValueError(error_message) if only_data: return resp['Data'] return resp
def _query_exchanges_rateapi(base: FiatAsset, quote: FiatAsset) -> Optional[Price]: log.debug( 'Querying api.exchangeratesapi.io fiat pair', base_currency=base, quote_currency=quote, ) querystr = f'https://api.exchangeratesapi.io/latest?base={base}&symbols={quote}' try: resp = request_get_dict(querystr) return Price(FVal(resp['rates'][quote])) except (ValueError, RemoteError, KeyError, requests.exceptions.TooManyRedirects): log.error( 'Querying api.exchangeratesapi.io for fiat pair failed', base_currency=base, quote_currency=quote, ) return None
def _query_currency_converterapi(base: FiatAsset, quote: FiatAsset) -> Optional[Price]: log.debug( 'Query free.currencyconverterapi.com fiat pair', base_currency=base, quote_currency=quote, ) pair = f'{base}_{quote}' querystr = (f'https://free.currencyconverterapi.com/api/v6/convert?' f'q={pair}&apiKey={CURRENCYCONVERTER_API_KEY}') try: resp = request_get_dict(querystr) return Price(FVal(resp['results'][pair]['val'])) except (ValueError, RemoteError, KeyError): log.error( 'Querying free.currencyconverterapi.com fiat pair failed', base_currency=base, quote_currency=quote, ) return None
def _query_currency_converterapi(base: Asset, quote: Asset) -> Optional[Price]: assert base.is_fiat(), 'fiat currency should have been provided' assert quote.is_fiat(), 'fiat currency should have been provided' log.debug( 'Query free.currencyconverterapi.com fiat pair', base_currency=base.identifier, quote_currency=quote.identifier, ) pair = f'{base.identifier}_{quote.identifier}' querystr = (f'https://free.currencyconverterapi.com/api/v6/convert?' f'q={pair}&apiKey={CURRENCYCONVERTER_API_KEY}') try: resp = request_get_dict(querystr) return Price(FVal(resp['results'][pair]['val'])) except (ValueError, RemoteError, KeyError, UnableToDecryptRemoteData): log.error( 'Querying free.currencyconverterapi.com fiat pair failed', base_currency=base.identifier, quote_currency=quote.identifier, ) return None
def query_ethereum_txlist( address: EthAddress, msg_aggregator: MessagesAggregator, internal: bool, from_block: Optional[int] = None, to_block: Optional[int] = None, ) -> List[EthereumTransaction]: """Query ethereum tx list""" log.debug( 'Querying etherscan for tx list', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, ) result = list() if internal: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlistinternal&address={}'.format(address)) else: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlist&address={}'.format(address)) if from_block: reqstring += '&startblock={}'.format(from_block) if to_block: reqstring += '&endblock={}'.format(to_block) resp = request_get_dict(reqstring) if 'status' not in resp or convert_to_int(resp['status']) != 1: status = convert_to_int(resp['status']) if status == 0 and resp['message'] == 'No transactions found': return list() log.error( 'Querying etherscan for tx list failed', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, error=resp['message'], ) # else unknown error raise ValueError( 'Failed to query txlist from etherscan with query: {} . ' 'Response was: {}'.format(reqstring, resp), ) log.debug('Etherscan tx list query result', results_num=len(resp['result'])) for v in resp['result']: try: tx = deserialize_transaction_from_etherscan(data=v, internal=internal) except DeserializationError as e: msg_aggregator.add_warning(f'{str(e)}. Skipping transaction') continue result.append(tx) return result
def get_multitoken_balance( self, token: EthereumToken, accounts: List[typing.EthAddress], ) -> Dict[typing.EthAddress, FVal]: """Return a dictionary with keys being accounts and value balances of token Balance value is normalized through the token decimals. """ balances = {} if self.connected: token_contract = self.web3.eth.contract( # pylint: disable=no-member address=token.ethereum_address, abi=self.token_abi, ) for account in accounts: log.debug( 'Ethereum node query for token balance', sensitive_log=True, eth_address=account, token_address=token.ethereum_address, token_symbol=token.decimals, ) token_amount = FVal(token_contract.functions.balanceOf(account).call()) if token_amount != 0: balances[account] = token_amount / (FVal(10) ** FVal(token.decimals)) log.debug( 'Ethereum node result for token balance', sensitive_log=True, eth_address=account, token_address=token.ethereum_address, token_symbol=token.symbol, amount=token_amount, ) else: for account in accounts: log.debug( 'Querying Etherscan for token balance', sensitive_log=True, eth_address=account, token_address=token.ethereum_address, token_symbol=token.symbol, ) resp = request_get_dict( 'https://api.etherscan.io/api?module=account&action=' 'tokenbalance&contractaddress={}&address={}'.format( token.ethereum_address, account, )) if resp['status'] != 1: raise ValueError( 'Failed to query etherscan for {} token balance of {}'.format( token.symbol, account, )) token_amount = FVal(resp['result']) if token_amount != 0: balances[account] = token_amount / (FVal(10) ** FVal(token.decimals)) log.debug( 'Etherscan result for token balance', sensitive_log=True, eth_address=account, token_address=token.ethereum_address, token_symbol=token.symbol, amount=token_amount, ) return balances
def query_ethereum_txlist( address: EthAddress, internal: bool, from_block: Optional[int] = None, to_block: Optional[int] = None, ) -> List[EthereumTransaction]: log.debug( 'Querying etherscan for tx list', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, ) result = list() if internal: reqstring = ( 'https://api.etherscan.io/api?module=account&action=' 'txlistinternal&address={}'.format(address) ) else: reqstring = ( 'https://api.etherscan.io/api?module=account&action=' 'txlist&address={}'.format(address) ) if from_block: reqstring += '&startblock={}'.format(from_block) if to_block: reqstring += '&endblock={}'.format(to_block) resp = request_get_dict(reqstring) if 'status' not in resp or convert_to_int(resp['status']) != 1: status = convert_to_int(resp['status']) if status == 0 and resp['message'] == 'No transactions found': return list() log.error( 'Querying etherscan for tx list failed', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, error=resp['message'], ) # else unknown error raise ValueError( 'Failed to query txlist from etherscan with query: {} . ' 'Response was: {}'.format(reqstring, resp), ) log.debug('Etherscan tx list query result', results_num=len(resp['result'])) for v in resp['result']: # internal tx list contains no gasprice gas_price = FVal(-1) if internal else FVal(v['gasPrice']) result.append(EthereumTransaction( timestamp=Timestamp(convert_to_int(v['timeStamp'])), block_number=convert_to_int(v['blockNumber']), hash=v['hash'], from_address=v['from'], to_address=v['to'], value=FVal(v['value']), gas=FVal(v['gas']), gas_price=gas_price, gas_used=FVal(v['gasUsed']), )) return result
def query_btc_accounts_balances( accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]: """Queries blockchain.info for the balance of account May raise: - RemotError if there is a problem querying blockchain.info or blockcypher """ source = 'blockchain.info' balances = {} try: if any(account.lower()[0:3] == 'bc1' for account in accounts): # if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work source = 'blockcypher.com' # the bech32 accounts have to be given lowercase to the # blockcypher query. No idea why. new_accounts = [] for x in accounts: lowered = x.lower() if lowered[0:3] == 'bc1': new_accounts.append(lowered) else: new_accounts.append(x) # blockcypher's batching takes up as many api queries as the batch, # and the api rate limit is 3 requests per second. So we should make # sure each batch is of max size 3 # https://www.blockcypher.com/dev/bitcoin/#batching batches = [ new_accounts[x:x + 3] for x in range(0, len(new_accounts), 3) ] total_idx = 0 for batch in batches: params = ';'.join(batch) url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance' response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4) if isinstance(response_data, dict): # If only one account was requested put it in a list so the # rest of the code works response_data = [response_data] for idx, entry in enumerate(response_data): # we don't use the returned address as it may be lowercased balances[accounts[total_idx + idx]] = satoshis_to_btc( FVal(entry['final_balance']), ) total_idx += len(batch) else: params = '|'.join(accounts) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) for idx, entry in enumerate(btc_resp['addresses']): balances[accounts[idx]] = satoshis_to_btc( FVal(entry['final_balance'])) except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e: raise RemoteError( f'bitcoin external API request failed due to {str(e)}') except KeyError as e: raise RemoteError( f'Malformed response when querying bitcoin blockchain via {source}.' f'Did not find key {e}', ) return balances