def get_multieth_balance( self, accounts: List[typing.EthAddress], ) -> Dict[typing.EthAddress, FVal]: """Returns a dict with keys being accounts and balances in ETH""" balances = {} if not self.connected: if len(accounts) > 20: new_accounts = [ accounts[x:x + 2] for x in range(0, len(accounts), 2) ] else: new_accounts = [accounts] for account_slice in new_accounts: eth_resp = request_get( 'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s' % ','.join(account_slice)) if eth_resp['status'] != 1: raise ValueError( 'Failed to query etherscan for accounts balance') eth_accounts = eth_resp['result'] for account_entry in eth_accounts: amount = FVal(account_entry['balance']) balances[account_entry['account']] = from_wei(amount) else: for account in accounts: amount = FVal(self.web3.eth.getBalance(account)) balances[account] = from_wei(amount) return balances
def get_multitoken_balance(self, token_symbol, token_address, token_decimals, accounts): """Return a dictionary with keys being accounts and value balances of token Balance value is normalized through the token decimals. """ balances = {} if self.connected: token_contract = self.web3.eth.contract( address=token_address, abi=self.token_abi ) for account in accounts: token_amount = FVal(token_contract.functions.balanceOf(account).call()) if token_amount != 0: balances[account] = token_amount / (FVal(10) ** FVal(token_decimals)) else: for account in accounts: print('Checking token {} for account {}'.format(token_symbol, account)) resp = request_get( 'https://api.etherscan.io/api?module=account&action=' 'tokenbalance&contractaddress={}&address={}'.format( token_address, account, )) if resp['status'] != 1: raise ValueError( 'Failed to query etherscan for {} token balance of {}'.format( token_symbol, account, )) token_amount = FVal(resp['result']) if token_amount != 0: balances[account] = token_amount / (FVal(10) ** FVal(token_decimals)) return balances
def get_eth_balance(self, account: typing.EthAddress) -> FVal: if not self.connected: log.debug( 'Querying etherscan for account balance', sensitive_log=True, eth_address=account, ) eth_resp = request_get( 'https://api.etherscan.io/api?module=account&action=balance&address=%s' % account) if eth_resp['status'] != 1: raise ValueError( 'Failed to query etherscan for accounts balance') amount = FVal(eth_resp['result']) log.debug( 'Etherscan account balance result', sensitive_log=True, eth_address=account, wei_amount=amount, ) return from_wei(amount) else: wei_amount = self.web3.eth.getBalance(account) # pylint: disable=no-member log.debug( 'Ethereum node account balance result', sensitive_log=True, eth_address=account, wei_amount=wei_amount, ) return from_wei(wei_amount)
def get_eth_balance(self, account): if not self.connected: eth_resp = request_get( 'https://api.etherscan.io/api?module=account&action=balance&address=%s' % account ) if eth_resp['status'] != 1: raise ValueError('Failed to query etherscan for accounts balance') amount = FVal(eth_resp['result']) return from_wei(amount) else: return from_wei(self.web3.eth.getBalance(account))
def query_eth_highest_block(self) -> int: """ Attempts to query blockcypher for the block height Returns the highest blockNumber""" url = 'https://api.blockcypher.com/v1/eth/main' log.debug('Querying ETH highest block', url=url) eth_resp = request_get(url) if 'height' not in eth_resp: return None block_number = int(eth_resp['height']) log.debug('ETH highest block result', block=block_number) return block_number
def query_ethereum_txlist( address: EthAddress, internal: bool, from_block: int = None, to_block: int = None, ) -> List[EthereumTransaction]: result = list() if internal: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlistinternal&address={}'.format(address)) else: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlist&address={}'.format(address)) if from_block: reqstring += '&startblock={}'.format(from_block) if to_block: reqstring += '&endblock={}'.format(to_block) resp = request_get(reqstring) if 'status' not in resp or convert_to_int(resp['status']) != 1: status = convert_to_int(resp['status']) if status == 0 and resp['message'] == 'No transactions found': return list() # else unknown error raise ValueError( 'Failed to query txlist from etherscan with query: {} . ' 'Response was: {}'.format(reqstring, resp)) for v in resp['result']: # internal tx list contains no gasprice gas_price = FVal(-1) if internal else FVal(v['gasPrice']) result.append( EthereumTransaction( timestamp=convert_to_int(v['timeStamp']), block_number=convert_to_int(v['blockNumber']), hash=v['hash'], from_address=v['from'], to_address=v['to'], value=FVal(v['value']), gas=FVal(v['gas']), gas_price=gas_price, gas_used=FVal(v['gasUsed']), )) return result
def get_multieth_balance(self, accounts): """Returns a dict with keys being accounts and balances in ETH""" balances = {} if not self.connected: # TODO: accounts.length should be less than 20. If more we gotta do # multiple calls eth_resp = request_get( 'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s' % ','.join(accounts) ) if eth_resp['status'] != 1: raise ValueError('Failed to query etherscan for accounts balance') eth_accounts = eth_resp['result'] for account_entry in eth_accounts: amount = FVal(account_entry['balance']) balances[account_entry['account']] = from_wei(amount) else: for account in accounts: amount = FVal(self.web3.eth.getBalance(account)) balances[account] = from_wei(amount) return balances
def query_btc_account_balance(self, account: typing.BTCAddress) -> FVal: btc_resp = request_get('https://blockchain.info/q/addressbalance/%s' % account) return FVal(btc_resp) * FVal('0.00000001') # result is in satoshis
def query_historical_price(self, from_asset, to_asset, timestamp): """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. Args: from_asset (str): The ticker symbol of the asset for which we want to know the price. to_asset (str): The ticker symbol of the asset against which we want to know the price. timestamp (int): The timestamp at which to query the price """ if from_asset == to_asset: return 1 if from_asset not in self.cryptocompare_coin_list: raise PriceQueryUnknownFromAsset(from_asset) data = self.get_historical_data(from_asset, to_asset, timestamp) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp assert timestamp > data[0]['time'] index = convert_to_int((timestamp - data[0]['time']) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index]['time'] - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1]['time'] - timestamp) if diff_p1 < diff: index = index + 1 if data[index]['high'] is None or data[index]['low'] is None: # If we get some None in the hourly set price to 0 so that we check daily price price = FVal(0) else: price = FVal((data[index]['high'] + data[index]['low'])) / 2 if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': # Just get the BTC price asset_btc_price = self.query_historical_price( from_asset, 'BTC', timestamp) btc_to_asset_price = self.query_historical_price( 'BTC', to_asset, timestamp) price = asset_btc_price * btc_to_asset_price else: # attempt to get the daily price by timestamp query_string = ( 'https://min-api.cryptocompare.com/data/pricehistorical?' 'fsym={}&tsyms={}&ts={}'.format(from_asset, to_asset, timestamp)) if to_asset == 'BTC': query_string += '&tryConversion=false' resp = request_get(query_string) if from_asset not in resp: error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) raise ValueError(error_message) price = FVal(resp[from_asset][to_asset]) if price == 0: raise NoPriceForGivenTimestamp( from_asset, to_asset, tsToDate(timestamp, formatstr='%d/%m/%Y, %H:%M:%S')) return price
def get_historical_data(self, from_asset, to_asset, timestamp): """Get historical price data from cryptocompare""" if from_asset not in self.cryptocompare_coin_list: raise ValueError('Attempted to query historical price data for ' 'unknown asset "{}"'.format(from_asset)) if to_asset not in self.cryptocompare_coin_list and to_asset not in FIAT_CURRENCIES: raise ValueError('Attempted to query historical price data for ' 'unknown asset "{}"'.format(to_asset)) cache_key = from_asset + '_' + to_asset got_cached_value = ( cache_key in self.price_history and self.price_history[cache_key]['start_time'] <= timestamp and self.price_history[cache_key]['end_time'] > timestamp) if got_cached_value: return self.price_history[cache_key]['data'] now_ts = int(time.time()) cryptocompare_hourquerylimit = 2000 calculated_history = list() if self.historical_data_start <= timestamp: end_date = self.historical_data_start else: end_date = timestamp while True: pr_end_date = end_date end_date = end_date + (cryptocompare_hourquerylimit) * 3600 query_string = ('https://min-api.cryptocompare.com/data/histohour?' 'fsym={}&tsym={}&limit={}&toTs={}'.format( from_asset, to_asset, cryptocompare_hourquerylimit, end_date)) resp = request_get(query_string) if 'Response' not in resp or resp['Response'] != 'Success': error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) raise ValueError(error_message) if pr_end_date != resp['TimeFrom']: # If we get more than we needed, since we are close to the now_ts # then skip all the already included entries diff = pr_end_date - resp['TimeFrom'] if resp['Data'][diff // 3600]['time'] != pr_end_date: raise ValueError( 'Expected to find the previous date timestamp during ' 'historical data fetching') # just add only the part from the previous timestamp and on resp['Data'] = resp['Data'][diff // 3600:] if end_date < now_ts and resp['TimeTo'] != end_date: raise ValueError('End dates no match') # If last time slot and first new are the same, skip the first new slot last_entry_equal_to_first = (len(calculated_history) != 0 and calculated_history[-1]['time'] == resp['Data'][0]['time']) if last_entry_equal_to_first: resp['Data'] = resp['Data'][1:] calculated_history += resp['Data'] if end_date >= now_ts: break # Let's always check for data sanity for the hourly prices. assert check_hourly_data_sanity(calculated_history, from_asset, to_asset) self.price_history[cache_key] = { 'data': calculated_history, 'start_time': self.historical_data_start, 'end_time': now_ts } # and now since we actually queried the data let's also save them locally write_history_data_in_file( calculated_history, os.path.join(self.data_directory, 'price_history_' + cache_key + '.json'), self.historical_data_start, now_ts) return calculated_history
def __init__(self, data_directory, history_date_start): self.data_directory = data_directory # get the start date for historical data self.historical_data_start = createTimeStamp(history_date_start, formatstr="%d/%m/%Y") self.price_history = dict() # TODO: Check if historical data is after the requested start date # Check the data folder and load any cached history prefix = os.path.join(self.data_directory, 'price_history_') regex = re.compile(prefix + '(.*)\.json') files_list = glob.glob(prefix + '*.json') for file_ in files_list: match = regex.match(file_) assert match cache_key = match.group(1) with open(file_, 'rb') as f: data = rlk_jsonloads(f.read()) self.price_history[cache_key] = data # Get coin list of crypto compare invalidate_cache = True coinlist_cache_path = os.path.join(self.data_directory, 'cryptocompare_coinlist.json') if os.path.isfile(coinlist_cache_path): with open(coinlist_cache_path, 'rb') as f: try: data = rlk_jsonloads(f.read()) now = ts_now() invalidate_cache = False # If we got a cache and its' over a month old then requery cryptocompare if data['time'] < now and now - data['time'] > 2629800: invalidate_cache = True data = data['data'] except JSONDecodeError: invalidate_cache = True if invalidate_cache: query_string = 'https://www.cryptocompare.com/api/data/coinlist/' resp = request_get(query_string) if 'Response' not in resp or resp['Response'] != 'Success': error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) raise ValueError(error_message) data = resp['Data'] # Also save the cache with open(coinlist_cache_path, 'w') as f: write_data = {'time': ts_now(), 'data': data} f.write(rlk_jsondumps(write_data)) else: # in any case take the data data = data['data'] self.cryptocompare_coin_list = data # For some reason even though price for the following assets is returned # it's not in the coinlist so let's add them here. self.cryptocompare_coin_list['DAO'] = object() self.cryptocompare_coin_list['USDT'] = object()
def query_historical_price(self, from_asset, to_asset, timestamp): """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. Args: from_asset (str): The ticker symbol of the asset for which we want to know the price. to_asset (str): The ticker symbol of the asset against which we want to know the price. timestamp (int): The timestamp at which to query the price """ log.debug( 'Querying historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if from_asset == to_asset: return 1 if from_asset in FIAT_CURRENCIES and to_asset in FIAT_CURRENCIES: # if we are querying historical forex data then try something other than cryptocompare price = self.inquirer.query_historical_fiat_exchange_rates( from_asset, to_asset, timestamp, ) if price is not None: return price # else cryptocompare also has historical fiat to fiat data if from_asset not in self.cryptocompare_coin_list and from_asset not in FIAT_CURRENCIES: raise PriceQueryUnknownFromAsset(from_asset) data = self.get_historical_data(from_asset, to_asset, timestamp) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0]['time']: index = convert_to_int((timestamp - data[0]['time']) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index]['time'] - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1]['time'] - timestamp) if diff_p1 < diff: index = index + 1 if data[index]['high'] is None or data[index]['low'] is None: # If we get some None in the hourly set price to 0 so that we check alternatives price = FVal(0) else: price = FVal((data[index]['high'] + data[index]['low'])) / 2 else: # no price found in the historical data from/to asset, try alternatives price = FVal(0) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Coudn't find historical price from {from_asset} to " f"{to_asset}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = self.query_historical_price( from_asset, 'BTC', timestamp) btc_to_asset_price = self.query_historical_price( 'BTC', to_asset, timestamp) price = asset_btc_price * btc_to_asset_price else: log.debug( f"Coudn't find historical price from {from_asset} to " f"{to_asset}. Attempting to get daily price...", ) # attempt to get the daily price by timestamp cc_from_asset = world_to_cryptocompare(from_asset) cc_to_asset = world_to_cryptocompare(to_asset) log.debug( 'Querying cryptocompare for daily historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) query_string = ( 'https://min-api.cryptocompare.com/data/pricehistorical?' 'fsym={}&tsyms={}&ts={}'.format( cc_from_asset, cc_to_asset, timestamp, )) if to_asset == 'BTC': query_string += '&tryConversion=false' resp = request_get(query_string) if cc_from_asset not in resp: error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) log.error( 'Cryptocompare query for daily historical price failed', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, error=error_message, ) raise ValueError(error_message) price = FVal(resp[cc_from_asset][cc_to_asset]) comparison_to_nonusd_fiat = ( (to_asset in FIAT_CURRENCIES and to_asset != 'USD') or (from_asset in FIAT_CURRENCIES and from_asset != 'USD')) if comparison_to_nonusd_fiat: price = self.adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset, to_asset, tsToDate(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def get_historical_data(self, from_asset, to_asset, timestamp): """Get historical price data from cryptocompare""" log.debug( 'Retrieving historical price data', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if from_asset not in self.cryptocompare_coin_list and from_asset not in FIAT_CURRENCIES: raise ValueError( 'Attempted to query historical price data for ' 'unknown asset "{}"'.format(from_asset), ) if to_asset not in self.cryptocompare_coin_list and to_asset not in FIAT_CURRENCIES: raise ValueError( 'Attempted to query historical price data for ' 'unknown asset "{}"'.format(to_asset), ) cache_key = from_asset + '_' + to_asset got_cached_value = self.got_cached_price(cache_key, timestamp) if got_cached_value: return self.price_history[cache_key]['data'] now_ts = int(time.time()) cryptocompare_hourquerylimit = 2000 calculated_history = list() if self.historical_data_start <= timestamp: end_date = self.historical_data_start else: end_date = timestamp while True: no_data_for_timestamp = False pr_end_date = end_date end_date = end_date + (cryptocompare_hourquerylimit) * 3600 log.debug( 'Querying cryptocompare for hourly historical price', from_asset=from_asset, to_asset=to_asset, cryptocompare_hourquerylimit=cryptocompare_hourquerylimit, end_date=end_date, ) query_string = ('https://min-api.cryptocompare.com/data/histohour?' 'fsym={}&tsym={}&limit={}&toTs={}'.format( world_to_cryptocompare(from_asset), world_to_cryptocompare(to_asset), cryptocompare_hourquerylimit, end_date, )) resp = request_get(query_string) if 'Response' not in resp or resp['Response'] != 'Success': msg = 'Unable to retrieve requested data at this time, please try again later' no_data_for_timestamp = (msg in resp['Message'] and resp['Type'] == 96) if no_data_for_timestamp: log.debug( 'No hourly cryptocompare historical data for pair', from_asset=from_asset, to_asset=to_asset, timestamp=end_date, ) continue error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) log.error( 'Cryptocompare hourly historical price query failed', error=error_message, ) raise ValueError(error_message) if pr_end_date != resp['TimeFrom']: # If we get more than we needed, since we are close to the now_ts # then skip all the already included entries diff = pr_end_date - resp['TimeFrom'] if resp['Data'][diff // 3600]['time'] != pr_end_date: raise ValueError( 'Expected to find the previous date timestamp during ' 'historical data fetching', ) # just add only the part from the previous timestamp and on resp['Data'] = resp['Data'][diff // 3600:] if end_date < now_ts and resp['TimeTo'] != end_date: raise ValueError('End dates no match') # If last time slot and first new are the same, skip the first new slot last_entry_equal_to_first = (len(calculated_history) != 0 and calculated_history[-1]['time'] == resp['Data'][0]['time']) if last_entry_equal_to_first: resp['Data'] = resp['Data'][1:] calculated_history += resp['Data'] if end_date >= now_ts: break # Let's always check for data sanity for the hourly prices. assert check_hourly_data_sanity(calculated_history, from_asset, to_asset) self.price_history[cache_key] = { 'data': calculated_history, 'start_time': self.historical_data_start, 'end_time': now_ts, } # and now since we actually queried the data let's also save them locally filename = os.path.join(self.data_directory, 'price_history_' + cache_key + '.json') log.info( 'Updating price history cache', filename=filename, from_asset=from_asset, to_asset=to_asset, ) write_history_data_in_file( calculated_history, filename, self.historical_data_start, now_ts, ) self.price_history_file[cache_key] = filename return calculated_history
def __init__(self, data_directory, history_date_start, inquirer): self.data_directory = data_directory # get the start date for historical data self.historical_data_start = createTimeStamp(history_date_start, formatstr="%d/%m/%Y") self.inquirer = inquirer self.price_history = dict() self.price_history_file = dict() # Check the data folder and remember the filenames of any cached history prefix = os.path.join(self.data_directory, 'price_history_') prefix = prefix.replace('\\', '\\\\') regex = re.compile(prefix + '(.*)\\.json') files_list = glob.glob(prefix + '*.json') for file_ in files_list: match = regex.match(file_) assert match cache_key = match.group(1) self.price_history_file[cache_key] = file_ # Get coin list of crypto compare invalidate_cache = True coinlist_cache_path = os.path.join(self.data_directory, 'cryptocompare_coinlist.json') if os.path.isfile(coinlist_cache_path): log.info('Found coinlist cache', path=coinlist_cache_path) with open(coinlist_cache_path, 'rb') as f: try: data = rlk_jsonloads(f.read()) now = ts_now() invalidate_cache = False # If we got a cache and its' over a month old then requery cryptocompare if data['time'] < now and now - data['time'] > 2629800: log.info('Coinlist cache is now invalidated') invalidate_cache = True data = data['data'] except JSONDecodeError: invalidate_cache = True if invalidate_cache: query_string = 'https://www.cryptocompare.com/api/data/coinlist/' log.debug('Querying cryptocompare', url=query_string) resp = request_get(query_string) if 'Response' not in resp or resp['Response'] != 'Success': error_message = 'Failed to query cryptocompare for: "{}"'.format( query_string) if 'Message' in resp: error_message += ". Error: {}".format(resp['Message']) log.error('Cryptocompare query failure', url=query_string, error=error_message) raise ValueError(error_message) data = resp['Data'] # Also save the cache with open(coinlist_cache_path, 'w') as f: now = ts_now() log.info('Writting coinlist cache', timestamp=now) write_data = {'time': now, 'data': data} f.write(rlk_jsondumps(write_data)) else: # in any case take the data data = data['data'] self.cryptocompare_coin_list = data # For some reason even though price for the following assets is returned # it's not in the coinlist so let's add them here. self.cryptocompare_coin_list['DAO'] = object() self.cryptocompare_coin_list['USDT'] = object()
def get_multitoken_balance( self, token_symbol: typing.EthToken, token_address: typing.EthAddress, token_decimals: int, accounts: List[typing.EthAddress], ) -> Dict[typing.EthAddress, FVal]: """Return a dictionary with keys being accounts and value balances of token Balance value is normalized through the token decimals. """ balances = {} if self.connected: token_contract = self.web3.eth.contract( # pylint: disable=no-member address=token_address, abi=self.token_abi) for account in accounts: log.debug( 'Ethereum node query for token balance', sensitive_log=True, eth_address=account, token_address=token_address, token_symbol=token_symbol, ) token_amount = FVal( token_contract.functions.balanceOf(account).call()) if token_amount != 0: balances[account] = token_amount / (FVal(10)** FVal(token_decimals)) log.debug( 'Ethereum node result for token balance', sensitive_log=True, eth_address=account, token_address=token_address, token_symbol=token_symbol, amount=token_amount, ) else: for account in accounts: log.debug( 'Querying Etherscan for token balance', sensitive_log=True, eth_address=account, token_address=token_address, token_symbol=token_symbol, ) resp = request_get( 'https://api.etherscan.io/api?module=account&action=' 'tokenbalance&contractaddress={}&address={}'.format( token_address, account, )) if resp['status'] != 1: raise ValueError( 'Failed to query etherscan for {} token balance of {}'. format( token_symbol, account, )) token_amount = FVal(resp['result']) if token_amount != 0: balances[account] = token_amount / (FVal(10)** FVal(token_decimals)) log.debug('Etherscan result for token balance', sensitive_log=True, eth_address=account, token_address=token_address, token_symbol=token_symbol, amount=token_amount) return balances