def transactions_from_dictlist( given_transactions: List[Dict[str, Any]], start_ts: Timestamp, end_ts: Timestamp, ) -> List[EthereumTransaction]: """ Gets a list of transaction, most probably read from the json files and a time period. Returns it as a list of the transaction tuples that are inside the time period May raise: - KeyError: If the given_transactions contain data in an unexpected format """ returned_transactions = list() for given_tx in given_transactions: if given_tx['timestamp'] < start_ts: continue if given_tx['timestamp'] > end_ts: break timestamp = Timestamp(convert_to_int(given_tx['timestamp'])) tx_hash = given_tx['hash'] from_address = given_tx['from_address'] to_address = given_tx['to_address'] value = FVal(given_tx['value']) gas = FVal(given_tx['gas']) gas_price = FVal(given_tx['gas_price']) gas_used = FVal(given_tx['gas_used']) log.debug( 'Processing eth transaction', sensitive_log=True, timestamp=timestamp, eth_tx_hash=tx_hash, from_eth_address=from_address, to_eth_address=to_address, tx_value=value, gas=gas, gas_price=gas_price, gas_used=gas_used, ) returned_transactions.append( EthereumTransaction( timestamp=timestamp, block_number=convert_to_int(given_tx['block_number']), hash=tx_hash, from_address=from_address, to_address=to_address, value=value, gas=gas, gas_price=gas_price, gas_used=gas_used, )) return returned_transactions
def deserialize_timestamp_from_kraken(time: Union[str, FVal]) -> Timestamp: """Deserializes a timestamp from a kraken api query result entry Kraken has timestamps in floating point strings. Example: '1561161486.3056'. If the dictionary has passed through rlk_jsonloads the entry can also be an Fval Can throw DeserializationError if the data is not as expected """ if not time: raise DeserializationError( 'Failed to deserialize a timestamp entry from a null entry in kraken', ) if isinstance(time, str): try: return Timestamp(convert_to_int(time, accept_only_exact=False)) except ConversionError: raise DeserializationError( f'Failed to deserialize {time} kraken timestamp entry') elif isinstance(time, FVal): try: return Timestamp(time.to_int(exact=False)) except ConversionError: raise DeserializationError( f'Failed to deserialize {time} kraken timestamp entry from an FVal', ) else: raise DeserializationError( f'Failed to deserialize a timestamp entry from a {type(time)} entry in kraken', )
def read_integer(data: Dict[str, Any], key: str) -> int: try: result = convert_to_int(data[key]) except ConversionError: raise DeserializationError( f'Failed to read {key} as an integer during etherscan transaction query', ) return result
def read_integer(data: Dict[str, Any], key: str, api: str = DEFAULT_API) -> int: try: result = convert_to_int(data[key]) except ConversionError as e: raise DeserializationError( f'Failed to read {key} as an integer during {api} transaction query', ) from e return result
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List: with self.lock: cache = self.check_trades_cache( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: movements.append( AssetMovement( exchange='kraken', category=movement['type'], # Kraken timestamps have floating point timestamp=convert_to_int(movement['time'], accept_only_exact=False), asset=asset_from_kraken(movement['asset']), amount=FVal(movement['amount']), fee=FVal(movement['fee']), )) return movements
def _retrieve_price_from_data( data: Optional[List[PriceHistoryEntry]], from_asset: Asset, to_asset: Asset, timestamp: Timestamp, ) -> Price: """Reads historical price data list returned from cryptocompare histohour or cache and returns a price. If nothing is found it returns Price(0). This can also happen if cryptocompare returns a list of 0s for the timerange. """ price = Price(ZERO) if data is None or len(data) == 0: return price # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: index_in_bounds = True # convert_to_int can't raise here due to its input index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) if index > len(data) - 1: # index out of bounds # Try to see if index - 1 is there and if yes take it if index > len(data): index = index - 1 else: # give up. This happened: https://github.com/rotki/rotki/issues/1534 log.error( f'Expected data index in cryptocompare historical hour price ' f'not found. Queried price of: {from_asset.identifier} in ' f'{to_asset.identifier} at {timestamp}. Data ' f'index: {index}. Length of returned data: {len(data)}. ' f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...', ) index_in_bounds = False if index_in_bounds: diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is not None and data[index].low is not None: price = Price((data[index].high + data[index].low) / 2) else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) return price
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade: """Turn a kraken trade returned from kraken trade history to our common trade history format - Can raise UnknownAsset due to kraken_to_world_pair - Can raise UnprocessableTradePair due to kraken_to_world_pair """ currency_pair = kraken_to_world_pair(kraken_trade['pair']) quote_currency = get_pair_position_asset(currency_pair, 'second') # Kraken timestamps have floating point timestamp = Timestamp( convert_to_int(kraken_trade['time'], accept_only_exact=False)) amount = FVal(kraken_trade['vol']) cost = FVal(kraken_trade['cost']) fee = FVal(kraken_trade['fee']) order_type = trade_type_from_string(kraken_trade['type']) rate = FVal(kraken_trade['price']) if cost != amount * rate: log.warning( 'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade' ) log.debug( 'Processing kraken Trade', sensitive_log=True, timestamp=timestamp, order_type=order_type, kraken_pair=kraken_trade['pair'], pair=currency_pair, quote_currency=quote_currency, amount=amount, cost=cost, fee=fee, rate=rate, ) return Trade( timestamp=timestamp, location='kraken', pair=currency_pair, trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=quote_currency, )
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. May raise: - PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from cryptocompare - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ try: data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) except UnsupportedAsset as e: raise PriceQueryUnsupportedAsset(e.asset_name) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: # convert_to_int can't raise here due to its input index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is None or data[index].low is None: # If we get some None in the hourly set price to 0 so that we check alternatives price = Price(ZERO) else: price = Price((data[index].high + data[index].low) / 2) else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = Price(asset_btc_price * btc_to_asset_price) else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset=from_asset, to_asset=to_asset, date=timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_list( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) result: List[Any] if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: movements.append( AssetMovement( exchange='kraken', category=movement['type'], # Kraken timestamps have floating point timestamp=Timestamp( convert_to_int(movement['time'], accept_only_exact=False)), asset=asset_from_kraken(movement['asset']), amount=FVal(movement['amount']), fee=Fee(FVal(movement['fee'])), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue return movements
def test_convert_to_int(): assert convert_to_int('5') == 5 with pytest.raises(ConversionError): assert convert_to_int(5.44, accept_only_exact=True) == 5 assert convert_to_int(5.44, accept_only_exact=False) == 5 assert convert_to_int(5.65, accept_only_exact=False) == 5 with pytest.raises(ConversionError): assert convert_to_int(FVal('5.44'), accept_only_exact=True) == 5 assert convert_to_int(FVal('5.44'), accept_only_exact=False) == 5 assert convert_to_int(FVal('5.65'), accept_only_exact=False) == 5 assert convert_to_int(FVal('4'), accept_only_exact=True) == 4 assert convert_to_int(3) == 3 with pytest.raises(ConversionError): assert convert_to_int('5.44', accept_only_exact=True) == 5 assert convert_to_int('5.44', accept_only_exact=False) == 5 assert convert_to_int('5.65', accept_only_exact=False) == 5 assert convert_to_int('4', accept_only_exact=False) == 4 with pytest.raises(ConversionError): assert convert_to_int(b'5.44', accept_only_exact=True) == 5 assert convert_to_int(b'5.44', accept_only_exact=False) == 5 assert convert_to_int(b'5.65', accept_only_exact=False) == 5 assert convert_to_int(b'4', accept_only_exact=False) == 4
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: if from_asset in KNOWN_TO_MISS_FROM_CRYPTOCOMPARE: raise PriceQueryUnknownFromAsset(from_asset) data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is None or data[index].low is None: # If we get some None in the hourly set price to 0 so that we check alternatives price = Price(ZERO) else: price = (data[index].high + data[index].low) / 2 else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = asset_btc_price * btc_to_asset_price else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset, to_asset, timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. May raise: - PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from cryptocompare - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ try: data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) except UnsupportedAsset as e: raise PriceQueryUnsupportedAsset(e.asset_name) price = Price(ZERO) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: index_in_bounds = True # convert_to_int can't raise here due to its input index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) if index > len(data) - 1: # index out of bounds # Try to see if index - 1 is there and if yes take it if index > len(data): index = index - 1 else: # give up. This happened: https://github.com/rotki/rotki/issues/1534 log.error( f'Expected data index in cryptocompare historical hour price ' f'not found. Queried price of: {from_asset.identifier} in ' f'{to_asset.identifier} at {timestamp}. Data ' f'index: {index}. Length of returned data: {len(data)}. ' f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...', ) index_in_bounds = False if index_in_bounds: diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is not None and data[index].low is not None: price = Price((data[index].high + data[index].low) / 2) else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = Price(asset_btc_price * btc_to_asset_price) else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset=from_asset, to_asset=to_asset, date=timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def test_convert_to_int(): assert convert_to_int('5') == 5 assert convert_to_int( '37451082560000003241000000000003221111111111' ) == 37451082560000003241000000000003221111111111 # noqa: E501 with pytest.raises(ConversionError): assert convert_to_int(5.44, accept_only_exact=True) == 5 assert convert_to_int(5.44, accept_only_exact=False) == 5 assert convert_to_int(5.65, accept_only_exact=False) == 5 with pytest.raises(ConversionError): assert convert_to_int(FVal('5.44'), accept_only_exact=True) == 5 assert convert_to_int(FVal('5.44'), accept_only_exact=False) == 5 assert convert_to_int(FVal('5.65'), accept_only_exact=False) == 5 assert convert_to_int(FVal('4'), accept_only_exact=True) == 4 assert convert_to_int(3) == 3 with pytest.raises(ConversionError): assert convert_to_int('5.44', accept_only_exact=True) == 5 assert convert_to_int('5.44', accept_only_exact=False) == 5 assert convert_to_int('5.65', accept_only_exact=False) == 5 assert convert_to_int('4', accept_only_exact=False) == 4 with pytest.raises(ConversionError): assert convert_to_int(b'5.44', accept_only_exact=True) == 5 assert convert_to_int(b'5.44', accept_only_exact=False) == 5 assert convert_to_int(b'5.65', accept_only_exact=False) == 5 assert convert_to_int(b'4', accept_only_exact=False) == 4
def query_ethereum_txlist( address: EthAddress, internal: bool, from_block: Optional[int] = None, to_block: Optional[int] = None, ) -> List[EthereumTransaction]: log.debug( 'Querying etherscan for tx list', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, ) result = list() if internal: reqstring = ( 'https://api.etherscan.io/api?module=account&action=' 'txlistinternal&address={}'.format(address) ) else: reqstring = ( 'https://api.etherscan.io/api?module=account&action=' 'txlist&address={}'.format(address) ) if from_block: reqstring += '&startblock={}'.format(from_block) if to_block: reqstring += '&endblock={}'.format(to_block) resp = request_get_dict(reqstring) if 'status' not in resp or convert_to_int(resp['status']) != 1: status = convert_to_int(resp['status']) if status == 0 and resp['message'] == 'No transactions found': return list() log.error( 'Querying etherscan for tx list failed', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, error=resp['message'], ) # else unknown error raise ValueError( 'Failed to query txlist from etherscan with query: {} . ' 'Response was: {}'.format(reqstring, resp), ) log.debug('Etherscan tx list query result', results_num=len(resp['result'])) for v in resp['result']: # internal tx list contains no gasprice gas_price = FVal(-1) if internal else FVal(v['gasPrice']) result.append(EthereumTransaction( timestamp=Timestamp(convert_to_int(v['timeStamp'])), block_number=convert_to_int(v['blockNumber']), hash=v['hash'], from_address=v['from'], to_address=v['to'], value=FVal(v['value']), gas=FVal(v['gas']), gas_price=gas_price, gas_used=FVal(v['gasUsed']), )) return result
def query_ethereum_txlist( address: EthAddress, msg_aggregator: MessagesAggregator, internal: bool, from_block: Optional[int] = None, to_block: Optional[int] = None, ) -> List[EthereumTransaction]: """Query ethereum tx list""" log.debug( 'Querying etherscan for tx list', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, ) result = list() if internal: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlistinternal&address={}'.format(address)) else: reqstring = ('https://api.etherscan.io/api?module=account&action=' 'txlist&address={}'.format(address)) if from_block: reqstring += '&startblock={}'.format(from_block) if to_block: reqstring += '&endblock={}'.format(to_block) resp = request_get_dict(reqstring) if 'status' not in resp or convert_to_int(resp['status']) != 1: status = convert_to_int(resp['status']) if status == 0 and resp['message'] == 'No transactions found': return list() log.error( 'Querying etherscan for tx list failed', sensitive_log=True, internal=internal, eth_address=address, from_block=from_block, to_block=to_block, error=resp['message'], ) # else unknown error raise ValueError( 'Failed to query txlist from etherscan with query: {} . ' 'Response was: {}'.format(reqstring, resp), ) log.debug('Etherscan tx list query result', results_num=len(resp['result'])) for v in resp['result']: try: tx = deserialize_transaction_from_etherscan(data=v, internal=internal) except DeserializationError as e: msg_aggregator.add_warning(f'{str(e)}. Skipping transaction') continue result.append(tx) return result