def asset_to_aave_reserve_address(asset: Asset) -> Optional[ChecksumEthAddress]: if asset == A_ETH: # for v2 this should be WETH return ETH_SPECIAL_ADDRESS token = EthereumToken.from_asset(asset) assert token, 'should not be a non token asset at this point' return token.ethereum_address
def _force_remote(cursor: sqlite3.Cursor, local_asset: Asset, full_insert: str) -> None: """Force the remote entry into the database by deleting old one and doing the full insert. May raise an sqlite3 error if something fails. """ cursor.executescript('PRAGMA foreign_keys = OFF;') if local_asset.asset_type == AssetType.ETHEREUM_TOKEN: token = EthereumToken.from_asset(local_asset) cursor.execute( 'DELETE FROM ethereum_tokens WHERE address=?;', (token.ethereum_address, ), # type: ignore # token != None ) else: cursor.execute( 'DELETE FROM common_asset_details WHERE asset_id=?;', (local_asset.identifier, ), ) cursor.execute( 'DELETE FROM assets WHERE identifier=?;', (local_asset.identifier, ), ) cursor.executescript('PRAGMA foreign_keys = ON;') # Insert new entry. Since identifiers are the same, no foreign key constrains should break executeall(cursor, full_insert) AssetResolver().clean_memory_cache(local_asset.identifier.lower())
def _add_account_defi_balances_to_token_and_totals( self, account: ChecksumEthAddress, balances: List[DefiProtocolBalances], ) -> None: """Add a single account's defi balances to per account and totals""" for entry in balances: skip_list = DEFI_PROTOCOLS_TO_SKIP_ASSETS.get(entry.protocol.name, None) double_entry = ( entry.balance_type == 'Asset' and skip_list and (skip_list is True or entry.base_balance.token_symbol in skip_list) # type: ignore ) # We have to filter out specific balances/protocols here to not get double entries if double_entry: continue if entry.balance_type == 'Asset' and entry.base_balance.token_symbol == 'ETH': # If ETH appears as asset here I am not sure how to handle, so ignore for now log.warning( f'Found ETH in DeFi balances for account: {account} and ' f'protocol: {entry.protocol.name}. Ignoring ...', ) continue try: asset = Asset(entry.base_balance.token_symbol) except UnknownAsset: log.warning( f'Found unknown asset {entry.base_balance.token_symbol} in DeFi ' f'balances for account: {account} and ' f'protocol: {entry.protocol.name}. Ignoring ...', ) continue token = EthereumToken.from_asset(asset) if token is not None and token.ethereum_address != entry.base_balance.token_address: log.warning( f'Found token {token.identifier} with address ' f'{entry.base_balance.token_address} instead of expected ' f'{token.ethereum_address} for account: {account} and ' f'protocol: {entry.protocol.name}. Ignoring ...', ) continue eth_balances = self.balances.eth if entry.balance_type == 'Asset': eth_balances[account].assets[asset] += entry.base_balance.balance self.totals.assets[asset] += entry.base_balance.balance elif entry.balance_type == 'Debt': eth_balances[account].liabilities[asset] += entry.base_balance.balance self.totals.liabilities[asset] += entry.base_balance.balance else: log.warning( # type: ignore # is an unreachable statement but we are defensive f'Zerion Defi Adapter returned unknown asset type {entry.balance_type}. ' f'Skipping ...', ) continue
def _get_reserve_address_decimals(asset: Asset) -> Tuple[ChecksumEthAddress, int]: """Get the reserve address and the number of decimals for symbol""" if asset == A_ETH: reserve_address = ETH_SPECIAL_ADDRESS decimals = 18 else: token = EthereumToken.from_asset(asset) assert token, 'should not be a non token asset at this point' reserve_address = token.ethereum_address decimals = token.decimals return reserve_address, decimals
def asset_to_aave_reserve(asset: Asset) -> Optional[ChecksumEthAddress]: if asset == A_ETH: return AAVE_ETH_RESERVE_ADDRESS token = EthereumToken.from_asset(asset) if token is None: # should not be called with non token asset except for A_ETH return None if token not in ASSET_TO_ATOKENV1: return None return token.ethereum_address
def asset_normalized_value(amount: int, asset: Asset) -> FVal: """Takes in an amount and an asset and returns its normalized value May raise: - UnsupportedAsset if the given asset is not ETH or an ethereum token """ if asset.identifier == 'ETH': decimals = 18 else: token = EthereumToken.from_asset(asset) if token is None: raise UnsupportedAsset(asset.identifier) decimals = token.decimals return token_normalized_value_decimals(amount, decimals)
def find_usd_price( asset: Asset, ignore_cache: bool = False, ) -> Price: """Returns the current USD price of the asset Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs """ if asset == A_USD: return Price(FVal(1)) instance = Inquirer() cache_key = (asset, A_USD) if ignore_cache is False: cache = instance.get_cached_current_price_entry( cache_key=cache_key) if cache is not None: return cache.price if asset.is_fiat(): try: return instance._query_fiat_pair(base=asset, quote=A_USD) except RemoteError: pass # continue, a price can be found by one of the oracles (CC for example) if asset in instance.special_tokens: ethereum = instance._ethereum assert ethereum, 'Inquirer should never be called before the injection of ethereum' token = EthereumToken.from_asset(asset) assert token, 'all assets in special tokens are already ethereum tokens' underlying_asset_price = get_underlying_asset_price(token) usd_price = handle_defi_price_query( ethereum=ethereum, token=token, underlying_asset_price=underlying_asset_price, ) if usd_price is None: price = Price(ZERO) else: price = Price(usd_price) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=price, time=ts_now()) # noqa: E501 return price return instance._query_oracle_instances(from_asset=asset, to_asset=A_USD)
def handle_protocols( self, protocol_name: str, token_symbol: str, normalized_balance: FVal, token_address: str, token_name: str, ) -> Optional[DefiBalance]: """Special handling for price for token/protocols which are easier to do onchain or need some kind of special treatment. This method can raise DeserializationError """ if protocol_name == 'PoolTogether': result = _handle_pooltogether(normalized_balance, token_name) if result is not None: return result asset = get_asset_by_symbol(token_symbol) if asset is None: return None token = EthereumToken.from_asset(asset) if token is None: return None underlying_asset_price = get_underlying_asset_price(token) usd_price = handle_defi_price_query(self.ethereum, token, underlying_asset_price) if usd_price is None: return None return DefiBalance( token_address=deserialize_ethereum_address(token_address), token_name=token_name, token_symbol=token_symbol, balance=Balance(amount=normalized_balance, usd_value=normalized_balance * usd_price), )
def find_usd_price( asset: Asset, ignore_cache: bool = False, ) -> Price: """Returns the current USD price of the asset Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs """ if asset == A_USD: return Price(FVal(1)) instance = Inquirer() cache_key = (asset, A_USD) if ignore_cache is False: cache = instance.get_cached_current_price_entry( cache_key=cache_key) if cache is not None: return cache.price if asset.is_fiat(): try: return instance._query_fiat_pair(base=asset, quote=A_USD) except RemoteError: pass # continue, a price can be found by one of the oracles (CC for example) # Try and check if it is an ethereum token with specified protocol or underlying tokens is_known_protocol = False underlying_tokens = None try: token = EthereumToken.from_asset(asset) if token is not None: if token.protocol is not None: is_known_protocol = token.protocol in KnownProtocolsAssets underlying_tokens = GlobalDBHandler( ).get_ethereum_token( # type: ignore token.ethereum_address, ).underlying_tokens except UnknownAsset: pass # Check if it is a special token if asset in instance.special_tokens: ethereum = instance._ethereum assert ethereum, 'Inquirer should never be called before the injection of ethereum' assert token, 'all assets in special tokens are already ethereum tokens' underlying_asset_price = get_underlying_asset_price(token) usd_price = handle_defi_price_query( ethereum=ethereum, token=token, underlying_asset_price=underlying_asset_price, ) if usd_price is None: price = Price(ZERO) else: price = Price(usd_price) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=price, time=ts_now()) # noqa: E501 return price if is_known_protocol is True or underlying_tokens is not None: assert token is not None result = get_underlying_asset_price(token) if result is None: usd_price = Price(ZERO) if instance._ethereum is not None: instance._ethereum.msg_aggregator.add_warning( f'Could not find price for {token}', ) else: usd_price = Price(result) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=usd_price, time=ts_now(), ) return usd_price # BSQ is a special asset that doesnt have oracle information but its custom API if asset == A_BSQ: try: price_in_btc = get_bisq_market_price(asset) btc_price = Inquirer().find_usd_price(A_BTC) usd_price = Price(price_in_btc * btc_price) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=usd_price, time=ts_now(), ) return usd_price except (RemoteError, DeserializationError) as e: msg = f'Could not find price for BSQ. {str(e)}' if instance._ethereum is not None: instance._ethereum.msg_aggregator.add_warning(msg) return Price(BTC_PER_BSQ * price_in_btc) if asset == A_KFEE: # KFEE is a kraken special asset where 1000 KFEE = 10 USD return Price(FVal(0.01)) return instance._query_oracle_instances(from_asset=asset, to_asset=A_USD)
def find_usd_price( asset: Asset, ignore_cache: bool = False, ) -> Price: """Returns the current USD price of the asset Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs """ if asset == A_USD: return Price(FVal(1)) instance = Inquirer() cache_key = (asset, A_USD) if ignore_cache is False: cache = instance.get_cached_current_price_entry( cache_key=cache_key) if cache is not None: return cache.price if asset.is_fiat(): try: return instance._query_fiat_pair(base=asset, quote=A_USD) except RemoteError: pass # continue, a price can be found by one of the oracles (CC for example) # Try and check if it is an ethereum token with specified protocol or underlying tokens is_known_protocol = False underlying_tokens = None try: token = EthereumToken.from_asset(asset) if token is not None: if token.protocol is not None: is_known_protocol = token.protocol in KnownProtocolsAssets underlying_tokens = GlobalDBHandler( ).get_ethereum_token( # type: ignore token.ethereum_address, ).underlying_tokens except UnknownAsset: pass # Check if it is a special token if asset in instance.special_tokens: ethereum = instance._ethereum assert ethereum, 'Inquirer should never be called before the injection of ethereum' assert token, 'all assets in special tokens are already ethereum tokens' underlying_asset_price = get_underlying_asset_price(token) usd_price = handle_defi_price_query( ethereum=ethereum, token=token, underlying_asset_price=underlying_asset_price, ) if usd_price is None: price = Price(ZERO) else: price = Price(usd_price) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=price, time=ts_now()) # noqa: E501 return price if is_known_protocol is True or underlying_tokens is not None: assert token is not None result = get_underlying_asset_price(token) usd_price = Price(ZERO) if result is None else Price(result) Inquirer._cached_current_price[cache_key] = CachedPriceEntry( price=usd_price, time=ts_now(), ) return usd_price return instance._query_oracle_instances(from_asset=asset, to_asset=A_USD)
def query_tokens_for_addresses( self, addresses: List[ChecksumEthAddress], force_detection: bool, ) -> TokensReturn: """Queries/detects token balances for a list of addresses If an address's tokens were recently autodetected they are not detected again but the balances are simply queried. Unless force_detection is True. Returns the token balances of each address and the usd prices of the tokens """ log.debug( 'Querying/detecting token balances for all addresses', force_detection=force_detection, ) ignored_assets = self.db.get_ignored_assets() exceptions = [ # Ignore the veCRV balance in token query. It's already detected by # defi SDK as part of locked CRV in Vote Escrowed CRV. Which is the right way # to approach it as there is no way to assign a price to 1 veCRV. It # can be 1 CRV locked for 4 years or 4 CRV locked for 1 year etc. string_to_ethereum_address( '0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2'), # Ignore for now xsushi since is queried by defi SDK. We'll do it for now # since the SDK entry might return other tokens from sushi and we don't # fully support sushi now. string_to_ethereum_address( '0x8798249c2E607446EfB7Ad49eC89dD1865Ff4272'), # Ignore the following tokens. They are old tokens of upgraded contracts which # duplicated the balances at upgrade instead of doing a token swap. # e.g.: https://github.com/rotki/rotki/issues/3548 # TODO: At some point we should actually remove them from the DB and # upgrade possible occurences in the user DB # # Old contract of Fetch.ai string_to_ethereum_address( '0x1D287CC25dAD7cCaF76a26bc660c5F7C8E2a05BD'), ] for asset in ignored_assets: # don't query for the ignored tokens if asset.is_eth_token( ): # type ignore since we know asset is a token exceptions.append( EthereumToken.from_asset( asset).ethereum_address) # type: ignore all_tokens = GlobalDBHandler().get_ethereum_tokens( exceptions=exceptions, except_protocols=['balancer'], ) # With etherscan with chunks > 120, we get request uri too large # so the limitation is not in the gas, but in the request uri length etherscan_chunks = list( get_chunks(all_tokens, n=ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH)) other_chunks = list( get_chunks(all_tokens, n=OTHER_MAX_TOKEN_CHUNK_LENGTH)) now = ts_now() token_usd_price: Dict[EthereumToken, Price] = {} result = {} for address in addresses: saved_list = self.db.get_tokens_for_address_if_time( address=address, current_time=now) if force_detection or saved_list is None: balances = self.detect_tokens_for_address( address=address, token_usd_price=token_usd_price, etherscan_chunks=etherscan_chunks, other_chunks=other_chunks, ) else: if len(saved_list) == 0: continue # Do not query if we know the address has no tokens balances = defaultdict(FVal) self._get_tokens_balance_and_price( address=address, tokens=saved_list, balances=balances, token_usd_price=token_usd_price, call_order=None, # use defaults ) result[address] = balances return result, token_usd_price
def get_price( self, from_asset: Asset, to_asset: Asset, block_identifier: BlockIdentifier, ) -> Price: """ Return the price of from_asset to to_asset at the block block_identifier. External oracles are used if non eth tokens are used. Can raise: - PriceQueryUnsupportedAsset - RemoteError """ log.debug( f'Searching price for {from_asset} to {to_asset} at ' f'{block_identifier!r} with {self.name}', ) # Uniswap V2 and V3 use in their contracts WETH instead of ETH if from_asset == A_ETH: from_asset = A_WETH if to_asset == A_ETH: to_asset = A_WETH if from_asset == to_asset: return Price(ONE) if not (from_asset.is_eth_token() and to_asset.is_eth_token()): raise PriceQueryUnsupportedAsset( f'Either {from_asset} or {to_asset} arent ethereum tokens for the uniswap oracle', ) # Could be that we are dealing with ethereum tokens as instances of Asset instead of # EthereumToken, handle the conversion from_asset_raw: Union[Asset, EthereumToken] = from_asset to_asset_raw: Union[Asset, EthereumToken] = to_asset if not isinstance(from_asset, EthereumToken): from_as_token = EthereumToken.from_asset(from_asset) if from_as_token is None: raise PriceQueryUnsupportedAsset( f'Unsupported asset for uniswap {from_asset_raw}') from_asset = from_as_token if not isinstance(to_asset, EthereumToken): to_as_token = EthereumToken.from_asset(to_asset) if to_as_token is None: raise PriceQueryUnsupportedAsset( f'Unsupported asset for uniswap {to_asset_raw}') to_asset = to_as_token route = self.find_route(from_asset, to_asset) if len(route) == 0: log.debug( f'Failed to find uniswap price for {from_asset} to {to_asset}') return Price(ZERO) log.debug( f'Found price route {route} for {from_asset} to {to_asset} using {self.name}' ) prices_and_tokens = [] for step in route: log.debug(f'Getting pool price for {step}') prices_and_tokens.append( self.get_pool_price( pool_addr=to_checksum_address(step), block_identifier=block_identifier, ), ) # Looking at which one is token0 and token1 we need to see if we need price or 1/price if prices_and_tokens[0].token_0 != from_asset: prices_and_tokens[0] = prices_and_tokens[0].swap_tokens() # For the possible intermediate steps also make sure that we use the correct price for pos, item in enumerate(prices_and_tokens[1:-1]): if item.token_0 != prices_and_tokens[pos - 1].token_1: prices_and_tokens[pos - 1] = prices_and_tokens[pos - 1].swap_tokens() # Finally for the tail query the price if prices_and_tokens[-1].token_1 != to_asset: prices_and_tokens[-1] = prices_and_tokens[-1].swap_tokens() price = FVal(reduce(mul, [item.price for item in prices_and_tokens], 1)) return Price(price)
def aave_event_from_db(event_tuple: AAVE_EVENT_DB_TUPLE) -> AaveEvent: """Turns a tuple read from the DB into an appropriate AaveEvent May raise a DeserializationError if something is wrong with the DB data """ event_type = event_tuple[1] block_number = event_tuple[2] timestamp = Timestamp(event_tuple[3]) tx_hash = event_tuple[4] log_index = event_tuple[5] asset2 = None if event_tuple[9] is not None: try: asset2 = Asset(event_tuple[9]) except UnknownAsset as e: raise DeserializationError( f'Unknown asset {event_tuple[6]} encountered during deserialization ' f'of Aave event from DB for asset2', ) from e try: asset1 = Asset(event_tuple[6]) except UnknownAsset as e: raise DeserializationError( f'Unknown asset {event_tuple[6]} encountered during deserialization ' f'of Aave event from DB for asset1', ) from e asset1_amount = FVal(event_tuple[7]) asset1_usd_value = FVal(event_tuple[8]) if event_type in ('deposit', 'withdrawal'): return AaveDepositWithdrawalEvent( event_type=event_type, block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, asset=asset1, atoken=EthereumToken.from_asset( asset2), # type: ignore # should be a token value=Balance(amount=asset1_amount, usd_value=asset1_usd_value), ) if event_type == 'interest': return AaveInterestEvent( event_type=event_type, block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, asset=asset1, value=Balance(amount=asset1_amount, usd_value=asset1_usd_value), ) if event_type == 'borrow': if event_tuple[12] not in ('stable', 'variable'): raise DeserializationError( f'Invalid borrow rate mode encountered in the DB: {event_tuple[12]}', ) borrow_rate_mode: Literal['stable', 'variable'] = event_tuple[12] # type: ignore borrow_rate = deserialize_optional_to_fval( value=event_tuple[10], name='borrow_rate', location='reading aave borrow event from DB', ) accrued_borrow_interest = deserialize_optional_to_fval( value=event_tuple[11], name='accrued_borrow_interest', location='reading aave borrow event from DB', ) return AaveBorrowEvent( event_type=event_type, block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, asset=asset1, value=Balance(amount=asset1_amount, usd_value=asset1_usd_value), borrow_rate_mode=borrow_rate_mode, borrow_rate=borrow_rate, accrued_borrow_interest=accrued_borrow_interest, ) if event_type == 'repay': fee_amount = deserialize_optional_to_fval( value=event_tuple[10], name='fee_amount', location='reading aave repay event from DB', ) fee_usd_value = deserialize_optional_to_fval( value=event_tuple[11], name='fee_usd_value', location='reading aave repay event from DB', ) return AaveRepayEvent( event_type=event_type, block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, asset=asset1, value=Balance(amount=asset1_amount, usd_value=asset1_usd_value), fee=Balance(amount=fee_amount, usd_value=fee_usd_value), ) if event_type == 'liquidation': if asset2 is None: raise DeserializationError( 'Did not find asset2 in an aave liquidation event fom the DB.', ) principal_amount = deserialize_optional_to_fval( value=event_tuple[10], name='principal_amount', location='reading aave liquidation event from DB', ) principal_usd_value = deserialize_optional_to_fval( value=event_tuple[11], name='principal_usd_value', location='reading aave liquidation event from DB', ) return AaveLiquidationEvent( event_type=event_type, block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, collateral_asset=asset1, collateral_balance=Balance(amount=asset1_amount, usd_value=asset1_usd_value), principal_asset=asset2, principal_balance=Balance( amount=principal_amount, usd_value=principal_usd_value, ), ) # else raise DeserializationError( f'Unknown event type {event_type} encountered during ' f'deserialization of Aave event from DB', )
def query_tokens_for_addresses( self, addresses: List[ChecksumEthAddress], force_detection: bool, ) -> TokensReturn: """Queries/detects token balances for a list of addresses If an address's tokens were recently autodetected they are not detected again but the balances are simply queried. Unless force_detection is True. Returns the token balances of each address and the usd prices of the tokens """ log.debug( 'Querying/detecting token balances for all addresses', force_detection=force_detection, ) ignored_assets = self.db.get_ignored_assets() exceptions = [ # Ignore the veCRV balance in token query. It's already detected by # defi SDK as part of locked CRV in Vote Escrowed CRV. Which is the right way # to approach it as there is no way to assign a price to 1 veCRV. It # can be 1 CRV locked for 4 years or 4 CRV locked for 1 year etc. string_to_ethereum_address('0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2'), ] for asset in ignored_assets: # don't query for the ignored tokens if asset.is_eth_token(): # type ignore since we know asset is a token exceptions.append(EthereumToken.from_asset(asset).ethereum_address) # type: ignore all_tokens = GlobalDBHandler().get_ethereum_tokens(exceptions=exceptions) # With etherscan with chunks > 120, we get request uri too large # so the limitation is not in the gas, but in the request uri length etherscan_chunks = list(get_chunks(all_tokens, n=ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH)) other_chunks = list(get_chunks(all_tokens, n=OTHER_MAX_TOKEN_CHUNK_LENGTH)) now = ts_now() token_usd_price: Dict[EthereumToken, Price] = {} result = {} for address in addresses: saved_list = self.db.get_tokens_for_address_if_time(address=address, current_time=now) if force_detection or saved_list is None: balances = self.detect_tokens_for_address( address=address, token_usd_price=token_usd_price, etherscan_chunks=etherscan_chunks, other_chunks=other_chunks, ) else: if len(saved_list) == 0: continue # Do not query if we know the address has no tokens balances = defaultdict(FVal) self._get_tokens_balance_and_price( address=address, tokens=saved_list, balances=balances, token_usd_price=token_usd_price, call_order=None, # use defaults ) result[address] = balances return result, token_usd_price