def asset_to_aave_reserve_address(asset: Asset) -> Optional[ChecksumEthAddress]:
if asset == A_ETH: # for v2 this should be WETH
return ETH_SPECIAL_ADDRESS
token = EthereumToken.from_asset(asset)
assert token, 'should not be a non token asset at this point'
return token.ethereum_address
def _force_remote(cursor: sqlite3.Cursor, local_asset: Asset,
full_insert: str) -> None:
"""Force the remote entry into the database by deleting old one and doing the full insert.
May raise an sqlite3 error if something fails.
"""
cursor.executescript('PRAGMA foreign_keys = OFF;')
if local_asset.asset_type == AssetType.ETHEREUM_TOKEN:
token = EthereumToken.from_asset(local_asset)
cursor.execute(
'DELETE FROM ethereum_tokens WHERE address=?;',
(token.ethereum_address, ), # type: ignore # token != None
)
else:
cursor.execute(
'DELETE FROM common_asset_details WHERE asset_id=?;',
(local_asset.identifier, ),
)
cursor.execute(
'DELETE FROM assets WHERE identifier=?;',
(local_asset.identifier, ),
)
cursor.executescript('PRAGMA foreign_keys = ON;')
# Insert new entry. Since identifiers are the same, no foreign key constrains should break
executeall(cursor, full_insert)
AssetResolver().clean_memory_cache(local_asset.identifier.lower())
def _add_account_defi_balances_to_token_and_totals(
self,
account: ChecksumEthAddress,
balances: List[DefiProtocolBalances],
) -> None:
"""Add a single account's defi balances to per account and totals"""
for entry in balances:
skip_list = DEFI_PROTOCOLS_TO_SKIP_ASSETS.get(entry.protocol.name, None)
double_entry = (
entry.balance_type == 'Asset' and
skip_list and
(skip_list is True or entry.base_balance.token_symbol in skip_list) # type: ignore
)
# We have to filter out specific balances/protocols here to not get double entries
if double_entry:
continue
if entry.balance_type == 'Asset' and entry.base_balance.token_symbol == 'ETH':
# If ETH appears as asset here I am not sure how to handle, so ignore for now
log.warning(
f'Found ETH in DeFi balances for account: {account} and '
f'protocol: {entry.protocol.name}. Ignoring ...',
)
continue
try:
asset = Asset(entry.base_balance.token_symbol)
except UnknownAsset:
log.warning(
f'Found unknown asset {entry.base_balance.token_symbol} in DeFi '
f'balances for account: {account} and '
f'protocol: {entry.protocol.name}. Ignoring ...',
)
continue
token = EthereumToken.from_asset(asset)
if token is not None and token.ethereum_address != entry.base_balance.token_address:
log.warning(
f'Found token {token.identifier} with address '
f'{entry.base_balance.token_address} instead of expected '
f'{token.ethereum_address} for account: {account} and '
f'protocol: {entry.protocol.name}. Ignoring ...',
)
continue
eth_balances = self.balances.eth
if entry.balance_type == 'Asset':
eth_balances[account].assets[asset] += entry.base_balance.balance
self.totals.assets[asset] += entry.base_balance.balance
elif entry.balance_type == 'Debt':
eth_balances[account].liabilities[asset] += entry.base_balance.balance
self.totals.liabilities[asset] += entry.base_balance.balance
else:
log.warning( # type: ignore # is an unreachable statement but we are defensive
f'Zerion Defi Adapter returned unknown asset type {entry.balance_type}. '
f'Skipping ...',
)
continue
def _get_reserve_address_decimals(asset: Asset) -> Tuple[ChecksumEthAddress, int]:
"""Get the reserve address and the number of decimals for symbol"""
if asset == A_ETH:
reserve_address = ETH_SPECIAL_ADDRESS
decimals = 18
else:
token = EthereumToken.from_asset(asset)
assert token, 'should not be a non token asset at this point'
reserve_address = token.ethereum_address
decimals = token.decimals
return reserve_address, decimals
def asset_to_aave_reserve(asset: Asset) -> Optional[ChecksumEthAddress]:
if asset == A_ETH:
return AAVE_ETH_RESERVE_ADDRESS
token = EthereumToken.from_asset(asset)
if token is None: # should not be called with non token asset except for A_ETH
return None
if token not in ASSET_TO_ATOKENV1:
return None
return token.ethereum_address
def asset_normalized_value(amount: int, asset: Asset) -> FVal:
"""Takes in an amount and an asset and returns its normalized value
May raise:
- UnsupportedAsset if the given asset is not ETH or an ethereum token
"""
if asset.identifier == 'ETH':
decimals = 18
else:
token = EthereumToken.from_asset(asset)
if token is None:
raise UnsupportedAsset(asset.identifier)
decimals = token.decimals
return token_normalized_value_decimals(amount, decimals)
def find_usd_price(
asset: Asset,
ignore_cache: bool = False,
) -> Price:
"""Returns the current USD price of the asset
Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs
"""
if asset == A_USD:
return Price(FVal(1))
instance = Inquirer()
cache_key = (asset, A_USD)
if ignore_cache is False:
cache = instance.get_cached_current_price_entry(
cache_key=cache_key)
if cache is not None:
return cache.price
if asset.is_fiat():
try:
return instance._query_fiat_pair(base=asset, quote=A_USD)
except RemoteError:
pass # continue, a price can be found by one of the oracles (CC for example)
if asset in instance.special_tokens:
ethereum = instance._ethereum
assert ethereum, 'Inquirer should never be called before the injection of ethereum'
token = EthereumToken.from_asset(asset)
assert token, 'all assets in special tokens are already ethereum tokens'
underlying_asset_price = get_underlying_asset_price(token)
usd_price = handle_defi_price_query(
ethereum=ethereum,
token=token,
underlying_asset_price=underlying_asset_price,
)
if usd_price is None:
price = Price(ZERO)
else:
price = Price(usd_price)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=price, time=ts_now()) # noqa: E501
return price
return instance._query_oracle_instances(from_asset=asset,
to_asset=A_USD)
def handle_protocols(
self,
protocol_name: str,
token_symbol: str,
normalized_balance: FVal,
token_address: str,
token_name: str,
) -> Optional[DefiBalance]:
"""Special handling for price for token/protocols which are easier to do onchain
or need some kind of special treatment.
This method can raise DeserializationError
"""
if protocol_name == 'PoolTogether':
result = _handle_pooltogether(normalized_balance, token_name)
if result is not None:
return result
asset = get_asset_by_symbol(token_symbol)
if asset is None:
return None
token = EthereumToken.from_asset(asset)
if token is None:
return None
underlying_asset_price = get_underlying_asset_price(token)
usd_price = handle_defi_price_query(self.ethereum, token,
underlying_asset_price)
if usd_price is None:
return None
return DefiBalance(
token_address=deserialize_ethereum_address(token_address),
token_name=token_name,
token_symbol=token_symbol,
balance=Balance(amount=normalized_balance,
usd_value=normalized_balance * usd_price),
)
def find_usd_price(
asset: Asset,
ignore_cache: bool = False,
) -> Price:
"""Returns the current USD price of the asset
Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs
"""
if asset == A_USD:
return Price(FVal(1))
instance = Inquirer()
cache_key = (asset, A_USD)
if ignore_cache is False:
cache = instance.get_cached_current_price_entry(
cache_key=cache_key)
if cache is not None:
return cache.price
if asset.is_fiat():
try:
return instance._query_fiat_pair(base=asset, quote=A_USD)
except RemoteError:
pass # continue, a price can be found by one of the oracles (CC for example)
# Try and check if it is an ethereum token with specified protocol or underlying tokens
is_known_protocol = False
underlying_tokens = None
try:
token = EthereumToken.from_asset(asset)
if token is not None:
if token.protocol is not None:
is_known_protocol = token.protocol in KnownProtocolsAssets
underlying_tokens = GlobalDBHandler(
).get_ethereum_token( # type: ignore
token.ethereum_address, ).underlying_tokens
except UnknownAsset:
pass
# Check if it is a special token
if asset in instance.special_tokens:
ethereum = instance._ethereum
assert ethereum, 'Inquirer should never be called before the injection of ethereum'
assert token, 'all assets in special tokens are already ethereum tokens'
underlying_asset_price = get_underlying_asset_price(token)
usd_price = handle_defi_price_query(
ethereum=ethereum,
token=token,
underlying_asset_price=underlying_asset_price,
)
if usd_price is None:
price = Price(ZERO)
else:
price = Price(usd_price)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=price, time=ts_now()) # noqa: E501
return price
if is_known_protocol is True or underlying_tokens is not None:
assert token is not None
result = get_underlying_asset_price(token)
if result is None:
usd_price = Price(ZERO)
if instance._ethereum is not None:
instance._ethereum.msg_aggregator.add_warning(
f'Could not find price for {token}', )
else:
usd_price = Price(result)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=usd_price,
time=ts_now(),
)
return usd_price
# BSQ is a special asset that doesnt have oracle information but its custom API
if asset == A_BSQ:
try:
price_in_btc = get_bisq_market_price(asset)
btc_price = Inquirer().find_usd_price(A_BTC)
usd_price = Price(price_in_btc * btc_price)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=usd_price,
time=ts_now(),
)
return usd_price
except (RemoteError, DeserializationError) as e:
msg = f'Could not find price for BSQ. {str(e)}'
if instance._ethereum is not None:
instance._ethereum.msg_aggregator.add_warning(msg)
return Price(BTC_PER_BSQ * price_in_btc)
if asset == A_KFEE:
# KFEE is a kraken special asset where 1000 KFEE = 10 USD
return Price(FVal(0.01))
return instance._query_oracle_instances(from_asset=asset,
to_asset=A_USD)
def find_usd_price(
asset: Asset,
ignore_cache: bool = False,
) -> Price:
"""Returns the current USD price of the asset
Returns Price(ZERO) if all options have been exhausted and errors are logged in the logs
"""
if asset == A_USD:
return Price(FVal(1))
instance = Inquirer()
cache_key = (asset, A_USD)
if ignore_cache is False:
cache = instance.get_cached_current_price_entry(
cache_key=cache_key)
if cache is not None:
return cache.price
if asset.is_fiat():
try:
return instance._query_fiat_pair(base=asset, quote=A_USD)
except RemoteError:
pass # continue, a price can be found by one of the oracles (CC for example)
# Try and check if it is an ethereum token with specified protocol or underlying tokens
is_known_protocol = False
underlying_tokens = None
try:
token = EthereumToken.from_asset(asset)
if token is not None:
if token.protocol is not None:
is_known_protocol = token.protocol in KnownProtocolsAssets
underlying_tokens = GlobalDBHandler(
).get_ethereum_token( # type: ignore
token.ethereum_address, ).underlying_tokens
except UnknownAsset:
pass
# Check if it is a special token
if asset in instance.special_tokens:
ethereum = instance._ethereum
assert ethereum, 'Inquirer should never be called before the injection of ethereum'
assert token, 'all assets in special tokens are already ethereum tokens'
underlying_asset_price = get_underlying_asset_price(token)
usd_price = handle_defi_price_query(
ethereum=ethereum,
token=token,
underlying_asset_price=underlying_asset_price,
)
if usd_price is None:
price = Price(ZERO)
else:
price = Price(usd_price)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=price, time=ts_now()) # noqa: E501
return price
if is_known_protocol is True or underlying_tokens is not None:
assert token is not None
result = get_underlying_asset_price(token)
usd_price = Price(ZERO) if result is None else Price(result)
Inquirer._cached_current_price[cache_key] = CachedPriceEntry(
price=usd_price,
time=ts_now(),
)
return usd_price
return instance._query_oracle_instances(from_asset=asset,
to_asset=A_USD)
def query_tokens_for_addresses(
self,
addresses: List[ChecksumEthAddress],
force_detection: bool,
) -> TokensReturn:
"""Queries/detects token balances for a list of addresses
If an address's tokens were recently autodetected they are not detected again but the
balances are simply queried. Unless force_detection is True.
Returns the token balances of each address and the usd prices of the tokens
"""
log.debug(
'Querying/detecting token balances for all addresses',
force_detection=force_detection,
)
ignored_assets = self.db.get_ignored_assets()
exceptions = [
# Ignore the veCRV balance in token query. It's already detected by
# defi SDK as part of locked CRV in Vote Escrowed CRV. Which is the right way
# to approach it as there is no way to assign a price to 1 veCRV. It
# can be 1 CRV locked for 4 years or 4 CRV locked for 1 year etc.
string_to_ethereum_address(
'0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2'),
# Ignore for now xsushi since is queried by defi SDK. We'll do it for now
# since the SDK entry might return other tokens from sushi and we don't
# fully support sushi now.
string_to_ethereum_address(
'0x8798249c2E607446EfB7Ad49eC89dD1865Ff4272'),
# Ignore the following tokens. They are old tokens of upgraded contracts which
# duplicated the balances at upgrade instead of doing a token swap.
# e.g.: https://github.com/rotki/rotki/issues/3548
# TODO: At some point we should actually remove them from the DB and
# upgrade possible occurences in the user DB
#
# Old contract of Fetch.ai
string_to_ethereum_address(
'0x1D287CC25dAD7cCaF76a26bc660c5F7C8E2a05BD'),
]
for asset in ignored_assets: # don't query for the ignored tokens
if asset.is_eth_token(
): # type ignore since we know asset is a token
exceptions.append(
EthereumToken.from_asset(
asset).ethereum_address) # type: ignore
all_tokens = GlobalDBHandler().get_ethereum_tokens(
exceptions=exceptions,
except_protocols=['balancer'],
)
# With etherscan with chunks > 120, we get request uri too large
# so the limitation is not in the gas, but in the request uri length
etherscan_chunks = list(
get_chunks(all_tokens, n=ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH))
other_chunks = list(
get_chunks(all_tokens, n=OTHER_MAX_TOKEN_CHUNK_LENGTH))
now = ts_now()
token_usd_price: Dict[EthereumToken, Price] = {}
result = {}
for address in addresses:
saved_list = self.db.get_tokens_for_address_if_time(
address=address, current_time=now)
if force_detection or saved_list is None:
balances = self.detect_tokens_for_address(
address=address,
token_usd_price=token_usd_price,
etherscan_chunks=etherscan_chunks,
other_chunks=other_chunks,
)
else:
if len(saved_list) == 0:
continue # Do not query if we know the address has no tokens
balances = defaultdict(FVal)
self._get_tokens_balance_and_price(
address=address,
tokens=saved_list,
balances=balances,
token_usd_price=token_usd_price,
call_order=None, # use defaults
)
result[address] = balances
return result, token_usd_price
def get_price(
self,
from_asset: Asset,
to_asset: Asset,
block_identifier: BlockIdentifier,
) -> Price:
"""
Return the price of from_asset to to_asset at the block block_identifier.
External oracles are used if non eth tokens are used.
Can raise:
- PriceQueryUnsupportedAsset
- RemoteError
"""
log.debug(
f'Searching price for {from_asset} to {to_asset} at '
f'{block_identifier!r} with {self.name}', )
# Uniswap V2 and V3 use in their contracts WETH instead of ETH
if from_asset == A_ETH:
from_asset = A_WETH
if to_asset == A_ETH:
to_asset = A_WETH
if from_asset == to_asset:
return Price(ONE)
if not (from_asset.is_eth_token() and to_asset.is_eth_token()):
raise PriceQueryUnsupportedAsset(
f'Either {from_asset} or {to_asset} arent ethereum tokens for the uniswap oracle',
)
# Could be that we are dealing with ethereum tokens as instances of Asset instead of
# EthereumToken, handle the conversion
from_asset_raw: Union[Asset, EthereumToken] = from_asset
to_asset_raw: Union[Asset, EthereumToken] = to_asset
if not isinstance(from_asset, EthereumToken):
from_as_token = EthereumToken.from_asset(from_asset)
if from_as_token is None:
raise PriceQueryUnsupportedAsset(
f'Unsupported asset for uniswap {from_asset_raw}')
from_asset = from_as_token
if not isinstance(to_asset, EthereumToken):
to_as_token = EthereumToken.from_asset(to_asset)
if to_as_token is None:
raise PriceQueryUnsupportedAsset(
f'Unsupported asset for uniswap {to_asset_raw}')
to_asset = to_as_token
route = self.find_route(from_asset, to_asset)
if len(route) == 0:
log.debug(
f'Failed to find uniswap price for {from_asset} to {to_asset}')
return Price(ZERO)
log.debug(
f'Found price route {route} for {from_asset} to {to_asset} using {self.name}'
)
prices_and_tokens = []
for step in route:
log.debug(f'Getting pool price for {step}')
prices_and_tokens.append(
self.get_pool_price(
pool_addr=to_checksum_address(step),
block_identifier=block_identifier,
), )
# Looking at which one is token0 and token1 we need to see if we need price or 1/price
if prices_and_tokens[0].token_0 != from_asset:
prices_and_tokens[0] = prices_and_tokens[0].swap_tokens()
# For the possible intermediate steps also make sure that we use the correct price
for pos, item in enumerate(prices_and_tokens[1:-1]):
if item.token_0 != prices_and_tokens[pos - 1].token_1:
prices_and_tokens[pos -
1] = prices_and_tokens[pos -
1].swap_tokens()
# Finally for the tail query the price
if prices_and_tokens[-1].token_1 != to_asset:
prices_and_tokens[-1] = prices_and_tokens[-1].swap_tokens()
price = FVal(reduce(mul, [item.price for item in prices_and_tokens],
1))
return Price(price)
def aave_event_from_db(event_tuple: AAVE_EVENT_DB_TUPLE) -> AaveEvent:
"""Turns a tuple read from the DB into an appropriate AaveEvent
May raise a DeserializationError if something is wrong with the DB data
"""
event_type = event_tuple[1]
block_number = event_tuple[2]
timestamp = Timestamp(event_tuple[3])
tx_hash = event_tuple[4]
log_index = event_tuple[5]
asset2 = None
if event_tuple[9] is not None:
try:
asset2 = Asset(event_tuple[9])
except UnknownAsset as e:
raise DeserializationError(
f'Unknown asset {event_tuple[6]} encountered during deserialization '
f'of Aave event from DB for asset2', ) from e
try:
asset1 = Asset(event_tuple[6])
except UnknownAsset as e:
raise DeserializationError(
f'Unknown asset {event_tuple[6]} encountered during deserialization '
f'of Aave event from DB for asset1', ) from e
asset1_amount = FVal(event_tuple[7])
asset1_usd_value = FVal(event_tuple[8])
if event_type in ('deposit', 'withdrawal'):
return AaveDepositWithdrawalEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
atoken=EthereumToken.from_asset(
asset2), # type: ignore # should be a token
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
)
if event_type == 'interest':
return AaveInterestEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
)
if event_type == 'borrow':
if event_tuple[12] not in ('stable', 'variable'):
raise DeserializationError(
f'Invalid borrow rate mode encountered in the DB: {event_tuple[12]}',
)
borrow_rate_mode: Literal['stable',
'variable'] = event_tuple[12] # type: ignore
borrow_rate = deserialize_optional_to_fval(
value=event_tuple[10],
name='borrow_rate',
location='reading aave borrow event from DB',
)
accrued_borrow_interest = deserialize_optional_to_fval(
value=event_tuple[11],
name='accrued_borrow_interest',
location='reading aave borrow event from DB',
)
return AaveBorrowEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
borrow_rate_mode=borrow_rate_mode,
borrow_rate=borrow_rate,
accrued_borrow_interest=accrued_borrow_interest,
)
if event_type == 'repay':
fee_amount = deserialize_optional_to_fval(
value=event_tuple[10],
name='fee_amount',
location='reading aave repay event from DB',
)
fee_usd_value = deserialize_optional_to_fval(
value=event_tuple[11],
name='fee_usd_value',
location='reading aave repay event from DB',
)
return AaveRepayEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
fee=Balance(amount=fee_amount, usd_value=fee_usd_value),
)
if event_type == 'liquidation':
if asset2 is None:
raise DeserializationError(
'Did not find asset2 in an aave liquidation event fom the DB.',
)
principal_amount = deserialize_optional_to_fval(
value=event_tuple[10],
name='principal_amount',
location='reading aave liquidation event from DB',
)
principal_usd_value = deserialize_optional_to_fval(
value=event_tuple[11],
name='principal_usd_value',
location='reading aave liquidation event from DB',
)
return AaveLiquidationEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
collateral_asset=asset1,
collateral_balance=Balance(amount=asset1_amount,
usd_value=asset1_usd_value),
principal_asset=asset2,
principal_balance=Balance(
amount=principal_amount,
usd_value=principal_usd_value,
),
)
# else
raise DeserializationError(
f'Unknown event type {event_type} encountered during '
f'deserialization of Aave event from DB', )
def query_tokens_for_addresses(
self,
addresses: List[ChecksumEthAddress],
force_detection: bool,
) -> TokensReturn:
"""Queries/detects token balances for a list of addresses
If an address's tokens were recently autodetected they are not detected again but the
balances are simply queried. Unless force_detection is True.
Returns the token balances of each address and the usd prices of the tokens
"""
log.debug(
'Querying/detecting token balances for all addresses',
force_detection=force_detection,
)
ignored_assets = self.db.get_ignored_assets()
exceptions = [
# Ignore the veCRV balance in token query. It's already detected by
# defi SDK as part of locked CRV in Vote Escrowed CRV. Which is the right way
# to approach it as there is no way to assign a price to 1 veCRV. It
# can be 1 CRV locked for 4 years or 4 CRV locked for 1 year etc.
string_to_ethereum_address('0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2'),
]
for asset in ignored_assets: # don't query for the ignored tokens
if asset.is_eth_token(): # type ignore since we know asset is a token
exceptions.append(EthereumToken.from_asset(asset).ethereum_address) # type: ignore
all_tokens = GlobalDBHandler().get_ethereum_tokens(exceptions=exceptions)
# With etherscan with chunks > 120, we get request uri too large
# so the limitation is not in the gas, but in the request uri length
etherscan_chunks = list(get_chunks(all_tokens, n=ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH))
other_chunks = list(get_chunks(all_tokens, n=OTHER_MAX_TOKEN_CHUNK_LENGTH))
now = ts_now()
token_usd_price: Dict[EthereumToken, Price] = {}
result = {}
for address in addresses:
saved_list = self.db.get_tokens_for_address_if_time(address=address, current_time=now)
if force_detection or saved_list is None:
balances = self.detect_tokens_for_address(
address=address,
token_usd_price=token_usd_price,
etherscan_chunks=etherscan_chunks,
other_chunks=other_chunks,
)
else:
if len(saved_list) == 0:
continue # Do not query if we know the address has no tokens
balances = defaultdict(FVal)
self._get_tokens_balance_and_price(
address=address,
tokens=saved_list,
balances=balances,
token_usd_price=token_usd_price,
call_order=None, # use defaults
)
result[address] = balances
return result, token_usd_price
