def _parse_atoken_balance_history(
history: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[ATokenBalanceHistory]:
result = []
for entry in history:
timestamp = entry['timestamp']
if timestamp < from_ts or timestamp > to_ts:
continue
entry_id = entry['id']
pairs = entry_id.split('0x')
if len(pairs) not in (4, 5):
log.error(
f'Expected to find 3-4 hashes in graph\'s aTokenBalanceHistory '
f'id but the encountered id does not match: {entry_id}. Skipping entry...',
)
continue
try:
address_s = '0x' + pairs[2]
reserve_address = deserialize_ethereum_address(address_s)
except DeserializationError:
log.error(f'Error deserializing reserve address {address_s}', )
continue
version = _get_version_from_reserveid(pairs, 3)
tx_hash = '0x' + pairs[4]
asset = aave_reserve_address_to_reserve_asset(reserve_address)
if asset is None:
log.error(
f'Unknown aave reserve address returned by atoken balance history '
f' graph query: {reserve_address}. Skipping entry ...', )
continue
_, decimals = _get_reserve_address_decimals(asset)
if 'currentATokenBalance' in entry:
balance = token_normalized_value_decimals(
int(entry['currentATokenBalance']),
token_decimals=decimals,
)
else:
balance = token_normalized_value_decimals(
int(entry['balance']),
token_decimals=decimals,
)
result.append(
ATokenBalanceHistory(
reserve_address=reserve_address,
balance=balance,
tx_hash=tx_hash,
timestamp=timestamp,
version=version,
))
return result
def _parse_repays(
self,
repays: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[AaveRepayEvent]:
events = []
for entry in repays:
common = _parse_common_event_data(entry, from_ts, to_ts)
if common is None:
continue # either timestamp out of range or error (logged in the function above)
timestamp, tx_hash, index = common
result = _get_reserve_asset_and_decimals(entry,
reserve_key='reserve')
if result is None:
continue # problem parsing, error already logged
asset, decimals = result
if 'amountAfterFee' in entry:
amount_after_fee = token_normalized_value_decimals(
int(entry['amountAfterFee']),
token_decimals=decimals,
)
fee = token_normalized_value_decimals(int(entry['fee']),
token_decimals=decimals)
else:
# In the V2 subgraph the amountAfterFee and Fee keys are replaced by amount
amount_after_fee = token_normalized_value_decimals(
int(entry['amount']),
token_decimals=decimals,
)
fee = ZERO
usd_price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=f'aave repay event {tx_hash} from graph query',
msg_aggregator=self.msg_aggregator,
)
events.append(
AaveRepayEvent(
event_type='repay',
asset=asset,
value=Balance(amount=amount_after_fee,
usd_value=amount_after_fee * usd_price),
fee=Balance(amount=fee, usd_value=fee * usd_price),
block_number=0, # can't get from graph query
timestamp=timestamp,
tx_hash=tx_hash,
log_index=
index, # not really the log index, but should also be unique
))
return events
def check_airdrops(
addresses: List[ChecksumEthAddress],
data_dir: Path,
) -> Dict[ChecksumEthAddress, Dict]:
"""Checks airdrop data for the given list of ethereum addresses
May raise:
- RemoteError if the remote request fails
"""
found_data: Dict[ChecksumEthAddress,
Dict] = defaultdict(lambda: defaultdict(dict))
for protocol_name, airdrop_data in AIRDROPS.items():
data, csvfile = get_airdrop_data(protocol_name, data_dir)
for addr, amount, *_ in data:
# not doing to_checksum_address() here since the file addresses are checksummed
# and doing to_checksum_address() so many times hits performance
if protocol_name in ('cornichon', 'tornado', 'grain', 'lido'):
amount = token_normalized_value_decimals(int(amount), 18)
if addr in addresses:
found_data[addr][protocol_name] = {
'amount': str(amount),
'asset': airdrop_data[1],
'link': airdrop_data[2],
}
csvfile.close()
return dict(found_data)
def _decode_token(entry: Tuple) -> TokenDetails:
decimals = entry[0][3]
return TokenDetails(
address=entry[0][0],
name=entry[0][1],
symbol=entry[0][2],
decimals=decimals,
amount=token_normalized_value_decimals(entry[1], decimals),
)
def check_airdrops(
addresses: List[ChecksumEthAddress],
data_dir: Path,
) -> Dict[ChecksumEthAddress, Dict]:
"""Checks airdrop data for the given list of ethereum addresses
May raise:
- RemoteError if the remote request fails
"""
found_data: Dict[ChecksumEthAddress,
Dict] = defaultdict(lambda: defaultdict(dict))
for protocol_name, airdrop_data in AIRDROPS.items():
data, csvfile = get_airdrop_data(protocol_name, data_dir)
for row in data:
if len(row) < 2:
raise UnableToDecryptRemoteData(
f'Airdrop CSV for {protocol_name} contains an invalid row: {row}',
)
addr, amount, *_ = row
# not doing to_checksum_address() here since the file addresses are checksummed
# and doing to_checksum_address() so many times hits performance
if protocol_name in (
'cornichon',
'tornado',
'grain',
'lido',
'sdl',
'cow_mainnet',
'cow_gnosis',
):
amount = token_normalized_value_decimals(int(amount), 18)
if addr in addresses:
found_data[addr][protocol_name] = {
'amount': str(amount),
'asset': airdrop_data[1],
'link': airdrop_data[2],
}
csvfile.close()
for protocol_name, poap_airdrop_data in POAP_AIRDROPS.items():
data_dict = get_poap_airdrop_data(protocol_name, data_dir)
for addr, assets in data_dict.items():
# not doing to_checksum_address() here since the file addresses are checksummed
# and doing to_checksum_address() so many times hits performance
if addr in addresses:
if 'poap' not in found_data[addr]:
found_data[addr]['poap'] = []
found_data[addr]['poap'].append({
'event': protocol_name,
'assets': assets,
'link': poap_airdrop_data[1],
'name': poap_airdrop_data[2],
})
return dict(found_data)
def on_account_addition(self, address: ChecksumEthAddress) -> Optional[List[AssetBalance]]:
"""When an account is added for adex check its balances"""
balance = self.staking_pool.call(self.ethereum, 'balanceOf', arguments=[address])
if balance == 0:
return None
# else the address has staked adex
usd_price = Inquirer().find_usd_price(A_ADX)
share_price = self.staking_pool.call(self.ethereum, 'shareValue')
amount = token_normalized_value_decimals(
token_amount=balance * share_price / (FVal(10) ** 18),
token_decimals=18,
)
return [AssetBalance(asset=A_ADX, balance=Balance(amount=amount, usd_value=amount * usd_price))] # noqa: E501
def query_defi_balances(
self,
addresses: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, List[DefiProtocolBalances]]:
defi_balances = defaultdict(list)
for account in addresses:
balances = self.zerion_sdk.all_balances_for_account(account)
if len(balances) != 0:
defi_balances[account] = balances
# and also query balances of tokens that are not detected by zerion adapter contract
result = multicall_specific(
ethereum=self.ethereum,
contract=VOTE_ESCROWED_CRV,
method_name='locked',
arguments=[[x] for x in addresses],
)
crv_price = Price(ZERO)
if any(x[0] != 0 for x in result):
crv_price = Inquirer().find_usd_price(A_CRV)
for idx, address in enumerate(addresses):
balance = result[idx][0]
if balance == 0:
continue
# else the address has vote escrowed CRV
amount = token_normalized_value_decimals(token_amount=balance,
token_decimals=18)
protocol_balance = DefiProtocolBalances(
protocol=DefiProtocol(
name='Curve • Vesting',
description='Curve vesting or locked in escrow for voting',
url='https://www.curve.fi/',
version=1,
),
balance_type='Asset',
base_balance=DefiBalance(
token_address=VOTE_ESCROWED_CRV.address,
token_name='Vote-escrowed CRV',
token_symbol='veCRV',
balance=Balance(
amount=amount,
usd_value=amount * crv_price,
),
),
underlying_balances=[],
)
defi_balances[address].append(protocol_balance)
return defi_balances
def _get_single_balance(
self,
protocol_name: str,
entry: Tuple[Tuple[str, str, str, int], int],
) -> DefiBalance:
"""
This method can raise DeserializationError while deserializing the token address
or handling the specific protocol.
"""
metadata = entry[0]
balance_value = entry[1]
decimals = metadata[3]
normalized_value = token_normalized_value_decimals(
balance_value, decimals)
token_symbol = metadata[2]
token_address = deserialize_ethereum_address(metadata[0])
token_name = metadata[1]
special_handling = self.handle_protocols(
protocol_name=protocol_name,
token_symbol=token_symbol,
normalized_balance=normalized_value,
token_address=token_address,
token_name=token_name,
)
if special_handling:
return special_handling
try:
token = EthereumToken(token_address)
usd_price = Inquirer().find_usd_price(token)
except (UnknownAsset, UnsupportedAsset):
if not _is_token_non_standard(token_symbol, token_address):
self.msg_aggregator.add_warning(
f'Unsupported asset {token_symbol} with address '
f'{token_address} encountered during DeFi protocol queries',
)
usd_price = Price(ZERO)
usd_value = normalized_value * usd_price
defi_balance = DefiBalance(
token_address=token_address,
token_name=token_name,
token_symbol=token_symbol,
balance=Balance(amount=normalized_value, usd_value=usd_value),
)
return defi_balance
def _parse_atoken_balance_history(
history: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[ATokenBalanceHistory]:
result = []
for entry in history:
timestamp = entry['timestamp']
if timestamp < from_ts or timestamp > to_ts:
continue
entry_id = entry['id']
pairs = entry_id.split('0x')
if len(pairs) != 4:
log.error(
f'Expected to find 3 hashes in graps\'s aTokenBalanceHistory '
f'id but the encountered id does not match: {entry_id}. Skipping entry...',
)
continue
reserve_address = to_checksum_address('0x' + pairs[2])
tx_hash = '0x' + pairs[3]
asset = AAVE_RESERVE_TO_ASSET.get(reserve_address, None)
if asset is None:
log.error(
f'Unknown aave reserve address returned by atoken balance history '
f' graph query: {reserve_address}. Skipping entry ...', )
continue
_, decimals = _get_reserve_address_decimals(asset.identifier)
balance = token_normalized_value_decimals(int(entry['balance']),
token_decimals=decimals)
result.append(
ATokenBalanceHistory(
reserve_address=reserve_address,
balance=balance,
tx_hash=tx_hash,
timestamp=timestamp,
))
return result
def get_balances(
self,
addresses: List[ChecksumAddress],
) -> Dict[ChecksumAddress, Balance]:
"""Return the addresses' balances (staked amount per pool) in the AdEx
protocol.
May raise:
- RemoteError: Problem querying the chain
"""
if len(addresses) == 0:
return {}
result = multicall_specific(
ethereum=self.ethereum,
contract=self.staking_pool,
method_name='balanceOf',
arguments=[[x] for x in addresses],
)
if all(x[0] == 0 for x in result):
return {} # no balances found
staking_balances = {}
usd_price = Inquirer().find_usd_price(A_ADX)
share_price = self.staking_pool.call(self.ethereum, 'shareValue')
for idx, address in enumerate(addresses):
balance = result[idx][0]
if balance == 0:
continue
# else the address has staked adex
amount = token_normalized_value_decimals(
token_amount=balance * share_price / (FVal(10)**18),
token_decimals=18,
)
staking_balances[address] = Balance(amount=amount,
usd_value=amount * usd_price)
return staking_balances
def _get_asset_and_balance(
self,
entry: Dict[str, Any],
timestamp: Timestamp,
reserve_key: str,
amount_key: str,
location: str,
) -> Optional[Tuple[Asset, Balance]]:
"""Utility function to parse asset from graph query amount and price and return balance"""
result = _get_reserve_asset_and_decimals(entry, reserve_key)
if result is None:
return None
asset, decimals = result
amount = token_normalized_value_decimals(
token_amount=int(entry[amount_key]),
token_decimals=decimals,
)
usd_price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=location,
msg_aggregator=self.msg_aggregator,
)
return asset, Balance(amount=amount, usd_value=amount * usd_price)
def find_curve_pool_price(
self,
lp_token: EthereumToken,
) -> Optional[Price]:
"""
1. Obtain the pool for this token
2. Obtain prices for assets in pool
3. Obtain the virtual price for share and the balances of each
token in the pool
4. Calc the price for a share
Returns the price of 1 LP token from the pool
"""
assert self._ethereum is not None, 'Inquirer ethereum manager should have been initialized' # noqa: E501
pools = get_curve_pools()
if lp_token.ethereum_address not in pools:
return None
pool = pools[lp_token.ethereum_address]
tokens = []
# Translate addresses to tokens
try:
for asset in pool.assets:
if asset == '0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE':
tokens.append(A_WETH)
else:
tokens.append(EthereumToken(asset))
except UnknownAsset:
return None
# Get price for each token in the pool
prices = []
for token in tokens:
price = self.find_usd_price(token)
if price == Price(ZERO):
log.error(
f'Could not calculate price for {lp_token} due to inability to '
f'fetch price for {token}.', )
return None
prices.append(price)
# Query virtual price of LP share and balances in the pool for each token
contract = EthereumContract(
address=pool.pool_address,
abi=CURVE_POOL_ABI,
deployed_block=0,
)
calls = [(pool.pool_address,
contract.encode(method_name='get_virtual_price'))]
calls += [(pool.pool_address,
contract.encode(method_name='balances', arguments=[i]))
for i in range(len(pool.assets))]
output = multicall_2(
ethereum=self._ethereum,
require_success=False,
calls=calls,
)
# Check that the output has the correct structure
if not all([len(call_result) == 2 for call_result in output]):
log.debug(
f'Failed to query contract methods while finding curve pool price. '
f'Not every outcome has length 2. {output}', )
return None
# Check that all the requests were successful
if not all([contract_output[0] for contract_output in output]):
log.debug(
f'Failed to query contract methods while finding curve price. {output}'
)
return None
# Deserialize information obtained in the multicall execution
data = []
# https://github.com/PyCQA/pylint/issues/4739
virtual_price_decoded = contract.decode(output[0][1],
'get_virtual_price') # pylint: disable=unsubscriptable-object # noqa: E501
if not _check_curve_contract_call(virtual_price_decoded):
log.debug(
f'Failed to decode get_virtual_price while finding curve price. {output}'
)
return None
data.append(FVal(virtual_price_decoded[0])) # pylint: disable=unsubscriptable-object
for i in range(len(pool.assets)):
amount_decoded = contract.decode(output[i + 1][1],
'balances',
arguments=[i])
if not _check_curve_contract_call(amount_decoded):
log.debug(
f'Failed to decode balances {i} while finding curve price. {output}'
)
return None
# https://github.com/PyCQA/pylint/issues/4739
amount = amount_decoded[0] # pylint: disable=unsubscriptable-object
normalized_amount = token_normalized_value_decimals(
amount, tokens[i].decimals)
data.append(normalized_amount)
# Prices and data should verify this relation for the following operations
if len(prices) != len(data) - 1:
log.debug(
f'Length of prices {len(prices)} does not match len of data {len(data)} '
f'while querying curve pool price.', )
return None
# Total number of assets price in the pool
total_assets_price = sum(map(operator.mul, data[1:], prices))
if total_assets_price == 0:
log.error(
f'Curve pool price returned unexpected data {data} that lead to a zero price.',
)
return None
# Calculate weight of each asset as the proportion of tokens value
weights = map(lambda x: data[x + 1] * prices[x] / total_assets_price,
range(len(tokens)))
assets_price = FVal(sum(map(operator.mul, weights, prices)))
return (assets_price * FVal(data[0])) / (10**lp_token.decimals)
def get_positions(
self,
addresses_list: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, Trove]:
contract = EthereumContract(
address=LIQUITY_TROVE_MANAGER.address,
abi=LIQUITY_TROVE_MANAGER.abi,
deployed_block=LIQUITY_TROVE_MANAGER.deployed_block,
)
# make a copy of the list to avoid modifications in the list that is passed as argument
addresses = list(addresses_list)
proxied_addresses = self._get_accounts_having_proxy()
proxies_to_address = {v: k for k, v in proxied_addresses.items()}
addresses += proxied_addresses.values()
calls = [(LIQUITY_TROVE_MANAGER.address,
contract.encode(method_name='Troves', arguments=[x]))
for x in addresses]
outputs = multicall_2(
ethereum=self.ethereum,
require_success=False,
calls=calls,
)
data: Dict[ChecksumEthAddress, Trove] = {}
eth_price = Inquirer().find_usd_price(A_ETH)
lusd_price = Inquirer().find_usd_price(A_LUSD)
for idx, output in enumerate(outputs):
status, result = output
if status is True:
try:
trove_info = contract.decode(result,
'Troves',
arguments=[addresses[idx]])
trove_is_active = bool(trove_info[3]) # pylint: disable=unsubscriptable-object
if not trove_is_active:
continue
collateral = deserialize_asset_amount(
token_normalized_value_decimals(trove_info[1], 18), # noqa: E501 pylint: disable=unsubscriptable-object
)
debt = deserialize_asset_amount(
token_normalized_value_decimals(trove_info[0], 18), # noqa: E501 pylint: disable=unsubscriptable-object
)
collateral_balance = AssetBalance(
asset=A_ETH,
balance=Balance(
amount=collateral,
usd_value=eth_price * collateral,
),
)
debt_balance = AssetBalance(
asset=A_LUSD,
balance=Balance(
amount=debt,
usd_value=lusd_price * debt,
),
)
# Avoid division errors
collateralization_ratio: Optional[FVal]
liquidation_price: Optional[FVal]
if debt > 0:
collateralization_ratio = eth_price * collateral / debt * 100
else:
collateralization_ratio = None
if collateral > 0:
liquidation_price = debt * lusd_price * FVal(
MIN_COLL_RATE) / collateral
else:
liquidation_price = None
account_address = addresses[idx]
if account_address in proxies_to_address:
account_address = proxies_to_address[account_address]
data[account_address] = Trove(
collateral=collateral_balance,
debt=debt_balance,
collateralization_ratio=collateralization_ratio,
liquidation_price=liquidation_price,
active=trove_is_active,
trove_id=trove_info[4], # pylint: disable=unsubscriptable-object
)
except DeserializationError as e:
self.msg_aggregator.add_warning(
f'Ignoring Liquity trove information. '
f'Failed to decode contract information. {str(e)}.', )
return data
def get_dill_balances(
self,
addresses: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, DillBalance]:
"""
Query information for amount locked, pending rewards and time until unlock
for Pickle's dill.
"""
api_output = {}
rewards_calls = [(
PICKLE_DILL_REWARDS.address,
self.rewards_contract.encode(method_name='claim', arguments=[x]),
) for x in addresses]
balance_calls = [(PICKLE_DILL.address,
self.dill_contract.encode(method_name='locked',
arguments=[x]))
for x in addresses]
outputs = multicall_2(
ethereum=self.ethereum,
require_success=False,
calls=rewards_calls + balance_calls,
)
reward_outputs, dill_outputs = outputs[:len(addresses)], outputs[
len(addresses):]
pickle_price = Inquirer().find_usd_price(A_PICKLE)
for idx, output in enumerate(reward_outputs):
status_rewards, result = output
status_dill, result_dill = dill_outputs[idx]
address = addresses[idx]
if all((status_rewards, status_dill)):
try:
rewards = self.rewards_contract.decode(result,
'claim',
arguments=[address])
dill_amounts = self.dill_contract.decode(
result_dill,
'locked',
arguments=[address],
)
dill_rewards = token_normalized_value_decimals(
token_amount=rewards[0], # pylint: disable=unsubscriptable-object
token_decimals=A_PICKLE.decimals,
)
dill_locked = token_normalized_value_decimals(
token_amount=dill_amounts[0], # pylint: disable=unsubscriptable-object
token_decimals=A_PICKLE.decimals,
)
balance = DillBalance(
dill_amount=AssetBalance(
asset=A_PICKLE,
balance=Balance(
amount=dill_locked,
usd_value=pickle_price * dill_locked,
),
),
pending_rewards=AssetBalance(
asset=A_PICKLE,
balance=Balance(
amount=dill_rewards,
usd_value=pickle_price * dill_rewards,
),
),
lock_time=deserialize_timestamp(dill_amounts[1]), # noqa: E501 pylint: disable=unsubscriptable-object
)
api_output[address] = balance
except (DeserializationError, IndexError) as e:
self.msg_aggregator.add_error(
f'Failed to query dill information for address {address}. {str(e)}',
)
return api_output
