def deserialize_transaction_from_etherscan(
data: Dict[str, Any],
internal: bool,
) -> EthereumTransaction:
"""Reads dict data of a transaction from etherscan and deserializes it
Can raise DeserializationError if something is wrong
"""
try:
# internal tx list contains no gasprice
gas_price = -1 if internal else read_integer(data, 'gasPrice')
tx_hash = read_hash(data, 'hash')
input_data = read_hash(data, 'input')
timestamp = deserialize_timestamp(data['timeStamp'])
block_number = read_integer(data, 'blockNumber')
nonce = -1 if internal else read_integer(data, 'nonce')
return EthereumTransaction(
timestamp=timestamp,
block_number=block_number,
tx_hash=tx_hash,
from_address=deserialize_ethereum_address(data['from']),
to_address=deserialize_ethereum_address(data['to']) if data['to'] != '' else None,
value=read_integer(data, 'value'),
gas=read_integer(data, 'gas'),
gas_price=gas_price,
gas_used=read_integer(data, 'gasUsed'),
input_data=input_data,
nonce=nonce,
)
except KeyError as e:
raise DeserializationError(
f'Etherscan ethereum transaction missing expected key {str(e)}',
) from e
def deserialize_bpt_event(
raw_event: Dict[str, Any],
event_type: Literal[BalancerBPTEventType.MINT, BalancerBPTEventType.BURN],
) -> BalancerBPTEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
raw_user_address = raw_event['user']['id']
amount = deserialize_asset_amount(raw_event['amount'])
raw_pool = raw_event['pool']
raw_pool_address = raw_pool['id']
raw_pool_tokens = raw_pool['tokens']
total_weight = deserialize_asset_amount(raw_pool['totalWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if total_weight == ZERO:
raise DeserializationError('Pool weight is zero.')
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
pool_tokens = []
for raw_token in raw_pool_tokens:
try:
raw_token_address = raw_token['address']
token_symbol = raw_token['symbol']
token_name = raw_token['name']
token_decimals = raw_token['decimals']
token_weight = deserialize_asset_amount(raw_token['denormWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(raw_token_address)
token = get_ethereum_token(
symbol=token_symbol,
ethereum_address=token_address,
name=token_name,
decimals=token_decimals,
)
pool_token = BalancerBPTEventPoolToken(
token=token,
weight=token_weight * 100 / total_weight,
)
pool_tokens.append(pool_token)
pool_tokens.sort(key=lambda pool_token: pool_token.token.ethereum_address)
bpt_event = BalancerBPTEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
event_type=event_type,
pool_address=pool_address,
pool_tokens=pool_tokens,
amount=amount,
)
return bpt_event
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=deserialize_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=deserialize_ethereum_address(event_tuple[5]),
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def _get_user_reserves(
self, address: ChecksumEthAddress) -> List[AaveUserReserve]:
query = self.graph.query(
querystr=USER_RESERVES_QUERY.format(address=address.lower()), )
query_v2 = self.graph_v2.query(
querystr=USER_RESERVES_QUERY.format(address=address.lower()), )
result = []
for entry in query['userReserves'] + query_v2['userReserves']:
reserve = entry['reserve']
try:
result.append(
AaveUserReserve(
# The ID of reserve is the address of the asset and the address of the market's LendingPoolAddressProvider, in lower case # noqa: E501
address=deserialize_ethereum_address(
reserve['id'][:42]),
symbol=reserve['symbol'],
))
except DeserializationError:
log.error(
f'Failed to deserialize reserve address {reserve["id"]} '
f'Skipping reserve address {reserve["id"]} for user address {address}',
)
continue
return result
def test_deserialize_deployment_ethereum_transaction():
data = {
'timeStamp': 0,
'blockNumber': 1,
'hash': '0xc5be14f87be25174846ed53ed239517e4c45c1fe024b184559c17d4f1fefa736',
'from': '0x568Ab4b8834646f97827bB966b13d60246157B8E',
'to': None,
'value': 0,
'gas': 1,
'gasPrice': 1,
'gasUsed': 1,
'input': '',
'nonce': 1,
}
tx = deserialize_ethereum_transaction(
data=data,
internal=False,
ethereum=None,
)
expected = EthereumTransaction(
timestamp=Timestamp(0),
block_number=1,
tx_hash=deserialize_evm_tx_hash(data['hash']),
from_address=deserialize_ethereum_address(data['from']),
to_address=None,
value=data['value'],
gas=data['gas'],
gas_price=data['gasPrice'],
gas_used=data['gasUsed'],
input_data=read_hash(data, 'input'),
nonce=data['nonce'],
)
assert tx == expected
def pairs_from_ethereum(ethereum: EthereumManager) -> Dict[str, Any]:
"""Detect the uniswap v2 pool tokens by using an ethereum node"""
contracts_file = Path(__file__).resolve().parent / 'contracts.json'
with contracts_file.open('r') as f:
contracts = json.loads(f.read())
univ2factory = EthereumContract(
address=contracts['UNISWAPV2FACTORY']['address'],
abi=contracts['UNISWAPV2FACTORY']['abi'],
deployed_block=0, # whatever
)
pairs_num = univ2factory.call(ethereum, 'allPairsLength')
chunks = list(get_chunks([[x] for x in range(pairs_num)], n=500))
pairs = []
for idx, chunk in enumerate(chunks):
print(f'Querying univ2 pairs chunk {idx + 1} / {len(chunks)}')
result = multicall_specific(ethereum, univ2factory, 'allPairs', chunk)
try:
pairs.extend([deserialize_ethereum_address(x[0]) for x in result])
except DeserializationError:
print(
'Error deserializing address while fetching uniswap v2 pool tokens'
)
sys.exit(1)
return pairs
def _get_accounts_proxy(
self,
addresses: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, ChecksumEthAddress]:
"""
Returns DSProxy if it exists for a list of addresses using only one call
to the chain.
May raise:
- RemoteError if query to the node failed
"""
output = multicall(
ethereum=self.ethereum,
calls=[(
DS_PROXY_REGISTRY.address,
DS_PROXY_REGISTRY.encode(method_name='proxies',
arguments=[address]),
) for address in addresses],
)
mapping = {}
for idx, result_encoded in enumerate(output):
address = addresses[idx]
result = DS_PROXY_REGISTRY.decode( # pylint: disable=unsubscriptable-object
result_encoded,
'proxies',
arguments=[address],
)[0]
if int(result, 16) != 0:
try:
proxy_address = deserialize_ethereum_address(result)
mapping[address] = proxy_address
except DeserializationError as e:
msg = f'Failed to deserialize {result} DSproxy for address {address}. {str(e)}'
log.error(msg)
return mapping
def deserialize_from_db(
cls,
deposit_tuple: Eth2DepositDBTuple,
) -> 'Eth2Deposit':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
Deposit_tuple index - Schema columns
------------------------------------
0 - tx_hash
1 - log_index
2 - from_address
3 - timestamp
4 - pubkey
5 - withdrawal_credentials
6 - value
7 - validator_index
"""
return cls(
tx_hash=deposit_tuple[0],
log_index=int(deposit_tuple[1]),
from_address=deserialize_ethereum_address(deposit_tuple[2]),
timestamp=Timestamp(int(deposit_tuple[3])),
pubkey=deposit_tuple[4],
withdrawal_credentials=deposit_tuple[5],
value=Balance(amount=FVal(deposit_tuple[6])),
validator_index=int(deposit_tuple[7]),
)
def deserialize_from_db(
cls,
trade_tuple: AMMSwapDBTuple,
) -> 'AMMSwap':
"""Turns a tuple read from DB into an appropriate Swap.
May raise a DeserializationError if something is wrong with the DB data
Trade_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - from_address
4 - to_address
5 - timestamp
6 - location
7 - token0_identifier
8 - token1_identifier
9 - amount0_in
10 - amount1_in
11 - amount0_out
12 - amount1_out
"""
address = deserialize_ethereum_address(trade_tuple[2])
from_address = deserialize_ethereum_address(trade_tuple[3])
to_address = deserialize_ethereum_address(trade_tuple[4])
token0 = deserialize_ethereum_token_from_db(identifier=trade_tuple[7])
token1 = deserialize_ethereum_token_from_db(identifier=trade_tuple[8])
return cls(
tx_hash=trade_tuple[0],
log_index=trade_tuple[1],
address=address,
from_address=from_address,
to_address=to_address,
timestamp=deserialize_timestamp(trade_tuple[5]),
location=Location.deserialize_from_db(trade_tuple[6]),
token0=token0,
token1=token1,
amount0_in=deserialize_asset_amount(trade_tuple[9]),
amount1_in=deserialize_asset_amount(trade_tuple[10]),
amount0_out=deserialize_asset_amount(trade_tuple[11]),
amount1_out=deserialize_asset_amount(trade_tuple[12]),
)
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[
BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY,
],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
try:
pool_address_token = EthereumToken(pool_address)
except UnknownAsset as e:
raise DeserializationError(
f'Balancer pool token with address {pool_address} should have been in the DB',
) from e
token_address = deserialize_ethereum_address(raw_token_address)
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address_token=pool_address_token,
token_address=token_address,
amount=amount,
)
return invest_event
def get_validator_deposits(
self,
indices_or_pubkeys: Union[List[int], List[Eth2PubKey]],
) -> List[Eth2Deposit]:
"""Get the deposits of all the validators given from the list of indices or pubkeys
Queries in chunks of 100 due to api limitations
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
chunks = _calculate_query_chunks(indices_or_pubkeys)
data = []
for chunk in chunks:
result = self._query(
module='validator',
endpoint='deposits',
encoded_args=','.join(str(x) for x in chunk),
)
if isinstance(result, list):
data.extend(result)
else:
data.append(result)
deposits = []
for entry in data:
try:
amount = from_gwei(FVal(entry['amount']))
timestamp = entry['block_ts']
usd_price = query_usd_price_zero_if_error(
asset=A_ETH,
time=timestamp,
location=f'Eth2 staking deposit at time {timestamp}',
msg_aggregator=self.msg_aggregator,
)
deposits.append(
Eth2Deposit(
from_address=deserialize_ethereum_address(
entry['from_address']),
pubkey=entry['publickey'],
withdrawal_credentials=entry['withdrawal_credentials'],
value=Balance(
amount=amount,
usd_value=amount * usd_price,
),
tx_hash=hexstring_to_bytes(entry['tx_hash']),
tx_index=entry['tx_index'],
timestamp=entry['block_ts'],
))
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
raise RemoteError(
f'Beaconchai.in deposits response processing error. {msg}',
) from e
return deposits
def _parse_atoken_balance_history(
history: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[ATokenBalanceHistory]:
result = []
for entry in history:
timestamp = entry['timestamp']
if timestamp < from_ts or timestamp > to_ts:
continue
entry_id = entry['id']
pairs = entry_id.split('0x')
if len(pairs) not in (4, 5):
log.error(
f'Expected to find 3-4 hashes in graph\'s aTokenBalanceHistory '
f'id but the encountered id does not match: {entry_id}. Skipping entry...',
)
continue
try:
address_s = '0x' + pairs[2]
reserve_address = deserialize_ethereum_address(address_s)
except DeserializationError:
log.error(f'Error deserializing reserve address {address_s}', )
continue
version = _get_version_from_reserveid(pairs, 3)
tx_hash = '0x' + pairs[4]
asset = aave_reserve_address_to_reserve_asset(reserve_address)
if asset is None:
log.error(
f'Unknown aave reserve address returned by atoken balance history '
f' graph query: {reserve_address}. Skipping entry ...', )
continue
_, decimals = _get_reserve_address_decimals(asset)
if 'currentATokenBalance' in entry:
balance = token_normalized_value_decimals(
int(entry['currentATokenBalance']),
token_decimals=decimals,
)
else:
balance = token_normalized_value_decimals(
int(entry['balance']),
token_decimals=decimals,
)
result.append(
ATokenBalanceHistory(
reserve_address=reserve_address,
balance=balance,
tx_hash=tx_hash,
timestamp=timestamp,
version=version,
))
return result
def modify_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
append_or_remove: str,
add_or_sub: Callable[[FVal, FVal], FVal],
) -> BlockchainBalancesUpdate:
"""Add or remove a list of blockchain account
May raise:
- InputError if accounts to remove do not exist.
- EthSyncError if there is a problem querying the ethereum chain
- RemoteError if there is a problem querying an external service such
as etherscan or blockchain.info
"""
if blockchain == SupportedBlockchain.BITCOIN:
for account in accounts:
self.modify_btc_account(
BTCAddress(account),
append_or_remove,
add_or_sub,
)
elif blockchain == SupportedBlockchain.ETHEREUM:
for account in accounts:
address = deserialize_ethereum_address(account)
try:
self.modify_eth_account(
account=address,
append_or_remove=append_or_remove,
add_or_sub=add_or_sub,
)
except BadFunctionCallOutput as e:
log.error(
'Assuming unsynced chain. Got web3 BadFunctionCallOutput '
'exception: {}'.format(str(e)),
)
raise EthSyncError(
'Tried to use the ethereum chain of a local client to edit '
'an eth account but the chain is not synced.',
)
for _, module in self.eth_modules.items():
if append_or_remove == 'append':
module.on_account_addition(address)
else: # remove
module.on_account_removal(address)
else:
# That should not happen. Should be checked by marshmallow
raise AssertionError(
'Unsupported blockchain {} provided at remove_blockchain_account'.format(
blockchain),
)
return self.get_balances_update()
def deserialize_ethereum_token(token_data: dict) -> CustomEthereumToken:
token = CustomEthereumToken(
address=deserialize_ethereum_address(token_data.get('address')),
name=token_data.get('name'),
symbol=token_data.get('symbol'),
decimals=token_data.get('decimals'),
coingecko=token_data.get('coingecko')
)
return token
def deserialize_token_day_data(
raw_token_day_data: Dict[str, Any], ) -> Tuple[ChecksumEthAddress, Price]:
"""May raise DeserializationError"""
try:
token_address = raw_token_day_data['token']['id']
usd_price = deserialize_price(raw_token_day_data['priceUSD'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(token_address)
return token_address, usd_price
def ens_reverse_lookup(self, reversed_addresses: List[ChecksumEthAddress]) -> Dict[ChecksumEthAddress, Optional[str]]: # noqa: E501
"""Performs a reverse ENS lookup on a list of addresses
Because a multicall is used, no exceptions are raised.
If any exceptions occur, they are logged and None is returned for that
"""
human_names: Dict[ChecksumEthAddress, Optional[str]] = {}
# Querying resolvers' addresses
resolver_params = [
EnsContractParams(address=addr, abi=ENS_ABI, method_name='resolver', arguments=_prepare_ens_call_arguments(addr)) # noqa: E501
for addr in reversed_addresses
]
resolvers_output = multicall(
ethereum=self,
calls=[(ENS_MAINNET_ADDR, _encode_ens_contract(params=params)) for params in resolver_params], # noqa: E501
)
resolvers = []
# We need a new list for reversed_addresses because not all addresses have resolver
filtered_reversed_addresses = []
# Processing resolvers query output
for reversed_addr, params, resolver_output in zip(reversed_addresses, resolver_params, resolvers_output): # noqa: E501
decoded_resolver = _decode_ens_contract(params=params, result_encoded=resolver_output)
if is_none_or_zero_address(decoded_resolver):
human_names[reversed_addr] = None
continue
try:
deserialized_resolver = deserialize_ethereum_address(decoded_resolver)
except DeserializationError:
log.error(
f'Error deserializing address {decoded_resolver} while doing reverse ens lookup', # noqa: E501
)
human_names[reversed_addr] = None
continue
resolvers.append(deserialized_resolver)
filtered_reversed_addresses.append(reversed_addr)
# Querying human names
human_names_params = [
EnsContractParams(address=resolver, abi=ENS_RESOLVER_ABI, method_name='name', arguments=_prepare_ens_call_arguments(addr)) # noqa: E501
for addr, resolver in zip(filtered_reversed_addresses, resolvers)]
human_names_output = multicall(
ethereum=self,
calls=[(params.address, _encode_ens_contract(params=params)) for params in human_names_params], # noqa: E501
)
# Processing human names query output
for addr, params, human_name_output in zip(filtered_reversed_addresses, human_names_params, human_names_output): # noqa: E501
human_names[addr] = _decode_ens_contract(params=params, result_encoded=human_name_output) # noqa: E501
return human_names
def _get_single_balance(
self,
protocol_name: str,
entry: Tuple[Tuple[str, str, str, int], int],
) -> DefiBalance:
"""
This method can raise DeserializationError while deserializing the token address
or handling the specific protocol.
"""
metadata = entry[0]
balance_value = entry[1]
decimals = metadata[3]
normalized_value = token_normalized_value_decimals(
balance_value, decimals)
token_symbol = metadata[2]
token_address = deserialize_ethereum_address(metadata[0])
token_name = metadata[1]
special_handling = self.handle_protocols(
protocol_name=protocol_name,
token_symbol=token_symbol,
normalized_balance=normalized_value,
token_address=token_address,
token_name=token_name,
)
if special_handling:
return special_handling
try:
token = EthereumToken(token_address)
usd_price = Inquirer().find_usd_price(token)
except (UnknownAsset, UnsupportedAsset):
if not _is_token_non_standard(token_symbol, token_address):
self.msg_aggregator.add_warning(
f'Unsupported asset {token_symbol} with address '
f'{token_address} encountered during DeFi protocol queries',
)
usd_price = Price(ZERO)
usd_value = normalized_value * usd_price
defi_balance = DefiBalance(
token_address=token_address,
token_name=token_name,
token_symbol=token_symbol,
balance=Balance(amount=normalized_value, usd_value=usd_value),
)
return defi_balance
def __init__(
self,
ethereum_manager: 'EthereumManager',
msg_aggregator: MessagesAggregator,
) -> None:
self.ethereum = ethereum_manager
self.msg_aggregator = msg_aggregator
result = self.ethereum.ens_lookup('api.zerion.eth')
if result is None:
self.msg_aggregator.add_error(
'Could not query api.zerion.eth address. Using last known address',
)
self.contract_address = deserialize_ethereum_address(
'0x06FE76B2f432fdfEcAEf1a7d4f6C3d41B5861672',
)
else:
self.contract_address = result
def _parse_ethereum_token_data(self, insert_text: str) -> Tuple[ChecksumEthAddress, Optional[int], Optional[str]]: # noqa: E501
match = self.ethereum_tokens_re.match(insert_text)
if match is None:
raise DeserializationError(
f'At asset DB update could not parse ethereum token data out '
f'of {insert_text}',
)
if len(match.groups()) != 3:
raise DeserializationError(
f'At asset DB update could not parse ethereum token data out of {insert_text}',
)
return (
deserialize_ethereum_address(self._parse_str(match.group(1), 'address', insert_text)),
self._parse_optional_int(match.group(2), 'decimals', insert_text),
self._parse_optional_str(match.group(3), 'protocol', insert_text),
)
def _get_account_proxy(self, address: ChecksumEthAddress) -> Optional[ChecksumEthAddress]:
"""Checks if a DS proxy exists for the given address and returns it if it does
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result. Also this error can be raised
if there is a problem deserializing the result address.
- BlockchainQueryError if an ethereum node is used and the contract call
queries fail for some reason
"""
result = DS_PROXY_REGISTRY.call(self.ethereum, 'proxies', arguments=[address])
if int(result, 16) != 0:
try:
return deserialize_ethereum_address(result)
except DeserializationError as e:
msg = f'Failed to deserialize {result} DS proxy for address {address}'
log.error(msg)
raise RemoteError(msg) from e
return None
def _get_vault_details(
self,
cdp_id: int,
) -> Tuple[ChecksumEthAddress, ChecksumEthAddress]:
"""
Queries the CDPManager to get the CDP details.
Returns a tuple with the CDP address and the CDP owner as of the
time of this call.
May raise:
- RemoteError if query to the node failed
- DeserializationError if the query returns unexpected output
"""
output = multicall(
ethereum=self.ethereum,
calls=[(
MAKERDAO_CDP_MANAGER.address,
MAKERDAO_CDP_MANAGER.encode(method_name='urns',
arguments=[cdp_id]),
),
(
MAKERDAO_CDP_MANAGER.address,
MAKERDAO_CDP_MANAGER.encode(method_name='owns',
arguments=[cdp_id]),
)],
)
mapping = {}
for result_encoded, method_name in zip(output, ('urns', 'owns')):
result = MAKERDAO_CDP_MANAGER.decode( # pylint: disable=unsubscriptable-object
result_encoded,
method_name,
arguments=[cdp_id],
)[0]
if int(result, 16) == 0:
raise DeserializationError(
'Could not deserialize {result} as address}')
address = deserialize_ethereum_address(result)
mapping[method_name] = address
return mapping['urns'], mapping['owns']
def _get_vaults_of_address(
self,
user_address: ChecksumEthAddress,
proxy_address: ChecksumEthAddress,
) -> List[MakerdaoVault]:
"""Gets the vaults of a single address
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result.
- BlockchainQueryError if an ethereum node is used and the contract call
queries fail for some reason
"""
result = MAKERDAO_GET_CDPS.call(
ethereum=self.ethereum,
method_name='getCdpsAsc',
arguments=[MAKERDAO_CDP_MANAGER.address, proxy_address],
)
vaults = []
for idx, identifier in enumerate(result[0]):
try:
urn = deserialize_ethereum_address(result[1][idx])
except DeserializationError as e:
raise RemoteError(
f'Failed to deserialize address {result[1][idx]} '
f'when processing vaults of {user_address}',
) from e
vault = self._query_vault_data(
identifier=identifier,
owner=user_address,
urn=urn,
ilk=result[2][idx],
)
if vault:
vaults.append(vault)
self.vault_mappings[user_address].append(vault)
return vaults
def _get_reserve_asset_and_decimals(
entry: Dict[str, Any],
reserve_key: str,
) -> Optional[Tuple[Asset, int]]:
try:
# The ID of reserve is the address of the asset and the address of the market's LendingPoolAddressProvider, in lower case # noqa: E501
reserve_address = deserialize_ethereum_address(
entry[reserve_key]['id'][:42])
except DeserializationError:
log.error(
f'Failed to Deserialize reserve address {entry[reserve_key]["id"]}'
)
return None
asset = aave_reserve_to_asset(reserve_address)
if asset is None:
log.error(
f'Unknown aave reserve address returned by graph query: '
f'{reserve_address}. Skipping entry ...', )
return None
_, decimals = _get_reserve_address_decimals(asset)
return asset, decimals
def handle_protocols(
self,
protocol_name: str,
token_symbol: str,
normalized_balance: FVal,
token_address: str,
token_name: str,
) -> Optional[DefiBalance]:
"""Special handling for price for token/protocols which are easier to do onchain
or need some kind of special treatment.
This method can raise DeserializationError
"""
if protocol_name == 'PoolTogether':
result = _handle_pooltogether(normalized_balance, token_name)
if result is not None:
return result
asset = get_asset_by_symbol(token_symbol)
if asset is None:
return None
token = EthereumToken.from_asset(asset)
if token is None:
return None
underlying_asset_price = get_underlying_asset_price(token)
usd_price = handle_defi_price_query(self.ethereum, token,
underlying_asset_price)
if usd_price is None:
return None
return DefiBalance(
token_address=deserialize_ethereum_address(token_address),
token_name=token_name,
token_symbol=token_symbol,
balance=Balance(amount=normalized_balance,
usd_value=normalized_balance * usd_price),
)
def _get_logs(
self,
web3: Optional[Web3],
contract_address: ChecksumEthAddress,
abi: List,
event_name: str,
argument_filters: Dict[str, Any],
from_block: int,
to_block: Union[int, Literal['latest']] = 'latest',
) -> List[Dict[str, Any]]:
"""Queries logs of an ethereum contract
May raise:
- RemoteError if etherscan is used and there is a problem with
reaching it or with the returned result
"""
event_abi = find_matching_event_abi(abi=abi, event_name=event_name)
_, filter_args = construct_event_filter_params(
event_abi=event_abi,
abi_codec=Web3().codec,
contract_address=contract_address,
argument_filters=argument_filters,
fromBlock=from_block,
toBlock=to_block,
)
if event_abi['anonymous']:
# web3.py does not handle the anonymous events correctly and adds the first topic
filter_args['topics'] = filter_args['topics'][1:]
events: List[Dict[str, Any]] = []
start_block = from_block
if web3 is not None:
events = _query_web3_get_logs(
web3=web3,
filter_args=filter_args,
from_block=from_block,
to_block=to_block,
contract_address=contract_address,
event_name=event_name,
argument_filters=argument_filters,
)
else: # etherscan
until_block = (self.etherscan.get_latest_block_number()
if to_block == 'latest' else to_block)
blocks_step = 300000
while start_block <= until_block:
while True: # loop to continuously reduce block range if need b
end_block = min(start_block + blocks_step, until_block)
try:
new_events = self.etherscan.get_logs(
contract_address=contract_address,
topics=filter_args['topics'], # type: ignore
from_block=start_block,
to_block=end_block,
)
except RemoteError as e:
if 'Please select a smaller result dataset' in str(e):
blocks_step = blocks_step // 2
if blocks_step < 100:
raise # stop trying
# else try with the smaller step
continue
# else some other error
raise
break # we must have a result
# Turn all Hex ints to ints
for e_idx, event in enumerate(new_events):
try:
block_number = deserialize_int_from_hex(
symbol=event['blockNumber'],
location='etherscan log query',
)
log_index = deserialize_int_from_hex(
symbol=event['logIndex'],
location='etherscan log query',
)
# Try to see if the event is a duplicate that got returned
# in the previous iteration
for previous_event in reversed(events):
if previous_event['blockNumber'] < block_number:
break
same_event = (previous_event['logIndex']
== log_index
and previous_event['transactionHash']
== event['transactionHash'])
if same_event:
events.pop()
new_events[e_idx][
'address'] = deserialize_ethereum_address(
event['address'], )
new_events[e_idx]['blockNumber'] = block_number
new_events[e_idx][
'timeStamp'] = deserialize_int_from_hex(
symbol=event['timeStamp'],
location='etherscan log query',
)
new_events[e_idx][
'gasPrice'] = deserialize_int_from_hex(
symbol=event['gasPrice'],
location='etherscan log query',
)
new_events[e_idx][
'gasUsed'] = deserialize_int_from_hex(
symbol=event['gasUsed'],
location='etherscan log query',
)
new_events[e_idx]['logIndex'] = log_index
new_events[e_idx][
'transactionIndex'] = deserialize_int_from_hex(
symbol=event['transactionIndex'],
location='etherscan log query',
)
except DeserializationError as e:
raise RemoteError(
'Couldnt decode an etherscan event due to {str(e)}}',
) from e
# etherscan will only return 1000 events in one go. If more than 1000
# are returned such as when no filter args are provided then continue
# the query from the last block
if len(new_events) == 1000:
start_block = new_events[-1]['blockNumber']
else:
start_block = end_block + 1
events.extend(new_events)
return events
def _ens_lookup(
self,
web3: Optional[Web3],
name: str,
blockchain: SupportedBlockchain = SupportedBlockchain.ETHEREUM,
) -> Optional[Union[ChecksumEthAddress, HexStr]]:
"""Performs an ENS lookup and returns address if found else None
TODO: currently web3.py 5.15.0 does not support multichain ENS domains
(EIP-2304), therefore requesting a non-Ethereum address won't use the
web3 ens library and will require to extend the library resolver ABI.
An issue in their repo (#1839) reporting the lack of support has been
created. This function will require refactoring once they include
support for EIP-2304.
https://github.com/ethereum/web3.py/issues/1839
May raise:
- RemoteError if Etherscan is used and there is a problem querying it or
parsing its response
- InputError if the given name is not a valid ENS name
"""
try:
normal_name = normalize_name(name)
except InvalidName as e:
raise InputError(str(e)) from e
resolver_addr = self._call_contract(
web3=web3,
contract_address=ENS_MAINNET_ADDR,
abi=ENS_ABI,
method_name='resolver',
arguments=[normal_name_to_hash(normal_name)],
)
if is_none_or_zero_address(resolver_addr):
return None
ens_resolver_abi = ENS_RESOLVER_ABI.copy()
arguments = [normal_name_to_hash(normal_name)]
if blockchain != SupportedBlockchain.ETHEREUM:
ens_resolver_abi.extend(ENS_RESOLVER_ABI_MULTICHAIN_ADDRESS)
arguments.append(blockchain.ens_coin_type())
try:
deserialized_resolver_addr = deserialize_ethereum_address(
resolver_addr)
except DeserializationError:
log.error(
f'Error deserializing address {resolver_addr} while doing'
f'ens lookup', )
return None
address = self._call_contract(
web3=web3,
contract_address=deserialized_resolver_addr,
abi=ens_resolver_abi,
method_name='addr',
arguments=arguments,
)
if is_none_or_zero_address(address):
return None
if blockchain != SupportedBlockchain.ETHEREUM:
return HexStr(address.hex())
try:
return deserialize_ethereum_address(address)
except DeserializationError:
log.error(f'Error deserializing address {address}')
return None
def pairs_and_token_details_from_graph() -> Dict[str, Any]:
"""Detect the uniswap v2 pool tokens by using the subgraph"""
step = 1000
querystr = """
pairs(first:$first, skip: $skip) {
id
token0{
id
symbol
name
decimals
}
token1{
id
symbol
name
decimals
}
}
}
"""
param_types = {'$first': 'Int!', '$skip': 'Int!'}
param_values = {'first': step, 'skip': 0}
graph = Graph('https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2')
contracts = []
total_pairs_num = 0
while True:
print(
f'Querying graph pairs batch {param_values["skip"]} - {param_values["skip"] + step}'
)
result = graph.query(querystr,
param_types=param_types,
param_values=param_values)
for entry in result['pairs']:
try:
deserialized_entry = deserialize_ethereum_address(entry['id'])
deserialized_token_0 = deserialize_ethereum_address(
entry['token0']['id'])
deserialized_token_1 = deserialize_ethereum_address(
entry['token1']['id'])
except DeserializationError:
print(
'Error deserializing address while fetching uniswap v2 pool tokens'
)
sys.exit(1)
contracts.append({
'address': deserialized_entry,
'token0': {
'address': deserialized_token_0,
'name': entry['token0']['name'],
'symbol': entry['token0']['symbol'],
'decimals': int(entry['token0']['decimals']),
},
'token1': {
'address': deserialized_token_1,
'name': entry['token1']['name'],
'symbol': entry['token1']['symbol'],
'decimals': int(entry['token1']['decimals']),
},
})
pairs_num = len(result['pairs'])
total_pairs_num += pairs_num
if pairs_num < step:
break
param_values['skip'] = total_pairs_num
return contracts
def mock_requests_get(url, *args, **kwargs):
if 'etherscan.io/api?module=account&action=balance&address' in url:
addr = url[67:109]
value = eth_map[addr].get('ETH', '0')
response = f'{{"status":"1","message":"OK","result":{value}}}'
elif 'etherscan.io/api?module=account&action=balancemulti' in url:
queried_accounts = []
length = 72
# process url and get the accounts
while True:
if len(url) < length:
break
potential_address = url[length:length + 42]
if 'apikey=' in potential_address:
break
queried_accounts.append(potential_address)
length += 43
accounts = []
for addr in queried_accounts:
value = eth_map[addr].get('ETH', '0')
accounts.append({'account': addr, 'balance': eth_map[addr]['ETH']})
response = f'{{"status":"1","message":"OK","result":{json.dumps(accounts)}}}'
elif 'api.etherscan.io/api?module=account&action=tokenbalance' in url:
token_address = url[80:122]
msg = 'token address missing from test mapping'
assert token_address in CONTRACT_ADDRESS_TO_TOKEN, msg
response = '{"status":"1","message":"OK","result":"0"}'
token = CONTRACT_ADDRESS_TO_TOKEN[token_address]
account = url[131:173]
value = eth_map[account].get(token.identifier, 0)
response = f'{{"status":"1","message":"OK","result":"{value}"}}'
elif 'api.etherscan.io/api?module=account&action=txlistinternal&' in url:
if 'transactions' in original_queries:
return original_requests_get(url, *args, **kwargs)
# By default when mocking, don't query for transactions
response = '{"status":"1","message":"OK","result":[]}'
elif 'api.etherscan.io/api?module=account&action=txlist&' in url:
if 'transactions' in original_queries:
return original_requests_get(url, *args, **kwargs)
# By default when mocking, don't query for transactions
response = '{"status":"1","message":"OK","result":[]}'
elif 'api.etherscan.io/api?module=logs&action=getLogs&' in url:
if 'logs' in original_queries:
return original_requests_get(url, *args, **kwargs)
# By default when mocking, don't query logs
response = '{"status":"1","message":"OK","result":[]}'
elif 'api.etherscan.io/api?module=block&action=getblocknobytime&' in url:
if 'blocknobytime' in original_queries:
return original_requests_get(url, *args, **kwargs)
# By default when mocking don't query blocknobytime
response = '{"status":"1","message":"OK","result":"1"}'
elif f'api.etherscan.io/api?module=proxy&action=eth_call&to={ZERION_ADAPTER_ADDRESS}' in url: # noqa: E501
if 'zerion' in original_queries:
return original_requests_get(url, *args, **kwargs)
web3 = Web3()
contract = web3.eth.contract(address=ZERION_ADAPTER_ADDRESS, abi=ZERION_ABI)
if 'data=0xc84aae17' in url: # getBalances
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='getBalances',
args=['address'],
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
# TODO: This here always returns empty response. If/when we want to
# mock it for etherscan, this is where we do it
args = []
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
elif 'data=0x85c6a7930' in url: # getProtocolBalances
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='getProtocolBalances',
args=['address', ['some', 'protocol', 'names']],
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
# TODO: This here always returns empty response. If/when we want to
# mock it for etherscan, this is where we do it
args = []
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
elif 'data=0x3b692f52' in url: # getProtocolNames
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='getProtocolNames',
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
# TODO: This here always returns empty response. If/when we want to
# mock it for etherscan, this is where we do it
args = []
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
else:
raise AssertionError(f'Unexpected etherscan call during tests: {url}')
elif 'api.etherscan.io/api?module=proxy&action=eth_call&to=0xB6456b57f03352bE48Bf101B46c1752a0813491a' in url: # noqa: E501 # ADEX Staking contract
if 'adex_staking' in original_queries:
return original_requests_get(url, *args, **kwargs)
if 'data=0x447b15f4' in url: # a mocked share value
response = '{"jsonrpc":"2.0","id":1,"result":"0x0000000000000000000000000000000000000000000000000fc4a48782d85b51"}' # noqa: E501
else:
raise AssertionError(f'Unknown call to Adex Staking pool during tests: {url}')
elif f'api.etherscan.io/api?module=proxy&action=eth_call&to={ETH_MULTICALL.address}' in url: # noqa: E501
web3 = Web3()
contract = web3.eth.contract(address=ETH_MULTICALL.address, abi=ETH_MULTICALL.abi)
if 'b6456b57f03352be48bf101b46c1752a0813491a' in url:
multicall_purpose = 'adex_staking'
elif '5f3b5dfeb7b28cdbd7faba78963ee202a494e2a2' in url:
multicall_purpose = 'vecrv'
else:
raise AssertionError('Unknown multicall in mocked tests')
if 'data=0x252dba42' in url: # aggregate
if multicall_purpose == 'adex_staking':
if 'adex_staking' in original_queries:
return original_requests_get(url, *args, **kwargs)
if 'mocked_adex_staking_balance' in extra_flags:
# mock adex staking balance for a single account
response = '{"jsonrpc": "2.0", "id": 1, "result": "0x0000000000000000000000000000000000000000000000000000000000bb45aa000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000152982285d2e4d5aeaa9"}' # noqa: E501
return MockResponse(200, response)
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
fn_abi = contract.functions.abi[1]
assert fn_abi['name'] == 'aggregate', 'Abi position of multicall aggregate changed'
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
# For now the only mocked multicall is 32 bytes for multicall balance
# of both veCRV and adex staking pool.
# When there is more we have to figure out a way to differentiate
# between them in mocking. Just return empty response here
# all pylint ignores below due to https://github.com/PyCQA/pylint/issues/4114
args = [1, [b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' for x in decoded_input[0]]] # pylint: disable=unsubscriptable-object # noqa: E501
result = '0x' + web3.codec.encode_abi(output_types, args).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
else:
raise AssertionError('Unexpected etherscan multicall during tests: {url}')
elif f'api.etherscan.io/api?module=proxy&action=eth_call&to={ETH_SCAN.address}' in url:
if 'ethscan' in original_queries:
return original_requests_get(url, *args, **kwargs)
web3 = Web3()
contract = web3.eth.contract(address=ETH_SCAN.address, abi=ETH_SCAN.abi)
if 'data=0xdbdbb51b' in url: # Eth balance query
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='etherBalances',
args=[list(eth_map.keys())],
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
args = []
for account_address in decoded_input[0]: # pylint: disable=unsubscriptable-object # noqa: E501
account_address = deserialize_ethereum_address(account_address)
args.append(int(eth_map[account_address]['ETH']))
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
elif 'data=0x06187b4f' in url: # Multi token multiaddress balance query
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
# not really the given args, but we just want the fn abi
args = [list(eth_map.keys()), list(eth_map.keys())]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='tokensBalances',
args=args,
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
args = []
for account_address in decoded_input[0]: # pylint: disable=unsubscriptable-object # noqa: E501
account_address = deserialize_ethereum_address(account_address)
x = []
for token_address in decoded_input[1]: # pylint: disable=unsubscriptable-object # noqa: E501
token_address = deserialize_ethereum_address(token_address)
value_to_add = 0
for given_asset, value in eth_map[account_address].items():
given_asset = _get_token(given_asset)
if given_asset is None:
# not a token
continue
if token_address != given_asset.ethereum_address:
continue
value_to_add = int(value)
break
x.append(value_to_add)
args.append(x)
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
elif 'data=0xe5da1b68' in url: # Multi token balance query
data = url.split('data=')[1]
if '&apikey' in data:
data = data.split('&apikey')[0]
# not really the given args, but we just want the fn abi
args = ['str', list(eth_map.keys())]
fn_abi = contract._find_matching_fn_abi(
fn_identifier='tokensBalance',
args=args,
)
input_types = get_abi_input_types(fn_abi)
output_types = get_abi_output_types(fn_abi)
decoded_input = web3.codec.decode_abi(input_types, bytes.fromhex(data[10:]))
args = []
account_address = deserialize_ethereum_address(decoded_input[0]) # pylint: disable=unsubscriptable-object # noqa: E501
x = []
for token_address in decoded_input[1]: # pylint: disable=unsubscriptable-object # noqa: E501
token_address = deserialize_ethereum_address(token_address)
value_to_add = 0
for given_asset, value in eth_map[account_address].items():
given_asset = _get_token(given_asset)
if given_asset is None:
# not a token
continue
if token_address != given_asset.ethereum_address:
continue
value_to_add = int(value)
break
args.append(value_to_add)
result = '0x' + web3.codec.encode_abi(output_types, [args]).hex()
response = f'{{"jsonrpc":"2.0","id":1,"result":"{result}"}}'
else:
raise AssertionError(f'Unexpected etherscan call during tests: {url}')
else:
return original_requests_get(url, *args, **kwargs)
return MockResponse(200, response)
from rotkehlchen.chain.ethereum.uniswap.typing import (
EventType,
LiquidityPool,
LiquidityPoolAsset,
LiquidityPoolEvent,
LiquidityPoolEventsBalance,
)
from rotkehlchen.constants import ZERO
from rotkehlchen.fval import FVal
from rotkehlchen.serialization.deserialize import deserialize_ethereum_address
from rotkehlchen.typing import AssetAmount, Price, Timestamp
# Logic: Get balances
# Addresses
TEST_ADDRESS_1 = deserialize_ethereum_address(
'0xfeF0E7635281eF8E3B705e9C5B86e1d3B0eAb397')
TEST_ADDRESS_2 = deserialize_ethereum_address(
'0xcf2B8EeC2A9cE682822b252a1e9B78EedebEFB02')
TEST_ADDRESS_3 = deserialize_ethereum_address(
'0x7777777777777777777777777777777777777777')
# Known tokens
ASSET_USDT = EthereumToken('USDT')
ASSET_WETH = EthereumToken('WETH')
TOKEN_BASED = EthereumToken('$BASED')
# Unknown tokens
ASSET_SHUF = UnknownEthereumToken(
ethereum_address=deserialize_ethereum_address(
'0x3A9FfF453d50D4Ac52A6890647b823379ba36B9E'),
symbol='SHUF',
return DefiBalance(
token_address=to_checksum_address(
'0xBD87447F48ad729C5c4b8bcb503e1395F62e8B98'),
token_name='Pool Together USDC token',
token_symbol='plUSDC',
balance=Balance(
amount=normalized_balance,
usd_value=normalized_balance * usdc_price,
),
)
# else
return None
# supported zerion adapter address
ZERION_ADAPTER_ADDRESS = deserialize_ethereum_address(
'0x06FE76B2f432fdfEcAEf1a7d4f6C3d41B5861672')
class ZerionSDK():
"""Adapter for the Zerion DeFi SDK https://github.com/zeriontech/defi-sdk"""
def __init__(
self,
ethereum_manager: 'EthereumManager',
msg_aggregator: MessagesAggregator,
) -> None:
self.ethereum = ethereum_manager
self.msg_aggregator = msg_aggregator
self.contract = EthereumContract(
address=ZERION_ADAPTER_ADDRESS,
abi=ZERION_ABI,
deployed_block=1586199170,
