def _parse_withdrawals(
self,
withdrawals: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[AaveSimpleEvent]:
events = []
for entry in withdrawals:
common = _parse_common_event_data(entry, from_ts, to_ts)
if common is None:
continue # either timestamp out of range or error (logged in the function above)
timestamp, tx_hash, index = common
result = self._get_asset_and_balance(
entry=entry,
timestamp=timestamp,
reserve_key='reserve',
amount_key='amount',
location='aave withdrawal from graph query',
)
if result is None:
continue # problem parsing, error already logged
asset, balance = result
events.append(AaveSimpleEvent(
event_type='withdrawal',
asset=asset,
value=balance,
block_number=0, # can't get from graph query
timestamp=timestamp,
tx_hash=tx_hash,
log_index=index, # not really the log index, but should also be unique
))
return events
def test_add_and_get_aave_events(data_dir, username):
"""Test that get aave events works fine and returns only events for what we need"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
addr1 = make_ethereum_address()
addr1_events = [AaveSimpleEvent(
event_type='deposit',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436',
log_index=1,
), AaveSimpleEvent(
event_type='withdrawal',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce',
log_index=2,
)]
data.db.add_aave_events(address=addr1, events=addr1_events)
addr2 = make_ethereum_address()
addr2_events = [AaveSimpleEvent(
event_type='deposit',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55',
log_index=1,
), AaveSimpleEvent(
event_type='withdrawal',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=2,
)]
data.db.add_aave_events(address=addr2, events=addr2_events)
# addr3 has all types of aave events so we test serialization/deserialization
addr3 = make_ethereum_address()
addr3_events = [AaveSimpleEvent(
event_type='deposit',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x9e394d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55',
log_index=1,
), AaveSimpleEvent(
event_type='withdrawal',
asset=A_DAI,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x4c167445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=2,
), AaveSimpleEvent(
event_type='interest',
asset=Asset('WBTC'),
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=4,
timestamp=Timestamp(4),
tx_hash='0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=4,
), AaveBorrowEvent(
event_type='borrow',
asset=Asset('ETH'),
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=5,
timestamp=Timestamp(5),
tx_hash='0x19c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=5,
borrow_rate_mode='stable',
borrow_rate=FVal('0.05233232323423432'),
accrued_borrow_interest=FVal('5.112234'),
), AaveRepayEvent(
event_type='repay',
asset=Asset('MANA'),
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=6,
timestamp=Timestamp(6),
tx_hash='0x29c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=6,
fee=Balance(amount=FVal('0.1'), usd_value=FVal('0.1')),
), AaveLiquidationEvent(
event_type='liquidation',
collateral_asset=Asset('ETH'),
collateral_balance=Balance(amount=FVal(1), usd_value=FVal(1)),
principal_asset=Asset('ETH'),
principal_balance=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=7,
log_index=7,
timestamp=Timestamp(7),
tx_hash='0x39c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
)]
data.db.add_aave_events(address=addr3, events=addr3_events)
events = data.db.get_aave_events(address=addr1, atoken=EthereumToken('aDAI'))
assert events == addr1_events
events = data.db.get_aave_events(address=addr2, atoken=EthereumToken('aDAI'))
assert events == addr2_events
events = data.db.get_aave_events(address=addr3)
assert events == addr3_events
# check that all aave events are properly hashable (aka can go in a set)
test_set = set()
for event in addr3_events:
test_set.add(event)
assert len(test_set) == len(addr3_events)
def get_events_for_atoken_and_address(
self,
user_address: ChecksumEthAddress,
atoken: EthereumToken,
deposit_events: List[Dict[str, Any]],
withdraw_events: List[Dict[str, Any]],
from_block: int,
to_block: int,
) -> List[AaveSimpleEvent]:
"""This function should be entered while holding the history_lock
semaphore"""
argument_filters = {
'from': ZERO_ADDRESS,
'to': user_address,
}
mint_events = self.ethereum.get_logs(
contract_address=atoken.ethereum_address,
abi=ATOKEN_ABI,
event_name='Transfer',
argument_filters=argument_filters,
from_block=from_block,
to_block=to_block,
)
mint_data = set()
mint_data_to_log_index = {}
for event in mint_events:
amount = hexstr_to_int(event['data'])
if amount == 0:
continue # first mint can be for 0. Ignore
entry = (
event['blockNumber'],
amount,
self.ethereum.get_event_timestamp(event),
event['transactionHash'],
)
mint_data.add(entry)
mint_data_to_log_index[entry] = event['logIndex']
reserve_asset = _atoken_to_reserve_asset(atoken)
reserve_address, decimals = _get_reserve_address_decimals(
reserve_asset.identifier)
aave_events = []
for event in deposit_events:
if hex_or_bytes_to_address(event['topics'][1]) == reserve_address:
# first 32 bytes of the data are the amount
deposit = hexstr_to_int(event['data'][:66])
block_number = event['blockNumber']
timestamp = self.ethereum.get_event_timestamp(event)
tx_hash = event['transactionHash']
log_index = event['logIndex']
# If there is a corresponding deposit event remove the minting event data
entry = (block_number, deposit, timestamp, tx_hash)
if entry in mint_data:
mint_data.remove(entry)
del mint_data_to_log_index[entry]
usd_price = query_usd_price_zero_if_error(
asset=reserve_asset,
time=timestamp,
location='aave deposit',
msg_aggregator=self.msg_aggregator,
)
deposit_amount = deposit / (FVal(10)**FVal(decimals))
aave_events.append(
AaveSimpleEvent(
event_type='deposit',
asset=reserve_asset,
value=Balance(
amount=deposit_amount,
usd_value=deposit_amount * usd_price,
),
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
))
for data in mint_data:
usd_price = query_usd_price_zero_if_error(
asset=atoken,
time=data[2],
location='aave interest profit',
msg_aggregator=self.msg_aggregator,
)
interest_amount = data[1] / (FVal(10)**FVal(decimals))
aave_events.append(
AaveSimpleEvent(
event_type='interest',
asset=atoken,
value=Balance(
amount=interest_amount,
usd_value=interest_amount * usd_price,
),
block_number=data[0],
timestamp=data[2],
tx_hash=data[3],
log_index=mint_data_to_log_index[data],
))
for event in withdraw_events:
if hex_or_bytes_to_address(event['topics'][1]) == reserve_address:
# first 32 bytes of the data are the amount
withdrawal = hexstr_to_int(event['data'][:66])
block_number = event['blockNumber']
timestamp = self.ethereum.get_event_timestamp(event)
tx_hash = event['transactionHash']
usd_price = query_usd_price_zero_if_error(
asset=reserve_asset,
time=timestamp,
location='aave withdrawal',
msg_aggregator=self.msg_aggregator,
)
withdrawal_amount = withdrawal / (FVal(10)**FVal(decimals))
aave_events.append(
AaveSimpleEvent(
event_type='withdrawal',
asset=reserve_asset,
value=Balance(
amount=withdrawal_amount,
usd_value=withdrawal_amount * usd_price,
),
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=event['logIndex'],
))
return aave_events
def _calculate_interest_and_profit(
self,
user_address: ChecksumEthAddress,
user_result: Dict[str, Any],
actions: List[AaveSimpleEvent],
balances: AaveBalances,
db_interest_events: Set[AaveSimpleEvent],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Tuple[List[AaveSimpleEvent], Dict[Asset, Balance]]:
reserve_history = {}
for reserve in user_result['reserves']:
pairs = reserve['id'].split('0x')
if len(pairs) != 3:
log.error(
f'Expected to find 2 hashes in graph\'s reserve history id '
f'but the encountered id does not match: {reserve["id"]}. Skipping entry...',
)
reserve_address = to_checksum_address('0x' + pairs[2])
atoken_history = _parse_atoken_balance_history(
history=reserve['aTokenBalanceHistory'],
from_ts=from_ts,
to_ts=to_ts,
)
reserve_history[reserve_address] = atoken_history
interest_events: List[AaveSimpleEvent] = []
atoken_balances: Dict[Asset, FVal] = defaultdict(FVal)
used_history_indices = set()
total_earned: Dict[Asset, Balance] = defaultdict(Balance)
# Go through the existing db interest events and add total earned
for interest_event in db_interest_events:
total_earned[interest_event.asset] += interest_event.value
# Create all new interest events in the query
actions.sort(key=lambda event: event.timestamp)
for action in actions:
if action.event_type == 'deposit':
atoken_balances[action.asset] += action.value.amount
else: # withdrawal
atoken_balances[action.asset] -= action.value.amount
reserve_address = ASSET_TO_AAVE_RESERVE_ADDRESS.get(action.asset.identifier, None) # type: ignore # noqa: E501
history = reserve_history.get(reserve_address, None)
if history is None:
log.error(
f'Could not find aTokenBalanceHistory for reserve '
f'{reserve_address} in an aave graph response.'
f' Skipping entry...',
)
continue
history.sort(key=lambda event: event.timestamp)
for idx, entry in enumerate(history):
if idx in used_history_indices:
continue
used_history_indices.add(idx)
if entry.tx_hash == action.tx_hash:
diff = entry.balance - atoken_balances[action.asset]
if diff != ZERO:
atoken_balances[action.asset] = entry.balance
try:
asset = EthereumToken('a' + action.asset.identifier)
except UnknownAsset:
log.error(
f'Could not find corresponding aToken to'
f'{action.asset.identifier} during an aave graph uery'
f' Skipping entry...',
)
continue
timestamp = entry.timestamp
usd_price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location='aave interest event from graph query',
msg_aggregator=self.msg_aggregator,
)
earned_balance = Balance(amount=diff, usd_value=diff * usd_price)
interest_event = AaveSimpleEvent(
event_type='interest',
asset=asset,
value=earned_balance,
block_number=0, # can't get from graph query
timestamp=timestamp,
tx_hash=entry.tx_hash,
# not really the log index, but should also be unique
log_index=action.log_index + 1,
)
if interest_event in db_interest_events:
# This should not really happen since we already query
# historical atoken balance history in the new range
log.warning(
f'During aave subgraph query interest and profit calculation '
f'tried to generate interest event {interest_event} that '
f'already existed in the DB ',
)
continue
interest_events.append(interest_event)
total_earned[asset] += earned_balance
# and once done break off the loop
break
# else this atoken history is not due to an action, so skip it.
# It's probably due to a simple transfer
atoken_balances[action.asset] = entry.balance
if action.event_type == 'deposit':
atoken_balances[action.asset] += action.value.amount
else: # withdrawal
atoken_balances[action.asset] -= action.value.amount
# Take aave unpaid interest into account
for symbol, lending_balance in balances.lending.items():
atoken = EthereumToken('a' + symbol)
principal_balance = self.ethereum.call_contract(
contract_address=atoken.ethereum_address,
abi=ATOKEN_ABI,
method_name='principalBalanceOf',
arguments=[user_address],
)
unpaid_interest = lending_balance.balance.amount - (principal_balance / (FVal(10) ** FVal(atoken.decimals))) # noqa: E501
usd_price = Inquirer().find_usd_price(atoken)
total_earned[atoken] += Balance(
amount=unpaid_interest,
usd_value=unpaid_interest * usd_price,
)
return interest_events, total_earned
1588430911: FVal('1.009'),
1592175763: FVal('1.013'),
1594502373: FVal('1.019'),
},
},
}
aave_mocked_current_prices = {'aDAI': FVal('1.017')}
expected_aave_deposit_test_events = [
AaveSimpleEvent(
event_type='deposit',
asset=A_DAI,
value=Balance(
amount=FVal('102.926986169186236436'),
usd_value=FVal('104.367963975554843746104'),
),
block_number=9963767,
timestamp=Timestamp(1588114293),
tx_hash=
'0x8b72307967c4f7a486c1cb1b6ebca5e549de06e02930ece0399e2096f1a132c5',
log_index=72,
),
AaveSimpleEvent(
event_type='deposit',
asset=A_DAI,
value=Balance(
amount=FVal('160'),
usd_value=FVal('161.440'),
),
block_number=9987395,
timestamp=Timestamp(1588430911),