def _parse_withdrawals( self, withdrawals: List[Dict[str, Any]], from_ts: Timestamp, to_ts: Timestamp, ) -> List[AaveSimpleEvent]: events = [] for entry in withdrawals: common = _parse_common_event_data(entry, from_ts, to_ts) if common is None: continue # either timestamp out of range or error (logged in the function above) timestamp, tx_hash, index = common result = self._get_asset_and_balance( entry=entry, timestamp=timestamp, reserve_key='reserve', amount_key='amount', location='aave withdrawal from graph query', ) if result is None: continue # problem parsing, error already logged asset, balance = result events.append(AaveSimpleEvent( event_type='withdrawal', asset=asset, value=balance, block_number=0, # can't get from graph query timestamp=timestamp, tx_hash=tx_hash, log_index=index, # not really the log index, but should also be unique )) return events
def test_add_and_get_aave_events(data_dir, username): """Test that get aave events works fine and returns only events for what we need""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) addr1 = make_ethereum_address() addr1_events = [AaveSimpleEvent( event_type='deposit', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436', log_index=1, ), AaveSimpleEvent( event_type='withdrawal', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce', log_index=2, )] data.db.add_aave_events(address=addr1, events=addr1_events) addr2 = make_ethereum_address() addr2_events = [AaveSimpleEvent( event_type='deposit', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55', log_index=1, ), AaveSimpleEvent( event_type='withdrawal', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=2, )] data.db.add_aave_events(address=addr2, events=addr2_events) # addr3 has all types of aave events so we test serialization/deserialization addr3 = make_ethereum_address() addr3_events = [AaveSimpleEvent( event_type='deposit', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x9e394d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55', log_index=1, ), AaveSimpleEvent( event_type='withdrawal', asset=A_DAI, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x4c167445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=2, ), AaveSimpleEvent( event_type='interest', asset=Asset('WBTC'), value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=4, timestamp=Timestamp(4), tx_hash='0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=4, ), AaveBorrowEvent( event_type='borrow', asset=Asset('ETH'), value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=5, timestamp=Timestamp(5), tx_hash='0x19c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=5, borrow_rate_mode='stable', borrow_rate=FVal('0.05233232323423432'), accrued_borrow_interest=FVal('5.112234'), ), AaveRepayEvent( event_type='repay', asset=Asset('MANA'), value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=6, timestamp=Timestamp(6), tx_hash='0x29c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=6, fee=Balance(amount=FVal('0.1'), usd_value=FVal('0.1')), ), AaveLiquidationEvent( event_type='liquidation', collateral_asset=Asset('ETH'), collateral_balance=Balance(amount=FVal(1), usd_value=FVal(1)), principal_asset=Asset('ETH'), principal_balance=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=7, log_index=7, timestamp=Timestamp(7), tx_hash='0x39c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', )] data.db.add_aave_events(address=addr3, events=addr3_events) events = data.db.get_aave_events(address=addr1, atoken=EthereumToken('aDAI')) assert events == addr1_events events = data.db.get_aave_events(address=addr2, atoken=EthereumToken('aDAI')) assert events == addr2_events events = data.db.get_aave_events(address=addr3) assert events == addr3_events # check that all aave events are properly hashable (aka can go in a set) test_set = set() for event in addr3_events: test_set.add(event) assert len(test_set) == len(addr3_events)
def get_events_for_atoken_and_address( self, user_address: ChecksumEthAddress, atoken: EthereumToken, deposit_events: List[Dict[str, Any]], withdraw_events: List[Dict[str, Any]], from_block: int, to_block: int, ) -> List[AaveSimpleEvent]: """This function should be entered while holding the history_lock semaphore""" argument_filters = { 'from': ZERO_ADDRESS, 'to': user_address, } mint_events = self.ethereum.get_logs( contract_address=atoken.ethereum_address, abi=ATOKEN_ABI, event_name='Transfer', argument_filters=argument_filters, from_block=from_block, to_block=to_block, ) mint_data = set() mint_data_to_log_index = {} for event in mint_events: amount = hexstr_to_int(event['data']) if amount == 0: continue # first mint can be for 0. Ignore entry = ( event['blockNumber'], amount, self.ethereum.get_event_timestamp(event), event['transactionHash'], ) mint_data.add(entry) mint_data_to_log_index[entry] = event['logIndex'] reserve_asset = _atoken_to_reserve_asset(atoken) reserve_address, decimals = _get_reserve_address_decimals( reserve_asset.identifier) aave_events = [] for event in deposit_events: if hex_or_bytes_to_address(event['topics'][1]) == reserve_address: # first 32 bytes of the data are the amount deposit = hexstr_to_int(event['data'][:66]) block_number = event['blockNumber'] timestamp = self.ethereum.get_event_timestamp(event) tx_hash = event['transactionHash'] log_index = event['logIndex'] # If there is a corresponding deposit event remove the minting event data entry = (block_number, deposit, timestamp, tx_hash) if entry in mint_data: mint_data.remove(entry) del mint_data_to_log_index[entry] usd_price = query_usd_price_zero_if_error( asset=reserve_asset, time=timestamp, location='aave deposit', msg_aggregator=self.msg_aggregator, ) deposit_amount = deposit / (FVal(10)**FVal(decimals)) aave_events.append( AaveSimpleEvent( event_type='deposit', asset=reserve_asset, value=Balance( amount=deposit_amount, usd_value=deposit_amount * usd_price, ), block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, )) for data in mint_data: usd_price = query_usd_price_zero_if_error( asset=atoken, time=data[2], location='aave interest profit', msg_aggregator=self.msg_aggregator, ) interest_amount = data[1] / (FVal(10)**FVal(decimals)) aave_events.append( AaveSimpleEvent( event_type='interest', asset=atoken, value=Balance( amount=interest_amount, usd_value=interest_amount * usd_price, ), block_number=data[0], timestamp=data[2], tx_hash=data[3], log_index=mint_data_to_log_index[data], )) for event in withdraw_events: if hex_or_bytes_to_address(event['topics'][1]) == reserve_address: # first 32 bytes of the data are the amount withdrawal = hexstr_to_int(event['data'][:66]) block_number = event['blockNumber'] timestamp = self.ethereum.get_event_timestamp(event) tx_hash = event['transactionHash'] usd_price = query_usd_price_zero_if_error( asset=reserve_asset, time=timestamp, location='aave withdrawal', msg_aggregator=self.msg_aggregator, ) withdrawal_amount = withdrawal / (FVal(10)**FVal(decimals)) aave_events.append( AaveSimpleEvent( event_type='withdrawal', asset=reserve_asset, value=Balance( amount=withdrawal_amount, usd_value=withdrawal_amount * usd_price, ), block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=event['logIndex'], )) return aave_events
def _calculate_interest_and_profit( self, user_address: ChecksumEthAddress, user_result: Dict[str, Any], actions: List[AaveSimpleEvent], balances: AaveBalances, db_interest_events: Set[AaveSimpleEvent], from_ts: Timestamp, to_ts: Timestamp, ) -> Tuple[List[AaveSimpleEvent], Dict[Asset, Balance]]: reserve_history = {} for reserve in user_result['reserves']: pairs = reserve['id'].split('0x') if len(pairs) != 3: log.error( f'Expected to find 2 hashes in graph\'s reserve history id ' f'but the encountered id does not match: {reserve["id"]}. Skipping entry...', ) reserve_address = to_checksum_address('0x' + pairs[2]) atoken_history = _parse_atoken_balance_history( history=reserve['aTokenBalanceHistory'], from_ts=from_ts, to_ts=to_ts, ) reserve_history[reserve_address] = atoken_history interest_events: List[AaveSimpleEvent] = [] atoken_balances: Dict[Asset, FVal] = defaultdict(FVal) used_history_indices = set() total_earned: Dict[Asset, Balance] = defaultdict(Balance) # Go through the existing db interest events and add total earned for interest_event in db_interest_events: total_earned[interest_event.asset] += interest_event.value # Create all new interest events in the query actions.sort(key=lambda event: event.timestamp) for action in actions: if action.event_type == 'deposit': atoken_balances[action.asset] += action.value.amount else: # withdrawal atoken_balances[action.asset] -= action.value.amount reserve_address = ASSET_TO_AAVE_RESERVE_ADDRESS.get(action.asset.identifier, None) # type: ignore # noqa: E501 history = reserve_history.get(reserve_address, None) if history is None: log.error( f'Could not find aTokenBalanceHistory for reserve ' f'{reserve_address} in an aave graph response.' f' Skipping entry...', ) continue history.sort(key=lambda event: event.timestamp) for idx, entry in enumerate(history): if idx in used_history_indices: continue used_history_indices.add(idx) if entry.tx_hash == action.tx_hash: diff = entry.balance - atoken_balances[action.asset] if diff != ZERO: atoken_balances[action.asset] = entry.balance try: asset = EthereumToken('a' + action.asset.identifier) except UnknownAsset: log.error( f'Could not find corresponding aToken to' f'{action.asset.identifier} during an aave graph uery' f' Skipping entry...', ) continue timestamp = entry.timestamp usd_price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location='aave interest event from graph query', msg_aggregator=self.msg_aggregator, ) earned_balance = Balance(amount=diff, usd_value=diff * usd_price) interest_event = AaveSimpleEvent( event_type='interest', asset=asset, value=earned_balance, block_number=0, # can't get from graph query timestamp=timestamp, tx_hash=entry.tx_hash, # not really the log index, but should also be unique log_index=action.log_index + 1, ) if interest_event in db_interest_events: # This should not really happen since we already query # historical atoken balance history in the new range log.warning( f'During aave subgraph query interest and profit calculation ' f'tried to generate interest event {interest_event} that ' f'already existed in the DB ', ) continue interest_events.append(interest_event) total_earned[asset] += earned_balance # and once done break off the loop break # else this atoken history is not due to an action, so skip it. # It's probably due to a simple transfer atoken_balances[action.asset] = entry.balance if action.event_type == 'deposit': atoken_balances[action.asset] += action.value.amount else: # withdrawal atoken_balances[action.asset] -= action.value.amount # Take aave unpaid interest into account for symbol, lending_balance in balances.lending.items(): atoken = EthereumToken('a' + symbol) principal_balance = self.ethereum.call_contract( contract_address=atoken.ethereum_address, abi=ATOKEN_ABI, method_name='principalBalanceOf', arguments=[user_address], ) unpaid_interest = lending_balance.balance.amount - (principal_balance / (FVal(10) ** FVal(atoken.decimals))) # noqa: E501 usd_price = Inquirer().find_usd_price(atoken) total_earned[atoken] += Balance( amount=unpaid_interest, usd_value=unpaid_interest * usd_price, ) return interest_events, total_earned
1588430911: FVal('1.009'), 1592175763: FVal('1.013'), 1594502373: FVal('1.019'), }, }, } aave_mocked_current_prices = {'aDAI': FVal('1.017')} expected_aave_deposit_test_events = [ AaveSimpleEvent( event_type='deposit', asset=A_DAI, value=Balance( amount=FVal('102.926986169186236436'), usd_value=FVal('104.367963975554843746104'), ), block_number=9963767, timestamp=Timestamp(1588114293), tx_hash= '0x8b72307967c4f7a486c1cb1b6ebca5e549de06e02930ece0399e2096f1a132c5', log_index=72, ), AaveSimpleEvent( event_type='deposit', asset=A_DAI, value=Balance( amount=FVal('160'), usd_value=FVal('161.440'), ), block_number=9987395, timestamp=Timestamp(1588430911),