def get_usd_price_zero_if_error( asset: Asset, time: Timestamp, location: str, msg_aggregator: MessagesAggregator, ) -> Price: """A special version of query_usd_price_zero_if_error using current price instead of historical token price for some assets. Since these assets are not supported by our price oracles we derive current price from the chain but without an archive node can't query old prices. TODO: MAke an issue about this This can be solved when we have an archive node. """ inquirer = Inquirer() if asset in inquirer.special_tokens: return inquirer.find_usd_price(asset) return query_usd_price_zero_if_error( asset=asset, time=time, location=location, msg_aggregator=msg_aggregator, )
def _get_eth2_staking_deposits_onchain( ethereum: 'EthereumManager', addresses: List[ChecksumEthAddress], has_premium: bool, msg_aggregator: MessagesAggregator, from_ts: Timestamp, to_ts: Timestamp, ) -> List[Eth2Deposit]: events = ETH2_DEPOSIT.get_logs( ethereum=ethereum, event_name='DepositEvent', argument_filters={}, from_block=ETH2_DEPOSIT.deployed_block, to_block='latest', ) transactions = ethereum.transactions.query( addresses=addresses, from_ts=from_ts, to_ts=to_ts, with_limit=False, recent_first=False, ) deposits: List[Eth2Deposit] = [] for transaction in transactions: if transaction.to_address != ETH2_DEPOSIT.address: continue tx_hash = '0x' + transaction.tx_hash.hex() for event in events: # Now find the corresponding event. If no event is found the transaction # probably failed or was something other than a deposit if event['transactionHash'] == tx_hash: decoded_data = decode_event_data(event['data'], EVENT_ABI) amount = int.from_bytes(decoded_data[2], byteorder='little') usd_price = ZERO if has_premium: # won't show this to non-premium so don't bother usd_price = query_usd_price_zero_if_error( asset=A_ETH, time=transaction.timestamp, location='Eth2 staking query', msg_aggregator=msg_aggregator, ) normalized_amount = from_gwei(FVal(amount)) deposits.append( Eth2Deposit( from_address=transaction.from_address, pubkey='0x' + decoded_data[0].hex(), withdrawal_credentials='0x' + decoded_data[1].hex(), value=Balance(normalized_amount, usd_price * normalized_amount), validator_index=int.from_bytes(decoded_data[4], byteorder='little'), tx_hash=tx_hash, log_index=event['logIndex'], timestamp=Timestamp(transaction.timestamp), )) break return deposits
def get_validator_deposits( self, indices_or_pubkeys: Union[List[int], List[Eth2PubKey]], ) -> List[Eth2Deposit]: """Get the deposits of all the validators given from the list of indices or pubkeys Queries in chunks of 100 due to api limitations May raise: - RemoteError due to problems querying beaconcha.in API """ chunks = _calculate_query_chunks(indices_or_pubkeys) data = [] for chunk in chunks: result = self._query( module='validator', endpoint='deposits', encoded_args=','.join(str(x) for x in chunk), ) if isinstance(result, list): data.extend(result) else: data.append(result) deposits = [] for entry in data: try: amount = from_gwei(FVal(entry['amount'])) timestamp = entry['block_ts'] usd_price = query_usd_price_zero_if_error( asset=A_ETH, time=timestamp, location=f'Eth2 staking deposit at time {timestamp}', msg_aggregator=self.msg_aggregator, ) deposits.append( Eth2Deposit( from_address=deserialize_ethereum_address( entry['from_address']), pubkey=entry['publickey'], withdrawal_credentials=entry['withdrawal_credentials'], value=Balance( amount=amount, usd_value=amount * usd_price, ), tx_hash=hexstring_to_bytes(entry['tx_hash']), tx_index=entry['tx_index'], timestamp=entry['block_ts'], )) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' raise RemoteError( f'Beaconchai.in deposits response processing error. {msg}', ) from e return deposits
def _get_comp_events( self, address: ChecksumEthAddress, from_ts: Timestamp, to_ts: Timestamp, ) -> List[CompoundEvent]: self.ethereum.get_blocknumber_by_time(from_ts) from_block = max( COMP_DEPLOYED_BLOCK, self.ethereum.get_blocknumber_by_time(from_ts), ) argument_filters = { 'from': COMPTROLLER_PROXY.address, 'to': address, } comp_events = self.ethereum.get_logs( contract_address=A_COMP.ethereum_address, abi=ERC20TOKEN_ABI, event_name='Transfer', argument_filters=argument_filters, from_block=from_block, to_block=self.ethereum.get_blocknumber_by_time(to_ts), ) events = [] for event in comp_events: timestamp = self.ethereum.get_event_timestamp(event) tx_hash = event['transactionHash'] amount = token_normalized_value(hexstr_to_int(event['data']), A_COMP) usd_price = query_usd_price_zero_if_error( asset=A_COMP, time=timestamp, location=f'comp_claim {tx_hash}', msg_aggregator=self.msg_aggregator, ) value = Balance(amount, amount * usd_price) events.append( CompoundEvent( event_type='comp', address=address, block_number=event['blockNumber'], timestamp=timestamp, asset=A_COMP, value=value, to_asset=None, to_value=None, realized_pnl=value, tx_hash=tx_hash, log_index=event['logIndex'], )) return events
def _parse_repays( self, repays: List[Dict[str, Any]], from_ts: Timestamp, to_ts: Timestamp, ) -> List[AaveRepayEvent]: events = [] for entry in repays: common = _parse_common_event_data(entry, from_ts, to_ts) if common is None: continue # either timestamp out of range or error (logged in the function above) timestamp, tx_hash, index = common result = _get_reserve_asset_and_decimals(entry, reserve_key='reserve') if result is None: continue # problem parsing, error already logged asset, decimals = result if 'amountAfterFee' in entry: amount_after_fee = token_normalized_value_decimals( int(entry['amountAfterFee']), token_decimals=decimals, ) fee = token_normalized_value_decimals(int(entry['fee']), token_decimals=decimals) else: # In the V2 subgraph the amountAfterFee and Fee keys are replaced by amount amount_after_fee = token_normalized_value_decimals( int(entry['amount']), token_decimals=decimals, ) fee = ZERO usd_price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location=f'aave repay event {tx_hash} from graph query', msg_aggregator=self.msg_aggregator, ) events.append( AaveRepayEvent( event_type='repay', asset=asset, value=Balance(amount=amount_after_fee, usd_value=amount_after_fee * usd_price), fee=Balance(amount=fee, usd_value=fee * usd_price), block_number=0, # can't get from graph query timestamp=timestamp, tx_hash=tx_hash, log_index= index, # not really the log index, but should also be unique )) return events
def _get_asset_and_balance( self, entry: Dict[str, Any], timestamp: Timestamp, reserve_key: str, amount_key: str, location: str, ) -> Optional[Tuple[Asset, Balance]]: """Utility function to parse asset from graph query amount and price and return balance""" result = _get_reserve_asset_and_decimals(entry, reserve_key) if result is None: return None asset, decimals = result amount = token_normalized_value_decimals( token_amount=int(entry[amount_key]), token_decimals=decimals, ) usd_price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location=location, msg_aggregator=self.msg_aggregator, ) return asset, Balance(amount=amount, usd_value=amount * usd_price)
def _process_events( self, events: List[CompoundEvent], given_defi_balances: GIVEN_DEFI_BALANCES, ) -> Tuple[ADDRESS_TO_ASSETS, ADDRESS_TO_ASSETS, ADDRESS_TO_ASSETS, ADDRESS_TO_ASSETS]: """Processes all events and returns a dictionary of earned balances totals""" assets: ADDRESS_TO_ASSETS = defaultdict(lambda: defaultdict(Balance)) loss_assets: ADDRESS_TO_ASSETS = defaultdict( lambda: defaultdict(Balance)) rewards_assets: ADDRESS_TO_ASSETS = defaultdict( lambda: defaultdict(Balance)) profit_so_far: ADDRESS_TO_ASSETS = defaultdict( lambda: defaultdict(Balance)) loss_so_far: ADDRESS_TO_ASSETS = defaultdict( lambda: defaultdict(Balance)) liquidation_profit: ADDRESS_TO_ASSETS = defaultdict( lambda: defaultdict(Balance)) balances = self.get_balances(given_defi_balances) for idx, event in enumerate(events): if event.event_type == 'mint': assets[event.address][event.asset] -= event.value elif event.event_type == 'redeem': assert event.to_asset, 'redeem events should have a to_asset' assert event.to_value, 'redeem events should have a to_value' profit_amount = ( assets[event.address][event.to_asset].amount + event.to_value.amount - profit_so_far[event.address][event.to_asset].amount) profit: Optional[Balance] if profit_amount >= 0: usd_price = query_usd_price_zero_if_error( asset=event.to_asset, time=event.timestamp, location='comp redeem event processing', msg_aggregator=self.msg_aggregator, ) profit = Balance(profit_amount, profit_amount * usd_price) profit_so_far[event.address][event.to_asset] += profit else: profit = None assets[event.address][event.to_asset] += event.to_value events[idx] = event._replace( realized_pnl=profit) # TODO: maybe not named tuple? elif event.event_type == 'borrow': loss_assets[event.address][event.asset] -= event.value elif event.event_type == 'repay': loss_amount = (loss_assets[event.address][event.asset].amount + event.value.amount - loss_so_far[event.address][event.asset].amount) loss: Optional[Balance] if loss_amount >= 0: usd_price = query_usd_price_zero_if_error( asset=event.asset, time=event.timestamp, location='comp repay event processing', msg_aggregator=self.msg_aggregator, ) loss = Balance(loss_amount, loss_amount * usd_price) loss_so_far[event.address][event.asset] += loss else: loss = None loss_assets[event.address][event.asset] += event.value events[idx] = event._replace( realized_pnl=loss) # TODO: maybe not named tuple? elif event.event_type == 'liquidation': assert event.to_asset, 'liquidation events should have a to_asset' # Liquidator covers part of the borrowed amount loss_assets[event.address][event.asset] += event.value liquidation_profit[event.address][event.asset] += event.value # Liquidator receives discounted to_asset loss_assets[event.address][event.to_asset] += event.to_value loss_so_far[event.address][event.to_asset] += event.to_value elif event.event_type == 'comp': rewards_assets[event.address][A_COMP] += event.value for address, bentry in balances.items(): for asset, entry in bentry['lending'].items(): profit_amount = (profit_so_far[address][asset].amount + entry.balance.amount + assets[address][asset].amount) if profit_amount < 0: log.error( f'In compound we calculated negative profit. Should not happen. ' f'address: {address} asset: {asset} ', ) else: usd_price = Inquirer().find_usd_price(Asset(asset)) profit_so_far[address][asset] = Balance( amount=profit_amount, usd_value=profit_amount * usd_price, ) for asset, entry in bentry['borrowing'].items(): remaining = entry.balance + loss_assets[address][asset] if remaining.amount < ZERO: continue loss_so_far[address][asset] += remaining if loss_so_far[address][asset].usd_value < ZERO: amount = loss_so_far[address][asset].amount loss_so_far[address][asset] = Balance( amount=amount, usd_value=amount * Inquirer().find_usd_price(Asset(asset)), ) for asset, entry in bentry['rewards'].items(): rewards_assets[address][asset] += entry.balance return profit_so_far, loss_so_far, liquidation_profit, rewards_assets
def _get_lend_events( self, event_type: Literal['mint', 'redeem'], address: ChecksumEthAddress, from_ts: Timestamp, to_ts: Timestamp, ) -> List[CompoundEvent]: param_types, param_values = get_common_params(from_ts, to_ts, address) if event_type == 'mint': graph_event_name = 'mintEvents' addr_position = 'to' elif event_type == 'redeem': graph_event_name = 'redeemEvents' addr_position = 'from' result = self.graph.query( # type: ignore querystr=LEND_EVENTS_QUERY_PREFIX.format( graph_event_name=graph_event_name, addr_position=addr_position, ), param_types=param_types, param_values=param_values, ) events = [] for entry in result[graph_event_name]: ctoken_symbol = entry['cTokenSymbol'] try: ctoken_asset = Asset(ctoken_symbol) except UnknownAsset: log.error( f'Found unexpected cTokenSymbol {ctoken_symbol} during graph query. Skipping.' ) continue underlying_symbol = ctoken_symbol[1:] try: underlying_asset = Asset(underlying_symbol) except UnknownAsset: log.error( f'Found unexpected token symbol {underlying_symbol} during ' f'graph query. Skipping.', ) continue timestamp = entry['blockTime'] usd_price = query_usd_price_zero_if_error( asset=underlying_asset, time=timestamp, location=f'compound {event_type}', msg_aggregator=self.msg_aggregator, ) underlying_amount = FVal(entry['underlyingAmount']) usd_value = underlying_amount * usd_price parse_result = _get_txhash_and_logidx(entry['id']) if parse_result is None: log.error( f'Found unprocessable mint id from the graph {entry["id"]}. Skipping' ) continue amount = FVal(entry['amount']) if event_type == 'mint': from_value = Balance(amount=underlying_amount, usd_value=usd_value) to_value = Balance(amount=amount, usd_value=usd_value) from_asset = underlying_asset to_asset = ctoken_asset else: # redeem from_value = Balance(amount=amount, usd_value=usd_value) to_value = Balance(amount=underlying_amount, usd_value=usd_value) from_asset = ctoken_asset to_asset = underlying_asset events.append( CompoundEvent( event_type=event_type, address=address, block_number=entry['blockNumber'], timestamp=timestamp, asset=from_asset, value=from_value, to_asset=to_asset, to_value=to_value, realized_pnl=None, tx_hash=parse_result[0], log_index=parse_result[1], )) return events
def _get_liquidation_events( self, address: ChecksumEthAddress, from_ts: Timestamp, to_ts: Timestamp, ) -> List[CompoundEvent]: """https://compound.finance/docs/ctokens#liquidate-borrow""" param_types, param_values = get_common_params(from_ts, to_ts, address) result = self.graph.query( # type: ignore querystr= """liquidationEvents (where: {blockTime_lte: $end_ts, blockTime_gte: $start_ts, from: $address}) { id amount from blockNumber blockTime cTokenSymbol underlyingSymbol underlyingRepayAmount }}""", param_types=param_types, param_values=param_values, ) events = [] for entry in result['liquidationEvents']: ctoken_symbol = entry['cTokenSymbol'] try: ctoken_asset = Asset(ctoken_symbol) except UnknownAsset: log.error( f'Found unexpected cTokenSymbol {ctoken_symbol} during graph query. Skipping.' ) continue underlying_symbol = entry['underlyingSymbol'] try: underlying_asset = Asset(underlying_symbol) except UnknownAsset: log.error( f'Found unexpected token symbol {underlying_symbol} during ' f'graph query. Skipping.', ) continue timestamp = entry['blockTime'] # Amount/value of underlying asset paid by liquidator # Essentially liquidator covers part of the debt of the user debt_amount = FVal(entry['underlyingRepayAmount']) underlying_usd_price = query_usd_price_zero_if_error( asset=underlying_asset, time=timestamp, location='compound liquidation underlying asset', msg_aggregator=self.msg_aggregator, ) debt_usd_value = debt_amount * underlying_usd_price # Amount/value of ctoken_asset lost to the liquidator # This is what the liquidator gains at a discount liquidated_amount = FVal(entry['amount']) liquidated_usd_price = query_usd_price_zero_if_error( asset=ctoken_asset, time=timestamp, location='compound liquidation ctoken asset', msg_aggregator=self.msg_aggregator, ) liquidated_usd_value = liquidated_amount * liquidated_usd_price parse_result = _get_txhash_and_logidx(entry['id']) if parse_result is None: log.error( f'Found unprocessable liquidation id from the graph {entry["id"]}. Skipping', ) continue gained_value = Balance(amount=debt_amount, usd_value=debt_usd_value) lost_value = Balance(amount=liquidated_amount, usd_value=liquidated_usd_value) events.append( CompoundEvent( event_type='liquidation', address=address, block_number=entry['blockNumber'], timestamp=timestamp, asset=underlying_asset, value=gained_value, to_asset=ctoken_asset, to_value=lost_value, realized_pnl=None, tx_hash=parse_result[0], log_index=parse_result[1], )) return events
def _get_borrow_events( self, event_type: Literal['borrow', 'repay'], address: ChecksumEthAddress, from_ts: Timestamp, to_ts: Timestamp, ) -> List[CompoundEvent]: param_types, param_values = get_common_params(from_ts, to_ts, address) if event_type == 'borrow': graph_event_name = 'borrowEvents' payer_or_empty = '' elif event_type == 'repay': graph_event_name = 'repayEvents' payer_or_empty = 'payer' result = self.graph.query( # type: ignore querystr=BORROW_EVENTS_QUERY_PREFIX.format( graph_event_name=graph_event_name, payer_or_empty=payer_or_empty, ), param_types=param_types, param_values=param_values, ) events = [] for entry in result[graph_event_name]: underlying_symbol = entry['underlyingSymbol'] try: underlying_asset = Asset(underlying_symbol) except UnknownAsset: log.error( f'Found unexpected token symbol {underlying_symbol} during ' f'graph query. Skipping.', ) continue timestamp = entry['blockTime'] usd_price = query_usd_price_zero_if_error( asset=underlying_asset, time=timestamp, location=f'compound {event_type}', msg_aggregator=self.msg_aggregator, ) amount = FVal(entry['amount']) parse_result = _get_txhash_and_logidx(entry['id']) if parse_result is None: log.error( f'Found unprocessable borrow/repay id from the graph {entry["id"]}. Skipping', ) continue events.append( CompoundEvent( event_type=event_type, address=address, block_number=entry['blockNumber'], timestamp=timestamp, asset=underlying_asset, value=Balance(amount=amount, usd_value=amount * usd_price), to_asset=None, to_value=None, realized_pnl=None, tx_hash=parse_result[0], log_index=parse_result[1], )) return events
def _calculate_interest_and_profit( self, user_address: ChecksumEthAddress, user_result: Dict[str, Any], actions: List[AaveDepositWithdrawalEvent], balances: AaveBalances, db_interest_events: Set[AaveInterestEvent], from_ts: Timestamp, to_ts: Timestamp, ) -> Tuple[List[AaveInterestEvent], Dict[Asset, Balance]]: reserve_history = {} for reserve in user_result['reserves']: pairs = reserve['id'].split('0x') if len(pairs) != 4: log.error( f'Expected to find 3 addresses in graph\'s reserve history id ' f'but the encountered id does not match: {reserve["id"]}. Skipping entry...', ) continue try: address_s = '0x' + pairs[2] reserve_address = deserialize_ethereum_address(address_s) except DeserializationError: log.error( f'Failed to deserialize reserve address {address_s} ' f'Skipping reserve address {address_s} for user address {user_address}', ) continue atoken_history = _parse_atoken_balance_history( history=reserve['aTokenBalanceHistory'], from_ts=from_ts, to_ts=to_ts, ) reserve_history[reserve_address] = atoken_history interest_events: List[AaveInterestEvent] = [] atoken_balances: Dict[Asset, FVal] = defaultdict(FVal) used_history_indices = set() total_earned: Dict[Asset, Balance] = defaultdict(Balance) # Go through the existing db interest events and add total earned for interest_event in db_interest_events: total_earned[interest_event.asset] += interest_event.value # Create all new interest events in the query actions.sort(key=lambda event: event.timestamp) for action in actions: if action.event_type == 'deposit': atoken_balances[action.asset] += action.value.amount else: # withdrawal atoken_balances[action.asset] -= action.value.amount action_reserve_address = asset_to_aave_reserve(action.asset) if action_reserve_address is None: log.error( f'Could not find aave reserve address for asset' f'{action.asset} in an aave graph response.' f' Skipping entry...', ) continue history = reserve_history.get(action_reserve_address, None) if history is None: log.error( f'Could not find aTokenBalanceHistory for reserve ' f'{action_reserve_address} in an aave graph response.' f' Skipping entry...', ) continue history.sort(key=lambda event: event.timestamp) for idx, entry in enumerate(history): if idx in used_history_indices: continue used_history_indices.add(idx) if entry.tx_hash == action.tx_hash: diff = entry.balance - atoken_balances[action.asset] if diff != ZERO: atoken_balances[action.asset] = entry.balance asset = ASSET_TO_ATOKENV1.get(action.asset, None) if asset is None: log.error( f'Could not find corresponding aToken to ' f'{action.asset.identifier} during an aave graph query' f' Skipping entry...', ) continue timestamp = entry.timestamp usd_price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location='aave interest event from graph query', msg_aggregator=self.msg_aggregator, ) earned_balance = Balance(amount=diff, usd_value=diff * usd_price) interest_event = AaveInterestEvent( event_type='interest', asset=asset, value=earned_balance, block_number=0, # can't get from graph query timestamp=timestamp, tx_hash=entry.tx_hash, # not really the log index, but should also be unique log_index=action.log_index + 1, ) if interest_event in db_interest_events: # This should not really happen since we already query # historical atoken balance history in the new range log.warning( f'During aave subgraph query interest and profit calculation ' f'tried to generate interest event {interest_event} that ' f'already existed in the DB ', ) continue interest_events.append(interest_event) total_earned[asset] += earned_balance # and once done break off the loop break # else this atoken history is not due to an action, so skip it. # It's probably due to a simple transfer atoken_balances[action.asset] = entry.balance if action.event_type == 'deposit': atoken_balances[action.asset] += action.value.amount else: # withdrawal atoken_balances[action.asset] -= action.value.amount # Take aave unpaid interest into account for balance_asset, lending_balance in balances.lending.items(): atoken = ASSET_TO_ATOKENV1.get(balance_asset, None) if atoken is None: log.error( f'Could not find corresponding aToken to ' f'{balance_asset.identifier} during an aave graph unpair interest ' f'query. Skipping entry...', ) continue principal_balance = self.ethereum.call_contract( contract_address=atoken.ethereum_address, abi=ATOKEN_ABI, method_name='principalBalanceOf', arguments=[user_address], ) unpaid_interest = lending_balance.balance.amount - ( principal_balance / (FVal(10)**FVal(atoken.decimals))) # noqa: E501 usd_price = Inquirer().find_usd_price(atoken) total_earned[atoken] += Balance( amount=unpaid_interest, usd_value=unpaid_interest * usd_price, ) return interest_events, total_earned
def get_events_for_atoken_and_address( self, user_address: ChecksumEthAddress, atoken: EthereumToken, deposit_events: List[Dict[str, Any]], withdraw_events: List[Dict[str, Any]], from_block: int, to_block: int, ) -> List[AaveEvent]: """This function should be entered while holding the history_lock semaphore""" argument_filters = { 'from': ZERO_ADDRESS, 'to': user_address, } mint_events = self.ethereum.get_logs( contract_address=atoken.ethereum_address, abi=ATOKEN_ABI, event_name='Transfer', argument_filters=argument_filters, from_block=from_block, to_block=to_block, ) mint_data = set() mint_data_to_log_index = {} for event in mint_events: amount = hex_or_bytes_to_int(event['data']) if amount == 0: continue # first mint can be for 0. Ignore entry = ( deserialize_blocknumber(event['blockNumber']), amount, self.ethereum.get_event_timestamp(event), event['transactionHash'], ) mint_data.add(entry) mint_data_to_log_index[entry] = deserialize_int_from_hex_or_int( event['logIndex'], 'aave log index', ) reserve_asset = _atoken_to_reserve_asset(atoken) reserve_address, decimals = _get_reserve_address_decimals( reserve_asset.identifier) aave_events = [] for event in deposit_events: if hex_or_bytes_to_address(event['topics'][1]) == reserve_address: # first 32 bytes of the data are the amount deposit = hex_or_bytes_to_int(event['data'][:66]) block_number = deserialize_blocknumber(event['blockNumber']) timestamp = self.ethereum.get_event_timestamp(event) tx_hash = event['transactionHash'] log_index = deserialize_int_from_hex_or_int( event['logIndex'], 'aave log index') # If there is a corresponding deposit event remove the minting event data entry = (block_number, deposit, timestamp, tx_hash) if entry in mint_data: mint_data.remove(entry) del mint_data_to_log_index[entry] usd_price = query_usd_price_zero_if_error( asset=reserve_asset, time=timestamp, location='aave deposit', msg_aggregator=self.msg_aggregator, ) deposit_amount = deposit / (FVal(10)**FVal(decimals)) aave_events.append( AaveEvent( event_type='deposit', asset=reserve_asset, value=Balance( amount=deposit_amount, usd_value=deposit_amount * usd_price, ), block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=log_index, )) for data in mint_data: usd_price = query_usd_price_zero_if_error( asset=atoken, time=data[2], location='aave interest profit', msg_aggregator=self.msg_aggregator, ) interest_amount = data[1] / (FVal(10)**FVal(decimals)) aave_events.append( AaveEvent( event_type='interest', asset=atoken, value=Balance( amount=interest_amount, usd_value=interest_amount * usd_price, ), block_number=data[0], timestamp=data[2], tx_hash=data[3], log_index=mint_data_to_log_index[data], )) for event in withdraw_events: if hex_or_bytes_to_address(event['topics'][1]) == reserve_address: # first 32 bytes of the data are the amount withdrawal = hex_or_bytes_to_int(event['data'][:66]) block_number = deserialize_blocknumber(event['blockNumber']) timestamp = self.ethereum.get_event_timestamp(event) tx_hash = event['transactionHash'] usd_price = query_usd_price_zero_if_error( asset=reserve_asset, time=timestamp, location='aave withdrawal', msg_aggregator=self.msg_aggregator, ) withdrawal_amount = withdrawal / (FVal(10)**FVal(decimals)) aave_events.append( AaveEvent( event_type='withdrawal', asset=reserve_asset, value=Balance( amount=withdrawal_amount, usd_value=withdrawal_amount * usd_price, ), block_number=block_number, timestamp=timestamp, tx_hash=tx_hash, log_index=deserialize_int_from_hex_or_int( event['logIndex'], 'aave log index'), )) return aave_events
def _get_eth2_staking_deposits_onchain( self, addresses: List[ChecksumEthAddress], msg_aggregator: MessagesAggregator, from_ts: Timestamp, to_ts: Timestamp, ) -> List[Eth2Deposit]: from_block = max( ETH2_DEPOSIT.deployed_block, self.ethereum.get_blocknumber_by_time(from_ts), ) to_block = self.ethereum.get_blocknumber_by_time(to_ts) events = ETH2_DEPOSIT.get_logs( ethereum=self.ethereum, event_name='DepositEvent', argument_filters={}, from_block=from_block, to_block=to_block, ) transactions = self.ethereum.transactions.query( addresses=addresses, from_ts=from_ts, to_ts=to_ts, with_limit=False, recent_first=False, ) deposits: List[Eth2Deposit] = [] for transaction in transactions: if transaction.to_address != ETH2_DEPOSIT.address: continue tx_hash = '0x' + transaction.tx_hash.hex() for event in events: # Now find the corresponding event. If no event is found the transaction # probably failed or was something other than a deposit if event['transactionHash'] == tx_hash: decoded_data = decode_event_data(event['data'], EVENT_ABI) # all pylint ignores below due to https://github.com/PyCQA/pylint/issues/4114 amount = int.from_bytes(decoded_data[2], byteorder='little') # pylint: disable=unsubscriptable-object # noqa: E501 usd_price = query_usd_price_zero_if_error( asset=A_ETH, time=transaction.timestamp, location='Eth2 staking query', msg_aggregator=msg_aggregator, ) normalized_amount = from_gwei(FVal(amount)) deposits.append( Eth2Deposit( from_address=transaction.from_address, pubkey='0x' + decoded_data[0].hex(), # pylint: disable=unsubscriptable-object # noqa: E501 withdrawal_credentials='0x' + decoded_data[1].hex(), # pylint: disable=unsubscriptable-object # noqa: E501 value=Balance(normalized_amount, usd_price * normalized_amount), deposit_index=int.from_bytes(decoded_data[4], byteorder='little'), # pylint: disable=unsubscriptable-object # noqa: E501 tx_hash=tx_hash, log_index=event['logIndex'], timestamp=Timestamp(transaction.timestamp), )) break return deposits
def _scrape_validator_daily_stats( validator_index: int, last_known_timestamp: Timestamp, msg_aggregator: MessagesAggregator, ) -> List[ValidatorDailyStats]: """Scrapes the website of beaconcha.in and parses the data directly out of the data table. The parser is very simple. And can break if they change stuff in the way it's displayed in https://beaconcha.in/validator/33710/stats. If that happpens we need to adjust here. If we also somehow programatically get the data in a CSV that would be swell. May raise: - RemoteError if we can't query beaconcha.in or if the data is not in the expected format """ url = f'https://beaconcha.in/validator/{validator_index}/stats' log.debug(f'Querying beaconchain stats: {url}') try: response = requests.get(url) except requests.exceptions.RequestException as e: raise RemoteError( f'Beaconcha.in api request {url} failed due to {str(e)}') from e if response.status_code != 200: raise RemoteError( f'Beaconcha.in api request {url} failed with code: {response.status_code}' f' and response: {response.text}', ) soup = BeautifulSoup(response.text, 'html.parser', parse_only=SoupStrainer('tbod')) if soup is None: raise RemoteError( 'Could not find <tbod> while parsing beaconcha.in stats page') try: tr = soup.tbod.tr except AttributeError as e: raise RemoteError( 'Could not find first <tr> while parsing beaconcha.in stats page' ) from e timestamp = Timestamp(0) pnl = ZERO start_amount = ZERO end_amount = ZERO missed_attestations = 0 orphaned_attestations = 0 proposed_blocks = 0 missed_blocks = 0 orphaned_blocks = 0 included_attester_slashings = 0 proposer_attester_slashings = 0 deposits_number = 0 amount_deposited = ZERO column_pos = 1 stats: List[ValidatorDailyStats] = [] while tr is not None: for column in tr.children: if column.name != 'td': continue if column_pos == 1: # date date = column.string try: timestamp = create_timestamp(date, formatstr='%d %b %Y') except ValueError as e: raise RemoteError( f'Failed to parse {date} to timestamp') from e if timestamp <= last_known_timestamp: return stats # we are done column_pos += 1 elif column_pos == 2: pnl = _parse_fval(column.string, 'income') column_pos += 1 elif column_pos == 3: start_amount = _parse_fval(column.string, 'start amount') column_pos += 1 elif column_pos == 4: end_amount = _parse_fval(column.string, 'end amount') column_pos += 1 elif column_pos == 5: missed_attestations = _parse_int(column.string, 'missed attestations') column_pos += 1 elif column_pos == 6: orphaned_attestations = _parse_int(column.string, 'orphaned attestations') column_pos += 1 elif column_pos == 7: proposed_blocks = _parse_int(column.string, 'proposed blocks') column_pos += 1 elif column_pos == 8: missed_blocks = _parse_int(column.string, 'missed blocks') column_pos += 1 elif column_pos == 9: orphaned_blocks = _parse_int(column.string, 'orphaned blocks') column_pos += 1 elif column_pos == 10: included_attester_slashings = _parse_int( column.string, 'included attester slashings') # noqa: E501 column_pos += 1 elif column_pos == 11: proposer_attester_slashings = _parse_int( column.string, 'proposer attester slashings') # noqa: E501 column_pos += 1 elif column_pos == 12: deposits_number = _parse_int(column.string, 'deposits number') column_pos += 1 elif column_pos == 13: amount_deposited = _parse_fval(column.string, 'amount deposited') column_pos += 1 column_pos = 1 prices = [ query_usd_price_zero_if_error( A_ETH, time=time, location='eth2 staking daily stats', msg_aggregator=msg_aggregator, ) for time in (timestamp, Timestamp(timestamp + DAY_IN_SECONDS)) ] stats.append( ValidatorDailyStats( validator_index=validator_index, timestamp=timestamp, start_usd_price=prices[0], end_usd_price=prices[1], pnl=pnl, start_amount=start_amount, end_amount=end_amount, missed_attestations=missed_attestations, orphaned_attestations=orphaned_attestations, proposed_blocks=proposed_blocks, missed_blocks=missed_blocks, orphaned_blocks=orphaned_blocks, included_attester_slashings=included_attester_slashings, proposer_attester_slashings=proposer_attester_slashings, deposits_number=deposits_number, amount_deposited=amount_deposited, )) tr = tr.find_next_sibling() return stats
def _consume_grant_entry(self, entry: Dict[str, Any]) -> Optional[LedgerAction]: """ Consumes a grant entry from the CSV and turns it into a LedgerAction May raise: - DeserializationError - KeyError - UnknownAsset """ if entry['Type'] != 'grant': return None timestamp = deserialize_timestamp_from_date( date=entry['date'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin CSV', skip_milliseconds=True, ) usd_value = deserialize_asset_amount(entry['Value In USD']) asset = get_asset_by_symbol(entry['token_name']) if asset is None: raise UnknownAsset(entry['token_name']) token_amount = deserialize_asset_amount(entry['token_value']) if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213 price = query_usd_price_zero_if_error( asset=asset, time=timestamp, location=f'Gitcoin CSV entry {entry["txid"]}', msg_aggregator=self.db.msg_aggregator, ) if price == ZERO: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to amount being zero and inability to find price. Skipping.', ) return None # calculate the amount from price and value token_amount = usd_value / price # type: ignore match = self.grantid_re.search(entry['url']) if match is None: self.db.msg_aggregator.add_warning( f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} ' f'due to inability to read grant id. Skipping.', ) return None grant_id = int(match.group(1)) rate = Price(usd_value / token_amount) raw_txid = entry['txid'] tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry) return LedgerAction( identifier=1, # whatever does not go in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=token_amount, asset=asset, rate=rate, rate_asset=A_USD, # let's use the rate gitcoin calculated link=raw_txid, notes=f'Gitcoin grant {grant_id} event', extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=None, # can't get round from CSV tx_type=tx_type, ), )