def _get_user_reserves( self, address: ChecksumEthAddress) -> List[AaveUserReserve]: query = self.graph.query( querystr=USER_RESERVES_QUERY.format(address=address.lower()), ) query_v2 = self.graph_v2.query( querystr=USER_RESERVES_QUERY.format(address=address.lower()), ) result = [] for entry in query['userReserves'] + query_v2['userReserves']: reserve = entry['reserve'] try: result.append( AaveUserReserve( # The ID of reserve is the address of the asset and the address of the market's LendingPoolAddressProvider, in lower case # noqa: E501 address=deserialize_ethereum_address( reserve['id'][:42]), symbol=reserve['symbol'], )) except DeserializationError: log.error( f'Failed to deserialize reserve address {reserve["id"]} ' f'Skipping reserve address {reserve["id"]} for user address {address}', ) continue return result
def _get_user_reserves(self, address: ChecksumEthAddress) -> List[AaveUserReserve]: query = self.graph.query( querystr=USER_RESERVES_QUERY.format(address=address.lower()), ) result = [] for entry in query['userReserves']: reserve = entry['reserve'] result.append(AaveUserReserve( address=to_checksum_address(reserve['id']), symbol=reserve['symbol'], )) return result
def get_common_params( from_ts: Timestamp, to_ts: Timestamp, address: ChecksumEthAddress, address_type: Literal['Bytes!', 'String!'] = 'Bytes!', ) -> Tuple[Dict[str, Any], Dict[str, Any]]: param_types = { '$start_ts': 'Int!', '$end_ts': 'Int!', '$address': address_type, } param_values = { 'start_ts': from_ts, 'end_ts': to_ts, 'address': address.lower(), } return param_types, param_values
def address_to_bytes32(address: ChecksumEthAddress) -> str: return '0x' + 24 * '0' + address.lower()[2:]
def _get_user_data( self, from_ts: Timestamp, to_ts: Timestamp, address: ChecksumEthAddress, balances: AaveBalances, ) -> AaveHistory: last_query = self.database.get_used_query_range( f'aave_events_{address}') db_events = self.database.get_aave_events(address=address) now = ts_now() last_query_ts = 0 if last_query is not None: last_query_ts = last_query[1] from_ts = Timestamp(last_query_ts + 1) deposits = withdrawals = borrows = repays = liquidation_calls = [] query = self.graph.query( querystr=USER_EVENTS_QUERY, param_types={'$address': 'ID!'}, param_values={'address': address.lower()}, ) user_result = query['users'][0] if now - last_query_ts > AAVE_GRAPH_RECENT_SECS: # In theory if these were individual queries we should do them only if # we have not queried recently. In practise since we only do 1 query above # this is useless for now, but keeping the mechanism in case we change # the way we query the subgraph deposits = self._parse_deposits(user_result['depositHistory'], from_ts, to_ts) withdrawals = self._parse_withdrawals( withdrawals=user_result['redeemUnderlyingHistory'], from_ts=from_ts, to_ts=to_ts, ) borrows = self._parse_borrows(user_result['borrowHistory'], from_ts, to_ts) repays = self._parse_repays(user_result['repayHistory'], from_ts, to_ts) liquidation_calls = self._parse_liquidations( user_result['liquidationCallHistory'], from_ts, to_ts, ) result = self._process_events( user_address=address, user_result=user_result, from_ts=from_ts, to_ts=to_ts, deposits=deposits, withdrawals=withdrawals, borrows=borrows, repays=repays, liquidations=liquidation_calls, db_events=db_events, balances=balances, ) # Add all new events to the DB new_events: List[ AaveEvent] = deposits + withdrawals + result.interest_events + borrows + repays + liquidation_calls # type: ignore # noqa: E501 self.database.add_aave_events(address, new_events) # After all events have been queried then also update the query range. # Even if no events are found for an address we need to remember the range self.database.update_used_query_range( name=f'aave_events_{address}', start_ts=Timestamp(0), end_ts=now, ) # Sort actions so that actions with same time are sorted deposit -> interest -> withdrawal all_events: List[AaveEvent] = new_events + db_events sort_map = { 'deposit': 0, 'interest': 0.1, 'withdrawal': 0.2, 'borrow': 0.3, 'repay': 0.4, 'liquidation': 0.5 } # noqa: E501 all_events.sort( key=lambda event: sort_map[event.event_type] + event.timestamp) return AaveHistory( events=all_events, total_earned_interest=result.total_earned_interest, total_lost=result.total_lost, total_earned_liquidations=result.total_earned_liquidations, )
def _get_trades_graph_for_address( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AMMTrade]: """Get the address' trades data querying the Uniswap subgraph Each trade (swap) instantiates an <AMMTrade>. The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`. Translated to Uniswap lingo: Trade type BUY: - `asset1In` (QUOTE, reserve1) is gt 0. - `asset0Out` (BASE, reserve0) is gt 0. Trade type SELL: - `asset0In` (BASE, reserve0) is gt 0. - `asset1Out` (QUOTE, reserve1) is gt 0. """ trades: List[AMMTrade] = [] param_types = { '$limit': 'Int!', '$offset': 'Int!', '$address': 'Bytes!', '$start_ts': 'BigInt!', '$end_ts': 'BigInt!', } param_values = { 'limit': GRAPH_QUERY_LIMIT, 'offset': 0, 'address': address.lower(), 'start_ts': str(start_ts), 'end_ts': str(end_ts), } querystr = format_query_indentation(SWAPS_QUERY.format()) while True: result = self.graph.query( # type: ignore # caller already checks querystr=querystr, param_types=param_types, param_values=param_values, ) result_data = result['swaps'] for entry in result_data: swaps = [] for swap in entry['transaction']['swaps']: timestamp = swap['timestamp'] swap_token0 = swap['pair']['token0'] swap_token1 = swap['pair']['token1'] token0 = get_ethereum_token( symbol=swap_token0['symbol'], ethereum_address=to_checksum_address(swap_token0['id']), name=swap_token0['name'], decimals=swap_token0['decimals'], ) token1 = get_ethereum_token( symbol=swap_token1['symbol'], ethereum_address=to_checksum_address(swap_token1['id']), name=swap_token1['name'], decimals=int(swap_token1['decimals']), ) amount0_in = FVal(swap['amount0In']) amount1_in = FVal(swap['amount1In']) amount0_out = FVal(swap['amount0Out']) amount1_out = FVal(swap['amount1Out']) swaps.append(AMMSwap( tx_hash=swap['id'].split('-')[0], log_index=int(swap['logIndex']), address=address, from_address=to_checksum_address(swap['sender']), to_address=to_checksum_address(swap['to']), timestamp=Timestamp(int(timestamp)), location=Location.UNISWAP, token0=token0, token1=token1, amount0_in=AssetAmount(amount0_in), amount1_in=AssetAmount(amount1_in), amount0_out=AssetAmount(amount0_out), amount1_out=AssetAmount(amount1_out), )) # Now that we got all swaps for a transaction, create the trade object trades.extend(self._tx_swaps_to_trades(swaps)) # Check whether an extra request is needed if len(result_data) < GRAPH_QUERY_LIMIT: break # Update pagination step param_values = { **param_values, 'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore } return trades
def _get_events_graph( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, event_type: EventType, ) -> List[LiquidityPoolEvent]: """Get the address' events (mints & burns) querying the Uniswap subgraph Each event data is stored in a <LiquidityPoolEvent>. """ address_events: List[LiquidityPoolEvent] = [] if event_type == EventType.MINT: query = MINTS_QUERY query_schema = 'mints' elif event_type == EventType.BURN: query = BURNS_QUERY query_schema = 'burns' else: log.error(f'Unexpected event_type: {event_type}. Skipping events query.') return address_events param_types = { '$limit': 'Int!', '$offset': 'Int!', '$address': 'Bytes!', '$start_ts': 'BigInt!', '$end_ts': 'BigInt!', } param_values = { 'limit': GRAPH_QUERY_LIMIT, 'offset': 0, 'address': address.lower(), 'start_ts': str(start_ts), 'end_ts': str(end_ts), } querystr = format_query_indentation(query.format()) while True: result = self.graph.query( # type: ignore # caller already checks querystr=querystr, param_types=param_types, param_values=param_values, ) result_data = result[query_schema] for event in result_data: token0_ = event['pair']['token0'] token1_ = event['pair']['token1'] token0 = get_ethereum_token( symbol=token0_['symbol'], ethereum_address=to_checksum_address(token0_['id']), name=token0_['name'], decimals=token0_['decimals'], ) token1 = get_ethereum_token( symbol=token1_['symbol'], ethereum_address=to_checksum_address(token1_['id']), name=token1_['name'], decimals=int(token1_['decimals']), ) lp_event = LiquidityPoolEvent( tx_hash=event['transaction']['id'], log_index=int(event['logIndex']), address=address, timestamp=Timestamp(int(event['timestamp'])), event_type=event_type, pool_address=to_checksum_address(event['pair']['id']), token0=token0, token1=token1, amount0=AssetAmount(FVal(event['amount0'])), amount1=AssetAmount(FVal(event['amount1'])), usd_price=Price(FVal(event['amountUSD'])), lp_amount=AssetAmount(FVal(event['liquidity'])), ) address_events.append(lp_event) # Check whether an extra request is needed if len(result_data) < GRAPH_QUERY_LIMIT: break # Update pagination step param_values = { **param_values, 'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore } return address_events
def _read_subgraph_trades( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AMMTrade]: """Get the address' trades data querying the AMM subgraph Each trade (swap) instantiates an <AMMTrade>. The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`. Translated to AMM lingo: Trade type BUY: - `asset1In` (QUOTE, reserve1) is gt 0. - `asset0Out` (BASE, reserve0) is gt 0. Trade type SELL: - `asset0In` (BASE, reserve0) is gt 0. - `asset1Out` (QUOTE, reserve1) is gt 0. May raise - RemoteError """ trades: List[AMMTrade] = [] query_id = '0' query_offset = 0 param_types = { '$limit': 'Int!', '$offset': 'Int!', '$address': 'Bytes!', '$start_ts': 'BigInt!', '$end_ts': 'BigInt!', '$id': 'ID!', } param_values = { 'limit': GRAPH_QUERY_LIMIT, 'offset': 0, 'address': address.lower(), 'start_ts': str(start_ts), 'end_ts': str(end_ts), 'id': query_id, } querystr = format_query_indentation(self.swaps_query.format()) while True: try: result = self.graph.query( querystr=querystr, param_types=param_types, param_values=param_values, ) except RemoteError as e: self.msg_aggregator.add_error( SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e), location=self.location), ) raise for entry in result['swaps']: swaps = [] try: for swap in entry['transaction']['swaps']: timestamp = swap['timestamp'] swap_token0 = swap['pair']['token0'] swap_token1 = swap['pair']['token1'] try: token0_deserialized = deserialize_ethereum_address( swap_token0['id']) token1_deserialized = deserialize_ethereum_address( swap_token1['id']) from_address_deserialized = deserialize_ethereum_address( swap['sender']) # noqa to_address_deserialized = deserialize_ethereum_address( swap['to']) except DeserializationError: msg = ( f'Failed to deserialize addresses in trade from {self.location} graph' # noqa f' with token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, ' # noqa f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}' ) log.error(msg) continue token0 = get_or_create_ethereum_token( userdb=self.database, symbol=swap_token0['symbol'], ethereum_address=token0_deserialized, name=swap_token0['name'], decimals=swap_token0['decimals'], ) token1 = get_or_create_ethereum_token( userdb=self.database, symbol=swap_token1['symbol'], ethereum_address=token1_deserialized, name=swap_token1['name'], decimals=int(swap_token1['decimals']), ) try: amount0_in = FVal(swap['amount0In']) amount1_in = FVal(swap['amount1In']) amount0_out = FVal(swap['amount0Out']) amount1_out = FVal(swap['amount1Out']) except ValueError as e: log.error( f'Failed to read amounts in {self.location} swap {str(swap)}. ' f'{str(e)}.', ) continue swaps.append( AMMSwap( tx_hash=swap['id'].split('-')[0], log_index=int(swap['logIndex']), address=address, from_address=from_address_deserialized, to_address=to_address_deserialized, timestamp=Timestamp(int(timestamp)), location=self.location, token0=token0, token1=token1, amount0_in=AssetAmount(amount0_in), amount1_in=AssetAmount(amount1_in), amount0_out=AssetAmount(amount0_out), amount1_out=AssetAmount(amount1_out), )) query_id = entry['id'] except KeyError as e: log.error( f'Failed to read trade in {self.location} swap {str(entry)}. ' f'{str(e)}.', ) continue # with the new logic the list of swaps can be empty, in that case don't try # to make trades from the swaps if len(swaps) == 0: continue # Now that we got all swaps for a transaction, create the trade object trades.extend(self._tx_swaps_to_trades(swaps)) # Check whether an extra request is needed if len(result['swaps']) < GRAPH_QUERY_LIMIT: break # Update pagination step if query_offset == GRAPH_QUERY_SKIP_LIMIT: query_offset = 0 new_query_id = query_id else: query_offset += GRAPH_QUERY_LIMIT new_query_id = '0' param_values = { **param_values, 'id': new_query_id, 'offset': query_offset, } return trades
def _get_events_graph( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, event_type: EventType, ) -> List[LiquidityPoolEvent]: """Get the address' events (mints & burns) querying the AMM's subgraph Each event data is stored in a <LiquidityPoolEvent>. """ address_events: List[LiquidityPoolEvent] = [] if event_type == self.mint_event: query = MINTS_QUERY query_schema = 'mints' elif event_type == self.burn_event: query = BURNS_QUERY query_schema = 'burns' else: log.error( f'Unexpected {self.location} event_type: {event_type}. Skipping events query.', ) return address_events query_id = '0' query_offset = 0 param_types = { '$limit': 'Int!', '$offset': 'Int!', '$address': 'Bytes!', '$start_ts': 'BigInt!', '$end_ts': 'BigInt!', '$id': 'ID!', } param_values = { 'limit': GRAPH_QUERY_LIMIT, 'offset': query_offset, 'address': address.lower(), 'start_ts': str(start_ts), 'end_ts': str(end_ts), 'id': query_id, } querystr = format_query_indentation(query.format()) while True: try: result = self.graph.query( querystr=querystr, param_types=param_types, param_values=param_values, ) except RemoteError as e: self.msg_aggregator.add_error( SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e), location=self.location), ) raise except AttributeError as e: raise ModuleInitializationFailure( f'{self.location} subgraph remote error') from e result_data = result[query_schema] for event in result_data: token0_ = event['pair']['token0'] token1_ = event['pair']['token1'] try: token0_deserialized = deserialize_ethereum_address( token0_['id']) token1_deserialized = deserialize_ethereum_address( token1_['id']) pool_deserialized = deserialize_ethereum_address( event['pair']['id']) except DeserializationError as e: msg = ( f'Failed to deserialize address involved in liquidity pool event for' f' {self.location}. Token 0: {token0_["id"]}, token 1: {token0_["id"]},' f' pair: {event["pair"]["id"]}.') log.error(msg) raise RemoteError(msg) from e token0 = get_or_create_ethereum_token( userdb=self.database, symbol=token0_['symbol'], ethereum_address=token0_deserialized, name=token0_['name'], decimals=token0_['decimals'], ) token1 = get_or_create_ethereum_token( userdb=self.database, symbol=token1_['symbol'], ethereum_address=token1_deserialized, name=token1_['name'], decimals=int(token1_['decimals']), ) lp_event = LiquidityPoolEvent( tx_hash=event['transaction']['id'], log_index=int(event['logIndex']), address=address, timestamp=Timestamp(int(event['timestamp'])), event_type=event_type, pool_address=pool_deserialized, token0=token0, token1=token1, amount0=AssetAmount(FVal(event['amount0'])), amount1=AssetAmount(FVal(event['amount1'])), usd_price=Price(FVal(event['amountUSD'])), lp_amount=AssetAmount(FVal(event['liquidity'])), ) address_events.append(lp_event) query_id = event['id'] # Check whether an extra request is needed if len(result_data) < GRAPH_QUERY_LIMIT: break # Update pagination step if query_offset == GRAPH_QUERY_SKIP_LIMIT: query_offset = 0 new_query_id = query_id else: query_offset += GRAPH_QUERY_LIMIT new_query_id = '0' param_values = { **param_values, 'id': new_query_id, 'offset': query_offset, } return address_events
def _get_trades_graph_v3_for_address( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AMMTrade]: """Get the address' trades data querying the Uniswap subgraph Each trade (swap) instantiates an <AMMTrade>. The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`. Translated to Uniswap lingo: Trade type BUY: - `amount1` (QUOTE, reserve1) is gt 0. - `amount0` (BASE, reserve0) is lt 0. Trade type SELL: - `amount0` (BASE, reserve0) is gt 0. - `amount1` (QUOTE, reserve1) is lt 0. May raise: - RemoteError """ trades: List[AMMTrade] = [] param_types = { '$limit': 'Int!', '$offset': 'Int!', '$address': 'Bytes!', '$start_ts': 'BigInt!', '$end_ts': 'BigInt!', } param_values = { 'limit': GRAPH_QUERY_LIMIT, 'offset': 0, 'address': address.lower(), 'start_ts': str(start_ts), 'end_ts': str(end_ts), } querystr = format_query_indentation(V3_SWAPS_QUERY.format()) while True: try: result = self.graph_v3.query( querystr=querystr, param_types=param_types, param_values=param_values, ) except RemoteError as e: self.msg_aggregator.add_error(SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e))) raise result_data = result['swaps'] for entry in result_data: swaps = [] for swap in entry['transaction']['swaps']: timestamp = swap['timestamp'] swap_token0 = swap['token0'] swap_token1 = swap['token1'] try: token0_deserialized = deserialize_ethereum_address(swap_token0['id']) token1_deserialized = deserialize_ethereum_address(swap_token1['id']) from_address_deserialized = deserialize_ethereum_address(swap['sender']) to_address_deserialized = deserialize_ethereum_address(swap['recipient']) except DeserializationError: msg = ( f'Failed to deserialize addresses in trade from uniswap graph with ' f'token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, ' f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}' ) log.error(msg) continue token0 = get_or_create_ethereum_token( userdb=self.database, symbol=swap_token0['symbol'], ethereum_address=token0_deserialized, name=swap_token0['name'], decimals=swap_token0['decimals'], ) token1 = get_or_create_ethereum_token( userdb=self.database, symbol=swap_token1['symbol'], ethereum_address=token1_deserialized, name=swap_token1['name'], decimals=int(swap_token1['decimals']), ) try: if swap['amount0'].startswith('-'): amount0_in = AssetAmount(FVal(ZERO)) amount0_out = deserialize_asset_amount_force_positive(swap['amount0']) amount1_in = deserialize_asset_amount_force_positive(swap['amount1']) amount1_out = AssetAmount(FVal(ZERO)) else: amount0_in = deserialize_asset_amount_force_positive(swap['amount0']) amount0_out = AssetAmount(FVal(ZERO)) amount1_in = AssetAmount(FVal(ZERO)) amount1_out = deserialize_asset_amount_force_positive(swap['amount1']) except ValueError as e: log.error( f'Failed to read amounts in Uniswap V3 swap {str(swap)}. ' f'{str(e)}.', ) continue swaps.append(AMMSwap( tx_hash=swap['id'].split('#')[0], log_index=int(swap['logIndex']), address=address, from_address=from_address_deserialized, to_address=to_address_deserialized, timestamp=Timestamp(int(timestamp)), location=Location.UNISWAP, token0=token0, token1=token1, amount0_in=amount0_in, amount1_in=amount1_in, amount0_out=amount0_out, amount1_out=amount1_out, )) # with the new logic the list of swaps can be empty, in that case don't try # to make trades from the swaps if len(swaps) == 0: continue # Now that we got all swaps for a transaction, create the trade object trades.extend(self._tx_swaps_to_trades(swaps)) # Check whether an extra request is needed if len(result_data) < GRAPH_QUERY_LIMIT: break # Update pagination step param_values = { **param_values, 'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore } return trades