def test_purge_ethereum_transaction_data(rotkehlchen_api_server): rotki = rotkehlchen_api_server.rest_api.rotkehlchen db = DBEthTx(rotki.data.db) db.add_ethereum_transactions([ EthereumTransaction( tx_hash=bytes(), timestamp=1, block_number=1, from_address=make_ethereum_address(), to_address=make_ethereum_address(), value=FVal(1), gas=FVal(1), gas_price=FVal(1), gas_used=FVal(1), input_data=bytes(), nonce=1, ) ], ) filter_ = ETHTransactionsFilterQuery.make() result, filter_count = db.get_ethereum_transactions(filter_) assert len(result) == 1 assert filter_count == 1 response = requests.delete( api_url_for( rotkehlchen_api_server, "ethereumtransactionsresource", ), ) assert_simple_ok_response(response) result, filter_count = db.get_ethereum_transactions(filter_) assert len(result) == 0 assert filter_count == 0
def get_or_query_transaction_receipt( self, tx_hash: str, ) -> Optional['EthereumTxReceipt']: """ Gets the receipt from the DB if it exist. If not queries the chain for it, saves it in the DB and then returns it. Also if the actual transaction does not exist in the DB it queries it and saves it there. May raise: - DeserializationError - RemoteError """ tx_hash_b = hexstring_to_bytes(tx_hash) dbethtx = DBEthTx(self.database) # If the transaction is not in the DB then query it and add it result, _ = dbethtx.get_ethereum_transactions(ETHTransactionsFilterQuery.make(tx_hash=tx_hash_b)) # noqa: E501 if len(result) == 0: transaction = self.ethereum.get_transaction_by_hash(tx_hash) if transaction is None: return None # hash does not correspond to a transaction dbethtx.add_ethereum_transactions([transaction]) tx_receipt = dbethtx.get_receipt(tx_hash_b) if tx_receipt is not None: return tx_receipt # not in the DB, so we need to query the chain for it tx_receipt_data = self.ethereum.get_transaction_receipt(tx_hash=tx_hash) dbethtx.add_receipt_data(tx_receipt_data) tx_receipt = dbethtx.get_receipt(tx_hash_b) return tx_receipt
def test_tx_decode(evm_transaction_decoder, database): dbethtx = DBEthTx(database) addr1 = '0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12' approve_tx_hash = deserialize_evm_tx_hash('0x5cc0e6e62753551313412492296d5e57bea0a9d1ce507cc96aa4aa076c5bde7a') # noqa: E501 transactions = dbethtx.get_ethereum_transactions( filter_=ETHTransactionsFilterQuery.make( addresses=[addr1], tx_hash=approve_tx_hash, ), has_premium=True, ) decoder = evm_transaction_decoder with patch.object(decoder, 'decode_transaction', wraps=decoder.decode_transaction) as decode_mock: # noqa: E501 for tx in transactions: receipt = dbethtx.get_receipt(tx.tx_hash) assert receipt is not None, 'all receipts should be queried in the test DB' events = decoder.get_or_decode_transaction_events(tx, receipt, ignore_cache=False) if tx.tx_hash == approve_tx_hash: assert len(events) == 2 assert_events_equal(events[0], HistoryBaseEntry( # The no-member is due to https://github.com/PyCQA/pylint/issues/3162 event_identifier=approve_tx_hash.hex(), # pylint: disable=no-member sequence_index=0, timestamp=1569924574000, location=Location.BLOCKCHAIN, location_label=addr1, asset=A_ETH, balance=Balance(amount=FVal('0.000030921')), # The no-member is due to https://github.com/PyCQA/pylint/issues/3162 notes=f'Burned 0.000030921 ETH in gas from {addr1}', event_type=HistoryEventType.SPEND, event_subtype=HistoryEventSubType.FEE, counterparty=CPT_GAS, )) assert_events_equal(events[1], HistoryBaseEntry( # The no-member is due to https://github.com/PyCQA/pylint/issues/3162 event_identifier=approve_tx_hash.hex(), # pylint: disable=no-member sequence_index=163, timestamp=1569924574000, location=Location.BLOCKCHAIN, location_label=addr1, asset=A_SAI, balance=Balance(amount=1), notes=f'Approve 1 SAI of {addr1} for spending by 0xdf869FAD6dB91f437B59F1EdEFab319493D4C4cE', # noqa: E501 event_type=HistoryEventType.INFORMATIONAL, event_subtype=HistoryEventSubType.APPROVE, counterparty='0xdf869FAD6dB91f437B59F1EdEFab319493D4C4cE', )) assert decode_mock.call_count == len(transactions) # now go again, and see that no more decoding happens as it's all pulled from the DB for tx in transactions: receipt = dbethtx.get_receipt(tx.tx_hash) assert receipt is not None, 'all receipts should be queried in the test DB' events = decoder.get_or_decode_transaction_events(tx, receipt, ignore_cache=False) assert decode_mock.call_count == len(transactions)
def query( self, filter_query: ETHTransactionsFilterQuery, with_limit: bool = False, only_cache: bool = False, ) -> Tuple[List[EthereumTransaction], int]: """Queries for all transactions (normal AND internal) of an ethereum address or of all addresses. Returns a list of all transactions filtered and sorted according to the parameters. If `with_limit` is true then the api limit is applied if `recent_first` is true then the transactions are returned with the most recent first on the list May raise: - RemoteError if etherscan is used and there is a problem with reaching it or with parsing the response. - pysqlcipher3.dbapi2.OperationalError if the SQL query fails due to invalid filtering arguments. """ query_addresses = filter_query.addresses if query_addresses is not None: accounts = query_addresses else: accounts = self.database.get_blockchain_accounts().eth if only_cache is False: f_from_ts = filter_query.from_ts f_to_ts = filter_query.to_ts from_ts = Timestamp(0) if f_from_ts is None else f_from_ts to_ts = ts_now() if f_to_ts is None else f_to_ts for address in accounts: self.single_address_query_transactions( address=address, start_ts=from_ts, end_ts=to_ts, ) dbethtx = DBEthTx(self.database) transactions, total_filter_count = dbethtx.get_ethereum_transactions(filter_=filter_query) return ( self._return_transactions_maybe_limit( requested_addresses=query_addresses, transactions=transactions, with_limit=with_limit, ), total_filter_count, )
def get_or_query_transaction_receipt( self, tx_hash: EVMTxHash, ) -> 'EthereumTxReceipt': """ Gets the receipt from the DB if it exists. If not queries the chain for it, saves it in the DB and then returns it. Also if the actual transaction does not exist in the DB it queries it and saves it there. May raise: - DeserializationError - RemoteError if the transaction hash can't be found in any of the connected nodes """ dbethtx = DBEthTx(self.database) # If the transaction is not in the DB then query it and add it result = dbethtx.get_ethereum_transactions( filter_=ETHTransactionsFilterQuery.make(tx_hash=tx_hash), has_premium=True, # we don't need any limiting here ) if len(result) == 0: transaction = self.ethereum.get_transaction_by_hash(tx_hash) dbethtx.add_ethereum_transactions([transaction], relevant_address=None) self._get_internal_transactions_for_ranges( address=transaction.from_address, start_ts=transaction.timestamp, end_ts=transaction.timestamp, ) self._get_erc20_transfers_for_ranges( address=transaction.from_address, start_ts=transaction.timestamp, end_ts=transaction.timestamp, ) tx_receipt = dbethtx.get_receipt(tx_hash) if tx_receipt is not None: return tx_receipt # not in the DB, so we need to query the chain for it tx_receipt_data = self.ethereum.get_transaction_receipt( tx_hash=tx_hash) dbethtx.add_receipt_data(tx_receipt_data) tx_receipt = dbethtx.get_receipt(tx_hash) return tx_receipt # type: ignore # tx_receipt was just added in the DB so should be there # noqa: E501
class EVMTransactionDecoder(): def __init__( self, database: 'DBHandler', ethereum_manager: 'EthereumManager', eth_transactions: 'EthTransactions', msg_aggregator: MessagesAggregator, ): self.database = database self.all_counterparties: Set[str] = set() self.ethereum_manager = ethereum_manager self.eth_transactions = eth_transactions self.msg_aggregator = msg_aggregator self.dbethtx = DBEthTx(self.database) self.dbevents = DBHistoryEvents(self.database) self.base = BaseDecoderTools(database=database) self.event_rules = [ # rules to try for all tx receipt logs decoding self._maybe_decode_erc20_approve, self._maybe_decode_erc20_721_transfer, self._maybe_enrich_transfers, self._maybe_decode_governance, ] self.token_enricher_rules: List[Callable] = [ ] # enrichers to run for token transfers self.initialize_all_decoders() self.undecoded_tx_query_lock = Semaphore() def _recursively_initialize_decoders( self, package: Union[str, ModuleType], ) -> Tuple[Dict[ChecksumEthAddress, Tuple[Any, ...]], List[Callable], List[Callable], ]: if isinstance(package, str): package = importlib.import_module(package) address_results = {} rules_results = [] enricher_results = [] for _, name, is_pkg in pkgutil.walk_packages(package.__path__): full_name = package.__name__ + '.' + name if full_name == __name__: continue # skip -- this is this source file if is_pkg: submodule = importlib.import_module(full_name) # take module name, transform it and find decoder if exists class_name = full_name[MODULES_PREFIX_LENGTH:].translate( {ord('.'): None}) parts = class_name.split('_') class_name = ''.join([x.capitalize() for x in parts]) submodule_decoder = getattr(submodule, f'{class_name}Decoder', None) if submodule_decoder: if class_name in self.decoders: raise ModuleLoadingError( f'Decoder with name {class_name} already loaded') self.decoders[class_name] = submodule_decoder( ethereum_manager=self.ethereum_manager, base_tools=self.base, msg_aggregator=self.msg_aggregator, ) address_results.update( self.decoders[class_name].addresses_to_decoders()) rules_results.extend( self.decoders[class_name].decoding_rules()) enricher_results.extend( self.decoders[class_name].enricher_rules()) self.all_counterparties.update( self.decoders[class_name].counterparties()) recursive_addrs, recursive_rules, recurisve_enricher_results = self._recursively_initialize_decoders( full_name) # noqa: E501 address_results.update(recursive_addrs) rules_results.extend(recursive_rules) enricher_results.extend(recurisve_enricher_results) return address_results, rules_results, enricher_results def initialize_all_decoders(self) -> None: """Recursively check all submodules to get all decoder address mappings and rules """ self.decoders: Dict[str, 'DecoderInterface'] = {} address_result, rules_result, enrichers_result = self._recursively_initialize_decoders( MODULES_PACKAGE) # noqa: E501 self.address_mappings = address_result self.event_rules.extend(rules_result) self.token_enricher_rules.extend(enrichers_result) # update with counterparties not in any module self.all_counterparties.update([CPT_GAS, CPT_GNOSIS_CHAIN]) def reload_from_db(self) -> None: """Reload all related settings from DB so that decoding happens with latest""" self.base.refresh_tracked_accounts() for _, decoder in self.decoders.items(): if isinstance(decoder, CustomizableDateMixin): decoder.reload_settings() def try_all_rules( self, token: Optional[EthereumToken], tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, decoded_events: List[HistoryBaseEntry], action_items: List[ActionItem], ) -> Optional[HistoryBaseEntry]: for rule in self.event_rules: event = rule(token=token, tx_log=tx_log, transaction=transaction, decoded_events=decoded_events, action_items=action_items) # noqa: E501 if event: return event return None def decode_by_address_rules( self, tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, decoded_events: List[HistoryBaseEntry], all_logs: List[EthereumTxReceiptLog], action_items: List[ActionItem], ) -> Tuple[Optional[HistoryBaseEntry], Optional[ActionItem]]: """ Sees if the log is on an address for which we have specific decoders and calls it Should catch all underlying errors these decoders will raise. So far known are: - DeserializationError - ConversionError - UnknownAsset """ mapping_result = self.address_mappings.get(tx_log.address) if mapping_result is None: return None, None method = mapping_result[0] try: if len(mapping_result) == 1: result = method(tx_log, transaction, decoded_events, all_logs, action_items) else: result = method(tx_log, transaction, decoded_events, all_logs, action_items, *mapping_result[1:]) # noqa: E501 except (DeserializationError, ConversionError, UnknownAsset) as e: log.debug( f'Decoding tx log with index {tx_log.log_index} of transaction ' f'{transaction.tx_hash.hex()} through {method.__name__} failed due to {str(e)}' ) return None, None return result def decode_transaction( self, transaction: EthereumTransaction, tx_receipt: EthereumTxReceipt, ) -> List[HistoryBaseEntry]: """Decodes an ethereum transaction and its receipt and saves result in the DB""" cursor = self.database.conn.cursor() self.base.reset_sequence_counter() # check if any eth transfer happened in the transaction, including in internal transactions events = self._maybe_decode_simple_transactions( transaction, tx_receipt) action_items: List[ActionItem] = [] # decode transaction logs from the receipt for tx_log in tx_receipt.logs: event, action_item = self.decode_by_address_rules( tx_log, transaction, events, tx_receipt.logs, action_items) # noqa: E501 if action_item: action_items.append(action_item) if event: events.append(event) continue token = GlobalDBHandler.get_ethereum_token(tx_log.address) event = self.try_all_rules(token=token, tx_log=tx_log, transaction=transaction, decoded_events=events, action_items=action_items) # noqa: E501 if event: events.append(event) self.dbevents.add_history_events(events) cursor.execute( 'INSERT OR IGNORE INTO evm_tx_mappings(tx_hash, blockchain, value) VALUES(?, ?, ?)', (transaction.tx_hash, 'ETH', HISTORY_MAPPING_DECODED), ) self.database.update_last_write() return sorted(events, key=lambda x: x.sequence_index, reverse=False) def get_and_decode_undecoded_transactions(self, limit: Optional[int] = None ) -> None: """Checks the DB for up to `limit` undecoded transactions and decodes them. This is protected by concurrent access from a lock""" with self.undecoded_tx_query_lock: hashes = self.dbethtx.get_transaction_hashes_not_decoded( limit=limit) self.decode_transaction_hashes(ignore_cache=False, tx_hashes=hashes) def decode_transaction_hashes( self, ignore_cache: bool, tx_hashes: Optional[List[EVMTxHash]] ) -> List[HistoryBaseEntry]: # noqa: E501 """Make sure that receipts are pulled + events decoded for the given transaction hashes. The transaction hashes must exist in the DB at the time of the call May raise: - DeserializationError if there is a problem with conacting a remote to get receipts - RemoteError if there is a problem with contacting a remote to get receipts - InputError if the transaction hash is not found in the DB """ events = [] self.reload_from_db() # If no transaction hashes are passed, decode all transactions. if tx_hashes is None: tx_hashes = [] cursor = self.database.conn.cursor() for entry in cursor.execute( 'SELECT tx_hash FROM ethereum_transactions'): tx_hashes.append(EVMTxHash(entry[0])) for tx_hash in tx_hashes: try: receipt = self.eth_transactions.get_or_query_transaction_receipt( tx_hash) except RemoteError as e: raise InputError( f'Hash {tx_hash.hex()} does not correspond to a transaction' ) from e # noqa: E501 # TODO: Change this if transaction filter query can accept multiple hashes txs = self.dbethtx.get_ethereum_transactions( filter_=ETHTransactionsFilterQuery.make(tx_hash=tx_hash), has_premium=True, # ignore limiting here ) events.extend( self.get_or_decode_transaction_events( transaction=txs[0], tx_receipt=receipt, ignore_cache=ignore_cache, )) return events def get_or_decode_transaction_events( self, transaction: EthereumTransaction, tx_receipt: EthereumTxReceipt, ignore_cache: bool, ) -> List[HistoryBaseEntry]: """Get a transaction's events if existing in the DB or decode them""" cursor = self.database.conn.cursor() if ignore_cache is True: # delete all decoded events self.dbevents.delete_events_by_tx_hash([transaction.tx_hash]) cursor.execute( 'DELETE from evm_tx_mappings WHERE tx_hash=? AND blockchain=? AND value=?', (transaction.tx_hash, 'ETH', HISTORY_MAPPING_DECODED), ) else: # see if events are already decoded and return them results = cursor.execute( 'SELECT COUNT(*) from evm_tx_mappings WHERE tx_hash=? AND blockchain=? AND value=?', # noqa: E501 (transaction.tx_hash, 'ETH', HISTORY_MAPPING_DECODED), ) if results.fetchone()[0] != 0: # already decoded and in the DB events = self.dbevents.get_history_events( filter_query=HistoryEventFilterQuery.make( event_identifier=transaction.tx_hash.hex(), ), has_premium= True, # for this function we don't limit anything ) return events # else we should decode now events = self.decode_transaction(transaction, tx_receipt) return events def _maybe_decode_internal_transactions( self, tx: EthereumTransaction, tx_receipt: EthereumTxReceipt, events: List[HistoryBaseEntry], tx_hash_hex: str, ts_ms: TimestampMS, ) -> None: """ check for internal transactions if the transaction is not canceled. This function mutates the events argument. """ if tx_receipt.status is False: return internal_txs = self.dbethtx.get_ethereum_internal_transactions( parent_tx_hash=tx.tx_hash, ) for internal_tx in internal_txs: if internal_tx.to_address is None: continue # can that happen? Internal transaction deploying a contract? direction_result = self.base.decode_direction( internal_tx.from_address, internal_tx.to_address) # noqa: E501 if direction_result is None: continue amount = ZERO if internal_tx.value == 0 else from_wei( FVal(internal_tx.value)) if amount == ZERO: continue event_type, location_label, counterparty, verb = direction_result events.append( HistoryBaseEntry( event_identifier=tx_hash_hex, sequence_index=self.base.get_next_sequence_counter(), timestamp=ts_ms, location=Location.BLOCKCHAIN, location_label=location_label, asset=A_ETH, balance=Balance(amount=amount), notes= f'{verb} {amount} ETH {internal_tx.from_address} -> {internal_tx.to_address}', # noqa: E501 event_type=event_type, event_subtype=HistoryEventSubType.NONE, counterparty=counterparty, )) def _maybe_decode_simple_transactions( self, tx: EthereumTransaction, tx_receipt: EthereumTxReceipt, ) -> List[HistoryBaseEntry]: """Decodes normal ETH transfers, internal transactions and gas cost payments""" events: List[HistoryBaseEntry] = [] tx_hash_hex = tx.tx_hash.hex() ts_ms = ts_sec_to_ms(tx.timestamp) # check for gas spent direction_result = self.base.decode_direction(tx.from_address, tx.to_address) if direction_result is not None: event_type, location_label, counterparty, verb = direction_result if event_type in (HistoryEventType.SPEND, HistoryEventType.TRANSFER): eth_burned_as_gas = from_wei(FVal(tx.gas_used * tx.gas_price)) events.append( HistoryBaseEntry( event_identifier=tx_hash_hex, sequence_index=self.base.get_next_sequence_counter(), timestamp=ts_ms, location=Location.BLOCKCHAIN, location_label=location_label, asset=A_ETH, balance=Balance(amount=eth_burned_as_gas), notes= f'Burned {eth_burned_as_gas} ETH in gas from {location_label}', event_type=HistoryEventType.SPEND, event_subtype=HistoryEventSubType.FEE, counterparty=CPT_GAS, )) # Decode internal transactions after gas so gas is always 0 indexed self._maybe_decode_internal_transactions( tx=tx, tx_receipt=tx_receipt, events=events, tx_hash_hex=tx_hash_hex, ts_ms=ts_ms, ) if tx_receipt.status is False or direction_result is None: # Not any other action to do for failed transactions or transaction where # any tracked address is not involved return events # now decode the actual transaction eth transfer itself amount = ZERO if tx.value == 0 else from_wei(FVal(tx.value)) if tx.to_address is None: if not self.base.is_tracked(tx.from_address): return events events.append( HistoryBaseEntry( # contract deployment event_identifier=tx_hash_hex, sequence_index=self.base.get_next_sequence_counter(), timestamp=ts_ms, location=Location.BLOCKCHAIN, location_label=tx.from_address, asset=A_ETH, balance=Balance(amount=amount), notes='Contract deployment', event_type=HistoryEventType.INFORMATIONAL, event_subtype=HistoryEventSubType.DEPLOY, counterparty=None, # TODO: Find out contract address )) return events if amount == ZERO: return events events.append( HistoryBaseEntry( event_identifier=tx_hash_hex, sequence_index=self.base.get_next_sequence_counter(), timestamp=ts_ms, location=Location.BLOCKCHAIN, location_label=location_label, asset=A_ETH, balance=Balance(amount=amount), notes= f'{verb} {amount} ETH {tx.from_address} -> {tx.to_address}', event_type=event_type, event_subtype=HistoryEventSubType.NONE, counterparty=counterparty, )) return events def _maybe_decode_erc20_approve( self, token: Optional[EthereumToken], tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument action_items: List[ActionItem], # pylint: disable=unused-argument ) -> Optional[HistoryBaseEntry]: if tx_log.topics[0] != ERC20_APPROVE or token is None: return None owner_address = hex_or_bytes_to_address(tx_log.topics[1]) spender_address = hex_or_bytes_to_address(tx_log.topics[2]) if not any( self.base.is_tracked(x) for x in (owner_address, spender_address)): return None amount_raw = hex_or_bytes_to_int(tx_log.data) amount = token_normalized_value(token_amount=amount_raw, token=token) notes = f'Approve {amount} {token.symbol} of {owner_address} for spending by {spender_address}' # noqa: E501 return HistoryBaseEntry( event_identifier=transaction.tx_hash.hex(), sequence_index=self.base.get_sequence_index(tx_log), timestamp=ts_sec_to_ms(transaction.timestamp), location=Location.BLOCKCHAIN, location_label=owner_address, asset=token, balance=Balance(amount=amount), notes=notes, event_type=HistoryEventType.INFORMATIONAL, event_subtype=HistoryEventSubType.APPROVE, counterparty=spender_address, ) def _maybe_decode_erc20_721_transfer( self, token: Optional[EthereumToken], tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument action_items: List[ActionItem], ) -> Optional[HistoryBaseEntry]: if tx_log.topics[0] != ERC20_OR_ERC721_TRANSFER: return None if token is None: try: found_token = get_or_create_ethereum_token( userdb=self.database, ethereum_address=tx_log.address, ethereum_manager=self.ethereum_manager, ) except NotERC20Conformant: return None # ignore non-ERC20 transfers for now else: found_token = token transfer = self.base.decode_erc20_721_transfer( token=found_token, tx_log=tx_log, transaction=transaction, ) if transfer is None: return None for idx, action_item in enumerate(action_items): if action_item.asset == found_token and action_item.amount == transfer.balance.amount and action_item.from_event_type == transfer.event_type and action_item.from_event_subtype == transfer.event_subtype: # noqa: E501 if action_item.action == 'skip': action_items.pop(idx) return None # else atm only transform if action_item.to_event_type is not None: transfer.event_type = action_item.to_event_type if action_item.to_event_subtype is not None: transfer.event_subtype = action_item.to_event_subtype if action_item.to_notes is not None: transfer.notes = action_item.to_notes if action_item.to_counterparty is not None: transfer.counterparty = action_item.to_counterparty if action_item.extra_data is not None: transfer.extra_data = action_item.extra_data if action_item.paired_event_data is not None: # If there is a paired event to this, take care of the order out_event = transfer in_event = action_item.paired_event_data[0] if action_item.paired_event_data[1] is True: out_event = action_item.paired_event_data[0] in_event = transfer maybe_reshuffle_events( out_event=out_event, in_event=in_event, events_list=decoded_events + [transfer], ) action_items.pop(idx) break # found an action item and acted on it # Add additional information to transfers for different protocols self._enrich_protocol_tranfers( token=found_token, tx_log=tx_log, transaction=transaction, event=transfer, action_items=action_items, ) return transfer def _maybe_enrich_transfers( # pylint: disable=no-self-use self, token: Optional[EthereumToken], # pylint: disable=unused-argument tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, # pylint: disable=unused-argument decoded_events: List[HistoryBaseEntry], action_items: List[ActionItem], # pylint: disable=unused-argument ) -> Optional[HistoryBaseEntry]: if tx_log.topics[ 0] == GTC_CLAIM and tx_log.address == '0xDE3e5a990bCE7fC60a6f017e7c4a95fc4939299E': # noqa: E501 for event in decoded_events: if event.asset == A_GTC and event.event_type == HistoryEventType.RECEIVE: event.event_subtype = HistoryEventSubType.AIRDROP event.notes = f'Claim {event.balance.amount} GTC from the GTC airdrop' return None if tx_log.topics[ 0] == ONEINCH_CLAIM and tx_log.address == '0xE295aD71242373C37C5FdA7B57F26f9eA1088AFe': # noqa: E501 for event in decoded_events: if event.asset == A_1INCH and event.event_type == HistoryEventType.RECEIVE: event.event_subtype = HistoryEventSubType.AIRDROP event.notes = f'Claim {event.balance.amount} 1INCH from the 1INCH airdrop' # noqa: E501 return None if tx_log.topics[ 0] == GNOSIS_CHAIN_BRIDGE_RECEIVE and tx_log.address == '0x88ad09518695c6c3712AC10a214bE5109a655671': # noqa: E501 for event in decoded_events: if event.event_type == HistoryEventType.RECEIVE: # user bridged from gnosis chain event.event_type = HistoryEventType.TRANSFER event.event_subtype = HistoryEventSubType.BRIDGE event.counterparty = CPT_GNOSIS_CHAIN event.notes = ( f'Bridge {event.balance.amount} {event.asset.symbol} from gnosis chain' ) return None def _enrich_protocol_tranfers( # pylint: disable=no-self-use self, token: EthereumToken, tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, event: HistoryBaseEntry, action_items: List[ActionItem], ) -> None: """ Decode special transfers made by contract execution for example at the moment of depositing assets or withdrawing. It assumes that the event being decoded has been already filtered and is a transfer. """ for enrich_call in self.token_enricher_rules: transfer_enriched = enrich_call( token=token, tx_log=tx_log, transaction=transaction, event=event, action_items=action_items, ) if transfer_enriched: break def _maybe_decode_governance( # pylint: disable=no-self-use self, token: Optional[EthereumToken], # pylint: disable=unused-argument tx_log: EthereumTxReceiptLog, transaction: EthereumTransaction, decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument action_items: List[ActionItem], # pylint: disable=unused-argument ) -> Optional[HistoryBaseEntry]: if tx_log.topics[0] == GOVERNORALPHA_PROPOSE: if tx_log.address == '0xDbD27635A534A3d3169Ef0498beB56Fb9c937489': governance_name = 'Gitcoin' else: governance_name = tx_log.address try: _, decoded_data = decode_event_data_abi_str( tx_log, GOVERNORALPHA_PROPOSE_ABI) except DeserializationError as e: log.debug( f'Failed to decode governor alpha event due to {str(e)}') return None proposal_id = decoded_data[0] proposal_text = decoded_data[8] notes = f'Create {governance_name} proposal {proposal_id}. {proposal_text}' return HistoryBaseEntry( event_identifier=transaction.tx_hash.hex(), sequence_index=self.base.get_sequence_index(tx_log), timestamp=ts_sec_to_ms(transaction.timestamp), location=Location.BLOCKCHAIN, location_label=transaction.from_address, # TODO: This should be null for proposals and other informational events asset=A_ETH, balance=Balance(), notes=notes, event_type=HistoryEventType.INFORMATIONAL, event_subtype=HistoryEventSubType.GOVERNANCE_PROPOSE, counterparty=governance_name, ) return None
def setup_ethereum_transactions_test( database: DBHandler, transaction_already_queried: bool, one_receipt_in_db: bool = False, ) -> Tuple[List[EthereumTransaction], List[EthereumTxReceipt]]: dbethtx = DBEthTx(database) tx_hash1 = '0x692f9a6083e905bdeca4f0293f3473d7a287260547f8cbccc38c5cb01591fcda' tx_hash1_b = hexstring_to_bytes(tx_hash1) transaction1 = EthereumTransaction( tx_hash=tx_hash1_b, timestamp=Timestamp(1630532276), block_number=13142218, from_address=string_to_ethereum_address( '0x443E1f9b1c866E54e914822B7d3d7165EdB6e9Ea'), to_address=string_to_ethereum_address( '0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D'), value=int(10 * 10**18), gas=194928, gas_price=int(0.000000204 * 10**18), gas_used=136675, input_data=hexstring_to_bytes( '0x7ff36ab5000000000000000000000000000000000000000000000367469995d0723279510000000000000000000000000000000000000000000000000000000000000080000000000000000000000000443e1f9b1c866e54e914822b7d3d7165edb6e9ea00000000000000000000000000000000000000000000000000000000612ff9b50000000000000000000000000000000000000000000000000000000000000002000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002a3bff78b79a009976eea096a51a948a3dc00e34' ), # noqa: E501 nonce=13, ) tx_hash2 = '0x6beab9409a8f3bd11f82081e99e856466a7daf5f04cca173192f79e78ed53a77' tx_hash2_b = hexstring_to_bytes(tx_hash2) transaction2 = EthereumTransaction( tx_hash=tx_hash2_b, timestamp=Timestamp(1631013757), block_number=13178342, from_address=string_to_ethereum_address( '0x442068F934BE670aDAb81242C87144a851d56d16'), to_address=string_to_ethereum_address( '0xEaDD9B69F96140283F9fF75DA5FD33bcF54E6296'), value=0, gas=77373, gas_price=int(0.000000100314697497 * 10**18), gas_used=46782, input_data=hexstring_to_bytes( '0xa9059cbb00000000000000000000000020c8032d4f7d4a380385f87aeadf05bed84504cb000000000000000000000000000000000000000000000000000000003b9deec6' ), # noqa: E501 nonce=3, ) transactions = [transaction1, transaction2] if transaction_already_queried is True: dbethtx.add_ethereum_transactions(ethereum_transactions=transactions) result, _ = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make()) assert result == transactions expected_receipt1 = EthereumTxReceipt( tx_hash=tx_hash1_b, contract_address=None, status=True, type=0, logs=[ EthereumTxReceiptLog( log_index=295, data=hexstring_to_bytes( '0x0000000000000000000000000000000000000000000000008ac7230489e80000' ), # noqa: E501 address=string_to_ethereum_address( '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'), removed=False, topics=[ hexstring_to_bytes( '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c' ), # noqa: E501 hexstring_to_bytes( '0x0000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488d' ), # noqa: E501 ], ), EthereumTxReceiptLog( log_index=296, data=hexstring_to_bytes( '0x0000000000000000000000000000000000000000000000008ac7230489e80000' ), # noqa: E501 address=string_to_ethereum_address( '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'), removed=False, topics=[ hexstring_to_bytes( '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' ), # noqa: E501 hexstring_to_bytes( '0x0000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488d' ), # noqa: E501 hexstring_to_bytes( '0x000000000000000000000000caa004418eb42cdf00cb057b7c9e28f0ffd840a5' ), # noqa: E501 ], ), EthereumTxReceiptLog( log_index=297, data=hexstring_to_bytes( '0x00000000000000000000000000000000000000000000036ba1d53baeeda5ed20' ), # noqa: E501 address=string_to_ethereum_address( '0x2a3bFF78B79A009976EeA096a51A948a3dC00e34'), removed=False, topics=[ hexstring_to_bytes( '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' ), # noqa: E501 hexstring_to_bytes( '0x000000000000000000000000caa004418eb42cdf00cb057b7c9e28f0ffd840a5' ), # noqa: E501 hexstring_to_bytes( '0x000000000000000000000000443e1f9b1c866e54e914822b7d3d7165edb6e9ea' ), # noqa: E501 ], ), EthereumTxReceiptLog( log_index=298, data=hexstring_to_bytes( '0x000000000000000000000000000000000000000000007b6ea033189ba7d047e30000000000000000000000000000000000000000000000140bc8194dd0f5e4be' ), # noqa: E501 address=string_to_ethereum_address( '0xcaA004418eB42cdf00cB057b7C9E28f0FfD840a5'), removed=False, topics=[ hexstring_to_bytes( '0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1' ) ], # noqa: E501 ), EthereumTxReceiptLog( log_index=299, data=hexstring_to_bytes( '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008ac7230489e8000000000000000000000000000000000000000000000000036ba1d53baeeda5ed200000000000000000000000000000000000000000000000000000000000000000' ), # noqa: E501 address=string_to_ethereum_address( '0xcaA004418eB42cdf00cB057b7C9E28f0FfD840a5'), removed=False, topics=[ hexstring_to_bytes( '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' ), # noqa: E501 hexstring_to_bytes( '0x0000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488d' ), # noqa: E501 hexstring_to_bytes( '0x000000000000000000000000443e1f9b1c866e54e914822b7d3d7165edb6e9ea' ), # noqa: E501 ], ), ], ) expected_receipt2 = EthereumTxReceipt( tx_hash=tx_hash2_b, contract_address=None, status=True, type=2, logs=[ EthereumTxReceiptLog( log_index=438, data=hexstring_to_bytes( '0x000000000000000000000000000000000000000000000000000000003b9deec6' ), # noqa: E501 address=string_to_ethereum_address( '0xEaDD9B69F96140283F9fF75DA5FD33bcF54E6296'), removed=False, topics=[ hexstring_to_bytes( '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' ), # noqa: E501 hexstring_to_bytes( '0x000000000000000000000000442068f934be670adab81242c87144a851d56d16' ), # noqa: E501 hexstring_to_bytes( '0x00000000000000000000000020c8032d4f7d4a380385f87aeadf05bed84504cb' ), # noqa: E501 ], ), ], ) if one_receipt_in_db: dbethtx.add_receipt_data(txreceipt_to_data(expected_receipt1)) return transactions, [expected_receipt1, expected_receipt2]
def _get_erc20_transfers_for_ranges( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, ) -> None: """Queries etherscan for all erc20 transfers of address in the given ranges. If any transfers are found, they are added in the DB """ location_string = f'{RANGE_PREFIX_ETHTOKENTX}_{address}' dbethtx = DBEthTx(self.database) ranges = DBQueryRanges(self.database) ranges_to_query = ranges.get_location_query_ranges( location_string=location_string, start_ts=start_ts, end_ts=end_ts, ) for query_start_ts, query_end_ts in ranges_to_query: log.debug( f'Querying ERC20 Transfers for {address} -> {query_start_ts} - {query_end_ts}' ) # noqa: E501 try: for erc20_tx_hashes in self.ethereum.etherscan.get_token_transaction_hashes( account=address, from_ts=query_start_ts, to_ts=query_end_ts, ): for tx_hash in erc20_tx_hashes: tx_hash_bytes = deserialize_evm_tx_hash(tx_hash) result = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make( tx_hash=tx_hash_bytes), has_premium=True, # ignore limiting here ) if len(result ) == 0: # if transaction is not there add it gevent.sleep(0) transaction = self.ethereum.get_transaction_by_hash( tx_hash_bytes) dbethtx.add_ethereum_transactions( [transaction], relevant_address=address, ) timestamp = transaction.timestamp else: timestamp = result[0].timestamp log.debug( f'ERC20 Transfers for {address} -> update range {query_start_ts} - {timestamp}' ) # noqa: E501 ranges.update_used_query_range( # update last queried time for the address location_string=location_string, queried_ranges=[(query_start_ts, timestamp)], ) self.msg_aggregator.add_message( message_type=WSMessageType. ETHEREUM_TRANSACTION_STATUS, data={ 'address': address, 'period': [query_start_ts, timestamp], 'status': str(TransactionStatusStep. QUERYING_ETHEREUM_TOKENS_TRANSACTIONS ), # noqa: E501 }, ) except RemoteError as e: self.ethereum.msg_aggregator.add_error( f'Got error "{str(e)}" while querying token transactions' f'from Etherscan. Transactions not added to the DB ' f'address: {address} ' f'from_ts: {query_start_ts} ' f'to_ts: {query_end_ts} ', ) log.debug( f'ERC20 Transfers done for address {address}. Update range {start_ts} - {end_ts}' ) # noqa: E501 ranges.update_used_query_range( # entire range is now considered queried location_string=location_string, queried_ranges=[(start_ts, end_ts)], )
def _get_internal_transactions_for_ranges( self, address: ChecksumEthAddress, start_ts: Timestamp, end_ts: Timestamp, ) -> None: """Queries etherscan for all internal transactions of address in the given ranges. If any internal transactions are found, they are added in the DB """ location_string = f'{RANGE_PREFIX_ETHINTERNALTX}_{address}' ranges = DBQueryRanges(self.database) ranges_to_query = ranges.get_location_query_ranges( location_string=location_string, start_ts=start_ts, end_ts=end_ts, ) dbethtx = DBEthTx(self.database) new_internal_txs = [] for query_start_ts, query_end_ts in ranges_to_query: log.debug( f'Querying Internal Transactions for {address} -> {query_start_ts} - {query_end_ts}' ) # noqa: E501 try: for new_internal_txs in self.ethereum.etherscan.get_transactions( account=address, from_ts=query_start_ts, to_ts=query_end_ts, action='txlistinternal', ): if len(new_internal_txs) != 0: for internal_tx in new_internal_txs: # make sure all internal transaction parent transactions are in the DB gevent.sleep(0) result = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make( tx_hash=internal_tx.parent_tx_hash ), # noqa: E501 has_premium=True, # ignore limiting here ) if len( result ) == 0: # parent transaction is not in the DB. Get it transaction = self.ethereum.get_transaction_by_hash( internal_tx.parent_tx_hash) # noqa: E501 gevent.sleep(0) dbethtx.add_ethereum_transactions( ethereum_transactions=[transaction], relevant_address=address, ) timestamp = transaction.timestamp else: timestamp = result[0].timestamp dbethtx.add_ethereum_internal_transactions( transactions=[internal_tx], relevant_address=address, ) log.debug( f'Internal Transactions for {address} -> update range {query_start_ts} - {timestamp}' ) # noqa: E501 ranges.update_used_query_range( # update last queried time for address location_string=location_string, queried_ranges=[(query_start_ts, timestamp)], ) self.msg_aggregator.add_message( message_type=WSMessageType. ETHEREUM_TRANSACTION_STATUS, data={ 'address': address, 'period': [query_start_ts, timestamp], 'status': str(TransactionStatusStep. QUERYING_INTERNAL_TRANSACTIONS ), # noqa: E501 }, ) except RemoteError as e: self.ethereum.msg_aggregator.add_error( f'Got error "{str(e)}" while querying internal ethereum transactions ' f'from Etherscan. Transactions not added to the DB ' f'address: {address} ' f'from_ts: {query_start_ts} ' f'to_ts: {query_end_ts} ', ) return log.debug( f'Internal Transactions for address {address} done. Update range {start_ts} - {end_ts}' ) # noqa: E501 ranges.update_used_query_range( # entire range is now considered queried location_string=location_string, queried_ranges=[(start_ts, end_ts)], )
def test_add_ethereum_transactions(data_dir, username): """Test that adding and retrieving ethereum transactions from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) tx2_hash_b = b'.h\xdd\x82\x85\x94\xeaq\xfe\n\xfc\xcf\xadwH\xc9\x0f\xfc\xd0\xf1\xad\xd4M\r$\x9b\xf7\x98\x87\xda\x93\x18' # noqa: E501 tx2_hash = '0x2e68dd828594ea71fe0afccfad7748c90ffcd0f1add44d0d249bf79887da9318' tx1 = EthereumTransaction( tx_hash=b'1', timestamp=Timestamp(1451606400), block_number=1, from_address=ETH_ADDRESS1, to_address=ETH_ADDRESS3, value=FVal('2000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=1, ) tx2 = EthereumTransaction( tx_hash=tx2_hash_b, timestamp=Timestamp(1451706400), block_number=3, from_address=ETH_ADDRESS2, to_address=ETH_ADDRESS3, value=FVal('4000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=1, ) tx3 = EthereumTransaction( tx_hash=b'3', timestamp=Timestamp(1452806400), block_number=5, from_address=ETH_ADDRESS3, to_address=ETH_ADDRESS1, value=FVal('1000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=3, ) # Add and retrieve the first 2 margins. All should be fine. dbethtx = DBEthTx(data.db) dbethtx.add_ethereum_transactions([tx1, tx2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 filter_query = ETHTransactionsFilterQuery.make() returned_transactions, _ = dbethtx.get_ethereum_transactions(filter_query) assert returned_transactions == [tx1, tx2] # Add the last 2 transactions. Since tx2 already exists in the DB it should be # ignored (no errors shown for attempting to add already existing transaction) dbethtx.add_ethereum_transactions([tx2, tx3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_transactions, _ = dbethtx.get_ethereum_transactions(filter_query) assert returned_transactions == [tx1, tx2, tx3] # try transaction query by tx_hash result, _ = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make(tx_hash=tx2_hash_b)) # noqa: E501 assert result == [tx2], 'querying transaction by hash in bytes failed' result, _ = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make(tx_hash=tx2_hash)) # noqa: E501 assert result == [tx2], 'querying transaction by hash string failed' result, _ = dbethtx.get_ethereum_transactions( ETHTransactionsFilterQuery.make(tx_hash=b'dsadsad')) # noqa: E501 assert result == []