class ExportErc20TokensJob(BaseJob): def __init__(self, web3, item_exporter, token_addresses_iterable, max_workers): self.item_exporter = item_exporter self.token_addresses_iterable = token_addresses_iterable self.batch_work_executor = BatchWorkExecutor(1, max_workers) self.erc20_token_service = EthErc20TokenService( web3, clean_user_provided_content) self.erc20_token_mapper = EthErc20TokenMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.token_addresses_iterable, self._export_tokens) def _export_tokens(self, token_addresses): for token_address in token_addresses: self._export_token(token_address) def _export_token(self, token_address): token = self.erc20_token_service.get_token(token_address) token_dict = self.erc20_token_mapper.erc20_token_to_dict(token) self.item_exporter.export_item(token_dict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportTokenTransfersJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.receipt_log_mapper = EthReceiptLogMapper() self.token_transfer_mapper = EthTokenTransferMapper() self.token_transfer_extractor = EthTokenTransferExtractor() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'fromBlock': block_number_batch[0], 'toBlock': block_number_batch[-1], 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) token_transfer = self.token_transfer_extractor.extract_transfer_from_log( log) if token_transfer is not None: self.item_exporter.export_item( self.token_transfer_mapper.token_transfer_to_dict( token_transfer)) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExtractGethTracesJob(BaseJob): def __init__(self, traces_iterable, batch_size, max_workers, item_exporter): self.traces_iterable = traces_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() self.geth_trace_mapper = EthGethTraceMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.traces_iterable, self._extract_geth_traces) def _extract_geth_traces(self, geth_traces): for geth_trace_dict in geth_traces: geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace( geth_trace_dict) traces = self.trace_mapper.geth_trace_to_traces(geth_trace) for trace in traces: self.item_exporter.export_item( self.trace_mapper.trace_to_dict(trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExtractErc20TransfersJob(BaseJob): def __init__(self, logs_iterable, batch_size, max_workers, item_exporter): self.logs_iterable = logs_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_transfer_extractor = EthErc20TransferExtractor() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.logs_iterable, self._extract_transfers) def _extract_transfers(self, log_dicts): for log_dict in log_dicts: self._extract_transfer(log_dict) def _extract_transfer(self, log_dict): log = self.receipt_log_mapper.dict_to_receipt_log(log_dict) erc20_transfer = self.erc20_transfer_extractor.extract_transfer_from_log( log) if erc20_transfer is not None: self.item_exporter.export_item( self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportContractsJob(BaseJob): def __init__(self, contract_addresses_iterable, batch_size, batch_web3_provider, max_workers, item_exporter): self.batch_web3_provider = batch_web3_provider self.contract_addresses_iterable = contract_addresses_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_service = EthContractService() self.contract_mapper = EthContractMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.contract_addresses_iterable, self._export_contracts) def _export_contracts(self, contract_addresses): contracts_code_rpc = list( generate_get_code_json_rpc(contract_addresses)) response_batch = self.batch_web3_provider.make_batch_request( json.dumps(contracts_code_rpc)) contracts = [] for response in response_batch: # request id is the index of the contract address in contract_addresses list request_id = response['id'] result = rpc_response_to_result(response) contract_address = contract_addresses[request_id] contract = self._get_contract(contract_address, result) contracts.append(contract) for contract in contracts: self.item_exporter.export_item( self.contract_mapper.contract_to_dict(contract)) def _get_contract(self, contract_address, rpc_result): contract = self.contract_mapper.rpc_result_to_contract( contract_address, rpc_result) bytecode = contract.bytecode function_sighashes = self.contract_service.get_function_sighashes( bytecode) contract.function_sighashes = function_sighashes contract.is_erc20 = self.contract_service.is_erc20_contract( function_sighashes) contract.is_erc721 = self.contract_service.is_erc721_contract( function_sighashes) return contract def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportReceiptsJob(BaseJob): def __init__(self, tx_hashes_iterable, batch_size, ipc_wrapper, max_workers, item_exporter, export_receipts=True, export_logs=True): self.ipc_wrapper = ipc_wrapper self.tx_hashes_iterable = tx_hashes_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_receipts = export_receipts self.export_logs = export_logs if not self.export_receipts and not self.export_logs: raise ValueError( 'At least one of export_receipts or export_logs must be True') self.receipt_mapper = EthReceiptMapper() self.receipt_log_mapper = EthReceiptLogMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.tx_hashes_iterable, self._export_receipts) def _export_receipts(self, tx_hashes): receipts_rpc = list( generate_get_receipt_by_tx_hash_json_rpc(tx_hashes)) response = self.ipc_wrapper.make_request(json.dumps(receipts_rpc)) results = rpc_response_batch_to_results(response) receipts = [ self.receipt_mapper.json_dict_to_receipt(result) for result in results ] for receipt in receipts: self._export_receipt(receipt) def _export_receipt(self, receipt): if self.export_receipts: self.item_exporter.export_item( self.receipt_mapper.receipt_to_dict(receipt)) if self.export_logs: for log in receipt.logs: self.item_exporter.export_item( self.receipt_log_mapper.receipt_log_to_dict(log)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportErc20TokensJob(BaseJob): def __init__(self, web3, item_exporter, token_addresses_iterable, max_workers): self.web3 = web3 self.item_exporter = item_exporter self.token_addresses_iterable = token_addresses_iterable self.batch_work_executor = BatchWorkExecutor(1, max_workers) self.erc20_token_mapper = EthErc20TokenMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.token_addresses_iterable, self._export_tokens) def _export_tokens(self, token_addresses): for token_address in token_addresses: self._export_token(token_address) def _export_token(self, token_address): checksum_address = self.web3.toChecksumAddress(token_address) contract = self.web3.eth.contract(address=checksum_address, abi=EIP20_ABI) symbol = self._call_contract_function(contract.functions.symbol()) name = self._call_contract_function(contract.functions.name()) decimals = self._call_contract_function(contract.functions.decimals()) total_supply = self._call_contract_function( contract.functions.totalSupply()) token = EthErc20Token() token.erc20_token_address = token_address token.erc20_token_symbol = symbol token.erc20_token_name = name token.erc20_token_decimals = decimals token.erc20_token_total_supply = total_supply token_dict = self.erc20_token_mapper.erc20_token_to_dict(token) self.item_exporter.export_item(token_dict) def _call_contract_function(self, func): # BadFunctionCallOutput exception happens if the token doesn't implement a particular function # or was self-destructed # OverflowError exception happens if the return type of the function doesn't match the expected type result = call_contract_function(func=func, ignore_errors=(BadFunctionCallOutput, OverflowError), default_value=None) return clean_user_provided_content(result) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExtractContractsJob(BaseJob): def __init__( self, traces_iterable, batch_size, max_workers, item_exporter): self.traces_iterable = traces_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_service = EthContractService() self.contract_mapper = EthContractMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.traces_iterable, self._extract_contracts) def _extract_contracts(self, traces): for trace in traces: trace['status'] = to_int_or_none(trace.get('status')) trace['block_number'] = to_int_or_none(trace.get('block_number')) contract_creation_traces = [trace for trace in traces if trace.get('trace_type') == 'create' and trace.get('to_address') is not None and len(trace.get('to_address')) > 0 and trace.get('status') == 1] contracts = [] for trace in contract_creation_traces: contract = EthContract() contract.address = trace.get('to_address') bytecode = trace.get('output') contract.bytecode = bytecode contract.block_number = trace.get('block_number') function_sighashes = self.contract_service.get_function_sighashes(bytecode) contract.function_sighashes = function_sighashes contract.is_erc20 = self.contract_service.is_erc20_contract(function_sighashes) contract.is_erc721 = self.contract_service.is_erc721_contract(function_sighashes) contracts.append(contract) for contract in contracts: self.item_exporter.export_item(self.contract_mapper.contract_to_dict(contract)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportTracesJob(BaseJob): def __init__( self, start_block, end_block, batch_size, web3, item_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1 ) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 filter_params = { 'fromBlock': hex(block_number_batch[0]), 'toBlock': hex(block_number_batch[-1]), } json_traces = self.web3.parity.traceFilter(filter_params) for json_trace in json_traces: trace = self.trace_mapper.json_dict_to_trace(json_trace) self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportGethTracesJob(BaseJob): def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.geth_trace_mapper = EthGethTraceMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): trace_block_rpc = list( generate_trace_block_by_number_json_rpc(block_number_batch)) response = self.batch_web3_provider.make_request( json.dumps(trace_block_rpc)) for response_item in response: block_number = response_item.get('id') result = rpc_response_to_result(response_item) geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace({ 'block_number': block_number, 'transaction_traces': [tx_trace.get('result') for tx_trace in result], }) self.item_exporter.export_item( self.geth_trace_mapper.geth_trace_to_dict(geth_trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportContractsJob(BaseJob): def __init__(self, contract_addresses_iterable, batch_size, batch_web3_provider, max_workers, item_exporter): self.batch_web3_provider = batch_web3_provider self.contract_addresses_iterable = contract_addresses_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_mapper = EthContractMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.contract_addresses_iterable, self._export_contracts) def _export_contracts(self, contract_addresses): contracts_code_rpc = list( generate_get_code_json_rpc(contract_addresses)) response_batch = self.batch_web3_provider.make_request( json.dumps(contracts_code_rpc)) contracts = [] for response in response_batch: # request id is the index of the contract address in contract_addresses list request_id = response['id'] result = response['result'] contract_address = contract_addresses[request_id] contracts.append( self.contract_mapper.rpc_result_to_receipt( contract_address, result)) for contract in contracts: self.item_exporter.export_item( self.contract_mapper.contract_to_dict(contract)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter, export_blocks=True, export_transactions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): blocks_rpc = list( generate_get_block_by_number_json_rpc(block_number_batch, self.export_transactions)) response = self.batch_web3_provider.make_request( json.dumps(blocks_rpc)) results = rpc_response_batch_to_results(response) blocks = [ self.block_mapper.json_dict_to_block(result) for result in results ] for block in blocks: self._export_block(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item( self.block_mapper.block_to_dict(block)) print("block=", self.block_mapper.block_to_dict(block)) print("number=", self.block_mapper.block_to_dict(block)['number']) if self.export_transactions: for tx in block.transactions: # print(self.transaction_mapper.transaction_to_dict(tx)) transactiondict = self.transaction_mapper.transaction_to_dict( tx) transactiondict['timestamp'] = self.block_mapper.block_to_dict( block)['timestamp'] self.item_exporter.export_item(transactiondict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportTracesJob(BaseJob): def __init__( self, start_block, end_block, batch_size, web3, item_exporter, max_workers, include_genesis_traces=False, include_daofork_traces=False): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 # TODO: use batch_size when this issue is fixed https://github.com/paritytech/parity-ethereum/issues/9822 self.batch_work_executor = BatchWorkExecutor(1, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() self.special_trace_service = EthSpecialTraceService() self.include_genesis_traces = include_genesis_traces self.include_daofork_traces = include_daofork_traces def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1 ) def _export_batch(self, block_number_batch): # TODO: Change to len(block_number_batch) > 0 when this issue is fixed # https://github.com/paritytech/parity-ethereum/issues/9822 assert len(block_number_batch) == 1 block_number = block_number_batch[0] if self.include_genesis_traces and 0 in block_number_batch: genesis_traces = self.special_trace_service.get_genesis_traces() for trace in genesis_traces: self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) if self.include_daofork_traces and DAOFORK_BLOCK_NUMBER in block_number_batch: daofork_traces = self.special_trace_service.get_daofork_traces() for trace in daofork_traces: self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) # TODO: Change to traceFilter when this issue is fixed # https://github.com/paritytech/parity-ethereum/issues/9822 json_traces = self.web3.parity.traceBlock(block_number) if json_traces is None: raise ValueError('Response from the node is None. Is the node fully synced?') for json_trace in json_traces: trace = self.trace_mapper.json_dict_to_trace(json_trace) self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportOriginJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, ipfs_client, marketplace_listing_exporter, shop_product_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.marketplace_listing_exporter = marketplace_listing_exporter self.shop_product_exporter = shop_product_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.event_extractor = OriginEventExtractor(ipfs_client) self.receipt_log_mapper = EthReceiptLogMapper() self.marketplace_listing_mapper = OriginMarketplaceListingMapper() self.shop_listing_mapper = OriginShopProductMapper() def _start(self): self.marketplace_listing_exporter.open() self.shop_product_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 from_block = block_number_batch[0] to_block = block_number_batch[-1] # Nothing to process if the block range is older than the V0 marketplace contract's epoch. if to_block < ORIGIN_MARKETPLACE_V0_BLOCK_NUMBER_EPOCH: return # Determine the version and address of the marketplace contract to query based on the block range. batches = [] if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH or from_block >= ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH: # The block range falls within a single version of the marketplace contract. version = '000' if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH else '001' address = ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS if version == '000' else ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS batches.append({ 'contract_address': address, 'contract_version': version, 'from_block': from_block, 'to_block': to_block }) else: # The block range spans across 2 versions of the marketplace contract. batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS, 'contract_version': '000', 'from_block': from_block, 'to_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH - 1 }) batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS, 'contract_version': '001', 'from_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH, 'to_block': to_block }) for batch in batches: # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'address': batch['contract_address'], 'fromBlock': batch['from_block'], 'toBlock': batch['to_block'] } event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) listing, shop_products = self.event_extractor.extract_event_from_log( log, batch['contract_version']) if listing: item = self.marketplace_listing_mapper.listing_to_dict( listing) self.marketplace_listing_exporter.export_item(item) for product in shop_products: item = self.shop_listing_mapper.product_to_dict(product) self.shop_product_exporter.export_item(item) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.batch_work_executor.shutdown() self.marketplace_listing_exporter.close() self.shop_product_exporter.close()