def __init__( self, contract_addresses_iterable, batch_size, ipc_wrapper, max_workers, item_exporter): self.ipc_wrapper = ipc_wrapper self.contract_addresses_iterable = contract_addresses_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_mapper = EthContractMapper()
def __init__( self, logs_iterable, batch_size, max_workers, item_exporter): self.logs_iterable = logs_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.token_transfer_mapper = EthTokenTransferMapper() self.token_transfer_extractor = EthTokenTransferExtractor()
def __init__( self, contract_addresses_iterable, batch_size, batch_web3_provider, max_workers, item_exporter): self.batch_web3_provider = batch_web3_provider self.contract_addresses_iterable = contract_addresses_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_service = EthContractService() self.contract_mapper = EthContractMapper()
class ExportReceiptsJob(BaseJob): def __init__( self, transaction_hashes_iterable, batch_size, batch_web3_provider, max_workers, item_exporter, export_receipts=True, export_logs=True): self.batch_web3_provider = batch_web3_provider self.transaction_hashes_iterable = transaction_hashes_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_receipts = export_receipts self.export_logs = export_logs if not self.export_receipts and not self.export_logs: raise ValueError('At least one of export_receipts or export_logs must be True') self.receipt_mapper = EthReceiptMapper() self.receipt_log_mapper = EthReceiptLogMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.transaction_hashes_iterable, self._export_receipts) def _export_receipts(self, transaction_hashes): receipts_rpc = list(generate_get_receipt_json_rpc(transaction_hashes)) response = self.batch_web3_provider.make_batch_request(json.dumps(receipts_rpc)) results = rpc_response_batch_to_results(response) receipts = [self.receipt_mapper.json_dict_to_receipt(result) for result in results] for receipt in receipts: self._export_receipt(receipt) def _export_receipt(self, receipt): if self.export_receipts: self.item_exporter.export_item(self.receipt_mapper.receipt_to_dict(receipt)) if self.export_logs: for log in receipt.logs: self.item_exporter.export_item(self.receipt_log_mapper.receipt_log_to_dict(log)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportTracesJob(BaseJob): def __init__( self, start_block, end_block, batch_size, web3, item_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1 ) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 filter_params = { 'fromBlock': hex(block_number_batch[0]), 'toBlock': hex(block_number_batch[-1]), } json_traces = self.web3.parity.traceFilter(filter_params) for json_trace in json_traces: trace = self.trace_mapper.json_dict_to_trace(json_trace) self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportGethTracesJob(BaseJob): def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.geth_trace_mapper = EthGethTraceMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): trace_block_rpc = list( generate_trace_block_by_number_json_rpc(block_number_batch)) response = self.batch_web3_provider.make_request( json.dumps(trace_block_rpc)) for response_item in response: block_number = response_item.get('id') result = rpc_response_to_result(response_item) geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace({ 'block_number': block_number, 'transaction_traces': [tx_trace.get('result') for tx_trace in result], }) self.item_exporter.export_item( self.geth_trace_mapper.geth_trace_to_dict(geth_trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
def __init__( self, start_block, end_block, batch_size, web3, item_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper()
def __init__(self, start_block, end_block, batch_size, web3, ipfs_client, marketplace_listing_exporter, shop_product_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.marketplace_listing_exporter = marketplace_listing_exporter self.shop_product_exporter = shop_product_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.event_extractor = OriginEventExtractor(ipfs_client) self.receipt_log_mapper = EthReceiptLogMapper() self.marketplace_listing_mapper = OriginMarketplaceListingMapper() self.shop_listing_mapper = OriginShopProductMapper()
class ExportContractsJob(BaseJob): def __init__(self, contract_addresses_iterable, batch_size, batch_web3_provider, max_workers, item_exporter): self.batch_web3_provider = batch_web3_provider self.contract_addresses_iterable = contract_addresses_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.contract_mapper = EthContractMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.contract_addresses_iterable, self._export_contracts) def _export_contracts(self, contract_addresses): contracts_code_rpc = list( generate_get_code_json_rpc(contract_addresses)) response_batch = self.batch_web3_provider.make_request( json.dumps(contracts_code_rpc)) contracts = [] for response in response_batch: # request id is the index of the contract address in contract_addresses list request_id = response['id'] result = response['result'] contract_address = contract_addresses[request_id] contracts.append( self.contract_mapper.rpc_result_to_receipt( contract_address, result)) for contract in contracts: self.item_exporter.export_item( self.contract_mapper.contract_to_dict(contract)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.receipt_log_mapper = EthReceiptLogMapper() self.token_transfer_mapper = EthTokenTransferMapper() self.token_transfer_extractor = EthTokenTransferExtractor()
def __init__(self, tx_hashes_iterable, batch_size, ipc_wrapper, max_workers, item_exporter, export_receipts=True, export_logs=True): self.ipc_wrapper = ipc_wrapper self.tx_hashes_iterable = tx_hashes_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_receipts = export_receipts self.export_logs = export_logs if not self.export_receipts and not self.export_logs: raise ValueError( 'At least one of export_receipts or export_logs must be True') self.receipt_mapper = EthReceiptMapper() self.receipt_log_mapper = EthReceiptLogMapper()
def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, include_genesis_traces=False, include_daofork_traces=False): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 # TODO: use batch_size when this issue is fixed https://github.com/paritytech/parity-ethereum/issues/9822 self.batch_work_executor = BatchWorkExecutor(1, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() self.special_trace_service = EthSpecialTraceService() self.include_genesis_traces = include_genesis_traces self.include_daofork_traces = include_daofork_traces
class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter, export_blocks=True, export_transactions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): blocks_rpc = list( generate_get_block_by_number_json_rpc(block_number_batch, self.export_transactions)) response = self.batch_web3_provider.make_request( json.dumps(blocks_rpc)) results = rpc_response_batch_to_results(response) blocks = [ self.block_mapper.json_dict_to_block(result) for result in results ] for block in blocks: self._export_block(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item( self.block_mapper.block_to_dict(block)) print("block=", self.block_mapper.block_to_dict(block)) print("number=", self.block_mapper.block_to_dict(block)['number']) if self.export_transactions: for tx in block.transactions: # print(self.transaction_mapper.transaction_to_dict(tx)) transactiondict = self.transaction_mapper.transaction_to_dict( tx) transactiondict['timestamp'] = self.block_mapper.block_to_dict( block)['timestamp'] self.item_exporter.export_item(transactiondict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportTracesJob(BaseJob): def __init__( self, start_block, end_block, batch_size, web3, item_exporter, max_workers, include_genesis_traces=False, include_daofork_traces=False): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 # TODO: use batch_size when this issue is fixed https://github.com/paritytech/parity-ethereum/issues/9822 self.batch_work_executor = BatchWorkExecutor(1, max_workers) self.item_exporter = item_exporter self.trace_mapper = EthTraceMapper() self.special_trace_service = EthSpecialTraceService() self.include_genesis_traces = include_genesis_traces self.include_daofork_traces = include_daofork_traces def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1 ) def _export_batch(self, block_number_batch): # TODO: Change to len(block_number_batch) > 0 when this issue is fixed # https://github.com/paritytech/parity-ethereum/issues/9822 assert len(block_number_batch) == 1 block_number = block_number_batch[0] if self.include_genesis_traces and 0 in block_number_batch: genesis_traces = self.special_trace_service.get_genesis_traces() for trace in genesis_traces: self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) if self.include_daofork_traces and DAOFORK_BLOCK_NUMBER in block_number_batch: daofork_traces = self.special_trace_service.get_daofork_traces() for trace in daofork_traces: self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) # TODO: Change to traceFilter when this issue is fixed # https://github.com/paritytech/parity-ethereum/issues/9822 json_traces = self.web3.parity.traceBlock(block_number) if json_traces is None: raise ValueError('Response from the node is None. Is the node fully synced?') for json_trace in json_traces: trace = self.trace_mapper.json_dict_to_trace(json_trace) self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportOriginJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, ipfs_client, marketplace_listing_exporter, shop_product_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.marketplace_listing_exporter = marketplace_listing_exporter self.shop_product_exporter = shop_product_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.event_extractor = OriginEventExtractor(ipfs_client) self.receipt_log_mapper = EthReceiptLogMapper() self.marketplace_listing_mapper = OriginMarketplaceListingMapper() self.shop_listing_mapper = OriginShopProductMapper() def _start(self): self.marketplace_listing_exporter.open() self.shop_product_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 from_block = block_number_batch[0] to_block = block_number_batch[-1] # Nothing to process if the block range is older than the V0 marketplace contract's epoch. if to_block < ORIGIN_MARKETPLACE_V0_BLOCK_NUMBER_EPOCH: return # Determine the version and address of the marketplace contract to query based on the block range. batches = [] if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH or from_block >= ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH: # The block range falls within a single version of the marketplace contract. version = '000' if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH else '001' address = ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS if version == '000' else ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS batches.append({ 'contract_address': address, 'contract_version': version, 'from_block': from_block, 'to_block': to_block }) else: # The block range spans across 2 versions of the marketplace contract. batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS, 'contract_version': '000', 'from_block': from_block, 'to_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH - 1 }) batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS, 'contract_version': '001', 'from_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH, 'to_block': to_block }) for batch in batches: # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'address': batch['contract_address'], 'fromBlock': batch['from_block'], 'toBlock': batch['to_block'] } event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) listing, shop_products = self.event_extractor.extract_event_from_log( log, batch['contract_version']) if listing: item = self.marketplace_listing_mapper.listing_to_dict( listing) self.marketplace_listing_exporter.export_item(item) for product in shop_products: item = self.shop_listing_mapper.product_to_dict(product) self.shop_product_exporter.export_item(item) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.batch_work_executor.shutdown() self.marketplace_listing_exporter.close() self.shop_product_exporter.close()