def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): self.start_block = start_block self.end_block = end_block self.batch_size = batch_size self.web3 = web3 self.output = output self.max_workers = max_workers self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.output_file = None self.exporter = None self.executor: FailSafeExecutor = None
def __init__(self, logs_iterable, batch_size, max_workers, item_exporter): self.logs_iterable = logs_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_transfer_extractor = EthErc20TransferExtractor()
class ExportTokenTransfersJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.receipt_log_mapper = EthReceiptLogMapper() self.token_transfer_mapper = EthTokenTransferMapper() self.token_transfer_extractor = EthTokenTransferExtractor() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'fromBlock': block_number_batch[0], 'toBlock': block_number_batch[-1], 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) token_transfer = self.token_transfer_extractor.extract_transfer_from_log( log) if token_transfer is not None: self.item_exporter.export_item( self.token_transfer_mapper.token_transfer_to_dict( token_transfer)) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor()
class ExtractErc20TransfersJob(BaseJob): def __init__(self, logs_iterable, batch_size, max_workers, item_exporter): self.logs_iterable = logs_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_transfer_extractor = EthErc20TransferExtractor() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.logs_iterable, self._extract_transfers) def _extract_transfers(self, log_dicts): for log_dict in log_dicts: self._extract_transfer(log_dict) def _extract_transfer(self, log_dict): log = self.receipt_log_mapper.dict_to_receipt_log(log_dict) erc20_transfer = self.erc20_transfer_extractor.extract_transfer_from_log( log) if erc20_transfer is not None: self.item_exporter.export_item( self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportErc20TransfersJob(BatchExportJob): def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.output = output self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.output_file = None self.exporter = None def _start(self): super()._start() self.output_file = get_file_handle(self.output, binary=True, create_parent_dirs=True) self.exporter = CsvItemExporter(self.output_file, fields_to_export=self.fields_to_export) def _export_batch(self, batch_start, batch_end): # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'fromBlock': batch_start, 'toBlock': batch_end, 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) erc20_transfer = self.erc20_processor.filter_transfer_from_log(log) if erc20_transfer is not None: self.exporter.export_item( self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer)) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): super()._end() close_silently(self.output_file)
def __init__(self, start_block, end_block, batch_size, web3, ipfs_client, marketplace_listing_exporter, shop_product_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.marketplace_listing_exporter = marketplace_listing_exporter self.shop_product_exporter = shop_product_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.event_extractor = OriginEventExtractor(ipfs_client) self.receipt_log_mapper = EthReceiptLogMapper() self.marketplace_listing_mapper = OriginMarketplaceListingMapper() self.shop_listing_mapper = OriginShopProductMapper()
class ExportReceiptsJob(BaseJob): def __init__(self, tx_hashes_iterable, batch_size, ipc_wrapper, max_workers, item_exporter, export_receipts=True, export_logs=True): self.ipc_wrapper = ipc_wrapper self.tx_hashes_iterable = tx_hashes_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_receipts = export_receipts self.export_logs = export_logs if not self.export_receipts and not self.export_logs: raise ValueError( 'At least one of export_receipts or export_logs must be True') self.receipt_mapper = EthReceiptMapper() self.receipt_log_mapper = EthReceiptLogMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.tx_hashes_iterable, self._export_receipts) def _export_receipts(self, tx_hashes): receipts_rpc = list( generate_get_receipt_by_tx_hash_json_rpc(tx_hashes)) response = self.ipc_wrapper.make_request(json.dumps(receipts_rpc)) results = rpc_response_batch_to_results(response) receipts = [ self.receipt_mapper.json_dict_to_receipt(result) for result in results ] for receipt in receipts: self._export_receipt(receipt) def _export_receipt(self, receipt): if self.export_receipts: self.item_exporter.export_item( self.receipt_mapper.receipt_to_dict(receipt)) if self.export_logs: for log in receipt.logs: self.item_exporter.export_item( self.receipt_log_mapper.receipt_log_to_dict(log)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.receipt_log_mapper = EthReceiptLogMapper() self.token_transfer_mapper = EthTokenTransferMapper() self.token_transfer_extractor = EthTokenTransferExtractor()
def __init__(self, tx_hashes_iterable, batch_size, ipc_wrapper, max_workers, item_exporter, export_receipts=True, export_logs=True): self.ipc_wrapper = ipc_wrapper self.tx_hashes_iterable = tx_hashes_iterable self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_receipts = export_receipts self.export_logs = export_logs if not self.export_receipts and not self.export_logs: raise ValueError( 'At least one of export_receipts or export_logs must be True') self.receipt_mapper = EthReceiptMapper() self.receipt_log_mapper = EthReceiptLogMapper()
class EthReceiptMapper(object): def __init__(self, receipt_log_mapper=None): if receipt_log_mapper is None: self.receipt_log_mapper = EthReceiptLogMapper() else: self.receipt_log_mapper = receipt_log_mapper def json_dict_to_receipt(self, json_dict): receipt = EthReceipt() receipt.transaction_hash = json_dict.get('transactionHash') receipt.transaction_index = hex_to_dec( json_dict.get('transactionIndex')) receipt.block_hash = json_dict.get('blockHash') receipt.block_number = hex_to_dec(json_dict.get('blockNumber')) receipt.cumulative_gas_used = hex_to_dec( json_dict.get('cumulativeGasUsed')) receipt.gas_used = hex_to_dec(json_dict.get('gasUsed')) receipt.contract_address = to_normalized_address( json_dict.get('contractAddress')) receipt.root = json_dict.get('root') receipt.status = hex_to_dec(json_dict.get('status')) receipt.effective_gas_price = hex_to_dec( json_dict.get('effectiveGasPrice')) if 'logs' in json_dict: receipt.logs = [ self.receipt_log_mapper.json_dict_to_receipt_log(log) for log in json_dict['logs'] ] return receipt def receipt_to_dict(self, receipt): return { 'type': 'receipt', 'transaction_hash': receipt.transaction_hash, 'transaction_index': receipt.transaction_index, 'block_hash': receipt.block_hash, 'block_number': receipt.block_number, 'cumulative_gas_used': receipt.cumulative_gas_used, 'gas_used': receipt.gas_used, 'contract_address': receipt.contract_address, 'root': receipt.root, 'status': receipt.status, 'effective_gas_price': receipt.effective_gas_price }
def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.output = output self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.ethTransactionMapper = EthTransactionMapper() self.output_file = None # self.exporter = None self.start_block = start_block
class ExportErc20TransfersJob(BatchExportJob): def __init__(self, start_block, end_block, batch_size, web3, item_exporter, max_workers, tokens=None): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.tokens = tokens self.item_exporter = item_exporter self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() def _start(self): super()._start() self.item_exporter.open() def _export_batch(self, batch_start, batch_end): # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'fromBlock': batch_start, 'toBlock': batch_end, 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) erc20_transfer = self.erc20_processor.filter_transfer_from_log(log) if erc20_transfer is not None: self.item_exporter.export_item( self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer)) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): super()._end() self.item_exporter.close()
class ExportOriginJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, ipfs_client, marketplace_listing_exporter, shop_product_exporter, max_workers): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.web3 = web3 self.marketplace_listing_exporter = marketplace_listing_exporter self.shop_product_exporter = shop_product_exporter self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.event_extractor = OriginEventExtractor(ipfs_client) self.receipt_log_mapper = EthReceiptLogMapper() self.marketplace_listing_mapper = OriginMarketplaceListingMapper() self.shop_listing_mapper = OriginShopProductMapper() def _start(self): self.marketplace_listing_exporter.open() self.shop_product_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): assert len(block_number_batch) > 0 from_block = block_number_batch[0] to_block = block_number_batch[-1] # Nothing to process if the block range is older than the V0 marketplace contract's epoch. if to_block < ORIGIN_MARKETPLACE_V0_BLOCK_NUMBER_EPOCH: return # Determine the version and address of the marketplace contract to query based on the block range. batches = [] if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH or from_block >= ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH: # The block range falls within a single version of the marketplace contract. version = '000' if to_block < ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH else '001' address = ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS if version == '000' else ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS batches.append({ 'contract_address': address, 'contract_version': version, 'from_block': from_block, 'to_block': to_block }) else: # The block range spans across 2 versions of the marketplace contract. batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V0_CONTRACT_ADDRESS, 'contract_version': '000', 'from_block': from_block, 'to_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH - 1 }) batches.append({ 'contract_address': ORIGIN_MARKETPLACE_V1_CONTRACT_ADDRESS, 'contract_version': '001', 'from_block': ORIGIN_MARKETPLACE_V1_BLOCK_NUMBER_EPOCH, 'to_block': to_block }) for batch in batches: # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'address': batch['contract_address'], 'fromBlock': batch['from_block'], 'toBlock': batch['to_block'] } event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) listing, shop_products = self.event_extractor.extract_event_from_log( log, batch['contract_version']) if listing: item = self.marketplace_listing_mapper.listing_to_dict( listing) self.marketplace_listing_exporter.export_item(item) for product in shop_products: item = self.shop_listing_mapper.product_to_dict(product) self.shop_product_exporter.export_item(item) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.batch_work_executor.shutdown() self.marketplace_listing_exporter.close() self.shop_product_exporter.close()
def __init__(self, receipt_log_mapper=None): if receipt_log_mapper is None: self.receipt_log_mapper = EthReceiptLogMapper() else: self.receipt_log_mapper = receipt_log_mapper
class ExportErc20TransfersJob(BaseJob): def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): self.start_block = start_block self.end_block = end_block self.batch_size = batch_size self.web3 = web3 self.output = output self.max_workers = max_workers self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.output_file = None self.exporter = None self.executor: FailSafeExecutor = None def _start(self): # Using bounded executor prevents unlimited queue growth # and allows monitoring in-progress futures and failing fast in case of errors. self.executor = FailSafeExecutor(BoundedExecutor(1, self.max_workers)) self.output_file = get_file_handle(self.output, binary=True) self.exporter = CsvItemExporter(self.output_file, fields_to_export=self.fields_to_export) def _export(self): for batch_start, batch_end in split_to_batches(self.start_block, self.end_block, self.batch_size): self.executor.submit(self._fail_safe_export_batch, batch_start, batch_end) def _fail_safe_export_batch(self, batch_start, batch_end): try: self._export_batch(batch_start, batch_end) except (Timeout, OSError): # try exporting one by one for block_number in range(batch_start, batch_end + 1): self._export_batch(block_number, block_number) def _export_batch(self, batch_start, batch_end): filter_params = { 'fromBlock': batch_start, 'toBlock': batch_end, 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) erc20_transfer = self.erc20_processor.filter_transfer_from_log(log) if erc20_transfer is not None: self.exporter.export_item( self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer)) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): self.executor.shutdown() close_silently(self.output_file)
class ExportErc20TransfersJob(BatchExportJob): def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.output = output self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.ethTransactionMapper = EthTransactionMapper() self.output_file = None # self.exporter = None self.start_block = start_block def _start(self): super()._start() self.output_file = get_file_handle(self.output, binary=True, create_parent_dirs=True) # self.exporter = CsvItemExporter(self.output_file, fields_to_export=self.fields_to_export) def _export_batch(self, batch_start, batch_end): # https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getfilterlogs filter_params = { 'fromBlock': batch_start, 'toBlock': batch_end, 'topics': [TRANSFER_EVENT_TOPIC] } if self.tokens is not None and len(self.tokens) > 0: filter_params['address'] = self.tokens event_filter = self.web3.eth.filter(filter_params) events = event_filter.get_all_entries() for event in events: log = self.receipt_log_mapper.web3_dict_to_receipt_log(event) erc20_transfer = self.erc20_processor.filter_transfer_from_log(log) if erc20_transfer is not None: # self.exporter.export_item(self.erc20_transfer_mapper.erc20_transfer_to_dict(erc20_transfer)) erc20_transfer_dict = self.erc20_transfer_mapper.erc20_transfer_to_dict( erc20_transfer) print(erc20_transfer_dict) erc20_transfer_dict["erc20_value"] = str( erc20_transfer_dict["erc20_value"]) block = self.web3.eth.getBlock( erc20_transfer_dict["erc20_block_number"]) erc20_transfer_dict["timestamp"] = block.timestamp receipt = self.web3.eth.getTransactionReceipt( erc20_transfer_dict.get("erc20_tx_hash")) receipt_dict = self.erc20_transfer_mapper.erc20_receipt_transfer_to_dict( receipt) transaction = self.web3.eth.getTransaction( erc20_transfer_dict.get("erc20_tx_hash")) transaction_dict = self.erc20_transfer_mapper.transaction_to_dict( transaction) erc20_transfer_dict["gasUsed"] = receipt_dict.get("gasUsed") erc20_transfer_dict["gasPrice"] = transaction_dict.get( "gasPrice") erc20_transfer_dict["status"] = 0 # 0 fail if receipt_dict.get( "blockNumber") is not None and receipt_dict.get( "gasUsed") <= transaction_dict.get("gas"): erc20_transfer_dict["status"] = 1 # 1 success if erc20_transfers.find_one( {"erc20_tx_hash": erc20_transfer_dict['erc20_tx_hash'] }) is None: erc20_transfers.insert(erc20_transfer_dict) erc20_receipt.insert(receipt_dict) erc20_transaction.insert(transaction_dict) self.web3.eth.uninstallFilter(event_filter.filter_id) def _end(self): super()._end() close_silently(self.output_file) blockConfig = eth_config.find_one({'config_id': 1}) blockConfig["export_flag"] = False blockConfig["blockid"] = self.start_block eth_config.save(blockConfig)