def __init__( self, start_block, end_block, batch_size, ipc_wrapper, web3, max_workers=5, blocks_output=None, transactions_output=None, block_fields_to_export=BLOCK_FIELDS_TO_EXPORT, transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.ipc_wrapper = ipc_wrapper self.blocks_output = blocks_output self.transactions_output = transactions_output self.block_fields_to_export = block_fields_to_export self.transaction_fields_to_export = transaction_fields_to_export self.export_blocks = blocks_output is not None self.export_transactions = transactions_output is not None # if not self.export_blocks and not self.export_transactions: # raise ValueError('Either blocks_output or transactions_output must be provided') self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() self.blocks_output_file = None self.transactions_output_file = None self.blocks_exporter = None self.transactions_exporter = None self.start_block = start_block
def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter, export_blocks=True, export_transactions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper()
class EthBlockMapper(object): def __init__(self, transaction_mapper=None): if transaction_mapper is None: self.transaction_mapper = EthTransactionMapper() else: self.transaction_mapper = transaction_mapper def json_dict_to_block(self, json_dict): block = EthBlock() block.number = hex_to_dec(json_dict.get('number')) block.hash = json_dict.get('hash') block.parent_hash = json_dict.get('parentHash') block.nonce = json_dict.get('nonce') block.sha3_uncles = json_dict.get('sha3Uncles') block.logs_bloom = json_dict.get('logsBloom') block.transactions_root = json_dict.get('transactionsRoot') block.state_root = json_dict.get('stateRoot') block.receipts_root = json_dict.get('receiptsRoot') block.miner = to_normalized_address(json_dict.get('miner')) block.difficulty = hex_to_dec(json_dict.get('difficulty')) block.total_difficulty = hex_to_dec(json_dict.get('totalDifficulty')) block.size = hex_to_dec(json_dict.get('size')) block.extra_data = json_dict.get('extraData') block.gas_limit = hex_to_dec(json_dict.get('gasLimit')) block.gas_used = hex_to_dec(json_dict.get('gasUsed')) block.timestamp = hex_to_dec(json_dict.get('timestamp')) if 'transactions' in json_dict: block.transactions = [ self.transaction_mapper.json_dict_to_transaction( tx, block_timestamp=block.timestamp) for tx in json_dict['transactions'] if isinstance(tx, dict) ] block.transaction_count = len(json_dict['transactions']) return block def block_to_dict(self, block): return { 'type': 'block', 'number': block.number, 'hash': block.hash, 'parent_hash': block.parent_hash, 'nonce': block.nonce, 'sha3_uncles': block.sha3_uncles, 'logs_bloom': block.logs_bloom, 'transactions_root': block.transactions_root, 'state_root': block.state_root, 'receipts_root': block.receipts_root, 'miner': block.miner, 'difficulty': block.difficulty, 'total_difficulty': block.total_difficulty, 'size': block.size, 'extra_data': block.extra_data, 'gas_limit': block.gas_limit, 'gas_used': block.gas_used, 'timestamp': block.timestamp, 'transaction_count': block.transaction_count, }
class EthBlockMapper(object): def __init__(self, transaction_mapper: EthTransactionMapper = None): if transaction_mapper is None: self.transaction_mapper = EthTransactionMapper() else: self.transaction_mapper = transaction_mapper def json_dict_to_block(self, json_dict) -> EthBlock: block = EthBlock() block.number = hex_to_dec(json_dict.get('number', None)) block.hash = json_dict.get('hash', None) block.parent_hash = json_dict.get('parentHash', None) block.nonce = json_dict.get('nonce', None) block.sha3_uncles = json_dict.get('sha3Uncles', None) block.logs_bloom = json_dict.get('logsBloom', None) block.transactions_root = json_dict.get('transactionsRoot', None) block.state_root = json_dict.get('stateRoot', None) block.miner = json_dict.get('miner', None) block.difficulty = hex_to_dec(json_dict.get('difficulty', None)) block.total_difficulty = hex_to_dec( json_dict.get('totalDifficulty', None)) block.size = hex_to_dec(json_dict.get('size', None)) block.extra_data = json_dict.get('extraData', None) block.gas_limit = hex_to_dec(json_dict.get('gasLimit', None)) block.gas_used = hex_to_dec(json_dict.get('gasUsed', None)) block.timestamp = hex_to_dec(json_dict.get('timestamp', None)) if 'transactions' in json_dict: block.transactions = [ self.transaction_mapper.json_dict_to_transaction(tx) for tx in json_dict['transactions'] if isinstance(tx, dict) ] block.transaction_count = len(json_dict['transactions']) return block def block_to_dict(self, block: EthBlock): return { 'block_number': block.number, 'block_hash': block.hash, 'block_parent_hash': block.parent_hash, 'block_nonce': block.nonce, 'block_sha3_uncles': block.sha3_uncles, 'block_logs_bloom': block.logs_bloom, 'block_transactions_root': block.transactions_root, 'block_state_root': block.state_root, 'block_miner': block.miner, 'block_difficulty': block.difficulty, 'block_total_difficulty': block.total_difficulty, 'block_size': block.size, 'block_extra_data': block.extra_data, 'block_gas_limit': block.gas_limit, 'block_gas_used': block.gas_used, 'block_timestamp': block.timestamp, 'block_transaction_count': block.transaction_count, }
class ExportBlocksJob(BatchExportJob): def __init__(self, start_block, end_block, batch_size, ipc_wrapper, max_workers, item_exporter, export_blocks=True, export_transactions=True): super().__init__(start_block, end_block, batch_size, max_workers) self.ipc_wrapper = ipc_wrapper self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() def _start(self): super()._start() self.item_exporter.open() def _export_batch(self, batch_start, batch_end): blocks_rpc = list( generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions)) response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc)) results = rpc_response_batch_to_results(response) blocks = [ self.block_mapper.json_dict_to_block(result) for result in results ] for block in blocks: self._export_block(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item( self.block_mapper.block_to_dict(block)) if self.export_transactions: for tx in block.transactions: self.item_exporter.export_item( self.transaction_mapper.transaction_to_dict(tx)) def _end(self): super()._end() self.item_exporter.close()
def __init__(self, start_block, end_block, batch_size, ipc_wrapper, max_workers, item_exporter, export_blocks=True, export_transactions=True): super().__init__(start_block, end_block, batch_size, max_workers) self.ipc_wrapper = ipc_wrapper self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper()
def __init__(self, start_block, end_block, batch_size, web3, output, max_workers=5, tokens=None, fields_to_export=FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.output = output self.tokens = tokens self.fields_to_export = fields_to_export self.receipt_log_mapper = EthReceiptLogMapper() self.erc20_transfer_mapper = EthErc20TransferMapper() self.erc20_processor = EthErc20Processor() self.ethTransactionMapper = EthTransactionMapper() self.output_file = None # self.exporter = None self.start_block = start_block
class ExportBlocksJob(BatchExportJob): def __init__( self, start_block, end_block, batch_size, ipc_wrapper, web3, max_workers=5, blocks_output=None, transactions_output=None, block_fields_to_export=BLOCK_FIELDS_TO_EXPORT, transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.web3 = web3 self.ipc_wrapper = ipc_wrapper self.blocks_output = blocks_output self.transactions_output = transactions_output self.block_fields_to_export = block_fields_to_export self.transaction_fields_to_export = transaction_fields_to_export self.export_blocks = blocks_output is not None self.export_transactions = transactions_output is not None # if not self.export_blocks and not self.export_transactions: # raise ValueError('Either blocks_output or transactions_output must be provided') self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() self.blocks_output_file = None self.transactions_output_file = None self.blocks_exporter = None self.transactions_exporter = None self.start_block = start_block def _start(self): super()._start() # self.blocks_output_file = get_file_handle(self.blocks_output, binary=True, create_parent_dirs=True) # self.blocks_exporter = CsvItemExporter( # self.blocks_output_file, fields_to_export=self.block_fields_to_export) # # self.transactions_output_file = get_file_handle(self.transactions_output, binary=True, create_parent_dirs=True) # self.transactions_exporter = CsvItemExporter( # self.transactions_output_file, fields_to_export=self.transaction_fields_to_export) def _export_batch(self, batch_start, batch_end): blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions)) response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc)) for response_item in response: result = response_item['result'] block = self.block_mapper.json_dict_to_block(result) self._export_block(block) def _export_block(self, block): # if self.export_blocks: # self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block)) block_dict = self.block_mapper.block_to_dict(block) if blocks_exporter.find_one({"block_hash": block_dict['block_hash']}) is None: block_timestamp = block_dict["block_timestamp"] blocks_exporter.insert(block_dict) # if self.export_transactions: # for tx in block.transactions: # self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx)) for tx in block.transactions: tx = self.transaction_mapper.transaction_to_dict(tx) if transactions_exporter.find_one({"tx_block_hash": tx['tx_block_hash']}) is None: tx["timestamp"] = block_timestamp receipt = self.web3.eth.getTransactionReceipt(tx.get("tx_hash")) gasUsed = receipt.gasUsed tx["status"] = 0 if int(gasUsed) <= int(tx.get("tx_gas")): tx["status"] = 1 # 1 success print(tx) transactions_exporter.insert(tx) def _end(self): super()._end() # close_silently(self.blocks_output_file) # close_silently(self.transactions_output_file) blockConfig = eth_config.find_one({'config_id': 1}) blockConfig["export_flag"] = False blockConfig["blockid"] = self.start_block eth_config.save(blockConfig)
def __init__(self, transaction_mapper=None): if transaction_mapper is None: self.transaction_mapper = EthTransactionMapper() else: self.transaction_mapper = transaction_mapper
class ExportBlocksJob(BatchExportJob): def __init__( self, start_block, end_block, batch_size, ipc_wrapper, max_workers=5, blocks_output=None, transactions_output=None, block_fields_to_export=BLOCK_FIELDS_TO_EXPORT, transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT): super().__init__(start_block, end_block, batch_size, max_workers) self.ipc_wrapper = ipc_wrapper self.blocks_output = blocks_output self.transactions_output = transactions_output self.block_fields_to_export = block_fields_to_export self.transaction_fields_to_export = transaction_fields_to_export self.export_blocks = blocks_output is not None self.export_transactions = transactions_output is not None if not self.export_blocks and not self.export_transactions: raise ValueError('Either blocks_output or transactions_output must be provided') self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() self.blocks_output_file = None self.transactions_output_file = None self.blocks_exporter = None self.transactions_exporter = None def _start(self): super()._start() self.blocks_output_file = get_file_handle(self.blocks_output, binary=True) self.blocks_exporter = CsvItemExporter( self.blocks_output_file, fields_to_export=self.block_fields_to_export) self.transactions_output_file = get_file_handle(self.transactions_output, binary=True) self.transactions_exporter = CsvItemExporter( self.transactions_output_file, fields_to_export=self.transaction_fields_to_export) def _export_batch(self, batch_start, batch_end): blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions)) response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc)) for response_item in response: result = response_item['result'] block = self.block_mapper.json_dict_to_block(result) self._export_block(block) def _export_block(self, block): if self.export_blocks: self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block)) if self.export_transactions: for tx in block.transactions: self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx)) def _end(self): super()._end() close_silently(self.blocks_output_file) close_silently(self.transactions_output_file)
class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, batch_web3_provider, max_workers, item_exporter, export_blocks=True, export_transactions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_web3_provider = batch_web3_provider self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): blocks_rpc = list( generate_get_block_by_number_json_rpc(block_number_batch, self.export_transactions)) response = self.batch_web3_provider.make_request( json.dumps(blocks_rpc)) results = rpc_response_batch_to_results(response) blocks = [ self.block_mapper.json_dict_to_block(result) for result in results ] for block in blocks: self._export_block(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item( self.block_mapper.block_to_dict(block)) print("block=", self.block_mapper.block_to_dict(block)) print("number=", self.block_mapper.block_to_dict(block)['number']) if self.export_transactions: for tx in block.transactions: # print(self.transaction_mapper.transaction_to_dict(tx)) transactiondict = self.transaction_mapper.transaction_to_dict( tx) transactiondict['timestamp'] = self.block_mapper.block_to_dict( block)['timestamp'] self.item_exporter.export_item(transactiondict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, ipc_wrapper, max_workers=5, blocks_output=None, transactions_output=None, block_fields_to_export=BLOCK_FIELDS_TO_EXPORT, transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT): self.start_block = start_block self.end_block = end_block self.batch_size = batch_size self.ipc_wrapper = ipc_wrapper self.max_workers = max_workers self.blocks_output = blocks_output self.transactions_output = transactions_output self.block_fields_to_export = block_fields_to_export self.transaction_fields_to_export = transaction_fields_to_export self.export_blocks = blocks_output is not None self.export_transactions = transactions_output is not None if not self.export_blocks and not self.export_transactions: raise ValueError( 'Either blocks_output or transactions_output must be provided') self.block_mapper = EthBlockMapper() self.transaction_mapper = EthTransactionMapper() self.blocks_output_file = None self.transactions_output_file = None self.blocks_exporter = None self.transactions_exporter = None self.executor: FailSafeExecutor = None def _start(self): # Using bounded executor prevents unlimited queue growth # and allows monitoring in-progress futures and failing fast in case of errors. self.executor = FailSafeExecutor(BoundedExecutor(1, self.max_workers)) self.blocks_output_file = get_file_handle(self.blocks_output, binary=True) self.blocks_exporter = CsvItemExporter( self.blocks_output_file, fields_to_export=self.block_fields_to_export) self.transactions_output_file = get_file_handle( self.transactions_output, binary=True) self.transactions_exporter = CsvItemExporter( self.transactions_output_file, fields_to_export=self.transaction_fields_to_export) def _export(self): for batch_start, batch_end in split_to_batches(self.start_block, self.end_block, self.batch_size): self.executor.submit(self._fail_safe_export_batch, batch_start, batch_end) def _fail_safe_export_batch(self, batch_start, batch_end): try: self._export_batch(batch_start, batch_end) except (Timeout, OSError): # try exporting blocks one by one for block_number in range(batch_start, batch_end + 1): self._export_batch(block_number, block_number) def _export_batch(self, batch_start, batch_end): blocks_rpc = list( generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions)) response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc)) for response_item in response: result = response_item['result'] block = self.block_mapper.json_dict_to_block(result) self._export_block(block) def _export_block(self, block): if self.export_blocks: self.blocks_exporter.export_item( self.block_mapper.block_to_dict(block)) if self.export_transactions: for tx in block.transactions: self.transactions_exporter.export_item( self.transaction_mapper.transaction_to_dict(tx)) def _end(self): self.executor.shutdown() close_silently(self.blocks_output_file) close_silently(self.transactions_output_file)