class ExportBlocksJob(BatchExportJob):
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
super().__init__(start_block, end_block, batch_size, max_workers)
self.ipc_wrapper = ipc_wrapper
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def _start(self):
super()._start()
self.item_exporter.open()
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(batch_start, batch_end,
self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
super()._end()
self.item_exporter.close()
class ExportBlocksJob(BatchExportJob):
def __init__(
self,
start_block,
end_block,
batch_size,
ipc_wrapper,
web3,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
# if not self.export_blocks and not self.export_transactions:
# raise ValueError('Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
self.start_block = start_block
def _start(self):
super()._start()
# self.blocks_output_file = get_file_handle(self.blocks_output, binary=True, create_parent_dirs=True)
# self.blocks_exporter = CsvItemExporter(
# self.blocks_output_file, fields_to_export=self.block_fields_to_export)
#
# self.transactions_output_file = get_file_handle(self.transactions_output, binary=True, create_parent_dirs=True)
# self.transactions_exporter = CsvItemExporter(
# self.transactions_output_file, fields_to_export=self.transaction_fields_to_export)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
# if self.export_blocks:
# self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block))
block_dict = self.block_mapper.block_to_dict(block)
if blocks_exporter.find_one({"block_hash": block_dict['block_hash']}) is None:
block_timestamp = block_dict["block_timestamp"]
blocks_exporter.insert(block_dict)
# if self.export_transactions:
# for tx in block.transactions:
# self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx))
for tx in block.transactions:
tx = self.transaction_mapper.transaction_to_dict(tx)
if transactions_exporter.find_one({"tx_block_hash": tx['tx_block_hash']}) is None:
tx["timestamp"] = block_timestamp
receipt = self.web3.eth.getTransactionReceipt(tx.get("tx_hash"))
gasUsed = receipt.gasUsed
tx["status"] = 0
if int(gasUsed) <= int(tx.get("tx_gas")):
tx["status"] = 1 # 1 success
print(tx)
transactions_exporter.insert(tx)
def _end(self):
super()._end()
# close_silently(self.blocks_output_file)
# close_silently(self.transactions_output_file)
blockConfig = eth_config.find_one({'config_id': 1})
blockConfig["export_flag"] = False
blockConfig["blockid"] = self.start_block
eth_config.save(blockConfig)
class ExportBlocksJob(BatchExportJob):
def __init__(
self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError('Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
def _start(self):
super()._start()
self.blocks_output_file = get_file_handle(self.blocks_output, binary=True)
self.blocks_exporter = CsvItemExporter(
self.blocks_output_file, fields_to_export=self.block_fields_to_export)
self.transactions_output_file = get_file_handle(self.transactions_output, binary=True)
self.transactions_exporter = CsvItemExporter(
self.transactions_output_file, fields_to_export=self.transaction_fields_to_export)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
super()._end()
close_silently(self.blocks_output_file)
close_silently(self.transactions_output_file)
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(block_number_batch,
self.export_transactions))
response = self.batch_web3_provider.make_request(
json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
print("block=", self.block_mapper.block_to_dict(block))
print("number=", self.block_mapper.block_to_dict(block)['number'])
if self.export_transactions:
for tx in block.transactions:
# print(self.transaction_mapper.transaction_to_dict(tx))
transactiondict = self.transaction_mapper.transaction_to_dict(
tx)
transactiondict['timestamp'] = self.block_mapper.block_to_dict(
block)['timestamp']
self.item_exporter.export_item(transactiondict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.ipc_wrapper = ipc_wrapper
self.max_workers = max_workers
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
self.executor: FailSafeExecutor = None
def _start(self):
# Using bounded executor prevents unlimited queue growth
# and allows monitoring in-progress futures and failing fast in case of errors.
self.executor = FailSafeExecutor(BoundedExecutor(1, self.max_workers))
self.blocks_output_file = get_file_handle(self.blocks_output,
binary=True)
self.blocks_exporter = CsvItemExporter(
self.blocks_output_file,
fields_to_export=self.block_fields_to_export)
self.transactions_output_file = get_file_handle(
self.transactions_output, binary=True)
self.transactions_exporter = CsvItemExporter(
self.transactions_output_file,
fields_to_export=self.transaction_fields_to_export)
def _export(self):
for batch_start, batch_end in split_to_batches(self.start_block,
self.end_block,
self.batch_size):
self.executor.submit(self._fail_safe_export_batch, batch_start,
batch_end)
def _fail_safe_export_batch(self, batch_start, batch_end):
try:
self._export_batch(batch_start, batch_end)
except (Timeout, OSError):
# try exporting blocks one by one
for block_number in range(batch_start, batch_end + 1):
self._export_batch(block_number, block_number)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(batch_start, batch_end,
self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.blocks_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.transactions_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.executor.shutdown()
close_silently(self.blocks_output_file)
close_silently(self.transactions_output_file)