def __init__(
self,
start_block,
end_block,
batch_size,
ipc_wrapper,
web3,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
# if not self.export_blocks and not self.export_transactions:
# raise ValueError('Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
self.start_block = start_block
def __init__(self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
class EthBlockMapper(object):
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = EthTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def json_dict_to_block(self, json_dict):
block = EthBlock()
block.number = hex_to_dec(json_dict.get('number'))
block.hash = json_dict.get('hash')
block.parent_hash = json_dict.get('parentHash')
block.nonce = json_dict.get('nonce')
block.sha3_uncles = json_dict.get('sha3Uncles')
block.logs_bloom = json_dict.get('logsBloom')
block.transactions_root = json_dict.get('transactionsRoot')
block.state_root = json_dict.get('stateRoot')
block.receipts_root = json_dict.get('receiptsRoot')
block.miner = to_normalized_address(json_dict.get('miner'))
block.difficulty = hex_to_dec(json_dict.get('difficulty'))
block.total_difficulty = hex_to_dec(json_dict.get('totalDifficulty'))
block.size = hex_to_dec(json_dict.get('size'))
block.extra_data = json_dict.get('extraData')
block.gas_limit = hex_to_dec(json_dict.get('gasLimit'))
block.gas_used = hex_to_dec(json_dict.get('gasUsed'))
block.timestamp = hex_to_dec(json_dict.get('timestamp'))
if 'transactions' in json_dict:
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(
tx, block_timestamp=block.timestamp)
for tx in json_dict['transactions'] if isinstance(tx, dict)
]
block.transaction_count = len(json_dict['transactions'])
return block
def block_to_dict(self, block):
return {
'type': 'block',
'number': block.number,
'hash': block.hash,
'parent_hash': block.parent_hash,
'nonce': block.nonce,
'sha3_uncles': block.sha3_uncles,
'logs_bloom': block.logs_bloom,
'transactions_root': block.transactions_root,
'state_root': block.state_root,
'receipts_root': block.receipts_root,
'miner': block.miner,
'difficulty': block.difficulty,
'total_difficulty': block.total_difficulty,
'size': block.size,
'extra_data': block.extra_data,
'gas_limit': block.gas_limit,
'gas_used': block.gas_used,
'timestamp': block.timestamp,
'transaction_count': block.transaction_count,
}
class EthBlockMapper(object):
def __init__(self, transaction_mapper: EthTransactionMapper = None):
if transaction_mapper is None:
self.transaction_mapper = EthTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
def json_dict_to_block(self, json_dict) -> EthBlock:
block = EthBlock()
block.number = hex_to_dec(json_dict.get('number', None))
block.hash = json_dict.get('hash', None)
block.parent_hash = json_dict.get('parentHash', None)
block.nonce = json_dict.get('nonce', None)
block.sha3_uncles = json_dict.get('sha3Uncles', None)
block.logs_bloom = json_dict.get('logsBloom', None)
block.transactions_root = json_dict.get('transactionsRoot', None)
block.state_root = json_dict.get('stateRoot', None)
block.miner = json_dict.get('miner', None)
block.difficulty = hex_to_dec(json_dict.get('difficulty', None))
block.total_difficulty = hex_to_dec(
json_dict.get('totalDifficulty', None))
block.size = hex_to_dec(json_dict.get('size', None))
block.extra_data = json_dict.get('extraData', None)
block.gas_limit = hex_to_dec(json_dict.get('gasLimit', None))
block.gas_used = hex_to_dec(json_dict.get('gasUsed', None))
block.timestamp = hex_to_dec(json_dict.get('timestamp', None))
if 'transactions' in json_dict:
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(tx)
for tx in json_dict['transactions'] if isinstance(tx, dict)
]
block.transaction_count = len(json_dict['transactions'])
return block
def block_to_dict(self, block: EthBlock):
return {
'block_number': block.number,
'block_hash': block.hash,
'block_parent_hash': block.parent_hash,
'block_nonce': block.nonce,
'block_sha3_uncles': block.sha3_uncles,
'block_logs_bloom': block.logs_bloom,
'block_transactions_root': block.transactions_root,
'block_state_root': block.state_root,
'block_miner': block.miner,
'block_difficulty': block.difficulty,
'block_total_difficulty': block.total_difficulty,
'block_size': block.size,
'block_extra_data': block.extra_data,
'block_gas_limit': block.gas_limit,
'block_gas_used': block.gas_used,
'block_timestamp': block.timestamp,
'block_transaction_count': block.transaction_count,
}
class ExportBlocksJob(BatchExportJob):
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
super().__init__(start_block, end_block, batch_size, max_workers)
self.ipc_wrapper = ipc_wrapper
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def _start(self):
super()._start()
self.item_exporter.open()
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(batch_start, batch_end,
self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
super()._end()
self.item_exporter.close()
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
super().__init__(start_block, end_block, batch_size, max_workers)
self.ipc_wrapper = ipc_wrapper
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def __init__(self,
start_block,
end_block,
batch_size,
web3,
output,
max_workers=5,
tokens=None,
fields_to_export=FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.output = output
self.tokens = tokens
self.fields_to_export = fields_to_export
self.receipt_log_mapper = EthReceiptLogMapper()
self.erc20_transfer_mapper = EthErc20TransferMapper()
self.erc20_processor = EthErc20Processor()
self.ethTransactionMapper = EthTransactionMapper()
self.output_file = None
# self.exporter = None
self.start_block = start_block
class ExportBlocksJob(BatchExportJob):
def __init__(
self,
start_block,
end_block,
batch_size,
ipc_wrapper,
web3,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.web3 = web3
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
# if not self.export_blocks and not self.export_transactions:
# raise ValueError('Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
self.start_block = start_block
def _start(self):
super()._start()
# self.blocks_output_file = get_file_handle(self.blocks_output, binary=True, create_parent_dirs=True)
# self.blocks_exporter = CsvItemExporter(
# self.blocks_output_file, fields_to_export=self.block_fields_to_export)
#
# self.transactions_output_file = get_file_handle(self.transactions_output, binary=True, create_parent_dirs=True)
# self.transactions_exporter = CsvItemExporter(
# self.transactions_output_file, fields_to_export=self.transaction_fields_to_export)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
# if self.export_blocks:
# self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block))
block_dict = self.block_mapper.block_to_dict(block)
if blocks_exporter.find_one({"block_hash": block_dict['block_hash']}) is None:
block_timestamp = block_dict["block_timestamp"]
blocks_exporter.insert(block_dict)
# if self.export_transactions:
# for tx in block.transactions:
# self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx))
for tx in block.transactions:
tx = self.transaction_mapper.transaction_to_dict(tx)
if transactions_exporter.find_one({"tx_block_hash": tx['tx_block_hash']}) is None:
tx["timestamp"] = block_timestamp
receipt = self.web3.eth.getTransactionReceipt(tx.get("tx_hash"))
gasUsed = receipt.gasUsed
tx["status"] = 0
if int(gasUsed) <= int(tx.get("tx_gas")):
tx["status"] = 1 # 1 success
print(tx)
transactions_exporter.insert(tx)
def _end(self):
super()._end()
# close_silently(self.blocks_output_file)
# close_silently(self.transactions_output_file)
blockConfig = eth_config.find_one({'config_id': 1})
blockConfig["export_flag"] = False
blockConfig["blockid"] = self.start_block
eth_config.save(blockConfig)
def __init__(self, transaction_mapper=None):
if transaction_mapper is None:
self.transaction_mapper = EthTransactionMapper()
else:
self.transaction_mapper = transaction_mapper
class ExportBlocksJob(BatchExportJob):
def __init__(
self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
super().__init__(start_block, end_block, batch_size, max_workers)
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError('Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
def _start(self):
super()._start()
self.blocks_output_file = get_file_handle(self.blocks_output, binary=True)
self.blocks_exporter = CsvItemExporter(
self.blocks_output_file, fields_to_export=self.block_fields_to_export)
self.transactions_output_file = get_file_handle(self.transactions_output, binary=True)
self.transactions_exporter = CsvItemExporter(
self.transactions_output_file, fields_to_export=self.transaction_fields_to_export)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.blocks_exporter.export_item(self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.transactions_exporter.export_item(self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
super()._end()
close_silently(self.blocks_output_file)
close_silently(self.transactions_output_file)
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(block_number_batch,
self.export_transactions))
response = self.batch_web3_provider.make_request(
json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
print("block=", self.block_mapper.block_to_dict(block))
print("number=", self.block_mapper.block_to_dict(block)['number'])
if self.export_transactions:
for tx in block.transactions:
# print(self.transaction_mapper.transaction_to_dict(tx))
transactiondict = self.transaction_mapper.transaction_to_dict(
tx)
transactiondict['timestamp'] = self.block_mapper.block_to_dict(
block)['timestamp']
self.item_exporter.export_item(transactiondict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
max_workers=5,
blocks_output=None,
transactions_output=None,
block_fields_to_export=BLOCK_FIELDS_TO_EXPORT,
transaction_fields_to_export=TRANSACTION_FIELDS_TO_EXPORT):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.ipc_wrapper = ipc_wrapper
self.max_workers = max_workers
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.block_fields_to_export = block_fields_to_export
self.transaction_fields_to_export = transaction_fields_to_export
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
self.executor: FailSafeExecutor = None
def _start(self):
# Using bounded executor prevents unlimited queue growth
# and allows monitoring in-progress futures and failing fast in case of errors.
self.executor = FailSafeExecutor(BoundedExecutor(1, self.max_workers))
self.blocks_output_file = get_file_handle(self.blocks_output,
binary=True)
self.blocks_exporter = CsvItemExporter(
self.blocks_output_file,
fields_to_export=self.block_fields_to_export)
self.transactions_output_file = get_file_handle(
self.transactions_output, binary=True)
self.transactions_exporter = CsvItemExporter(
self.transactions_output_file,
fields_to_export=self.transaction_fields_to_export)
def _export(self):
for batch_start, batch_end in split_to_batches(self.start_block,
self.end_block,
self.batch_size):
self.executor.submit(self._fail_safe_export_batch, batch_start,
batch_end)
def _fail_safe_export_batch(self, batch_start, batch_end):
try:
self._export_batch(batch_start, batch_end)
except (Timeout, OSError):
# try exporting blocks one by one
for block_number in range(batch_start, batch_end + 1):
self._export_batch(block_number, block_number)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(batch_start, batch_end,
self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.blocks_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.transactions_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.executor.shutdown()
close_silently(self.blocks_output_file)
close_silently(self.transactions_output_file)