def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
blocks_output=None,
transactions_output=None):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
class EthBlockMapper(object):
transaction_mapper = EthTransactionMapper()
def json_dict_to_block(self, json_dict):
# type: ({}) -> EthBlock
block = EthBlock()
block.number = hex_to_dec(json_dict.get('number', None))
block.hash = json_dict.get('hash', None)
block.parent_hash = json_dict.get('parentHash', None)
block.nonce = json_dict.get('nonce', None)
block.sha3_uncles = json_dict.get('sha3Uncles', None)
block.logs_bloom = json_dict.get('logsBloom', None)
block.transactions_root = json_dict.get('transactionsRoot', None)
block.state_root = json_dict.get('stateRoot', None)
block.miner = to_checksum_address(json_dict.get('miner', None))
block.difficulty = hex_to_dec(json_dict.get('difficulty', None))
block.total_difficulty = hex_to_dec(
json_dict.get('totalDifficulty', None))
block.size = hex_to_dec(json_dict.get('size', None))
block.extra_data = json_dict.get('extraData', None)
block.gas_limit = hex_to_dec(json_dict.get('gasLimit', None))
block.gas_used = hex_to_dec(json_dict.get('gasUsed', None))
block.timestamp = hex_to_dec(json_dict.get('timestamp', None))
if 'transactions' in json_dict:
block.transactions = [
self.transaction_mapper.json_dict_to_transaction(tx)
for tx in json_dict['transactions'] if isinstance(tx, dict)
]
block.transaction_count = len(json_dict['transactions'])
return block
def block_to_dict(self, block):
# type: (EthBlock) -> {}
return {
'block_number': block.number,
'block_hash': block.hash,
'block_parent_hash': block.parent_hash,
'block_nonce': block.nonce,
'block_sha3_uncles': block.sha3_uncles,
'block_logs_bloom': block.logs_bloom,
'block_transactions_root': block.transactions_root,
'block_state_root': block.state_root,
'block_miner': block.miner,
'block_difficulty': block.difficulty,
'block_total_difficulty': block.total_difficulty,
'block_size': block.size,
'block_extra_data': block.extra_data,
'block_gas_limit': block.gas_limit,
'block_gas_used': block.gas_used,
'block_timestamp': block.timestamp,
'block_transaction_count': block.transaction_count,
}
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
ipc_wrapper,
blocks_output=None,
transactions_output=None):
self.start_block = start_block
self.end_block = end_block
self.batch_size = batch_size
self.ipc_wrapper = ipc_wrapper
self.blocks_output = blocks_output
self.transactions_output = transactions_output
self.export_blocks = blocks_output is not None
self.export_transactions = transactions_output is not None
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'Either blocks_output or transactions_output must be provided')
self.block_mapper = EthBlockMapper()
self.transaction_mapper = EthTransactionMapper()
self.blocks_output_file = None
self.transactions_output_file = None
self.blocks_exporter = None
self.transactions_exporter = None
def _start(self):
self.blocks_output_file = get_file_handle(self.blocks_output,
binary=True)
self.transactions_output_file = get_file_handle(
self.transactions_output, binary=True)
self.blocks_exporter = CsvItemExporter(self.blocks_output_file)
self.transactions_exporter = CsvItemExporter(
self.transactions_output_file)
def _export(self):
for batch_start, batch_end in split_to_batches(self.start_block,
self.end_block,
self.batch_size):
try:
self._export_batch(batch_start, batch_end)
except (Timeout, SocketTimeoutException):
# try exporting blocks one by one
for block_number in range(batch_start, batch_end + 1):
self._export_batch(block_number, block_number)
def _export_batch(self, batch_start, batch_end):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(batch_start, batch_end,
self.export_transactions))
response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
for response_item in response:
result = response_item['result']
block = self.block_mapper.json_dict_to_block(result)
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.blocks_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.transactions_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
if self.blocks_output_file is not None:
self.blocks_output_file.close()
if self.transactions_output_file is not None:
self.transactions_output_file.close()
default=True,
type=bool,
help='Whether or not to extract transactions.')
parser.add_argument(
'--transactions-output',
default=None,
type=str,
help='The output file for transactions. If not specified stdout is used.')
args = parser.parse_args()
with smart_open(args.input, 'r') as input_file, \
smart_open(args.blocks_output, binary=True) if args.extract_blocks else None as blocks_output_file , \
smart_open(args.transactions_output, binary=True) if args.extract_transactions else None as tx_output_file:
block_mapper = EthBlockMapper()
tx_mapper = EthTransactionMapper()
blocks_exporter = CsvItemExporter(
blocks_output_file) if blocks_output_file is not None else None
tx_exporter = CsvItemExporter(
tx_output_file) if tx_output_file is not None else None
for line in input_file:
json_line = json.loads(line)
result = json_line.get('result', None)
if result is None:
continue
block = block_mapper.json_dict_to_block(result)
if blocks_exporter is not None:
blocks_exporter.export_item(block_mapper.block_to_dict(block))
parser = argparse.ArgumentParser(
description='Extract blocks and transactions from eth_getBlockByNumber JSON RPC output')
parser.add_argument('--input', default=None, type=str, help='The input file. If not specified stdin is used.')
parser.add_argument('--blocks-output', default=None, type=str,
help='The output file for blocks. If not specified stdout is used.')
parser.add_argument('--transactions-output', default=None, type=str,
help='The output file for transactions. If not specified stdout is used.')
args = parser.parse_args()
with smart_open(args.input, 'r') as input_file, \
smart_open(args.blocks_output, binary=True) as blocks_output_file, \
smart_open(args.transactions_output, binary=True) as tx_output_file:
block_mapper = EthBlockMapper()
tx_mapper = EthTransactionMapper()
blocks_exporter = CsvItemExporter(blocks_output_file)
tx_exporter = CsvItemExporter(tx_output_file)
for line in input_file:
json_line = json.loads(line)
result = json_line.get('result', None)
if result is None:
continue
block = block_mapper.json_dict_to_block(result)
blocks_exporter.export_item(block_mapper.block_to_dict(block))
if block.transactions is not None:
for transaction in block.transactions:
tx_exporter.export_item(tx_mapper.transaction_to_dict(transaction))