import json import argparse from builtins import str from ethereumetl.utils import smart_open parser = argparse.ArgumentParser( description='Generate Ethereum eth_getBlockByNumber JSON RPC call inputs for a block range') parser.add_argument('--start-block', default=0, type=int, help='Start block') parser.add_argument('--end-block', required=True, type=int, help='End block') parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.') args = parser.parse_args() def generate_get_block_by_number_json_rpc(start_block, end_block): for block_number in range(start_block, end_block + 1): yield { 'jsonrpc': '2.0', 'method': 'eth_getBlockByNumber', 'params': [hex(block_number), True], 'id': 1, } with smart_open(args.output) as output_file: for data in generate_get_block_by_number_json_rpc(args.start_block, args.end_block): output_file.write(json.dumps(data) + '\n')
from ethereumetl.service.erc20_processor import EthErc20Processor, TRANSFER_EVENT_TOPIC from ethereumetl.utils import smart_open parser = argparse.ArgumentParser( description='Exports ERC20 transfers using eth_newFilter and eth_getFilterLogs JSON RPC APIs.') parser.add_argument('--start-block', default=0, type=int, help='Start block') parser.add_argument('--end-block', required=True, type=int, help='End block') parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.') parser.add_argument('--ipc-path', required=True, type=str, help='The full path to the ipc socket file.') parser.add_argument('--ipc-timeout', default=300, type=int, help='The timeout in seconds for ipc calls.') parser.add_argument('--batch-size', default=100, type=int, help='The number of blocks to filter at a time.') args = parser.parse_args() with smart_open(args.output, binary=True) as output_file: transaction_receipt_log_mapper = EthTransactionReceiptLogMapper() erc20_transfer_mapper = EthErc20TransferMapper() erc20_processor = EthErc20Processor() exporter = CsvItemExporter(output_file) web3 = Web3(IPCProvider(args.ipc_path, timeout=args.ipc_timeout)) for batch_start_block in range(args.start_block, args.end_block + 1, args.batch_size): batch_end_block = min(batch_start_block + args.batch_size - 1, args.end_block) event_filter = web3.eth.filter({ "fromBlock": batch_start_block, "toBlock": batch_end_block, "topics": [TRANSFER_EVENT_TOPIC] })
default=None, type=str, help='The output file for blocks. If not specified stdout is used.') parser.add_argument('--extract-transactions', default=True, type=bool, help='Whether or not to extract transactions.') parser.add_argument( '--transactions-output', default=None, type=str, help='The output file for transactions. If not specified stdout is used.') args = parser.parse_args() with smart_open(args.input, 'r') as input_file, \ smart_open(args.blocks_output, binary=True) if args.extract_blocks else None as blocks_output_file , \ smart_open(args.transactions_output, binary=True) if args.extract_transactions else None as tx_output_file: block_mapper = EthBlockMapper() tx_mapper = EthTransactionMapper() blocks_exporter = CsvItemExporter( blocks_output_file) if blocks_output_file is not None else None tx_exporter = CsvItemExporter( tx_output_file) if tx_output_file is not None else None for line in input_file: json_line = json.loads(line) result = json_line.get('result', None) if result is None: continue
from ethereumetl.utils import smart_open parser = argparse.ArgumentParser( description='Extract blocks from eth_getBlockByNumber JSON RPC output') parser.add_argument('--input', default=None, type=str, help='The input file. If not specified stdin is used.') parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.') args = parser.parse_args() with smart_open(args.input, 'r') as input_file, smart_open(args.output, binary=True) as output_file: block_mapper = EthBlockMapper() exporter = CsvItemExporter(output_file) exporter.start_exporting() for line in input_file: json_line = json.loads(line) result = json_line.get('result', None) if result is None: continue block = block_mapper.json_dict_to_block(result) exporter.export_item(block_mapper.block_to_dict(block)) exporter.finish_exporting()
from ethereumetl.exporters import CsvItemExporter from ethereumetl.mapper.block_mapper import EthBlockMapper from ethereumetl.mapper.transaction_mapper import EthTransactionMapper from ethereumetl.utils import smart_open parser = argparse.ArgumentParser( description='Extract blocks and transactions from eth_getBlockByNumber JSON RPC output') parser.add_argument('--input', default=None, type=str, help='The input file. If not specified stdin is used.') parser.add_argument('--blocks-output', default=None, type=str, help='The output file for blocks. If not specified stdout is used.') parser.add_argument('--transactions-output', default=None, type=str, help='The output file for transactions. If not specified stdout is used.') args = parser.parse_args() with smart_open(args.input, 'r') as input_file, \ smart_open(args.blocks_output, binary=True) as blocks_output_file, \ smart_open(args.transactions_output, binary=True) as tx_output_file: block_mapper = EthBlockMapper() tx_mapper = EthTransactionMapper() blocks_exporter = CsvItemExporter(blocks_output_file) tx_exporter = CsvItemExporter(tx_output_file) for line in input_file: json_line = json.loads(line) result = json_line.get('result', None) if result is None: continue block = block_mapper.json_dict_to_block(result) blocks_exporter.export_item(block_mapper.block_to_dict(block))
'Sends the given input to the ipc socket and outputs its response') parser.add_argument('--input', default=None, type=str, help='The input file. If not specified stdin is used.') parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.') parser.add_argument('--ipc-path', required=True, type=str, help='The full path to the ipc socket file.') parser.add_argument('--ipc-timeout', default=10, type=int, help='The timeout for ipc calls.') parser.add_argument('--batch-size', default=100, type=int, help='The number of lines in the batch to send to socket.') args = parser.parse_args() with smart_open(args.input, 'r') as input_file, smart_open(args.output) as output_file: for line_batch in batch_readlines(input_file, args.batch_size): response = socket_exchange(args.ipc_path, ''.join(line_batch), args.ipc_timeout) output_file.write(response)