import json
import argparse
from builtins import str
from ethereumetl.utils import smart_open
parser = argparse.ArgumentParser(
description='Generate Ethereum eth_getBlockByNumber JSON RPC call inputs for a block range')
parser.add_argument('--start-block', default=0, type=int, help='Start block')
parser.add_argument('--end-block', required=True, type=int, help='End block')
parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.')
args = parser.parse_args()
def generate_get_block_by_number_json_rpc(start_block, end_block):
for block_number in range(start_block, end_block + 1):
yield {
'jsonrpc': '2.0',
'method': 'eth_getBlockByNumber',
'params': [hex(block_number), True],
'id': 1,
}
with smart_open(args.output) as output_file:
for data in generate_get_block_by_number_json_rpc(args.start_block, args.end_block):
output_file.write(json.dumps(data) + '\n')
from ethereumetl.service.erc20_processor import EthErc20Processor, TRANSFER_EVENT_TOPIC
from ethereumetl.utils import smart_open
parser = argparse.ArgumentParser(
description='Exports ERC20 transfers using eth_newFilter and eth_getFilterLogs JSON RPC APIs.')
parser.add_argument('--start-block', default=0, type=int, help='Start block')
parser.add_argument('--end-block', required=True, type=int, help='End block')
parser.add_argument('--output', default=None, type=str, help='The output file. If not specified stdout is used.')
parser.add_argument('--ipc-path', required=True, type=str, help='The full path to the ipc socket file.')
parser.add_argument('--ipc-timeout', default=300, type=int, help='The timeout in seconds for ipc calls.')
parser.add_argument('--batch-size', default=100, type=int, help='The number of blocks to filter at a time.')
args = parser.parse_args()
with smart_open(args.output, binary=True) as output_file:
transaction_receipt_log_mapper = EthTransactionReceiptLogMapper()
erc20_transfer_mapper = EthErc20TransferMapper()
erc20_processor = EthErc20Processor()
exporter = CsvItemExporter(output_file)
web3 = Web3(IPCProvider(args.ipc_path, timeout=args.ipc_timeout))
for batch_start_block in range(args.start_block, args.end_block + 1, args.batch_size):
batch_end_block = min(batch_start_block + args.batch_size - 1, args.end_block)
event_filter = web3.eth.filter({
"fromBlock": batch_start_block,
"toBlock": batch_end_block,
"topics": [TRANSFER_EVENT_TOPIC]
})
default=None,
type=str,
help='The output file for blocks. If not specified stdout is used.')
parser.add_argument('--extract-transactions',
default=True,
type=bool,
help='Whether or not to extract transactions.')
parser.add_argument(
'--transactions-output',
default=None,
type=str,
help='The output file for transactions. If not specified stdout is used.')
args = parser.parse_args()
with smart_open(args.input, 'r') as input_file, \
smart_open(args.blocks_output, binary=True) if args.extract_blocks else None as blocks_output_file , \
smart_open(args.transactions_output, binary=True) if args.extract_transactions else None as tx_output_file:
block_mapper = EthBlockMapper()
tx_mapper = EthTransactionMapper()
blocks_exporter = CsvItemExporter(
blocks_output_file) if blocks_output_file is not None else None
tx_exporter = CsvItemExporter(
tx_output_file) if tx_output_file is not None else None
for line in input_file:
json_line = json.loads(line)
result = json_line.get('result', None)
if result is None:
continue
from ethereumetl.utils import smart_open
parser = argparse.ArgumentParser(
description='Extract blocks from eth_getBlockByNumber JSON RPC output')
parser.add_argument('--input',
default=None,
type=str,
help='The input file. If not specified stdin is used.')
parser.add_argument('--output',
default=None,
type=str,
help='The output file. If not specified stdout is used.')
args = parser.parse_args()
with smart_open(args.input,
'r') as input_file, smart_open(args.output,
binary=True) as output_file:
block_mapper = EthBlockMapper()
exporter = CsvItemExporter(output_file)
exporter.start_exporting()
for line in input_file:
json_line = json.loads(line)
result = json_line.get('result', None)
if result is None:
continue
block = block_mapper.json_dict_to_block(result)
exporter.export_item(block_mapper.block_to_dict(block))
exporter.finish_exporting()
from ethereumetl.exporters import CsvItemExporter
from ethereumetl.mapper.block_mapper import EthBlockMapper
from ethereumetl.mapper.transaction_mapper import EthTransactionMapper
from ethereumetl.utils import smart_open
parser = argparse.ArgumentParser(
description='Extract blocks and transactions from eth_getBlockByNumber JSON RPC output')
parser.add_argument('--input', default=None, type=str, help='The input file. If not specified stdin is used.')
parser.add_argument('--blocks-output', default=None, type=str,
help='The output file for blocks. If not specified stdout is used.')
parser.add_argument('--transactions-output', default=None, type=str,
help='The output file for transactions. If not specified stdout is used.')
args = parser.parse_args()
with smart_open(args.input, 'r') as input_file, \
smart_open(args.blocks_output, binary=True) as blocks_output_file, \
smart_open(args.transactions_output, binary=True) as tx_output_file:
block_mapper = EthBlockMapper()
tx_mapper = EthTransactionMapper()
blocks_exporter = CsvItemExporter(blocks_output_file)
tx_exporter = CsvItemExporter(tx_output_file)
for line in input_file:
json_line = json.loads(line)
result = json_line.get('result', None)
if result is None:
continue
block = block_mapper.json_dict_to_block(result)
blocks_exporter.export_item(block_mapper.block_to_dict(block))
'Sends the given input to the ipc socket and outputs its response')
parser.add_argument('--input',
default=None,
type=str,
help='The input file. If not specified stdin is used.')
parser.add_argument('--output',
default=None,
type=str,
help='The output file. If not specified stdout is used.')
parser.add_argument('--ipc-path',
required=True,
type=str,
help='The full path to the ipc socket file.')
parser.add_argument('--ipc-timeout',
default=10,
type=int,
help='The timeout for ipc calls.')
parser.add_argument('--batch-size',
default=100,
type=int,
help='The number of lines in the batch to send to socket.')
args = parser.parse_args()
with smart_open(args.input,
'r') as input_file, smart_open(args.output) as output_file:
for line_batch in batch_readlines(input_file, args.batch_size):
response = socket_exchange(args.ipc_path, ''.join(line_batch),
args.ipc_timeout)
output_file.write(response)