def test_export_blocks_job(tmpdir, start_block, end_block, batch_size,
resource_group, provider_type, chain):
blocks_output_file = str(tmpdir.join('actual_block.json'))
transactions_output_file = str(tmpdir.join("actual_transactions.json"))
job = ExportBlocksJob(
start_block=start_block,
end_block=end_block,
batch_size=batch_size,
bitcoin_rpc=ThreadLocalProxy(
lambda: get_bitcoin_rpc(provider_type,
read_resource_lambda=lambda file:
read_resource(resource_group, file),
chain=chain)),
max_workers=5,
item_exporter=blocks_and_transactions_item_exporter(
blocks_output_file, transactions_output_file),
chain=chain,
export_blocks=blocks_output_file is not None,
export_transactions=transactions_output_file is not None)
job.run()
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'),
read_file(blocks_output_file))
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'),
read_file(transactions_output_file))
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, provider_type, chain):
try:
os.remove('last_synced_block.txt')
except OSError:
pass
blocks_output_file = str(tmpdir.join('actual_block.json'))
transactions_output_file = str(tmpdir.join("actual_transactions.json"))
stream(
bitcoin_rpc=ThreadLocalProxy(
lambda: get_bitcoin_rpc(
provider_type,
read_resource_lambda=lambda file: read_resource(resource_group, file),
chain=chain)),
start_block=start_block,
end_block=end_block,
batch_size=batch_size,
item_exporter=blocks_and_transactions_item_exporter(blocks_output_file, transactions_output_file)
)
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)
)
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'), read_file(transactions_output_file)
)
def enrich_transactions(batch_size, provider_uri, max_workers,
transactions_input, transactions_output, chain):
"""Enrich transactions."""
with smart_open(transactions_input, 'r') as transactions_input_file:
job = EnrichTransactionsJob(
transactions_iterable=(json.loads(transaction)
for transaction in transactions_input_file),
batch_size=batch_size,
bitcoin_rpc=ThreadLocalProxy(lambda: BitcoinRpc(provider_uri)),
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter(
None, transactions_output),
chain=chain)
job.run()
def export_blocks_and_transactions(start_block, end_block, batch_size,
provider_uri, max_workers, blocks_output,
transactions_output, chain):
"""Export blocks and transactions."""
if blocks_output is None and transactions_output is None:
raise ValueError(
'Either --blocks-output or --transactions-output options must be provided'
)
job = ExportBlocksJob(
start_block=start_block,
end_block=end_block,
batch_size=batch_size,
bitcoin_rpc=ThreadLocalProxy(lambda: BitcoinRpc(provider_uri)),
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter(
blocks_output, transactions_output),
chain=chain,
export_blocks=blocks_output is not None,
export_transactions=transactions_output is not None)
job.run()
def export_all(chain, partitions, output_dir, provider_uri, max_workers,
batch_size, enrich):
for batch_start_block, batch_end_block, partition_dir, *args in partitions:
# # # start # # #
start_time = time()
padded_batch_start_block = str(batch_start_block).zfill(8)
padded_batch_end_block = str(batch_end_block).zfill(8)
block_range = '{padded_batch_start_block}-{padded_batch_end_block}'.format(
padded_batch_start_block=padded_batch_start_block,
padded_batch_end_block=padded_batch_end_block,
)
file_name_suffix = '{padded_batch_start_block}_{padded_batch_end_block}'.format(
padded_batch_start_block=padded_batch_start_block,
padded_batch_end_block=padded_batch_end_block,
)
# # # blocks_and_transactions # # #
blocks_output_dir = '{output_dir}/blocks{partition_dir}'.format(
output_dir=output_dir,
partition_dir=partition_dir,
)
os.makedirs(os.path.dirname(blocks_output_dir), exist_ok=True)
transactions_output_dir = '{output_dir}/transactions{partition_dir}'.format(
output_dir=output_dir,
partition_dir=partition_dir,
)
os.makedirs(os.path.dirname(transactions_output_dir), exist_ok=True)
blocks_file = '{blocks_output_dir}/blocks_{file_name_suffix}.json'.format(
blocks_output_dir=blocks_output_dir,
file_name_suffix=file_name_suffix,
)
transactions_file = '{transactions_output_dir}/transactions_{file_name_suffix}.json'.format(
transactions_output_dir=transactions_output_dir,
file_name_suffix=file_name_suffix,
)
enriched_transactions_file = '{transactions_output_dir}/enriched_transactions_{file_name_suffix}.json'.format(
transactions_output_dir=transactions_output_dir,
file_name_suffix=file_name_suffix,
)
logger.info('Exporting blocks {block_range} to {blocks_file}'.format(
block_range=block_range,
blocks_file=blocks_file,
))
logger.info(
'Exporting transactions from blocks {block_range} to {transactions_file}'
.format(
block_range=block_range,
transactions_file=transactions_file,
))
job = ExportBlocksJob(
chain=chain,
start_block=batch_start_block,
end_block=batch_end_block,
batch_size=batch_size,
bitcoin_rpc=ThreadLocalProxy(lambda: BitcoinRpc(provider_uri)),
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter(
blocks_file, transactions_file),
export_blocks=blocks_file is not None,
export_transactions=transactions_file is not None)
job.run()
if enrich == True:
with smart_open(transactions_file, 'r') as transactions_file:
job = EnrichTransactionsJob(
transactions_iterable=(
json.loads(transaction)
for transaction in transactions_file),
batch_size=batch_size,
bitcoin_rpc=ThreadLocalProxy(
lambda: BitcoinRpc(provider_uri)),
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter(
None, enriched_transactions_file),
chain=chain)
job.run()
if args is not None and len(args) > 0:
date = args[0]
logger.info('Filtering blocks {blocks_file} by date {date}'.format(
blocks_file=blocks_file,
date=date,
))
def filter_by_date(item, field):
return datetime.datetime.fromtimestamp(item[field]).astimezone(datetime.timezone.utc) \
.strftime('%Y-%m-%d') == date.strftime('%Y-%m-%d')
filtered_blocks_file = blocks_file + '.filtered'
filter_items(blocks_file, filtered_blocks_file,
lambda item: filter_by_date(item, 'timestamp'))
shutil.move(filtered_blocks_file, blocks_file)
logger.info(
'Filtering transactions {transactions_file} by date {date}'.
format(
transactions_file=transactions_file,
date=date,
))
filtered_transactions_file = transactions_file + '.filtered'
filter_items(transactions_file, filtered_transactions_file,
lambda item: filter_by_date(item, 'block_timestamp'))
shutil.move(filtered_transactions_file, transactions_file)
# # # finish # # #
end_time = time()
time_diff = round(end_time - start_time, 5)
logger.info(
'Exporting blocks {block_range} took {time_diff} seconds'.format(
block_range=block_range,
time_diff=time_diff,
))