def export_receipts_and_logs( batch_size, transaction_hashes, provider_uri, max_workers, receipts_output, logs_output, ): """Exports receipts and logs.""" with smart_open(transaction_hashes, "r") as transaction_hashes_file: job = ExportReceiptsJob( transaction_hashes_iterable=( transaction_hash.strip() for transaction_hash in transaction_hashes_file), batch_size=batch_size, batch_web3_provider=ThreadLocalProxy( lambda: get_provider_from_uri(provider_uri, batch=True)), max_workers=max_workers, item_exporter=receipts_and_logs_item_exporter( receipts_output, logs_output), export_receipts=receipts_output is not None, export_logs=logs_output is not None, ) job.run()
def test_export_blocks_job(tmpdir, start_block, end_block, batch_size, resource_group, web3_provider_type): blocks_output_file = str(tmpdir.join("actual_blocks.csv")) transactions_output_file = str(tmpdir.join("actual_transactions.csv")) job = ExportBlocksJob( start_block=start_block, end_block=end_block, batch_size=batch_size, batch_web3_provider=ThreadLocalProxy(lambda: get_web3_provider( web3_provider_type, lambda file: read_resource(resource_group, file), batch=True, )), max_workers=5, item_exporter=blocks_and_transactions_item_exporter( blocks_output_file, transactions_output_file), export_blocks=blocks_output_file is not None, export_transactions=transactions_output_file is not None, ) job.run() compare_lines_ignore_order( read_resource(resource_group, "expected_blocks.csv"), read_file(blocks_output_file), ) compare_lines_ignore_order( read_resource(resource_group, "expected_transactions.csv"), read_file(transactions_output_file), )
def export_blocks_and_transactions( start_block, end_block, batch_size, provider_uri, max_workers, blocks_output, transactions_output, ): """Exports blocks and transactions.""" if blocks_output is None and transactions_output is None: raise ValueError( "Either --blocks-output or --transactions-output options must be provided" ) job = ExportBlocksJob( start_block=start_block, end_block=end_block, batch_size=batch_size, batch_web3_provider=ThreadLocalProxy( lambda: get_provider_from_uri(provider_uri, batch=True) ), max_workers=max_workers, item_exporter=blocks_and_transactions_item_exporter( blocks_output, transactions_output ), export_blocks=blocks_output is not None, export_transactions=transactions_output is not None, ) job.run()
def export_transaction_logs(start_block, end_block, provider_uri, max_workers, output_dir, output_format): """Exports transaction logs.""" job = ExportTransactionLogsJob( start_block=start_block, end_block=end_block, iotex_rpc=ThreadLocalProxy(lambda: IotexRpc(provider_uri)), max_workers=max_workers, item_exporter=IotexItemExporter(output_dir, output_format=output_format), batch_size=10 ) job.run()
def export_ds_blocks(start_block, end_block, provider_uri, max_workers, output_dir, output_format): """Exports ds blocks.""" job = ExportDsBlocksJob( start_block=start_block, end_block=end_block, zilliqa_api=ThreadLocalProxy(lambda: ZilliqaAPI(provider_uri)), max_workers=max_workers, item_exporter=ZilliqaItemExporter(output_dir, output_format=output_format), ) job.run()
def export(start_block, end_block, provider_uri, max_workers, output_dir, output_format): """Exports blocks, balance updates, and operations.""" job = ExportJob( start_block=start_block, end_block=end_block, tezos_rpc=ThreadLocalProxy(lambda: TezosRpc(provider_uri)), max_workers=max_workers, item_exporter=TezosItemExporter(output_dir, output_format=output_format), ) job.run()
def test_export_transaction_logs_job(tmpdir, start_block, end_block, resource_group, provider_type): job = ExportTransactionLogsJob( start_block=start_block, end_block=end_block, iotex_rpc=ThreadLocalProxy( lambda: get_iotex_rpc( provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), max_workers=5, item_exporter=IotexItemExporter(str(tmpdir)), ) job.run() compare_lines_ignore_order( read_resource(resource_group, 'expected_transaction_logs.json'), read_file(str(tmpdir.join('transaction_logs.json'))) )
def export_blocks(start_block, end_block, provider_uri, testnet, max_workers, batch_size, output_dir, output_format): """Exports blocks, actions, receipts, and logs.""" if testnet: set_iotex_utils_context(address_prefix='it') job = ExportBlocksJob( start_block=start_block, end_block=end_block, iotex_rpc=ThreadLocalProxy(lambda: IotexRpc(provider_uri)), max_workers=max_workers, item_exporter=IotexItemExporter(output_dir, output_format=output_format), batch_size=batch_size) job.run()
def export_partitioned(partitions, output_dir, output_format, provider_uri, max_workers, batch_size): for batch_start_block, batch_end_block, partition_dir, *args in partitions: # # # start # # # start_time = time() padded_batch_start_block = str(batch_start_block).zfill(8) padded_batch_end_block = str(batch_end_block).zfill(8) block_range = '{padded_batch_start_block}-{padded_batch_end_block}'.format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) partition_output_dir = '{output_dir}/blocks{partition_dir}'.format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(partition_output_dir), exist_ok=True) logger.info( 'Exporting blocks {block_range} to {partition_output_dir}'.format( block_range=block_range, partition_output_dir=partition_output_dir, )) job = ExportJob( start_block=batch_start_block, end_block=batch_end_block, batch_size=batch_size, tezos_rpc=ThreadLocalProxy(lambda: TezosRpc(provider_uri)), max_workers=max_workers, item_exporter=TezosItemExporter(partition_output_dir, output_format=output_format), ) job.run() # # # finish # # # end_time = time() time_diff = round(end_time - start_time, 5) logger.info( 'Exporting blocks {block_range} took {time_diff} seconds'.format( block_range=block_range, time_diff=time_diff, ))
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, provider_type): try: os.remove('last_synced_block.txt') except OSError: pass blocks_output_file = str(tmpdir.join('actual_block.json')) transactions_output_file = str(tmpdir.join("actual_transactions.json")) actions_output_file = str(tmpdir.join("actual_actions.json")) streamer_adapter = EosStreamerAdapter( eos_rpc=ThreadLocalProxy( lambda: get_eos_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), batch_size=batch_size, item_exporter=blocks_item_exporter(blocks_output_file, transactions_output_file, actions_output_file), ) streamer = Streamer(blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False) streamer.stream() print('=====================') print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)) print('=====================') print(read_file(transactions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_transactions.json'), read_file(transactions_output_file)) print('=====================') print(read_file(actions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_actions.json'), read_file(actions_output_file))
def export_blocks(start_block, end_block, provider_uri, max_workers, blocks_output, transactions_output, actions_output): """Export blocks, transactions and actions.""" if blocks_output is None and transactions_output is None and actions_output is None: raise ValueError( 'Either --blocks-output or --transactions-output or --actions-output options must be provided' ) job = ExportBlocksJob( start_block=start_block, end_block=end_block, eos_rpc=ThreadLocalProxy(lambda: EosRpc(provider_uri)), max_workers=max_workers, item_exporter=blocks_item_exporter(blocks_output, transactions_output, actions_output), export_blocks=blocks_output is not None, export_transactions=transactions_output is not None, export_actions=actions_output is not None) job.run()
def test_export_blocks_job(tmpdir, start_block, end_block, resource_group, provider_type): job = ExportBlocksJob( start_block=start_block, end_block=end_block, iotex_rpc=ThreadLocalProxy( lambda: get_iotex_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), max_workers=5, item_exporter=IotexItemExporter(str(tmpdir)), ) job.run() all_files = ['blocks.json', 'actions.json', 'logs.json'] for file in all_files: print(read_file(str(tmpdir.join(file)))) compare_lines_ignore_order( read_resource(resource_group, f'expected_{file}'), read_file(str(tmpdir.join(file))))
def test_export_tx_blocks_job(tmpdir, start_block, end_block, resource_group, provider_type): job = ExportTxBlocksJob( start_block=start_block, end_block=end_block, zilliqa_api=ThreadLocalProxy( lambda: get_zilliqa_api( provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), max_workers=5, item_exporter=ZilliqaItemExporter(str(tmpdir)), ) job.run() all_files = ['tx_blocks.json', 'ds_blocks.json', 'transactions.json', 'event_logs.json', 'transitions.json', 'exceptions.json'] for file in all_files: print(read_file(str(tmpdir.join(file)))) compare_lines_ignore_order( read_resource(resource_group, f'expected_{file}'), read_file(str(tmpdir.join(file))) )
def export_tx_blocks(start_block, end_block, provider_uri, max_workers, output_dir, output_format, rate_limit=None): """Exports tx blocks.""" zilliqa_api = ThreadLocalProxy(lambda: ZilliqaAPI(provider_uri)) if rate_limit is not None and rate_limit > 0: zilliqa_api = RateLimitingProxy(zilliqa_api, max_per_second=rate_limit) job = ExportTxBlocksJob( start_block=start_block, end_block=end_block, zilliqa_api=zilliqa_api, max_workers=max_workers, item_exporter=ZilliqaItemExporter(output_dir, output_format=output_format), ) job.run()
def test_export_blocks_job(tmpdir, start_block, end_block, batch_size, resource_group, provider_type): blocks_output_file = str(tmpdir.join('actual_block.json')) transactions_output_file = str(tmpdir.join("actual_transactions.json")) actions_output_file = str(tmpdir.join("actual_actions.json")) job = ExportBlocksJob( start_block=start_block, end_block=end_block, batch_size=batch_size, eos_rpc=ThreadLocalProxy( lambda: get_eos_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), max_workers=5, item_exporter=blocks_item_exporter(blocks_output_file, transactions_output_file, actions_output_file), export_blocks=blocks_output_file is not None, export_transactions=transactions_output_file is not None, export_actions=actions_output_file is not None) job.run() print('=====================') print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)) print('=====================') print(read_file(transactions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_transactions.json'), read_file(transactions_output_file)) print('=====================') print(read_file(actions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_actions.json'), read_file(actions_output_file))
def test_export_job(tmpdir, start_block, end_block, resource_group, provider_type): job = ExportJob( start_block=start_block, end_block=end_block, tezos_rpc=ThreadLocalProxy( lambda: get_tezos_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), max_workers=5, item_exporter=TezosItemExporter(str(tmpdir)), ) job.run() all_files = ['blocks.json', 'balance_updates.json'] + \ [f'{operation_kind}_operations.json' for operation_kind in OperationKind.ALL] for file in all_files: print('=====================') print(read_file(str(tmpdir.join(file)))) compare_lines_ignore_order( read_resource(resource_group, f'expected_{file}'), read_file(str(tmpdir.join(file))))
def stream(last_synced_block_file, lag, provider_uri, output, start_block, period_seconds=10, batch_size=1, block_batch_size=1, max_workers=5, log_file=None, pid_file=None): """Streams all data types to console or Google Pub/Sub.""" configure_logging(log_file) configure_signals() from eosetl.streaming.eos_streamer_adapter import EosStreamerAdapter from blockchainetl_common.streaming.streamer import Streamer # TODO: Implement fallback mechanism for provider uris instead of picking randomly provider_uri = pick_random_provider_uri(provider_uri) logging.info('Using ' + provider_uri) streamer_adapter = EosStreamerAdapter( eos_rpc=ThreadLocalProxy(lambda: EosRpc(provider_uri)), item_exporter=get_item_exporter(output), batch_size=batch_size, max_workers=max_workers) streamer = Streamer( blockchain_streamer_adapter=streamer_adapter, last_synced_block_file=last_synced_block_file, lag=lag, start_block=start_block, period_seconds=period_seconds, block_batch_size=block_batch_size, pid_file=pid_file, ) streamer.stream()
def get_partitions(start, end, partition_batch_size, provider_uri): """Yield partitions based on input data type.""" if is_date_range(start, end): start_date = datetime.strptime(start, '%Y-%m-%d').date() end_date = datetime.strptime(end, '%Y-%m-%d').date() day = timedelta(days=1) eos_service = EosBlockRangeService( eos_rpc=ThreadLocalProxy(lambda: EosRpc(provider_uri)) ) while start_date <= end_date: batch_start_block, batch_end_block = eos_service.get_block_range_for_date(start_date) partition_dir = '/date={start_date!s}/'.format(start_date=start_date) yield batch_start_block, batch_end_block, partition_dir, start_date start_date += day elif is_block_range(start, end): start_block = int(start) end_block = int(end) for batch_start_block in range(start_block, end_block + 1, partition_batch_size): batch_end_block = batch_start_block + partition_batch_size - 1 if batch_end_block > end_block: batch_end_block = end_block padded_batch_start_block = str(batch_start_block).zfill(8) padded_batch_end_block = str(batch_end_block).zfill(8) partition_dir = '/start_block={padded_batch_start_block}/end_block={padded_batch_end_block}'.format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) yield batch_start_block, batch_end_block, partition_dir else: raise ValueError('start and end must be either block numbers or ISO dates')
def test_export_receipts_job( tmpdir, batch_size, transaction_hashes, output_format, resource_group, web3_provider_type, ): receipts_output_file = str(tmpdir.join("actual_receipts." + output_format)) logs_output_file = str(tmpdir.join("actual_logs." + output_format)) job = ExportReceiptsJob( transaction_hashes_iterable=transaction_hashes, batch_size=batch_size, batch_web3_provider=ThreadLocalProxy(lambda: get_web3_provider( web3_provider_type, lambda file: read_resource(resource_group, file), batch=True, )), max_workers=5, item_exporter=receipts_and_logs_item_exporter(receipts_output_file, logs_output_file), export_receipts=receipts_output_file is not None, export_logs=logs_output_file is not None, ) job.run() compare_lines_ignore_order( read_resource(resource_group, "expected_receipts." + output_format), read_file(receipts_output_file), ) compare_lines_ignore_order( read_resource(resource_group, "expected_logs." + output_format), read_file(logs_output_file), )
def mock_stream( input_directory, output, start_block, end_block, entity_types, kafka_blocks_topic, kafka_transactions_topic, kafka_logs_topic, kafka_compression_type, kafka_schema_registry_url, kafka_use_schema_registry, batch_size=2, max_workers=5, log_file=None, ): # Clear existing last_synced_block.txt if os.path.exists("last_synced_block.txt"): os.remove("last_synced_block.txt") """Streams all data types to console or Google Pub/Sub.""" configure_logging(log_file) configure_signals() entity_types = parse_entity_types(entity_types) validate_entity_types(entity_types, output) from blockchainetl_common.streaming.streamer import Streamer from iconetl.streaming.icx_streamer_adapter import IcxStreamerAdapter from iconetl.streaming.item_exporter_creator import create_item_exporter from tests.iconetl.job.utils import get_web3_provider kafka_settings = { "topic_map": { "block": kafka_blocks_topic, "transaction": kafka_transactions_topic, "log": kafka_logs_topic, }, "compression_type": kafka_compression_type, "enable_schema_registry": kafka_use_schema_registry, "schema_registry_url": kafka_schema_registry_url, } streamer_adapter = IcxStreamerAdapter( batch_web3_provider=ThreadLocalProxy(lambda: get_web3_provider( provider_type="mock", read_resource_lambda=lambda file: read_resource( input_directory, file), batch=True, )), item_exporter=create_item_exporter(output, kafka_settings), batch_size=batch_size, max_workers=max_workers, entity_types=entity_types, ) streamer = Streamer( blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False, ) streamer.stream()
def export_all_common(partitions, output_dir, provider_uri, max_workers, batch_size): for batch_start_block, batch_end_block, partition_dir in partitions: start_time = time() padded_batch_start_block = str(batch_start_block).zfill(8) padded_batch_end_block = str(batch_end_block).zfill(8) block_range = "{padded_batch_start_block}-{padded_batch_end_block}".format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) file_name_suffix = "{padded_batch_start_block}_{padded_batch_end_block}".format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) blocks_output_dir = "{output_dir}/blocks{partition_dir}".format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(blocks_output_dir), exist_ok=True) transactions_output_dir = "{output_dir}/transactions{partition_dir}".format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(transactions_output_dir), exist_ok=True) blocks_file = "{blocks_output_dir}/blocks_{file_name_suffix}.csv".format( blocks_output_dir=blocks_output_dir, file_name_suffix=file_name_suffix, ) transactions_file = "{transactions_output_dir}/transactions_{file_name_suffix}.csv".format( transactions_output_dir=transactions_output_dir, file_name_suffix=file_name_suffix, ) logger.info("Exporting blocks {block_range} to {blocks_file}".format( block_range=block_range, blocks_file=blocks_file, )) logger.info( "Exporting transactions from blocks {block_range} to {transactions_file}" .format( block_range=block_range, transactions_file=transactions_file, )) job = ExportBlocksJob( start_block=batch_start_block, end_block=batch_end_block, batch_size=batch_size, batch_web3_provider=ThreadLocalProxy( lambda: get_provider_from_uri(provider_uri, batch=True)), max_workers=max_workers, item_exporter=blocks_and_transactions_item_exporter( blocks_file, transactions_file), export_blocks=blocks_file is not None, export_transactions=transactions_file is not None, ) job.run() cache_output_dir = "{output_dir}/.tmp{partition_dir}".format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(cache_output_dir), exist_ok=True) transaction_hashes_file = "{cache_output_dir}/transaction_hashes_{file_name_suffix}.csv".format( cache_output_dir=cache_output_dir, file_name_suffix=file_name_suffix, ) logger.info( "Extracting hash column from transaction file {transactions_file}". format(transactions_file=transactions_file, )) extract_csv_column_unique(transactions_file, transaction_hashes_file, "hash") receipts_output_dir = "{output_dir}/receipts{partition_dir}".format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(receipts_output_dir), exist_ok=True) logs_output_dir = "{output_dir}/logs{partition_dir}".format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(logs_output_dir), exist_ok=True) receipts_file = "{receipts_output_dir}/receipts_{file_name_suffix}.csv".format( receipts_output_dir=receipts_output_dir, file_name_suffix=file_name_suffix, ) logs_file = "{logs_output_dir}/logs_{file_name_suffix}.csv".format( logs_output_dir=logs_output_dir, file_name_suffix=file_name_suffix, ) logger.info( "Exporting receipts and logs from blocks {block_range} to {receipts_file} and {logs_file}" .format( block_range=block_range, receipts_file=receipts_file, logs_file=logs_file, )) with smart_open(transaction_hashes_file, "r") as transaction_hashes: job = ExportReceiptsJob( transaction_hashes_iterable=( transaction_hash.strip() for transaction_hash in transaction_hashes), batch_size=batch_size, batch_web3_provider=ThreadLocalProxy( lambda: get_provider_from_uri(provider_uri, batch=True)), max_workers=max_workers, item_exporter=receipts_and_logs_item_exporter( receipts_file, logs_file), export_receipts=receipts_file is not None, export_logs=logs_file is not None, ) job.run() shutil.rmtree(os.path.dirname(cache_output_dir)) end_time = time() time_diff = round(end_time - start_time, 5) logger.info( "Exporting blocks {block_range} took {time_diff} seconds".format( block_range=block_range, time_diff=time_diff, ))
def test_stream( tmpdir, start_block, end_block, batch_size, resource_group, entity_types, provider_type, ): try: os.remove("last_synced_block.txt") except OSError: pass blocks_output_file = str(tmpdir.join("actual_blocks.json")) transactions_output_file = str(tmpdir.join("actual_transactions.json")) logs_output_file = str(tmpdir.join("actual_logs.json")) streamer_adapter = IcxStreamerAdapter( batch_web3_provider=ThreadLocalProxy( lambda: get_web3_provider( provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file), batch=True, ) ), batch_size=batch_size, item_exporter=CompositeItemExporter( filename_mapping={ "block": blocks_output_file, "transaction": transactions_output_file, "log": logs_output_file, } ), entity_types=entity_types, ) streamer = Streamer( blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False, ) streamer.stream() if "block" in entity_types: print("=====================") print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, "expected_blocks.json"), read_file(blocks_output_file), ) if "transaction" in entity_types: print("=====================") print(read_file(transactions_output_file)) compare_lines_ignore_order( read_resource(resource_group, "expected_transactions.json"), read_file(transactions_output_file), ) if "log" in entity_types: print("=====================") print(read_file(logs_output_file)) compare_lines_ignore_order( read_resource(resource_group, "expected_logs.json"), read_file(logs_output_file), )
def export_all(partitions, output_dir, provider_uri, max_workers, batch_size): for batch_start_block, batch_end_block, partition_dir, *args in partitions: # # # start # # # start_time = time() padded_batch_start_block = str(batch_start_block).zfill(8) padded_batch_end_block = str(batch_end_block).zfill(8) block_range = '{padded_batch_start_block}-{padded_batch_end_block}'.format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) file_name_suffix = '{padded_batch_start_block}_{padded_batch_end_block}'.format( padded_batch_start_block=padded_batch_start_block, padded_batch_end_block=padded_batch_end_block, ) # # # blocks_and_transactions # # # blocks_output_dir = '{output_dir}/blocks{partition_dir}'.format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(blocks_output_dir), exist_ok=True) transactions_output_dir = '{output_dir}/transactions{partition_dir}'.format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(transactions_output_dir), exist_ok=True) actions_output_dir = '{output_dir}/actions{partition_dir}'.format( output_dir=output_dir, partition_dir=partition_dir, ) os.makedirs(os.path.dirname(actions_output_dir), exist_ok=True) blocks_file = '{blocks_output_dir}/blocks_{file_name_suffix}.json'.format( blocks_output_dir=blocks_output_dir, file_name_suffix=file_name_suffix, ) transactions_file = '{transactions_output_dir}/transactions_{file_name_suffix}.json'.format( transactions_output_dir=transactions_output_dir, file_name_suffix=file_name_suffix, ) actions_file = '{actions_output_dir}/actions_{file_name_suffix}.json'.format( actions_output_dir=actions_output_dir, file_name_suffix=file_name_suffix, ) logger.info('Exporting blocks {block_range} to {blocks_file}'.format( block_range=block_range, blocks_file=blocks_file, )) logger.info('Exporting transactions from blocks {block_range} to {transactions_file}'.format( block_range=block_range, transactions_file=transactions_file, )) logger.info('Exporting actions from blocks {block_range} to {actions_file}'.format( block_range=block_range, actions_file=actions_file, )) job = ExportBlocksJob( start_block=batch_start_block, end_block=batch_end_block, batch_size=batch_size, eos_rpc=ThreadLocalProxy(lambda: EosRpc(provider_uri)), max_workers=max_workers, item_exporter=blocks_item_exporter(blocks_file, transactions_file, actions_file), export_blocks=blocks_file is not None, export_transactions=transactions_file is not None) job.run() # # # finish # # # end_time = time() time_diff = round(end_time - start_time, 5) logger.info('Exporting blocks {block_range} took {time_diff} seconds'.format( block_range=block_range, time_diff=time_diff, ))
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, entity_types, provider_type): try: os.remove('last_synced_block.txt') except OSError: pass blocks_output_file = str(tmpdir.join('actual_blocks.json')) actions_output_file = str(tmpdir.join('actual_actions.json')) logs_output_file = str(tmpdir.join('actual_logs.json')) transaction_logs_output_file = str( tmpdir.join('actual_transaction_logs.json')) streamer_adapter = IotexStreamerAdapter( iotex_rpc=ThreadLocalProxy( lambda: get_iotex_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file))), batch_size=batch_size, item_exporter=CompositeItemExporter( filename_mapping={ 'block': blocks_output_file, 'action': actions_output_file, 'log': logs_output_file, 'transaction_log': transaction_logs_output_file, }), entity_types=entity_types, ) streamer = Streamer(blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False) streamer.stream() if EntityType.BLOCK in entity_types: print('=====================') print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)) if EntityType.ACTION in entity_types: print('=====================') print(read_file(actions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_actions.json'), read_file(actions_output_file)) if EntityType.LOG in entity_types: print('=====================') print(read_file(logs_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_logs.json'), read_file(logs_output_file)) if EntityType.TRANSACTION_LOG in entity_types: print('=====================') print(read_file(logs_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_transaction_logs.json'), read_file(transaction_logs_output_file))