def contracts_item_exporter(contracts_output): return CompositeItemExporter( filename_mapping={ 'contract': contracts_output }, field_mapping={ 'contract': FIELDS_TO_EXPORT } )
def traces_item_exporter(traces_output): return CompositeItemExporter( filename_mapping={ 'trace': traces_output }, field_mapping={ 'trace': FIELDS_TO_EXPORT } )
def receipts_and_logs_item_exporter(receipts_output=None, logs_output=None): return CompositeItemExporter(filename_mapping={ 'receipt': receipts_output, 'log': logs_output }, field_mapping={ 'receipt': RECEIPT_FIELDS_TO_EXPORT, 'log': LOG_FIELDS_TO_EXPORT })
def blocks_and_transactions_item_exporter(blocks_output=None, transactions_output=None): return CompositeItemExporter( filename_mapping={ 'block': blocks_output, 'transaction': transactions_output }, field_mapping={ 'block': BLOCK_FIELDS_TO_EXPORT, 'transaction': TRANSACTION_FIELDS_TO_EXPORT } )
def blocks_and_transactions_item_exporter(blocks_output=None, transactions_output=None): filename_mapping = {} field_mapping = {} if blocks_output is not None: filename_mapping['block'] = blocks_output field_mapping['block'] = BLOCK_FIELDS_TO_EXPORT if transactions_output is not None: filename_mapping['transaction'] = transactions_output field_mapping['transaction'] = TRANSACTION_FIELDS_TO_EXPORT return CompositeItemExporter(filename_mapping=filename_mapping, field_mapping=field_mapping)
def enrich_block_group_item_exporter(blocks_output=None, transactions_output=None, logs_output=None, token_transfers_output=None): return CompositeItemExporter(filename_mapping={ 'block': blocks_output, 'transaction': transactions_output, 'log': logs_output, 'token_transfer': token_transfers_output }, field_mapping={ 'block': BLOCK_FIELDS_TO_EXPORT, 'transaction': TRANSACTION_FIELDS_TO_EXPORT, 'log': LOG_FIELDS_TO_EXPORT, 'token_transfer': TOKEN_TRANSFER_FIELDS_TO_EXPORT })
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, provider_type, chain): try: os.remove('last_synced_block.txt') except OSError: pass blocks_output_file = str(tmpdir.join('actual_block.json')) transactions_output_file = str(tmpdir.join("actual_transactions.json")) streamer_adapter = BtcStreamerAdapter( bitcoin_rpc=ThreadLocalProxy( lambda: get_bitcoin_rpc(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file), chain=chain)), batch_size=batch_size, item_exporter=CompositeItemExporter( filename_mapping={ 'block': blocks_output_file, 'transaction': transactions_output_file, }), ) streamer = Streamer(blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False) streamer.stream() print('=====================') print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)) print('=====================') print(read_file(transactions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_transactions.json'), read_file(transactions_output_file))
def tokens_item_exporter(tokens_output, converters=()): return CompositeItemExporter(filename_mapping={'token': tokens_output}, field_mapping={'token': FIELDS_TO_EXPORT}, converters=converters)
def origin_shop_product_item_exporter(output): return CompositeItemExporter( filename_mapping={'origin_shop_product': output}, field_mapping={'origin_shop_product': SHOP_FIELDS_TO_EXPORT})
def origin_marketplace_listing_item_exporter(output): return CompositeItemExporter( filename_mapping={'origin_marketplace_listing': output}, field_mapping={ 'origin_marketplace_listing': MARKETPLACE_FIELDS_TO_EXPORT })
def token_transfers_item_exporter(token_transfer_output): return CompositeItemExporter( filename_mapping={'token_transfer': token_transfer_output}, field_mapping={'token_transfer': FIELDS_TO_EXPORT})
def events_item_exporter(events_output=None): return CompositeItemExporter( filename_mapping={'event': events_output}, field_mapping={'event': EVENT_FIELDS_TO_EXPORT})
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group, entity_types, provider_type): try: os.remove('last_synced_block.txt') except OSError: pass blocks_output_file = str(tmpdir.join('actual_blocks.json')) transactions_output_file = str(tmpdir.join('actual_transactions.json')) logs_output_file = str(tmpdir.join('actual_logs.json')) token_transfers_output_file = str( tmpdir.join('actual_token_transfers.json')) traces_output_file = str(tmpdir.join('actual_traces.json')) contracts_output_file = str(tmpdir.join('actual_contracts.json')) tokens_output_file = str(tmpdir.join('actual_tokens.json')) streamer_adapter = EthStreamerAdapter( batch_web3_provider=ThreadLocalProxy( lambda: get_web3_provider(provider_type, read_resource_lambda=lambda file: read_resource(resource_group, file), batch=True)), batch_size=batch_size, item_exporter=CompositeItemExporter( filename_mapping={ 'block': blocks_output_file, 'transaction': transactions_output_file, 'log': logs_output_file, 'token_transfer': token_transfers_output_file, 'trace': traces_output_file, 'contract': contracts_output_file, 'token': tokens_output_file, }), entity_types=entity_types, ) streamer = Streamer(blockchain_streamer_adapter=streamer_adapter, start_block=start_block, end_block=end_block, retry_errors=False) streamer.stream() if 'block' in entity_types: print('=====================') print(read_file(blocks_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_blocks.json'), read_file(blocks_output_file)) if 'transaction' in entity_types: print('=====================') print(read_file(transactions_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_transactions.json'), read_file(transactions_output_file)) if 'log' in entity_types: print('=====================') print(read_file(logs_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_logs.json'), read_file(logs_output_file)) if 'token_transfer' in entity_types: print('=====================') print(read_file(token_transfers_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_token_transfers.json'), read_file(token_transfers_output_file)) if 'trace' in entity_types: print('=====================') print(read_file(traces_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_traces.json'), read_file(traces_output_file)) if 'contract' in entity_types: print('=====================') print(read_file(contracts_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_contracts.json'), read_file(contracts_output_file)) if 'token' in entity_types: print('=====================') print(read_file(tokens_output_file)) compare_lines_ignore_order( read_resource(resource_group, 'expected_tokens.json'), read_file(tokens_output_file))