def contracts_item_exporter(contracts_output):
return CompositeItemExporter(
filename_mapping={
'contract': contracts_output
},
field_mapping={
'contract': FIELDS_TO_EXPORT
}
)
def traces_item_exporter(traces_output):
return CompositeItemExporter(
filename_mapping={
'trace': traces_output
},
field_mapping={
'trace': FIELDS_TO_EXPORT
}
)
def receipts_and_logs_item_exporter(receipts_output=None, logs_output=None):
return CompositeItemExporter(filename_mapping={
'receipt': receipts_output,
'log': logs_output
},
field_mapping={
'receipt': RECEIPT_FIELDS_TO_EXPORT,
'log': LOG_FIELDS_TO_EXPORT
})
def blocks_and_transactions_item_exporter(blocks_output=None, transactions_output=None):
return CompositeItemExporter(
filename_mapping={
'block': blocks_output,
'transaction': transactions_output
},
field_mapping={
'block': BLOCK_FIELDS_TO_EXPORT,
'transaction': TRANSACTION_FIELDS_TO_EXPORT
}
)
def blocks_and_transactions_item_exporter(blocks_output=None,
transactions_output=None):
filename_mapping = {}
field_mapping = {}
if blocks_output is not None:
filename_mapping['block'] = blocks_output
field_mapping['block'] = BLOCK_FIELDS_TO_EXPORT
if transactions_output is not None:
filename_mapping['transaction'] = transactions_output
field_mapping['transaction'] = TRANSACTION_FIELDS_TO_EXPORT
return CompositeItemExporter(filename_mapping=filename_mapping,
field_mapping=field_mapping)
def enrich_block_group_item_exporter(blocks_output=None,
transactions_output=None,
logs_output=None,
token_transfers_output=None):
return CompositeItemExporter(filename_mapping={
'block': blocks_output,
'transaction': transactions_output,
'log': logs_output,
'token_transfer': token_transfers_output
},
field_mapping={
'block':
BLOCK_FIELDS_TO_EXPORT,
'transaction':
TRANSACTION_FIELDS_TO_EXPORT,
'log':
LOG_FIELDS_TO_EXPORT,
'token_transfer':
TOKEN_TRANSFER_FIELDS_TO_EXPORT
})
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group,
provider_type, chain):
try:
os.remove('last_synced_block.txt')
except OSError:
pass
blocks_output_file = str(tmpdir.join('actual_block.json'))
transactions_output_file = str(tmpdir.join("actual_transactions.json"))
streamer_adapter = BtcStreamerAdapter(
bitcoin_rpc=ThreadLocalProxy(
lambda: get_bitcoin_rpc(provider_type,
read_resource_lambda=lambda file:
read_resource(resource_group, file),
chain=chain)),
batch_size=batch_size,
item_exporter=CompositeItemExporter(
filename_mapping={
'block': blocks_output_file,
'transaction': transactions_output_file,
}),
)
streamer = Streamer(blockchain_streamer_adapter=streamer_adapter,
start_block=start_block,
end_block=end_block,
retry_errors=False)
streamer.stream()
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'),
read_file(blocks_output_file))
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'),
read_file(transactions_output_file))
def tokens_item_exporter(tokens_output, converters=()):
return CompositeItemExporter(filename_mapping={'token': tokens_output},
field_mapping={'token': FIELDS_TO_EXPORT},
converters=converters)
def origin_shop_product_item_exporter(output):
return CompositeItemExporter(
filename_mapping={'origin_shop_product': output},
field_mapping={'origin_shop_product': SHOP_FIELDS_TO_EXPORT})
def origin_marketplace_listing_item_exporter(output):
return CompositeItemExporter(
filename_mapping={'origin_marketplace_listing': output},
field_mapping={
'origin_marketplace_listing': MARKETPLACE_FIELDS_TO_EXPORT
})
def token_transfers_item_exporter(token_transfer_output):
return CompositeItemExporter(
filename_mapping={'token_transfer': token_transfer_output},
field_mapping={'token_transfer': FIELDS_TO_EXPORT})
def events_item_exporter(events_output=None):
return CompositeItemExporter(
filename_mapping={'event': events_output},
field_mapping={'event': EVENT_FIELDS_TO_EXPORT})
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group,
entity_types, provider_type):
try:
os.remove('last_synced_block.txt')
except OSError:
pass
blocks_output_file = str(tmpdir.join('actual_blocks.json'))
transactions_output_file = str(tmpdir.join('actual_transactions.json'))
logs_output_file = str(tmpdir.join('actual_logs.json'))
token_transfers_output_file = str(
tmpdir.join('actual_token_transfers.json'))
traces_output_file = str(tmpdir.join('actual_traces.json'))
contracts_output_file = str(tmpdir.join('actual_contracts.json'))
tokens_output_file = str(tmpdir.join('actual_tokens.json'))
streamer_adapter = EthStreamerAdapter(
batch_web3_provider=ThreadLocalProxy(
lambda: get_web3_provider(provider_type,
read_resource_lambda=lambda file:
read_resource(resource_group, file),
batch=True)),
batch_size=batch_size,
item_exporter=CompositeItemExporter(
filename_mapping={
'block': blocks_output_file,
'transaction': transactions_output_file,
'log': logs_output_file,
'token_transfer': token_transfers_output_file,
'trace': traces_output_file,
'contract': contracts_output_file,
'token': tokens_output_file,
}),
entity_types=entity_types,
)
streamer = Streamer(blockchain_streamer_adapter=streamer_adapter,
start_block=start_block,
end_block=end_block,
retry_errors=False)
streamer.stream()
if 'block' in entity_types:
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'),
read_file(blocks_output_file))
if 'transaction' in entity_types:
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'),
read_file(transactions_output_file))
if 'log' in entity_types:
print('=====================')
print(read_file(logs_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_logs.json'),
read_file(logs_output_file))
if 'token_transfer' in entity_types:
print('=====================')
print(read_file(token_transfers_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_token_transfers.json'),
read_file(token_transfers_output_file))
if 'trace' in entity_types:
print('=====================')
print(read_file(traces_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_traces.json'),
read_file(traces_output_file))
if 'contract' in entity_types:
print('=====================')
print(read_file(contracts_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_contracts.json'),
read_file(contracts_output_file))
if 'token' in entity_types:
print('=====================')
print(read_file(tokens_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_tokens.json'),
read_file(tokens_output_file))