Beispiel #1
0
def test_export_blocks_job(tmpdir, start_block, end_block, batch_size,
                           resource_group, web3_provider_type):
    blocks_output_file = str(tmpdir.join('actual_blocks.csv'))
    transactions_output_file = str(tmpdir.join('actual_transactions.csv'))

    job = ExportBlocksJob(
        start_block=start_block,
        end_block=end_block,
        batch_size=batch_size,
        batch_web3_provider=ThreadLocalProxy(lambda: get_web3_provider(
            web3_provider_type,
            lambda file: read_resource(resource_group, file),
            batch=True)),
        max_workers=5,
        item_exporter=blocks_and_transactions_item_exporter(
            blocks_output_file, transactions_output_file),
        export_blocks=blocks_output_file is not None,
        export_transactions=transactions_output_file is not None)
    job.run()

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_blocks.csv'),
        read_file(blocks_output_file))

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_transactions.csv'),
        read_file(transactions_output_file))
Beispiel #2
0
def test_export_tokens_job(tmpdir, token_addresses, resource_group,
                           web3_provider_type):
    output_file = str(tmpdir.join('tokens.csv'))

    job = ExportTokensJob(
        token_addresses_iterable=token_addresses,
        web3=ThreadLocalProxy(lambda: Web3(
            get_web3_provider(web3_provider_type, lambda file: read_resource(
                resource_group, file)))),
        item_exporter=tokens_item_exporter(output_file),
        max_workers=5)
    job.run()

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_tokens.csv'),
        read_file(output_file))
Beispiel #3
0
def test_export_traces_job(tmpdir, start_block, end_block, resource_group, web3_provider_type):
    traces_output_file = str(tmpdir.join('actual_traces.csv'))

    job = ExportTracesJob(
        start_block=start_block, end_block=end_block, batch_size=1,
        web3=ThreadLocalProxy(
            lambda: Web3(get_web3_provider(web3_provider_type, lambda file: read_resource(resource_group, file)))
        ),
        max_workers=5,
        item_exporter=traces_item_exporter(traces_output_file),
    )
    job.run()

    print('=====================')
    print(read_file(traces_output_file))
    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_traces.csv'), read_file(traces_output_file)
    )
Beispiel #4
0
def test_export_token_transfers_job(tmpdir, start_block, end_block, batch_size,
                                    resource_group, web3_provider_type):
    output_file = str(tmpdir.join('token_transfers.csv'))

    job = ExportTokenTransfersJob(
        start_block=start_block,
        end_block=end_block,
        batch_size=batch_size,
        web3=ThreadLocalProxy(lambda: Web3(
            get_web3_provider(web3_provider_type, lambda file: read_resource(
                resource_group, file)))),
        item_exporter=token_transfers_item_exporter(output_file),
        max_workers=5)
    job.run()

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_token_transfers.csv'),
        read_file(output_file))
Beispiel #5
0
def test_export_contracts_job(tmpdir, batch_size, contract_addresses, output_format, resource_group,
                              web3_provider_type):
    contracts_output_file = str(tmpdir.join('actual_contracts.' + output_format))

    job = ExportContractsJob(
        contract_addresses_iterable=contract_addresses,
        batch_size=batch_size,
        batch_web3_provider=ThreadLocalProxy(
            lambda: get_web3_provider(web3_provider_type, lambda file: read_resource(resource_group, file), batch=True)
        ),
        max_workers=5,
        item_exporter=contracts_item_exporter(contracts_output_file)
    )
    job.run()

    print('=====================')
    print(read_file(contracts_output_file))
    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_contracts.' + output_format), read_file(contracts_output_file)
    )
Beispiel #6
0
def test_export_geth_traces_job(tmpdir, start_block, end_block, resource_group,
                                web3_provider_type):
    traces_output_file = str(tmpdir.join('actual_geth_traces.json'))

    job = ExportGethTracesJob(
        start_block=start_block,
        end_block=end_block,
        batch_size=1,
        batch_web3_provider=ThreadLocalProxy(lambda: get_web3_provider(
            web3_provider_type,
            lambda file: read_resource(resource_group, file),
            batch=True)),
        max_workers=5,
        item_exporter=geth_traces_item_exporter(traces_output_file),
    )
    job.run()

    compare_lines_ignore_order(
        read_resource(resource_group, 'geth_traces.json'),
        read_file(traces_output_file))
def test_export_receipts_job(tmpdir, batch_size, transaction_hashes, output_format, resource_group, web3_provider_type):
    receipts_output_file = str(tmpdir.join('actual_receipts.' + output_format))
    logs_output_file = str(tmpdir.join('actual_logs.' + output_format))

    job = ExportReceiptsJob(
        transaction_hashes_iterable=transaction_hashes,
        batch_size=batch_size,
        batch_web3_provider=ThreadLocalProxy(
            lambda: get_web3_provider(web3_provider_type, lambda file: read_resource(resource_group, file), batch=True)
        ),
        max_workers=5,
        item_exporter=receipts_and_logs_item_exporter(receipts_output_file, logs_output_file),
        export_receipts=receipts_output_file is not None,
        export_logs=logs_output_file is not None
    )
    job.run()

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_receipts.' + output_format), read_file(receipts_output_file)
    )

    compare_lines_ignore_order(
        read_resource(resource_group, 'expected_logs.' + output_format), read_file(logs_output_file)
    )
Beispiel #8
0
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group,
                entity_types, provider_type):
    try:
        os.remove('last_synced_block.txt')
    except OSError:
        pass

    blocks_output_file = str(tmpdir.join('actual_blocks.json'))
    transactions_output_file = str(tmpdir.join('actual_transactions.json'))
    logs_output_file = str(tmpdir.join('actual_logs.json'))
    token_transfers_output_file = str(
        tmpdir.join('actual_token_transfers.json'))
    traces_output_file = str(tmpdir.join('actual_traces.json'))
    contracts_output_file = str(tmpdir.join('actual_contracts.json'))
    tokens_output_file = str(tmpdir.join('actual_tokens.json'))

    streamer_adapter = EthStreamerAdapter(
        batch_web3_provider=ThreadLocalProxy(
            lambda: get_web3_provider(provider_type,
                                      read_resource_lambda=lambda file:
                                      read_resource(resource_group, file),
                                      batch=True)),
        batch_size=batch_size,
        item_exporter=CompositeItemExporter(
            filename_mapping={
                'block': blocks_output_file,
                'transaction': transactions_output_file,
                'log': logs_output_file,
                'token_transfer': token_transfers_output_file,
                'trace': traces_output_file,
                'contract': contracts_output_file,
                'token': tokens_output_file,
            }),
        entity_types=entity_types,
    )
    streamer = Streamer(blockchain_streamer_adapter=streamer_adapter,
                        start_block=start_block,
                        end_block=end_block,
                        retry_errors=False)
    streamer.stream()

    if 'block' in entity_types:
        print('=====================')
        print(read_file(blocks_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_blocks.json'),
            read_file(blocks_output_file))

    if 'transaction' in entity_types:
        print('=====================')
        print(read_file(transactions_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_transactions.json'),
            read_file(transactions_output_file))

    if 'log' in entity_types:
        print('=====================')
        print(read_file(logs_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_logs.json'),
            read_file(logs_output_file))

    if 'token_transfer' in entity_types:
        print('=====================')
        print(read_file(token_transfers_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_token_transfers.json'),
            read_file(token_transfers_output_file))

    if 'trace' in entity_types:
        print('=====================')
        print(read_file(traces_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_traces.json'),
            read_file(traces_output_file))

    if 'contract' in entity_types:
        print('=====================')
        print(read_file(contracts_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_contracts.json'),
            read_file(contracts_output_file))

    if 'token' in entity_types:
        print('=====================')
        print(read_file(tokens_output_file))
        compare_lines_ignore_order(
            read_resource(resource_group, 'expected_tokens.json'),
            read_file(tokens_output_file))