def stream(last_synced_block_file, lag, provider_uri, output, start_block, chain=Chain.BITCOIN,
period_seconds=10, batch_size=2, block_batch_size=10, max_workers=5, log_file=None, pid_file=None,
enrich=True):
"""Streams all data types to console or Google Pub/Sub."""
configure_logging(log_file)
configure_signals()
from bitcoinetl.streaming.streaming_utils import get_item_exporter
from bitcoinetl.streaming.btc_streamer_adapter import BtcStreamerAdapter
from blockchainetl.streaming.streamer import Streamer
streamer_adapter = BtcStreamerAdapter(
bitcoin_rpc=ThreadLocalProxy(lambda: BitcoinRpc(provider_uri)),
item_exporter=get_item_exporter(output),
chain=chain,
batch_size=batch_size,
enable_enrich=enrich,
max_workers=max_workers
)
streamer = Streamer(
blockchain_streamer_adapter=streamer_adapter,
last_synced_block_file=last_synced_block_file,
lag=lag,
start_block=start_block,
period_seconds=period_seconds,
block_batch_size=block_batch_size,
pid_file=pid_file,
)
streamer.stream()
def stream(last_synced_block_file, lag, provider_uri, output, start_block, entity_types,
period_seconds=10, batch_size=2, block_batch_size=10, max_workers=5, log_file=None, pid_file=None):
"""Streams all data types to console or Google Pub/Sub."""
configure_logging(log_file)
configure_signals()
entity_types = parse_entity_types(entity_types)
validate_entity_types(entity_types, output)
from thetaetl.streaming.item_exporter_creator import create_item_exporter
from thetaetl.streaming.theta_streamer_adapter import ThetaStreamerAdapter
from blockchainetl.streaming.streamer import Streamer
# TODO: Implement fallback mechanism for provider uris instead of picking randomly
provider_uri = pick_random_provider_uri(provider_uri)
logging.info('Using ' + provider_uri)
streamer_adapter = ThetaStreamerAdapter(
theta_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
item_exporter=create_item_exporter(output),
batch_size=batch_size,
max_workers=max_workers,
entity_types=entity_types
)
streamer = Streamer(
blockchain_streamer_adapter=streamer_adapter,
last_synced_block_file=last_synced_block_file,
lag=lag,
start_block=start_block,
period_seconds=period_seconds,
block_batch_size=block_batch_size,
pid_file=pid_file
)
streamer.stream()
def stream(last_synced_block_file, lag, provider_uri, output, start_block, entity_types,
period_seconds=10, batch_size=2, block_batch_size=10, max_workers=5, log_file=None, pid_file=None):
"""Streams all data types to console or Google Pub/Sub."""
configure_logging(log_file)
configure_signals()
entity_types = parse_entity_types(entity_types)
from blockchainetl.streaming.streaming_utils import get_item_exporter
from ethereumpoaetl.streaming.eth_streamer_adapter import EthStreamerAdapter
from blockchainetl.streaming.streamer import Streamer
streamer_adapter = EthStreamerAdapter(
batch_web3_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
item_exporter=get_item_exporter(output),
batch_size=batch_size,
max_workers=max_workers,
entity_types=entity_types
)
streamer = Streamer(
blockchain_streamer_adapter=streamer_adapter,
last_synced_block_file=last_synced_block_file,
lag=lag,
start_block=start_block,
period_seconds=period_seconds,
block_batch_size=block_batch_size,
pid_file=pid_file
)
streamer.stream()
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group,
provider_type, chain):
try:
os.remove('last_synced_block.txt')
except OSError:
pass
blocks_output_file = str(tmpdir.join('actual_block.json'))
transactions_output_file = str(tmpdir.join("actual_transactions.json"))
streamer_adapter = BtcStreamerAdapter(
bitcoin_rpc=ThreadLocalProxy(
lambda: get_bitcoin_rpc(provider_type,
read_resource_lambda=lambda file:
read_resource(resource_group, file),
chain=chain)),
batch_size=batch_size,
item_exporter=CompositeItemExporter(
filename_mapping={
'block': blocks_output_file,
'transaction': transactions_output_file,
}),
)
streamer = Streamer(blockchain_streamer_adapter=streamer_adapter,
start_block=start_block,
end_block=end_block,
retry_errors=False)
streamer.stream()
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'),
read_file(blocks_output_file))
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'),
read_file(transactions_output_file))
def test_stream(tmpdir, start_block, end_block, batch_size, resource_group,
entity_types, provider_type):
try:
os.remove('last_synced_block.txt')
except OSError:
pass
blocks_output_file = str(tmpdir.join('actual_blocks.json'))
transactions_output_file = str(tmpdir.join('actual_transactions.json'))
logs_output_file = str(tmpdir.join('actual_logs.json'))
token_transfers_output_file = str(
tmpdir.join('actual_token_transfers.json'))
traces_output_file = str(tmpdir.join('actual_traces.json'))
contracts_output_file = str(tmpdir.join('actual_contracts.json'))
tokens_output_file = str(tmpdir.join('actual_tokens.json'))
streamer_adapter = EthStreamerAdapter(
batch_web3_provider=ThreadLocalProxy(
lambda: get_web3_provider(provider_type,
read_resource_lambda=lambda file:
read_resource(resource_group, file),
batch=True)),
batch_size=batch_size,
item_exporter=CompositeItemExporter(
filename_mapping={
'block': blocks_output_file,
'transaction': transactions_output_file,
'log': logs_output_file,
'token_transfer': token_transfers_output_file,
'trace': traces_output_file,
'contract': contracts_output_file,
'token': tokens_output_file,
}),
entity_types=entity_types,
)
streamer = Streamer(blockchain_streamer_adapter=streamer_adapter,
start_block=start_block,
end_block=end_block,
retry_errors=False)
streamer.stream()
if 'block' in entity_types:
print('=====================')
print(read_file(blocks_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_blocks.json'),
read_file(blocks_output_file))
if 'transaction' in entity_types:
print('=====================')
print(read_file(transactions_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_transactions.json'),
read_file(transactions_output_file))
if 'log' in entity_types:
print('=====================')
print(read_file(logs_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_logs.json'),
read_file(logs_output_file))
if 'token_transfer' in entity_types:
print('=====================')
print(read_file(token_transfers_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_token_transfers.json'),
read_file(token_transfers_output_file))
if 'trace' in entity_types:
print('=====================')
print(read_file(traces_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_traces.json'),
read_file(traces_output_file))
if 'contract' in entity_types:
print('=====================')
print(read_file(contracts_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_contracts.json'),
read_file(contracts_output_file))
if 'token' in entity_types:
print('=====================')
print(read_file(tokens_output_file))
compare_lines_ignore_order(
read_resource(resource_group, 'expected_tokens.json'),
read_file(tokens_output_file))