def create_item_exporter(output):
item_exporter_type = determine_item_exporter_type(output)
if item_exporter_type == ItemExporterType.PUBSUB:
from blockchainetl_common.jobs.exporters.google_pubsub_item_exporter import GooglePubSubItemExporter
enable_message_ordering = 'sorted' in output
item_exporter = GooglePubSubItemExporter(
item_type_to_topic_mapping={
'block': output + '.blocks',
'transaction': output + '.transactions',
'log': output + '.logs',
'token_transfer': output + '.token_transfers',
'trace': output + '.traces',
'contract': output + '.contracts',
'token': output + '.tokens',
},
message_attributes=('item_id', ),
batch_max_bytes=1024 * 1024 * 5,
batch_max_latency=5,
batch_max_messages=1000,
enable_message_ordering=enable_message_ordering)
elif item_exporter_type == ItemExporterType.POSTGRES:
from blockchainetl_common.jobs.exporters.postgres_item_exporter import PostgresItemExporter
from blockchainetl_common.streaming.postgres_utils import create_insert_statement_for_table
from blockchainetl_common.jobs.exporters.converters.unix_timestamp_item_converter import UnixTimestampItemConverter
from blockchainetl_common.jobs.exporters.converters.int_to_decimal_item_converter import IntToDecimalItemConverter
from blockchainetl_common.jobs.exporters.converters.list_field_item_converter import ListFieldItemConverter
from polygonetl.streaming.postgres_tables import BLOCKS, TRANSACTIONS, LOGS, TOKEN_TRANSFERS, TRACES
item_exporter = PostgresItemExporter(
output,
item_type_to_insert_stmt_mapping={
'block':
create_insert_statement_for_table(BLOCKS),
'transaction':
create_insert_statement_for_table(TRANSACTIONS),
'log':
create_insert_statement_for_table(LOGS),
'token_transfer':
create_insert_statement_for_table(TOKEN_TRANSFERS),
'traces':
create_insert_statement_for_table(TRACES),
},
converters=[
UnixTimestampItemConverter(),
IntToDecimalItemConverter(),
ListFieldItemConverter('topics', 'topic', fill=4)
])
elif item_exporter_type == ItemExporterType.GCS:
from blockchainetl_common.jobs.exporters.gcs_item_exporter import GcsItemExporter
bucket, path = get_bucket_and_path_from_gcs_output(output)
item_exporter = GcsItemExporter(bucket=bucket, path=path)
elif item_exporter_type == ItemExporterType.CONSOLE:
item_exporter = ConsoleItemExporter()
else:
raise ValueError('Unable to determine item exporter type for output ' +
output)
return item_exporter
def __init__(self,
eos_rpc,
item_exporter=ConsoleItemExporter(),
batch_size=1,
max_workers=5):
self.eos_rpc = eos_rpc
self.eos_service = EosService(eos_rpc)
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.item_id_calculator = EosItemIdCalculator()
def __init__(
self,
iotex_rpc,
item_exporter=ConsoleItemExporter(),
batch_size=100,
max_workers=5,
entity_types=tuple(EntityType.ALL_FOR_STREAMING)):
self.iotex_rpc = iotex_rpc
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.entity_types = entity_types
self.item_id_calculator = ItemIdCalculator()
def __init__(self,
batch_web3_provider,
item_exporter=ConsoleItemExporter(),
batch_size=100,
max_workers=5,
entity_types=tuple(EntityType.ALL_FOR_STREAMING)):
self.batch_web3_provider = batch_web3_provider
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.entity_types = entity_types
self.item_id_calculator = EthItemIdCalculator()
self.item_timestamp_calculator = EthItemTimestampCalculator()
self.web3 = Web3(self.batch_web3_provider)
self.web3.middleware_stack.inject(geth_poa_middleware, layer=0)
def __init__(
self,
batch_web3_provider,
item_exporter=ConsoleItemExporter(),
batch_size=100,
max_workers=5,
entity_types=tuple(EntityType.ALL_FOR_STREAMING),
):
self.batch_web3_provider = batch_web3_provider
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.entity_types = entity_types
self.item_id_calculator = IcxItemIdCalculator()
self.item_timestamp_calculator = IcxItemTimestampCalculator()
def get_item_exporter(output):
if output is not None:
from blockchainetl_common.jobs.exporters.google_pubsub_item_exporter import GooglePubSubItemExporter
item_exporter = GooglePubSubItemExporter(
item_type_to_topic_mapping={
'block': output + '.blocks',
'transaction': output + '.transactions',
'log': output + '.logs',
'token_transfer': output + '.token_transfers',
'trace': output + '.traces',
'contract': output + '.contracts',
'token': output + '.tokens',
})
else:
item_exporter = ConsoleItemExporter()
return item_exporter
def get_item_exporter(output):
if output is not None:
from blockchainetl_common.jobs.exporters.google_pubsub_item_exporter import GooglePubSubItemExporter
item_exporter = GooglePubSubItemExporter(
item_type_to_topic_mapping={
'block': output + '.blocks',
'transaction': output + '.transactions',
'action': output + '.actions'
},
message_attributes=('item_id', ),
batch_max_bytes=1024 * 1024,
batch_max_latency=3,
batch_max_messages=1000)
else:
item_exporter = ConsoleItemExporter()
return item_exporter
def create_item_exporter(output):
item_exporter_type = determine_item_exporter_type(output)
if item_exporter_type == ItemExporterType.PUBSUB:
from blockchainetl_common.jobs.exporters.google_pubsub_item_exporter import GooglePubSubItemExporter
item_exporter = GooglePubSubItemExporter(
item_type_to_topic_mapping={
'block': output + '.blocks',
'action': output + '.actions',
'log': output + '.logs',
'transaction_log': output + '.transaction_logs',
},
message_attributes=('item_id', ))
elif item_exporter_type == ItemExporterType.CONSOLE:
item_exporter = ConsoleItemExporter()
else:
raise ValueError('Unable to determine item exporter type for output ' +
output)
return item_exporter