def __init__(self,
batch_web3_provider,
item_exporter=ConsoleItemExporter(),
batch_size=100,
max_workers=5,
entity_types=tuple(EntityType.ALL_FOR_STREAMING)):
self.batch_web3_provider = batch_web3_provider
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.entity_types = entity_types
self.item_id_calculator = EthItemIdCalculator()
self.item_timestamp_calculator = EthItemTimestampCalculator()
def __init__(
self,
batch_web3_provider: ThreadLocalProxy,
batch_size: int = 100,
max_workers: int = 5,
) -> None:
self.batch_web3_provider = batch_web3_provider
self.batch_size = batch_size
self.max_workers = max_workers
self.item_id_calculator = EthItemIdCalculator()
self.item_timestamp_calculator = EthItemTimestampCalculator()
class EthStreamerAdapter:
def __init__(self,
batch_web3_provider,
item_exporter=ConsoleItemExporter(),
batch_size=100,
max_workers=5,
entity_types=tuple(EntityType.ALL_FOR_STREAMING)):
self.batch_web3_provider = batch_web3_provider
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
self.entity_types = entity_types
self.item_id_calculator = EthItemIdCalculator()
self.item_timestamp_calculator = EthItemTimestampCalculator()
def open(self):
self.item_exporter.open()
def get_current_block_number(self):
w3 = Web3(self.batch_web3_provider)
w3.middleware_stack.inject(geth_poa_middleware, layer=0)
return int(w3.eth.getBlock("latest").number)
def export_all(self, start_block, end_block):
# Export blocks and transactions
blocks, transactions = [], []
if self._should_export(EntityType.BLOCK) or self._should_export(
EntityType.TRANSACTION):
blocks, transactions = self._export_blocks_and_transactions(
start_block, end_block)
# Export receipts and logs
receipts, logs = [], []
if self._should_export(EntityType.RECEIPT) or self._should_export(
EntityType.LOG):
receipts, logs = self._export_receipts_and_logs(transactions)
# Extract token transfers
token_transfers = []
if self._should_export(EntityType.TOKEN_TRANSFER):
token_transfers = self._extract_token_transfers(logs)
# Export traces
traces = []
if self._should_export(EntityType.TRACE):
traces = self._export_traces(start_block, end_block)
# Export contracts
contracts = []
if self._should_export(EntityType.CONTRACT):
contracts = self._export_contracts(traces)
# Export tokens
tokens = []
if self._should_export(EntityType.TOKEN):
tokens = self._extract_tokens(contracts)
enriched_blocks = blocks \
if EntityType.BLOCK in self.entity_types else []
enriched_transactions = enrich_transactions(transactions, receipts) \
if EntityType.TRANSACTION in self.entity_types else []
enriched_logs = enrich_logs(blocks, logs) \
if EntityType.LOG in self.entity_types else []
enriched_token_transfers = enrich_token_transfers(blocks, token_transfers) \
if EntityType.TOKEN_TRANSFER in self.entity_types else []
enriched_traces = enrich_traces(blocks, traces) \
if EntityType.TRACE in self.entity_types else []
enriched_contracts = enrich_contracts(blocks, contracts) \
if EntityType.CONTRACT in self.entity_types else []
enriched_tokens = enrich_tokens(blocks, tokens) \
if EntityType.TOKEN in self.entity_types else []
logging.info('Exporting with ' + type(self.item_exporter).__name__)
all_items = enriched_blocks + \
enriched_transactions + \
enriched_logs + \
enriched_token_transfers + \
enriched_traces + \
enriched_contracts + \
enriched_tokens
self.calculate_item_ids(all_items)
self.calculate_item_timestamps(all_items)
self.item_exporter.export_items(all_items)
def _export_blocks_and_transactions(self, start_block, end_block):
blocks_and_transactions_item_exporter = InMemoryItemExporter(
item_types=['block', 'transaction'])
blocks_and_transactions_job = ExportBlocksJob(
start_block=start_block,
end_block=end_block,
batch_size=self.batch_size,
batch_web3_provider=self.batch_web3_provider,
max_workers=self.max_workers,
item_exporter=blocks_and_transactions_item_exporter,
export_blocks=self._should_export(EntityType.BLOCK),
export_transactions=self._should_export(EntityType.TRANSACTION))
blocks_and_transactions_job.run()
blocks = blocks_and_transactions_item_exporter.get_items('block')
transactions = blocks_and_transactions_item_exporter.get_items(
'transaction')
return blocks, transactions
def _export_receipts_and_logs(self, transactions):
exporter = InMemoryItemExporter(item_types=['receipt', 'log'])
job = ExportReceiptsJob(
transaction_hashes_iterable=(transaction['hash']
for transaction in transactions),
batch_size=self.batch_size,
batch_web3_provider=self.batch_web3_provider,
max_workers=self.max_workers,
item_exporter=exporter,
export_receipts=self._should_export(EntityType.RECEIPT),
export_logs=self._should_export(EntityType.LOG))
job.run()
receipts = exporter.get_items('receipt')
logs = exporter.get_items('log')
return receipts, logs
def _extract_token_transfers(self, logs):
exporter = InMemoryItemExporter(item_types=['token_transfer'])
job = ExtractTokenTransfersJob(logs_iterable=logs,
batch_size=self.batch_size,
max_workers=self.max_workers,
item_exporter=exporter)
job.run()
token_transfers = exporter.get_items('token_transfer')
return token_transfers
def _export_traces(self, start_block, end_block):
exporter = InMemoryItemExporter(item_types=['trace'])
job = ExportTracesJob(
start_block=start_block,
end_block=end_block,
batch_size=self.batch_size,
web3=ThreadLocalProxy(
lambda: Web3(self.batch_web3_provider).middleware_stack.inject(
geth_poa_middleware, layer=0)),
max_workers=self.max_workers,
item_exporter=exporter)
job.run()
traces = exporter.get_items('trace')
return traces
def _export_contracts(self, traces):
exporter = InMemoryItemExporter(item_types=['contract'])
job = ExtractContractsJob(traces_iterable=traces,
batch_size=self.batch_size,
max_workers=self.max_workers,
item_exporter=exporter)
job.run()
contracts = exporter.get_items('contract')
return contracts
def _extract_tokens(self, contracts):
exporter = InMemoryItemExporter(item_types=['token'])
job = ExtractTokensJob(
contracts_iterable=contracts,
web3=ThreadLocalProxy(
lambda: Web3(self.batch_web3_provider).middleware_stack.inject(
geth_poa_middleware, layer=0)),
max_workers=self.max_workers,
item_exporter=exporter)
job.run()
tokens = exporter.get_items('token')
return tokens
def _should_export(self, entity_type):
if entity_type == EntityType.BLOCK:
return True
if entity_type == EntityType.TRANSACTION:
return EntityType.TRANSACTION in self.entity_types or self._should_export(
EntityType.LOG)
if entity_type == EntityType.RECEIPT:
return EntityType.TRANSACTION in self.entity_types or self._should_export(
EntityType.TOKEN_TRANSFER)
if entity_type == EntityType.LOG:
return EntityType.LOG in self.entity_types or self._should_export(
EntityType.TOKEN_TRANSFER)
if entity_type == EntityType.TOKEN_TRANSFER:
return EntityType.TOKEN_TRANSFER in self.entity_types
if entity_type == EntityType.TRACE:
return EntityType.TRACE in self.entity_types or self._should_export(
EntityType.CONTRACT)
if entity_type == EntityType.CONTRACT:
return EntityType.CONTRACT in self.entity_types or self._should_export(
EntityType.TOKEN)
if entity_type == EntityType.TOKEN:
return EntityType.TOKEN in self.entity_types
raise ValueError('Unexpected entity type ' + entity_type)
def calculate_item_ids(self, items):
for item in items:
item['item_id'] = self.item_id_calculator.calculate(item)
def calculate_item_timestamps(self, items):
for item in items:
item['item_timestamp'] = self.item_timestamp_calculator.calculate(
item)
def close(self):
self.item_exporter.close()