def __init__(
self,
start_block,
end_block,
eos_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_transactions=True,
export_actions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
self.export_actions = export_actions
if not self.export_blocks and not self.export_transactions and not self.export_actions:
raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True')
self.eos_service = EosService(eos_rpc)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
class ExportDsBlocksJob(BaseJob):
def __init__(self, start_block, end_block, zilliqa_api, max_workers,
item_exporter):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(
1, max_workers, retry_exceptions=RETRY_EXCEPTIONS)
self.item_exporter = item_exporter
self.zilliqa_service = ZilliqaService(zilliqa_api)
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
for block_number in block_number_batch:
block = self.zilliqa_service.get_ds_block(block_number)
self.item_exporter.export_item(map_ds_block(block))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(
self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True,
):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
"At least one of export_blocks or export_transactions must be True"
)
self.block_mapper = IcxBlockMapper()
self.transaction_mapper = IcxTransactionMapper()
class ExportReceiptsJob(BaseJob):
def __init__(
self,
transaction_hashes_iterable,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_receipts=True,
export_logs=True,
):
self.batch_web3_provider = batch_web3_provider
self.transaction_hashes_iterable = transaction_hashes_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_receipts = export_receipts
self.export_logs = export_logs
if not self.export_receipts and not self.export_logs:
raise ValueError(
"At least one of export_receipts or export_logs must be True")
self.receipt_mapper = IcxReceiptMapper()
self.receipt_log_mapper = IcxReceiptLogMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transaction_hashes_iterable,
self._export_receipts)
def _export_receipts(self, transaction_hashes):
receipts_rpc = list(generate_get_receipt_json_rpc(transaction_hashes))
response = self.batch_web3_provider.make_batch_request(
json.dumps(receipts_rpc))
results = rpc_response_batch_to_results(response)
receipts = [
self.receipt_mapper.json_dict_to_receipt(result)
for result in results
]
for receipt in receipts:
self._export_receipt(receipt)
def _export_receipt(self, receipt):
if self.export_receipts:
self.item_exporter.export_item(
self.receipt_mapper.receipt_to_dict(receipt))
if self.export_logs:
for log in receipt.logs:
self.item_exporter.export_item(
self.receipt_log_mapper.receipt_log_to_dict(log))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self, start_block, end_block, zilliqa_api, max_workers,
item_exporter):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(
1, max_workers, retry_exceptions=RETRY_EXCEPTIONS)
self.item_exporter = item_exporter
self.zilliqa_service = ZilliqaService(zilliqa_api)
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
iotex_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_actions=True,
export_logs=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(
batch_size, max_workers, retry_exceptions=RETRY_EXCEPTIONS)
self.item_exporter = item_exporter
self.iotex_service = IotexService(iotex_rpc)
self.export_blocks = export_blocks
self.export_actions = export_actions
self.export_logs = export_logs
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.iotex_service.get_blocks(block_number_batch)
block_metas = self.iotex_service.get_block_metas(block_number_batch)
for block, block_meta in zip(blocks, block_metas):
if self.export_blocks:
self.item_exporter.export_item(map_block(block, block_meta))
if self.export_actions:
for action in map_action(block):
self.item_exporter.export_item(action)
if self.export_logs:
for receipt in block.receipts:
for log in [
map_log(block.block, log) for log in receipt.logs
]:
self.item_exporter.export_item(log)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
start_block,
end_block,
tezos_rpc,
max_workers,
item_exporter,
batch_size=1):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.tezos_service = TezosService(tezos_rpc)
class ExportJob(BaseJob):
def __init__(self,
start_block,
end_block,
tezos_rpc,
max_workers,
item_exporter,
batch_size=1):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.tezos_service = TezosService(tezos_rpc)
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
responses = self.tezos_service.get_blocks(block_number_batch)
for response in responses:
block = map_block(response)
self.item_exporter.export_item(block)
self.item_exporter.export_items(
map_balance_updates(block, response))
self.item_exporter.export_items(map_operations(block, response))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
start_block,
end_block,
iotex_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_actions=True,
export_logs=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(
batch_size, max_workers, retry_exceptions=RETRY_EXCEPTIONS)
self.item_exporter = item_exporter
self.iotex_service = IotexService(iotex_rpc)
self.export_blocks = export_blocks
self.export_actions = export_actions
self.export_logs = export_logs
def __init__(
self,
transaction_hashes_iterable,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_receipts=True,
export_logs=True,
):
self.batch_web3_provider = batch_web3_provider
self.transaction_hashes_iterable = transaction_hashes_iterable
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_receipts = export_receipts
self.export_logs = export_logs
if not self.export_receipts and not self.export_logs:
raise ValueError(
"At least one of export_receipts or export_logs must be True")
self.receipt_mapper = IcxReceiptMapper()
self.receipt_log_mapper = IcxReceiptLogMapper()
class ExportTxBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
zilliqa_api,
max_workers,
item_exporter,
export_transactions=True,
export_event_logs=True,
export_exceptions=True,
export_transitions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(
1, max_workers, retry_exceptions=RETRY_EXCEPTIONS)
self.item_exporter = item_exporter
self.zilliqa_service = ZilliqaService(zilliqa_api)
self.export_transactions = export_transactions
self.export_event_logs = export_event_logs
self.export_exceptions = export_exceptions
self.export_transitions = export_transitions
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
items = []
for number in block_number_batch:
tx_block = map_tx_block(self.zilliqa_service.get_tx_block(number))
txns = list(self.zilliqa_service.get_transactions(
number)) if tx_block.get('num_transactions') > 0 else []
if self._should_export_transactions():
for txn in txns:
items.append(map_transaction(tx_block, txn))
if self._should_export_event_logs(txn):
items.extend(map_event_logs(tx_block, txn))
if self._should_export_exceptions(txn):
items.extend(map_exceptions(tx_block, txn))
if self._should_export_transitions(txn):
items.extend(map_transitions(tx_block, txn))
tx_block['num_present_transactions'] = len(txns)
items.append(tx_block)
for item in items:
self.item_exporter.export_item(item)
def _should_export_transactions(self):
return self.export_transactions
def _should_export_event_logs(self, txn):
return self.export_event_logs and txn.get('receipt')
def _should_export_exceptions(self, txn):
return self.export_exceptions and txn.get('receipt')
def _should_export_transitions(self, txn):
return self.export_transitions and txn.get('receipt')
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(
self,
start_block,
end_block,
eos_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_transactions=True,
export_actions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
self.export_actions = export_actions
if not self.export_blocks and not self.export_transactions and not self.export_actions:
raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True')
self.eos_service = EosService(eos_rpc)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1
)
def _export_batch(self, block_number_batch):
blocks = self.eos_service.get_blocks(block_number_batch)
for block in blocks:
self._export_block(block)
self._export_transactions(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(self.block_mapper.block_to_dict(block))
def _export_transactions(self, block):
if self.export_transactions:
for transaction in block['transactions']:
self._export_transaction(transaction, block)
def _export_transaction(self, transaction, block):
transaction_dict = self.transaction_mapper.transaction_to_dict(transaction, block)
self.item_exporter.export_item(transaction_dict)
if self.export_actions \
and isinstance(transaction.get('trx'), dict) \
and transaction.get('trx').get('transaction') is not None \
and transaction.get('trx').get('transaction').get('actions') is not None:
for action in transaction.get('trx').get('transaction').get('actions'):
action_dict = self.action_mapper.action_to_dict(action, transaction_dict)
self.item_exporter.export_item(action_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(
self,
start_block,
end_block,
batch_size,
batch_web3_provider,
max_workers,
item_exporter,
export_blocks=True,
export_transactions=True,
):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_web3_provider = batch_web3_provider
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
"At least one of export_blocks or export_transactions must be True"
)
self.block_mapper = IcxBlockMapper()
self.transaction_mapper = IcxTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1,
)
def _export_batch(self, block_number_batch):
blocks_rpc = list(
generate_get_block_by_number_json_rpc(block_number_batch))
response = self.batch_web3_provider.make_batch_request(
json.dumps(blocks_rpc))
results = rpc_response_batch_to_results(response)
blocks = [
self.block_mapper.json_dict_to_block(result) for result in results
]
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
