class EnrichTransactionsJob(BaseJob):
def __init__(
self,
transactions_iterable,
batch_size,
eos_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = EosService(eos_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers, exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = EosTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transactions_iterable, self._enrich_transactions)
def _enrich_transactions(self, transactions):
transactions = [self.transaction_mapper.dict_to_transaction(transaction) for transaction in transactions]
all_inputs = [transaction.inputs for transaction in transactions]
flat_inputs = [input for inputs in all_inputs for input in inputs]
for transaction_input_batch in dynamic_batch_iterator(flat_inputs, lambda: self.batch_size):
input_transactions_map = self._get_input_transactions_as_map(transaction_input_batch)
for input in transaction_input_batch:
output = self._get_output_for_input(input, input_transactions_map) \
if input.spent_transaction_hash is not None else None
if output is not None:
input.required_signatures = output.required_signatures
input.type = output.type
input.addresses = output.addresses
input.value = output.value
for transaction in transactions:
tx_dict = self.transaction_mapper.transaction_to_dict(transaction)
if tx_dict: # skip in None returned
self.item_exporter.export_item(tx_dict)
def _get_input_transactions_as_map(self, transaction_inputs):
transaction_hashes = [input.spent_transaction_hash for input in transaction_inputs
if input.spent_transaction_hash is not None]
transaction_hashes = set(transaction_hashes)
if len(transaction_hashes) > 0:
transactions = self.btc_service.get_transactions_by_hashes(transaction_hashes)
return {transaction.hash: transaction for transaction in transactions}
else:
return {}
def _get_output_for_input(self, transaction_input, input_transactions_map):
spent_transaction_hash = transaction_input.spent_transaction_hash
input_transaction = input_transactions_map.get(spent_transaction_hash)
if input_transaction is None:
raise ValueError('Input transaction with hash {} not found'.format(spent_transaction_hash))
spent_output_index = transaction_input.spent_output_index
if input_transaction.outputs is None or len(input_transaction.outputs) < (spent_output_index + 1):
raise ValueError(
'There is no output with index {} in transaction with hash {}'.format(
spent_output_index, spent_transaction_hash))
output = input_transaction.outputs[spent_output_index]
return output
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(
self,
start_block,
end_block,
eos_rpc,
max_workers,
item_exporter,
batch_size=1,
export_blocks=True,
export_transactions=True,
export_actions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
self.export_actions = export_actions
if not self.export_blocks and not self.export_transactions and not self.export_actions:
raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True')
self.eos_service = EosService(eos_rpc)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1
)
def _export_batch(self, block_number_batch):
blocks = self.eos_service.get_blocks(block_number_batch)
for block in blocks:
self._export_block(block)
self._export_transactions(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(self.block_mapper.block_to_dict(block))
def _export_transactions(self, block):
if self.export_transactions:
for transaction in block['transactions']:
self._export_transaction(transaction, block)
def _export_transaction(self, transaction, block):
transaction_dict = self.transaction_mapper.transaction_to_dict(transaction, block)
self.item_exporter.export_item(transaction_dict)
if self.export_actions \
and isinstance(transaction.get('trx'), dict) \
and transaction.get('trx').get('transaction') is not None \
and transaction.get('trx').get('transaction').get('actions') is not None:
for action in transaction.get('trx').get('transaction').get('actions'):
action_dict = self.action_mapper.action_to_dict(action, transaction_dict)
self.item_exporter.export_item(action_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
eos_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.eos_service = EosService(eos_rpc, chain)
self.block_mapper = EosBlockMapper()
self.transaction_mapper = EosTransactionMapper()
self.action_mapper = EosActionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.eos_service.get_blocks(block_number_batch,
self.export_transactions)
for block in blocks:
self._export_block(block)
self._export_txs(block)
def _export_block(self, block):
if not self.export_blocks:
return
self.item_exporter.export_item(self.block_mapper.block_to_dict(block))
def _export_txs(self, block):
if not self.export_transactions:
return
for tx in block["transactions"]:
self._export_tx(tx, block)
def _export_tx(self, tx, block):
tx_dict = self.transaction_mapper.transaction_to_dict(tx, block)
if not tx_dict: # skip in None returned
return
self.item_exporter.export_item(tx_dict)
if tx_dict.get("trx.transaction.actions") is None:
return
for action in tx_dict["trx.transaction.actions"]:
action_dict = self.action_mapper.action_to_dict(
action, tx_dict, block)
self.item_exporter.export_item(action_dict)
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()