class EnrichTransactionsJob(BaseJob): def __init__( self, transactions_iterable, batch_size, eos_rpc, max_workers, item_exporter, chain=Chain.BITCOIN): self.transactions_iterable = transactions_iterable self.btc_service = EosService(eos_rpc, chain) self.batch_size = batch_size self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers, exponential_backoff=False) self.item_exporter = item_exporter self.transaction_mapper = EosTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.transactions_iterable, self._enrich_transactions) def _enrich_transactions(self, transactions): transactions = [self.transaction_mapper.dict_to_transaction(transaction) for transaction in transactions] all_inputs = [transaction.inputs for transaction in transactions] flat_inputs = [input for inputs in all_inputs for input in inputs] for transaction_input_batch in dynamic_batch_iterator(flat_inputs, lambda: self.batch_size): input_transactions_map = self._get_input_transactions_as_map(transaction_input_batch) for input in transaction_input_batch: output = self._get_output_for_input(input, input_transactions_map) \ if input.spent_transaction_hash is not None else None if output is not None: input.required_signatures = output.required_signatures input.type = output.type input.addresses = output.addresses input.value = output.value for transaction in transactions: tx_dict = self.transaction_mapper.transaction_to_dict(transaction) if tx_dict: # skip in None returned self.item_exporter.export_item(tx_dict) def _get_input_transactions_as_map(self, transaction_inputs): transaction_hashes = [input.spent_transaction_hash for input in transaction_inputs if input.spent_transaction_hash is not None] transaction_hashes = set(transaction_hashes) if len(transaction_hashes) > 0: transactions = self.btc_service.get_transactions_by_hashes(transaction_hashes) return {transaction.hash: transaction for transaction in transactions} else: return {} def _get_output_for_input(self, transaction_input, input_transactions_map): spent_transaction_hash = transaction_input.spent_transaction_hash input_transaction = input_transactions_map.get(spent_transaction_hash) if input_transaction is None: raise ValueError('Input transaction with hash {} not found'.format(spent_transaction_hash)) spent_output_index = transaction_input.spent_output_index if input_transaction.outputs is None or len(input_transaction.outputs) < (spent_output_index + 1): raise ValueError( 'There is no output with index {} in transaction with hash {}'.format( spent_output_index, spent_transaction_hash)) output = input_transaction.outputs[spent_output_index] return output def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportBlocksJob(BaseJob): def __init__( self, start_block, end_block, eos_rpc, max_workers, item_exporter, batch_size=1, export_blocks=True, export_transactions=True, export_actions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions self.export_actions = export_actions if not self.export_blocks and not self.export_transactions and not self.export_actions: raise ValueError('At least one of export_blocks or export_transactions or export_actions must be True') self.eos_service = EosService(eos_rpc) self.block_mapper = EosBlockMapper() self.transaction_mapper = EosTransactionMapper() self.action_mapper = EosActionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1 ) def _export_batch(self, block_number_batch): blocks = self.eos_service.get_blocks(block_number_batch) for block in blocks: self._export_block(block) self._export_transactions(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item(self.block_mapper.block_to_dict(block)) def _export_transactions(self, block): if self.export_transactions: for transaction in block['transactions']: self._export_transaction(transaction, block) def _export_transaction(self, transaction, block): transaction_dict = self.transaction_mapper.transaction_to_dict(transaction, block) self.item_exporter.export_item(transaction_dict) if self.export_actions \ and isinstance(transaction.get('trx'), dict) \ and transaction.get('trx').get('transaction') is not None \ and transaction.get('trx').get('transaction').get('actions') is not None: for action in transaction.get('trx').get('transaction').get('actions'): action_dict = self.action_mapper.action_to_dict(action, transaction_dict) self.item_exporter.export_item(action_dict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, eos_rpc, max_workers, item_exporter, chain, export_blocks=True, export_transactions=True): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.eos_service = EosService(eos_rpc, chain) self.block_mapper = EosBlockMapper() self.transaction_mapper = EosTransactionMapper() self.action_mapper = EosActionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): blocks = self.eos_service.get_blocks(block_number_batch, self.export_transactions) for block in blocks: self._export_block(block) self._export_txs(block) def _export_block(self, block): if not self.export_blocks: return self.item_exporter.export_item(self.block_mapper.block_to_dict(block)) def _export_txs(self, block): if not self.export_transactions: return for tx in block["transactions"]: self._export_tx(tx, block) def _export_tx(self, tx, block): tx_dict = self.transaction_mapper.transaction_to_dict(tx, block) if not tx_dict: # skip in None returned return self.item_exporter.export_item(tx_dict) if tx_dict.get("trx.transaction.actions") is None: return for action in tx_dict["trx.transaction.actions"]: action_dict = self.action_mapper.action_to_dict( action, tx_dict, block) self.item_exporter.export_item(action_dict) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()