class ExportBlocksJob(BaseJob): def __init__(self, start_block, end_block, batch_size, bitcoin_rpc, max_workers, item_exporter, chain, export_blocks=True, export_transactions=True, coin_price_type=CoinPriceType.empty): validate_range(start_block, end_block) self.start_block = start_block self.end_block = end_block self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers) self.item_exporter = item_exporter self.export_blocks = export_blocks self.export_transactions = export_transactions if not self.export_blocks and not self.export_transactions: raise ValueError( 'At least one of export_blocks or export_transactions must be True' ) self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type) self.block_mapper = BtcBlockMapper() self.transaction_mapper = BtcTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute( range(self.start_block, self.end_block + 1), self._export_batch, total_items=self.end_block - self.start_block + 1) def _export_batch(self, block_number_batch): blocks = self.btc_service.get_blocks(block_number_batch, self.export_transactions) for block in blocks: self._export_block(block) def _export_block(self, block): if self.export_blocks: self.item_exporter.export_item( self.block_mapper.block_to_dict(block)) if self.export_transactions: for tx in block.transactions: self.item_exporter.export_item( self.transaction_mapper.transaction_to_dict(tx)) def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()
class EnrichTransactionsJob(BaseJob): def __init__(self, transactions_iterable, batch_size, bitcoin_rpc, max_workers, item_exporter, chain=Chain.BITCOIN): self.transactions_iterable = transactions_iterable self.btc_service = BtcService(bitcoin_rpc, chain) self.batch_size = batch_size self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers, exponential_backoff=False) self.item_exporter = item_exporter self.transaction_mapper = BtcTransactionMapper() def _start(self): self.item_exporter.open() def _export(self): self.batch_work_executor.execute(self.transactions_iterable, self._enrich_transactions) def _enrich_transactions(self, transactions): transactions = [ self.transaction_mapper.dict_to_transaction(transaction) for transaction in transactions ] all_inputs = [transaction.inputs for transaction in transactions] flat_inputs = [input for inputs in all_inputs for input in inputs] for transaction_input_batch in dynamic_batch_iterator( flat_inputs, lambda: self.batch_size): input_transactions_map = self._get_input_transactions_as_map( transaction_input_batch) for input in transaction_input_batch: output = self._get_output_for_input(input, input_transactions_map) \ if input.spent_transaction_hash is not None else None if output is not None: input.required_signatures = output.required_signatures input.type = output.type input.addresses = output.addresses input.value = output.value for transaction in transactions: self.item_exporter.export_item( self.transaction_mapper.transaction_to_dict(transaction)) def _get_input_transactions_as_map(self, transaction_inputs): transaction_hashes = [ input.spent_transaction_hash for input in transaction_inputs if input.spent_transaction_hash is not None ] transaction_hashes = set(transaction_hashes) if len(transaction_hashes) > 0: transactions = self.btc_service.get_transactions_by_hashes( transaction_hashes) return { transaction.hash: transaction for transaction in transactions } else: return {} def _get_output_for_input(self, transaction_input, input_transactions_map): spent_transaction_hash = transaction_input.spent_transaction_hash input_transaction = input_transactions_map.get(spent_transaction_hash) if input_transaction is None: raise ValueError('Input transaction with hash {} not found'.format( spent_transaction_hash)) spent_output_index = transaction_input.spent_output_index if input_transaction.outputs is None or len( input_transaction.outputs) < (spent_output_index + 1): raise ValueError( 'There is no output with index {} in transaction with hash {}'. format(spent_output_index, spent_transaction_hash)) output = input_transaction.outputs[spent_output_index] return output def _end(self): self.batch_work_executor.shutdown() self.item_exporter.close()