class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True,
coin_price_type=CoinPriceType.empty):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.btc_service.get_blocks(block_number_batch,
self.export_transactions)
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
class EnrichTransactionsJob(BaseJob):
def __init__(self,
transactions_iterable,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = BtcService(bitcoin_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size,
max_workers,
exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transactions_iterable,
self._enrich_transactions)
def _enrich_transactions(self, transactions):
transactions = [
self.transaction_mapper.dict_to_transaction(transaction)
for transaction in transactions
]
all_inputs = [transaction.inputs for transaction in transactions]
flat_inputs = [input for inputs in all_inputs for input in inputs]
for transaction_input_batch in dynamic_batch_iterator(
flat_inputs, lambda: self.batch_size):
input_transactions_map = self._get_input_transactions_as_map(
transaction_input_batch)
for input in transaction_input_batch:
output = self._get_output_for_input(input, input_transactions_map) \
if input.spent_transaction_hash is not None else None
if output is not None:
input.required_signatures = output.required_signatures
input.type = output.type
input.addresses = output.addresses
input.value = output.value
for transaction in transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(transaction))
def _get_input_transactions_as_map(self, transaction_inputs):
transaction_hashes = [
input.spent_transaction_hash for input in transaction_inputs
if input.spent_transaction_hash is not None
]
transaction_hashes = set(transaction_hashes)
if len(transaction_hashes) > 0:
transactions = self.btc_service.get_transactions_by_hashes(
transaction_hashes)
return {
transaction.hash: transaction
for transaction in transactions
}
else:
return {}
def _get_output_for_input(self, transaction_input, input_transactions_map):
spent_transaction_hash = transaction_input.spent_transaction_hash
input_transaction = input_transactions_map.get(spent_transaction_hash)
if input_transaction is None:
raise ValueError('Input transaction with hash {} not found'.format(
spent_transaction_hash))
spent_output_index = transaction_input.spent_output_index
if input_transaction.outputs is None or len(
input_transaction.outputs) < (spent_output_index + 1):
raise ValueError(
'There is no output with index {} in transaction with hash {}'.
format(spent_output_index, spent_transaction_hash))
output = input_transaction.outputs[spent_output_index]
return output
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
