def __init__(self,
start_block,
end_block,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True,
coin_price_type=CoinPriceType.empty):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
def __init__(self,
bitcoin_rpc,
item_exporter=ConsoleItemExporter(),
chain=Chain.BITCOIN,
batch_size=2,
max_workers=5):
self.bitcoin_rpc = bitcoin_rpc
self.chain = chain
self.btc_service = BtcService(bitcoin_rpc, chain)
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
def __init__(self,
bitcoin_rpc,
item_exporter=ConsoleItemExporter(),
chain=Chain.BITCOIN,
batch_size=2,
enable_enrich=True,
max_workers=5):
self.bitcoin_rpc = bitcoin_rpc
self.chain = chain
self.btc_service = BtcService(bitcoin_rpc, chain)
self.item_exporter = item_exporter
self.batch_size = batch_size
self.enable_enrich = enable_enrich
self.max_workers = max_workers
self.item_id_calculator = BtcItemIdCalculator()
class ExportBlocksJob(BaseJob):
def __init__(self,
start_block,
end_block,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain,
export_blocks=True,
export_transactions=True,
coin_price_type=CoinPriceType.empty):
validate_range(start_block, end_block)
self.start_block = start_block
self.end_block = end_block
self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
self.item_exporter = item_exporter
self.export_blocks = export_blocks
self.export_transactions = export_transactions
if not self.export_blocks and not self.export_transactions:
raise ValueError(
'At least one of export_blocks or export_transactions must be True'
)
self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
self.block_mapper = BtcBlockMapper()
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(
range(self.start_block, self.end_block + 1),
self._export_batch,
total_items=self.end_block - self.start_block + 1)
def _export_batch(self, block_number_batch):
blocks = self.btc_service.get_blocks(block_number_batch,
self.export_transactions)
for block in blocks:
self._export_block(block)
def _export_block(self, block):
if self.export_blocks:
self.item_exporter.export_item(
self.block_mapper.block_to_dict(block))
if self.export_transactions:
for tx in block.transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(tx))
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self,
transactions_iterable,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = BtcService(bitcoin_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size,
max_workers,
exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = BtcTransactionMapper()
class BlockTimestampGraph(object):
def __init__(self, bitcoin_rpc):
self._btc_service = BtcService(bitcoin_rpc)
def get_first_point(self):
block = self._btc_service.get_genesis_block()
return block_to_point(block)
def get_last_point(self):
block = self._btc_service.get_latest_block()
return block_to_point(block)
def get_point(self, block_number):
block = self._btc_service.get_block(block_number)
return block_to_point(block)
def get_points(self, block_numbers):
blocks = self._btc_service.get_blocks(block_numbers)
return [block_to_point(block) for block in blocks]
class BtcStreamerAdapter:
def __init__(self,
bitcoin_rpc,
item_exporter=ConsoleItemExporter(),
chain=Chain.BITCOIN,
batch_size=2,
max_workers=5):
self.bitcoin_rpc = bitcoin_rpc
self.chain = chain
self.btc_service = BtcService(bitcoin_rpc, chain)
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
def open(self):
self.item_exporter.open()
def get_current_block_number(self):
return int(self.btc_service.get_latest_block().number)
def export_all(self, start_block, end_block):
# Export blocks and transactions
blocks_and_transactions_item_exporter = InMemoryItemExporter(
item_types=['block', 'transaction'])
blocks_and_transactions_job = ExportBlocksJob(
start_block=start_block,
end_block=end_block,
batch_size=self.batch_size,
bitcoin_rpc=self.bitcoin_rpc,
max_workers=self.max_workers,
item_exporter=blocks_and_transactions_item_exporter,
chain=self.chain,
export_blocks=True,
export_transactions=True)
blocks_and_transactions_job.run()
blocks = blocks_and_transactions_item_exporter.get_items('block')
transactions = blocks_and_transactions_item_exporter.get_items(
'transaction')
# Enrich transactions
enriched_transactions_item_exporter = InMemoryItemExporter(
item_types=['transaction'])
enrich_transactions_job = EnrichTransactionsJob(
transactions_iterable=transactions,
batch_size=self.batch_size,
bitcoin_rpc=self.bitcoin_rpc,
max_workers=self.max_workers,
item_exporter=enriched_transactions_item_exporter,
chain=self.chain)
enrich_transactions_job.run()
enriched_transactions = enriched_transactions_item_exporter.get_items(
'transaction')
if len(enriched_transactions) != len(transactions):
raise ValueError('The number of transactions is wrong ' +
str(transactions))
logging.info('Exporting with ' + type(self.item_exporter).__name__)
self.item_exporter.export_items(blocks + enriched_transactions)
def close(self):
self.item_exporter.close()
class EnrichTransactionsJob(BaseJob):
def __init__(self,
transactions_iterable,
batch_size,
bitcoin_rpc,
max_workers,
item_exporter,
chain=Chain.BITCOIN):
self.transactions_iterable = transactions_iterable
self.btc_service = BtcService(bitcoin_rpc, chain)
self.batch_size = batch_size
self.batch_work_executor = BatchWorkExecutor(batch_size,
max_workers,
exponential_backoff=False)
self.item_exporter = item_exporter
self.transaction_mapper = BtcTransactionMapper()
def _start(self):
self.item_exporter.open()
def _export(self):
self.batch_work_executor.execute(self.transactions_iterable,
self._enrich_transactions)
def _enrich_transactions(self, transactions):
transactions = [
self.transaction_mapper.dict_to_transaction(transaction)
for transaction in transactions
]
all_inputs = [transaction.inputs for transaction in transactions]
flat_inputs = [input for inputs in all_inputs for input in inputs]
for transaction_input_batch in dynamic_batch_iterator(
flat_inputs, lambda: self.batch_size):
input_transactions_map = self._get_input_transactions_as_map(
transaction_input_batch)
for input in transaction_input_batch:
output = self._get_output_for_input(input, input_transactions_map) \
if input.spent_transaction_hash is not None else None
if output is not None:
input.required_signatures = output.required_signatures
input.type = output.type
input.addresses = output.addresses
input.value = output.value
for transaction in transactions:
self.item_exporter.export_item(
self.transaction_mapper.transaction_to_dict(transaction))
def _get_input_transactions_as_map(self, transaction_inputs):
transaction_hashes = [
input.spent_transaction_hash for input in transaction_inputs
if input.spent_transaction_hash is not None
]
transaction_hashes = set(transaction_hashes)
if len(transaction_hashes) > 0:
transactions = self.btc_service.get_transactions_by_hashes(
transaction_hashes)
return {
transaction.hash: transaction
for transaction in transactions
}
else:
return {}
def _get_output_for_input(self, transaction_input, input_transactions_map):
spent_transaction_hash = transaction_input.spent_transaction_hash
input_transaction = input_transactions_map.get(spent_transaction_hash)
if input_transaction is None:
raise ValueError('Input transaction with hash {} not found'.format(
spent_transaction_hash))
spent_output_index = transaction_input.spent_output_index
if input_transaction.outputs is None or len(
input_transaction.outputs) < (spent_output_index + 1):
raise ValueError(
'There is no output with index {} in transaction with hash {}'.
format(spent_output_index, spent_transaction_hash))
output = input_transaction.outputs[spent_output_index]
return output
def _end(self):
self.batch_work_executor.shutdown()
self.item_exporter.close()
def __init__(self, bitcoin_rpc):
self._btc_service = BtcService(bitcoin_rpc)
def stream(bitcoin_rpc,
last_synced_block_file='last_synced_block.txt',
lag=0,
item_exporter=ConsoleItemExporter(),
start_block=None,
end_block=None,
chain=Chain.BITCOIN,
period_seconds=10,
batch_size=2,
block_batch_size=10,
max_workers=5):
if start_block is not None or not os.path.isfile(last_synced_block_file):
init_last_synced_block_file((start_block or 0) - 1,
last_synced_block_file)
last_synced_block = read_last_synced_block(last_synced_block_file)
btc_service = BtcService(bitcoin_rpc, chain)
item_exporter.open()
while True and (end_block is None or last_synced_block < end_block):
blocks_to_sync = 0
try:
current_block = int(btc_service.get_latest_block().number)
target_block = current_block - lag
target_block = min(target_block,
last_synced_block + block_batch_size)
target_block = min(
target_block,
end_block) if end_block is not None else target_block
blocks_to_sync = max(target_block - last_synced_block, 0)
logging.info(
'Current block {}, target block {}, last synced block {}, blocks to sync {}'
.format(current_block, target_block, last_synced_block,
blocks_to_sync))
if blocks_to_sync == 0:
logging.info(
'Nothing to sync. Sleeping for {} seconds...'.format(
period_seconds))
time.sleep(period_seconds)
continue
# Export blocks and transactions
blocks_and_transactions_item_exporter = InMemoryItemExporter(
item_types=['block', 'transaction'])
blocks_and_transactions_job = ExportBlocksJob(
start_block=last_synced_block + 1,
end_block=target_block,
batch_size=batch_size,
bitcoin_rpc=bitcoin_rpc,
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter,
chain=chain,
export_blocks=True,
export_transactions=True)
blocks_and_transactions_job.run()
blocks = blocks_and_transactions_item_exporter.get_items('block')
transactions = blocks_and_transactions_item_exporter.get_items(
'transaction')
# Enrich transactions
enriched_transactions_item_exporter = InMemoryItemExporter(
item_types=['transaction'])
enrich_transactions_job = EnrichTransactionsJob(
transactions_iterable=transactions,
batch_size=batch_size,
bitcoin_rpc=bitcoin_rpc,
max_workers=max_workers,
item_exporter=enriched_transactions_item_exporter,
chain=chain)
enrich_transactions_job.run()
enriched_transactions = enriched_transactions_item_exporter.get_items(
'transaction')
if len(enriched_transactions) != len(transactions):
raise ValueError('The number of transactions is wrong ' +
str(transactions))
logging.info('Exporting with ' + type(item_exporter).__name__)
item_exporter.export_items(blocks + enriched_transactions)
logging.info('Writing last synced block {}'.format(target_block))
write_last_synced_block(last_synced_block_file, target_block)
last_synced_block = target_block
except Exception as e:
# https://stackoverflow.com/a/4992124/1580227
logging.exception(
'An exception occurred while fetching block data.')
if blocks_to_sync != block_batch_size and last_synced_block != end_block:
logging.info('Sleeping {} seconds...'.format(period_seconds))
time.sleep(period_seconds)
item_exporter.close()