class BlockTimestampGraph(object):
def __init__(self, bitcoin_rpc):
self._btc_service = BtcService(bitcoin_rpc)
def get_first_point(self):
block = self._btc_service.get_genesis_block()
return block_to_point(block)
def get_last_point(self):
block = self._btc_service.get_latest_block()
return block_to_point(block)
def get_point(self, block_number):
block = self._btc_service.get_block(block_number)
return block_to_point(block)
def get_points(self, block_numbers):
blocks = self._btc_service.get_blocks(block_numbers)
return [block_to_point(block) for block in blocks]
class BtcStreamerAdapter:
def __init__(self,
bitcoin_rpc,
item_exporter=ConsoleItemExporter(),
chain=Chain.BITCOIN,
batch_size=2,
max_workers=5):
self.bitcoin_rpc = bitcoin_rpc
self.chain = chain
self.btc_service = BtcService(bitcoin_rpc, chain)
self.item_exporter = item_exporter
self.batch_size = batch_size
self.max_workers = max_workers
def open(self):
self.item_exporter.open()
def get_current_block_number(self):
return int(self.btc_service.get_latest_block().number)
def export_all(self, start_block, end_block):
# Export blocks and transactions
blocks_and_transactions_item_exporter = InMemoryItemExporter(
item_types=['block', 'transaction'])
blocks_and_transactions_job = ExportBlocksJob(
start_block=start_block,
end_block=end_block,
batch_size=self.batch_size,
bitcoin_rpc=self.bitcoin_rpc,
max_workers=self.max_workers,
item_exporter=blocks_and_transactions_item_exporter,
chain=self.chain,
export_blocks=True,
export_transactions=True)
blocks_and_transactions_job.run()
blocks = blocks_and_transactions_item_exporter.get_items('block')
transactions = blocks_and_transactions_item_exporter.get_items(
'transaction')
# Enrich transactions
enriched_transactions_item_exporter = InMemoryItemExporter(
item_types=['transaction'])
enrich_transactions_job = EnrichTransactionsJob(
transactions_iterable=transactions,
batch_size=self.batch_size,
bitcoin_rpc=self.bitcoin_rpc,
max_workers=self.max_workers,
item_exporter=enriched_transactions_item_exporter,
chain=self.chain)
enrich_transactions_job.run()
enriched_transactions = enriched_transactions_item_exporter.get_items(
'transaction')
if len(enriched_transactions) != len(transactions):
raise ValueError('The number of transactions is wrong ' +
str(transactions))
logging.info('Exporting with ' + type(self.item_exporter).__name__)
self.item_exporter.export_items(blocks + enriched_transactions)
def close(self):
self.item_exporter.close()
def stream(bitcoin_rpc,
last_synced_block_file='last_synced_block.txt',
lag=0,
item_exporter=ConsoleItemExporter(),
start_block=None,
end_block=None,
chain=Chain.BITCOIN,
period_seconds=10,
batch_size=2,
block_batch_size=10,
max_workers=5):
if start_block is not None or not os.path.isfile(last_synced_block_file):
init_last_synced_block_file((start_block or 0) - 1,
last_synced_block_file)
last_synced_block = read_last_synced_block(last_synced_block_file)
btc_service = BtcService(bitcoin_rpc, chain)
item_exporter.open()
while True and (end_block is None or last_synced_block < end_block):
blocks_to_sync = 0
try:
current_block = int(btc_service.get_latest_block().number)
target_block = current_block - lag
target_block = min(target_block,
last_synced_block + block_batch_size)
target_block = min(
target_block,
end_block) if end_block is not None else target_block
blocks_to_sync = max(target_block - last_synced_block, 0)
logging.info(
'Current block {}, target block {}, last synced block {}, blocks to sync {}'
.format(current_block, target_block, last_synced_block,
blocks_to_sync))
if blocks_to_sync == 0:
logging.info(
'Nothing to sync. Sleeping for {} seconds...'.format(
period_seconds))
time.sleep(period_seconds)
continue
# Export blocks and transactions
blocks_and_transactions_item_exporter = InMemoryItemExporter(
item_types=['block', 'transaction'])
blocks_and_transactions_job = ExportBlocksJob(
start_block=last_synced_block + 1,
end_block=target_block,
batch_size=batch_size,
bitcoin_rpc=bitcoin_rpc,
max_workers=max_workers,
item_exporter=blocks_and_transactions_item_exporter,
chain=chain,
export_blocks=True,
export_transactions=True)
blocks_and_transactions_job.run()
blocks = blocks_and_transactions_item_exporter.get_items('block')
transactions = blocks_and_transactions_item_exporter.get_items(
'transaction')
# Enrich transactions
enriched_transactions_item_exporter = InMemoryItemExporter(
item_types=['transaction'])
enrich_transactions_job = EnrichTransactionsJob(
transactions_iterable=transactions,
batch_size=batch_size,
bitcoin_rpc=bitcoin_rpc,
max_workers=max_workers,
item_exporter=enriched_transactions_item_exporter,
chain=chain)
enrich_transactions_job.run()
enriched_transactions = enriched_transactions_item_exporter.get_items(
'transaction')
if len(enriched_transactions) != len(transactions):
raise ValueError('The number of transactions is wrong ' +
str(transactions))
logging.info('Exporting with ' + type(item_exporter).__name__)
item_exporter.export_items(blocks + enriched_transactions)
logging.info('Writing last synced block {}'.format(target_block))
write_last_synced_block(last_synced_block_file, target_block)
last_synced_block = target_block
except Exception as e:
# https://stackoverflow.com/a/4992124/1580227
logging.exception(
'An exception occurred while fetching block data.')
if blocks_to_sync != block_batch_size and last_synced_block != end_block:
logging.info('Sleeping {} seconds...'.format(period_seconds))
time.sleep(period_seconds)
item_exporter.close()