Beispiel #1
0
    def __init__(self,
                 start_block,
                 end_block,
                 batch_size,
                 bitcoin_rpc,
                 max_workers,
                 item_exporter,
                 chain,
                 export_blocks=True,
                 export_transactions=True,
                 coin_price_type=CoinPriceType.empty):
        validate_range(start_block, end_block)

        self.start_block = start_block
        self.end_block = end_block

        self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
        self.item_exporter = item_exporter

        self.export_blocks = export_blocks
        self.export_transactions = export_transactions
        if not self.export_blocks and not self.export_transactions:
            raise ValueError(
                'At least one of export_blocks or export_transactions must be True'
            )

        self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
        self.block_mapper = BtcBlockMapper()
        self.transaction_mapper = BtcTransactionMapper()
 def __init__(self,
              bitcoin_rpc,
              item_exporter=ConsoleItemExporter(),
              chain=Chain.BITCOIN,
              batch_size=2,
              max_workers=5):
     self.bitcoin_rpc = bitcoin_rpc
     self.chain = chain
     self.btc_service = BtcService(bitcoin_rpc, chain)
     self.item_exporter = item_exporter
     self.batch_size = batch_size
     self.max_workers = max_workers
Beispiel #3
0
 def __init__(self,
              bitcoin_rpc,
              item_exporter=ConsoleItemExporter(),
              chain=Chain.BITCOIN,
              batch_size=2,
              enable_enrich=True,
              max_workers=5):
     self.bitcoin_rpc = bitcoin_rpc
     self.chain = chain
     self.btc_service = BtcService(bitcoin_rpc, chain)
     self.item_exporter = item_exporter
     self.batch_size = batch_size
     self.enable_enrich = enable_enrich
     self.max_workers = max_workers
     self.item_id_calculator = BtcItemIdCalculator()
Beispiel #4
0
class ExportBlocksJob(BaseJob):
    def __init__(self,
                 start_block,
                 end_block,
                 batch_size,
                 bitcoin_rpc,
                 max_workers,
                 item_exporter,
                 chain,
                 export_blocks=True,
                 export_transactions=True,
                 coin_price_type=CoinPriceType.empty):
        validate_range(start_block, end_block)

        self.start_block = start_block
        self.end_block = end_block

        self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
        self.item_exporter = item_exporter

        self.export_blocks = export_blocks
        self.export_transactions = export_transactions
        if not self.export_blocks and not self.export_transactions:
            raise ValueError(
                'At least one of export_blocks or export_transactions must be True'
            )

        self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
        self.block_mapper = BtcBlockMapper()
        self.transaction_mapper = BtcTransactionMapper()

    def _start(self):
        self.item_exporter.open()

    def _export(self):
        self.batch_work_executor.execute(
            range(self.start_block, self.end_block + 1),
            self._export_batch,
            total_items=self.end_block - self.start_block + 1)

    def _export_batch(self, block_number_batch):
        blocks = self.btc_service.get_blocks(block_number_batch,
                                             self.export_transactions)

        for block in blocks:
            self._export_block(block)

    def _export_block(self, block):
        if self.export_blocks:
            self.item_exporter.export_item(
                self.block_mapper.block_to_dict(block))
        if self.export_transactions:
            for tx in block.transactions:
                self.item_exporter.export_item(
                    self.transaction_mapper.transaction_to_dict(tx))

    def _end(self):
        self.batch_work_executor.shutdown()
        self.item_exporter.close()
Beispiel #5
0
    def __init__(self,
                 transactions_iterable,
                 batch_size,
                 bitcoin_rpc,
                 max_workers,
                 item_exporter,
                 chain=Chain.BITCOIN):
        self.transactions_iterable = transactions_iterable
        self.btc_service = BtcService(bitcoin_rpc, chain)

        self.batch_size = batch_size
        self.batch_work_executor = BatchWorkExecutor(batch_size,
                                                     max_workers,
                                                     exponential_backoff=False)
        self.item_exporter = item_exporter

        self.transaction_mapper = BtcTransactionMapper()
Beispiel #6
0
class BlockTimestampGraph(object):
    def __init__(self, bitcoin_rpc):
        self._btc_service = BtcService(bitcoin_rpc)

    def get_first_point(self):
        block = self._btc_service.get_genesis_block()
        return block_to_point(block)

    def get_last_point(self):
        block = self._btc_service.get_latest_block()
        return block_to_point(block)

    def get_point(self, block_number):
        block = self._btc_service.get_block(block_number)
        return block_to_point(block)

    def get_points(self, block_numbers):
        blocks = self._btc_service.get_blocks(block_numbers)
        return [block_to_point(block) for block in blocks]
class BtcStreamerAdapter:
    def __init__(self,
                 bitcoin_rpc,
                 item_exporter=ConsoleItemExporter(),
                 chain=Chain.BITCOIN,
                 batch_size=2,
                 max_workers=5):
        self.bitcoin_rpc = bitcoin_rpc
        self.chain = chain
        self.btc_service = BtcService(bitcoin_rpc, chain)
        self.item_exporter = item_exporter
        self.batch_size = batch_size
        self.max_workers = max_workers

    def open(self):
        self.item_exporter.open()

    def get_current_block_number(self):
        return int(self.btc_service.get_latest_block().number)

    def export_all(self, start_block, end_block):
        # Export blocks and transactions
        blocks_and_transactions_item_exporter = InMemoryItemExporter(
            item_types=['block', 'transaction'])

        blocks_and_transactions_job = ExportBlocksJob(
            start_block=start_block,
            end_block=end_block,
            batch_size=self.batch_size,
            bitcoin_rpc=self.bitcoin_rpc,
            max_workers=self.max_workers,
            item_exporter=blocks_and_transactions_item_exporter,
            chain=self.chain,
            export_blocks=True,
            export_transactions=True)
        blocks_and_transactions_job.run()

        blocks = blocks_and_transactions_item_exporter.get_items('block')
        transactions = blocks_and_transactions_item_exporter.get_items(
            'transaction')

        # Enrich transactions
        enriched_transactions_item_exporter = InMemoryItemExporter(
            item_types=['transaction'])

        enrich_transactions_job = EnrichTransactionsJob(
            transactions_iterable=transactions,
            batch_size=self.batch_size,
            bitcoin_rpc=self.bitcoin_rpc,
            max_workers=self.max_workers,
            item_exporter=enriched_transactions_item_exporter,
            chain=self.chain)
        enrich_transactions_job.run()
        enriched_transactions = enriched_transactions_item_exporter.get_items(
            'transaction')
        if len(enriched_transactions) != len(transactions):
            raise ValueError('The number of transactions is wrong ' +
                             str(transactions))

        logging.info('Exporting with ' + type(self.item_exporter).__name__)
        self.item_exporter.export_items(blocks + enriched_transactions)

    def close(self):
        self.item_exporter.close()
Beispiel #8
0
class EnrichTransactionsJob(BaseJob):
    def __init__(self,
                 transactions_iterable,
                 batch_size,
                 bitcoin_rpc,
                 max_workers,
                 item_exporter,
                 chain=Chain.BITCOIN):
        self.transactions_iterable = transactions_iterable
        self.btc_service = BtcService(bitcoin_rpc, chain)

        self.batch_size = batch_size
        self.batch_work_executor = BatchWorkExecutor(batch_size,
                                                     max_workers,
                                                     exponential_backoff=False)
        self.item_exporter = item_exporter

        self.transaction_mapper = BtcTransactionMapper()

    def _start(self):
        self.item_exporter.open()

    def _export(self):
        self.batch_work_executor.execute(self.transactions_iterable,
                                         self._enrich_transactions)

    def _enrich_transactions(self, transactions):
        transactions = [
            self.transaction_mapper.dict_to_transaction(transaction)
            for transaction in transactions
        ]

        all_inputs = [transaction.inputs for transaction in transactions]
        flat_inputs = [input for inputs in all_inputs for input in inputs]

        for transaction_input_batch in dynamic_batch_iterator(
                flat_inputs, lambda: self.batch_size):
            input_transactions_map = self._get_input_transactions_as_map(
                transaction_input_batch)
            for input in transaction_input_batch:
                output = self._get_output_for_input(input, input_transactions_map) \
                    if input.spent_transaction_hash is not None else None
                if output is not None:
                    input.required_signatures = output.required_signatures
                    input.type = output.type
                    input.addresses = output.addresses
                    input.value = output.value

        for transaction in transactions:
            self.item_exporter.export_item(
                self.transaction_mapper.transaction_to_dict(transaction))

    def _get_input_transactions_as_map(self, transaction_inputs):
        transaction_hashes = [
            input.spent_transaction_hash for input in transaction_inputs
            if input.spent_transaction_hash is not None
        ]

        transaction_hashes = set(transaction_hashes)
        if len(transaction_hashes) > 0:
            transactions = self.btc_service.get_transactions_by_hashes(
                transaction_hashes)
            return {
                transaction.hash: transaction
                for transaction in transactions
            }
        else:
            return {}

    def _get_output_for_input(self, transaction_input, input_transactions_map):
        spent_transaction_hash = transaction_input.spent_transaction_hash
        input_transaction = input_transactions_map.get(spent_transaction_hash)
        if input_transaction is None:
            raise ValueError('Input transaction with hash {} not found'.format(
                spent_transaction_hash))

        spent_output_index = transaction_input.spent_output_index
        if input_transaction.outputs is None or len(
                input_transaction.outputs) < (spent_output_index + 1):
            raise ValueError(
                'There is no output with index {} in transaction with hash {}'.
                format(spent_output_index, spent_transaction_hash))

        output = input_transaction.outputs[spent_output_index]
        return output

    def _end(self):
        self.batch_work_executor.shutdown()
        self.item_exporter.close()
Beispiel #9
0
 def __init__(self, bitcoin_rpc):
     self._btc_service = BtcService(bitcoin_rpc)
Beispiel #10
0
def stream(bitcoin_rpc,
           last_synced_block_file='last_synced_block.txt',
           lag=0,
           item_exporter=ConsoleItemExporter(),
           start_block=None,
           end_block=None,
           chain=Chain.BITCOIN,
           period_seconds=10,
           batch_size=2,
           block_batch_size=10,
           max_workers=5):
    if start_block is not None or not os.path.isfile(last_synced_block_file):
        init_last_synced_block_file((start_block or 0) - 1,
                                    last_synced_block_file)

    last_synced_block = read_last_synced_block(last_synced_block_file)
    btc_service = BtcService(bitcoin_rpc, chain)

    item_exporter.open()

    while True and (end_block is None or last_synced_block < end_block):
        blocks_to_sync = 0

        try:
            current_block = int(btc_service.get_latest_block().number)
            target_block = current_block - lag
            target_block = min(target_block,
                               last_synced_block + block_batch_size)
            target_block = min(
                target_block,
                end_block) if end_block is not None else target_block
            blocks_to_sync = max(target_block - last_synced_block, 0)
            logging.info(
                'Current block {}, target block {}, last synced block {}, blocks to sync {}'
                .format(current_block, target_block, last_synced_block,
                        blocks_to_sync))

            if blocks_to_sync == 0:
                logging.info(
                    'Nothing to sync. Sleeping for {} seconds...'.format(
                        period_seconds))
                time.sleep(period_seconds)
                continue

            # Export blocks and transactions
            blocks_and_transactions_item_exporter = InMemoryItemExporter(
                item_types=['block', 'transaction'])

            blocks_and_transactions_job = ExportBlocksJob(
                start_block=last_synced_block + 1,
                end_block=target_block,
                batch_size=batch_size,
                bitcoin_rpc=bitcoin_rpc,
                max_workers=max_workers,
                item_exporter=blocks_and_transactions_item_exporter,
                chain=chain,
                export_blocks=True,
                export_transactions=True)
            blocks_and_transactions_job.run()

            blocks = blocks_and_transactions_item_exporter.get_items('block')
            transactions = blocks_and_transactions_item_exporter.get_items(
                'transaction')

            # Enrich transactions
            enriched_transactions_item_exporter = InMemoryItemExporter(
                item_types=['transaction'])

            enrich_transactions_job = EnrichTransactionsJob(
                transactions_iterable=transactions,
                batch_size=batch_size,
                bitcoin_rpc=bitcoin_rpc,
                max_workers=max_workers,
                item_exporter=enriched_transactions_item_exporter,
                chain=chain)
            enrich_transactions_job.run()
            enriched_transactions = enriched_transactions_item_exporter.get_items(
                'transaction')
            if len(enriched_transactions) != len(transactions):
                raise ValueError('The number of transactions is wrong ' +
                                 str(transactions))

            logging.info('Exporting with ' + type(item_exporter).__name__)
            item_exporter.export_items(blocks + enriched_transactions)

            logging.info('Writing last synced block {}'.format(target_block))
            write_last_synced_block(last_synced_block_file, target_block)
            last_synced_block = target_block
        except Exception as e:
            # https://stackoverflow.com/a/4992124/1580227
            logging.exception(
                'An exception occurred while fetching block data.')

        if blocks_to_sync != block_batch_size and last_synced_block != end_block:
            logging.info('Sleeping {} seconds...'.format(period_seconds))
            time.sleep(period_seconds)

    item_exporter.close()