def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: ExtensionTransactionService ) -> CompactBlockCompressionResult: compress_start_datetime = datetime.utcnow() tsk = self.compact_mapping_tasks.borrow_task() tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy, compact_block.magic()) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compact_mapping_tasks.return_task(tsk) raise message_conversion_error.btc_compact_block_compression_error( compact_block.block_hash(), e) success = tsk.success() recovered_item = ExtensionCompactBlockRecoveryData( transaction_service, tsk) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if success: result = CompactBlockCompressionResult( False, block_info, None, None, [], create_recovered_transactions()) return self._extension_recovered_compact_block_to_bx_block( result, recovered_item) else: recovery_index = self._last_recovery_idx self._extension_recovered_items[recovery_index] = recovered_item self._last_recovery_idx += 1 return CompactBlockCompressionResult( False, block_info, None, recovery_index, tsk.missing_indices(), create_recovered_transactions())
def test_msg_compact_block_handler(self): message = CompactBlockBtcMessage(buf=bytearray(convert.hex_to_bytes(self.COMPACT_BLOCK_BYTES_HEX))) message._timestamp = int(time.time()) self.assertFalse(message.block_hash() in self.connection.node.blocks_seen.contents) self.node.block_processing_service.process_compact_block.assert_not_called() self.sut.msg_compact_block(message) self.assertTrue(message.block_hash() in self.connection.node.blocks_seen.contents) self.node.block_processing_service.process_compact_block.assert_called_once()
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: TransactionService ) -> CompactBlockCompressionResult: """ Handle decompression of Bitcoin compact block. Decompression converts compact block message to full block message. """ compress_start_datetime = datetime.utcnow() block_header = compact_block.block_header() sha256_hash = hashlib.sha256() sha256_hash.update(block_header) sha256_hash.update(compact_block.short_nonce_buf()) hex_digest = sha256_hash.digest() key = hex_digest[0:16] short_ids = compact_block.short_ids() short_id_to_tx_contents = {} for tx_hash in transaction_service.iter_transaction_hashes(): tx_hash_binary = tx_hash.binary[::-1] tx_short_id = compute_short_id(key, tx_hash_binary) if tx_short_id in short_ids: tx_content = transaction_service.get_transaction_by_hash( tx_hash) if tx_content is None: logger.debug( "Hash {} is known by transactions service but content is missing.", tx_hash) else: short_id_to_tx_contents[tx_short_id] = tx_content if len(short_id_to_tx_contents) == len(short_ids): break block_transactions = [] missing_transactions_indices = [] pre_filled_transactions = compact_block.pre_filled_transactions() total_txs_count = len(pre_filled_transactions) + len(short_ids) size = 0 block_msg_parts = deque() block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size short_ids_iter = iter(short_ids.keys()) for index in range(total_txs_count): if index not in pre_filled_transactions: short_id = next(short_ids_iter) if short_id in short_id_to_tx_contents: short_tx = short_id_to_tx_contents[short_id] block_msg_parts.append(short_tx) block_transactions.append(short_tx) size += len(short_tx) else: missing_transactions_indices.append(index) block_transactions.append(None) else: pre_filled_transaction = pre_filled_transactions[index] block_msg_parts.append(pre_filled_transaction) block_transactions.append(pre_filled_transaction) size += len(pre_filled_transaction) recovered_item = CompactBlockRecoveryData(block_transactions, block_header, compact_block.magic(), transaction_service) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if len(missing_transactions_indices) > 0: recovery_index = self._last_recovery_idx self._last_recovery_idx += 1 self._recovery_items[ recovery_index] = recovered_item # pyre-ignore return CompactBlockCompressionResult(False, block_info, None, recovery_index, missing_transactions_indices, []) result = CompactBlockCompressionResult(False, block_info, None, None, [], []) return self._recovered_compact_block_to_bx_block( result, recovered_item)
def process_compact_block( self, block_message: CompactBlockBtcMessage, connection: BtcNodeConnection ) -> CompactBlockCompressionResult: """ Process compact block for processing on timeout if hold message received. If no hold exists, compress and broadcast block immediately. :param block_message: compact block message to process :param connection: receiving connection (AbstractBlockchainConnection) """ block_hash = block_message.block_hash() parse_result = self._node.message_converter.compact_block_to_bx_block( # pyre-ignore block_message, self._node.get_tx_service() ) block_info = parse_result.block_info if parse_result.success: block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType.COMPACT_BLOCK_COMPRESSED_SUCCESS, network_num=connection.network_num, start_date_time=block_info.start_datetime, end_date_time=block_info.end_datetime, duration=block_info.duration_ms / 1000, success=parse_result.success, txs_count=parse_result.block_info.txn_count, prev_block_hash=parse_result.block_info.prev_block_hash, original_size=block_info.original_size, more_info="Compression: {}->{} bytes, {}, {}; Tx count: {}".format( block_info.original_size, block_info.compressed_size, stats_format.percentage(block_info.compression_rate), stats_format.duration(block_info.duration_ms), block_info.txn_count) ) self._node.block_cleanup_service.on_new_block_received(block_hash, block_message.prev_block_hash()) self._process_and_broadcast_compressed_block( parse_result.bx_block, connection, parse_result.block_info, block_hash ) else: missing_indices = parse_result.missing_indices missing_indices_count = 0 if missing_indices is None else len(missing_indices) start_datetime = block_info.start_datetime end_datetime = datetime.utcnow() duration = (end_datetime - start_datetime).total_seconds() block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType.COMPACT_BLOCK_COMPRESSED_FAILED, network_num=connection.network_num, start_date_time=start_datetime, end_date_time=end_datetime, duration=duration, success=parse_result.success, missing_short_id_count=missing_indices_count, more_info="{:.2f}ms".format( duration * 1000 ) ) logger.warning(log_messages.UNKNOWN_SHORT_IDS, missing_indices_count) return parse_result
def msg_compact_block(self, msg: CompactBlockBtcMessage) -> None: """ Handle COMPACT BLOCK message from Bitcoin node :param msg: COMPACT BLOCK message """ block_hash = msg.block_hash() if not self.node.should_process_block_hash(block_hash): return short_ids_count = len(msg.short_ids()) block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType.COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE, network_num=self.connection.network_num, peer=self.connection.peer_desc, more_info="{} short ids".format(short_ids_count)) if block_hash in self.node.blocks_seen.contents: self.node.on_block_seen_by_blockchain_node(block_hash) block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType. COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN, network_num=self.connection.network_num, peer=self.connection.peer_desc) self.connection.log_info( "Discarding duplicate block {} from local blockchain node.", block_hash) return max_time_offset = self.node.opts.blockchain_block_interval * self.node.opts.blockchain_ignore_block_interval_count if time.time() - msg.timestamp() >= max_time_offset: self.connection.log_trace( "Received block {} more than {} seconds after it was created ({}). Ignoring.", block_hash, max_time_offset, msg.timestamp()) return self.node.track_block_from_node_handling_started(block_hash) if short_ids_count < self.node.opts.compact_block_min_tx_count: self.connection.log_debug( "Compact block {} contains {} short transactions, less than limit {}. Requesting full block.", convert.bytes_to_hex(msg.block_hash().binary), short_ids_count, btc_constants.BTC_COMPACT_BLOCK_DECOMPRESS_MIN_TX_COUNT) get_data_msg = GetDataBtcMessage(magic=self.magic, inv_vects=[ (InventoryType.MSG_BLOCK, msg.block_hash()) ]) self.node.send_msg_to_node(get_data_msg) block_stats.add_block_event_by_block_hash( block_hash, BlockStatEventType.COMPACT_BLOCK_REQUEST_FULL, network_num=self.connection.network_num) return self.node.block_cleanup_service.on_new_block_received( msg.block_hash(), msg.prev_block_hash()) self.node.on_block_seen_by_blockchain_node(block_hash) self.connection.log_info( "Processing compact block {} from local Bitcoin node.", block_hash) try: parse_result = self.node.block_processing_service.process_compact_block( msg, self.connection) except MessageConversionError as e: block_stats.add_block_event_by_block_hash( e.msg_hash, BlockStatEventType.BLOCK_CONVERSION_FAILED, network_num=self.connection.network_num, conversion_type=e.conversion_type.value) self.connection.log_warning(log_messages.PROCESS_BLOCK_FAILURE, e.msg_hash, e) get_data_msg = GetDataBtcMessage(magic=self.magic, inv_vects=[ (InventoryType.MSG_BLOCK, msg.block_hash()) ]) self.node.send_msg_to_node(get_data_msg) return if not parse_result.success: self._recovery_compact_blocks.add(block_hash, parse_result) get_block_txs_msg = GetBlockTransactionsBtcMessage( magic=self.magic, block_hash=block_hash, indices=parse_result.missing_indices) self.node.send_msg_to_node(get_block_txs_msg)
def get_sample_compact_block(): buf = bytearray(convert.hex_to_bytes(COMPACT_BLOCK_BYTES_HEX)) parsed_block = CompactBlockBtcMessage(buf=buf) return parsed_block