def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: ExtensionTransactionService
) -> CompactBlockCompressionResult:
compress_start_datetime = datetime.utcnow()
tsk = self.compact_mapping_tasks.borrow_task()
tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy,
compact_block.magic())
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compact_mapping_tasks.return_task(tsk)
raise message_conversion_error.btc_compact_block_compression_error(
compact_block.block_hash(), e)
success = tsk.success()
recovered_item = ExtensionCompactBlockRecoveryData(
transaction_service, tsk)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if success:
result = CompactBlockCompressionResult(
False, block_info, None, None, [],
create_recovered_transactions())
return self._extension_recovered_compact_block_to_bx_block(
result, recovered_item)
else:
recovery_index = self._last_recovery_idx
self._extension_recovered_items[recovery_index] = recovered_item
self._last_recovery_idx += 1
return CompactBlockCompressionResult(
False, block_info, None, recovery_index, tsk.missing_indices(),
create_recovered_transactions())
def test_msg_compact_block_handler(self):
message = CompactBlockBtcMessage(buf=bytearray(convert.hex_to_bytes(self.COMPACT_BLOCK_BYTES_HEX)))
message._timestamp = int(time.time())
self.assertFalse(message.block_hash() in self.connection.node.blocks_seen.contents)
self.node.block_processing_service.process_compact_block.assert_not_called()
self.sut.msg_compact_block(message)
self.assertTrue(message.block_hash() in self.connection.node.blocks_seen.contents)
self.node.block_processing_service.process_compact_block.assert_called_once()
def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: TransactionService
) -> CompactBlockCompressionResult:
"""
Handle decompression of Bitcoin compact block.
Decompression converts compact block message to full block message.
"""
compress_start_datetime = datetime.utcnow()
block_header = compact_block.block_header()
sha256_hash = hashlib.sha256()
sha256_hash.update(block_header)
sha256_hash.update(compact_block.short_nonce_buf())
hex_digest = sha256_hash.digest()
key = hex_digest[0:16]
short_ids = compact_block.short_ids()
short_id_to_tx_contents = {}
for tx_hash in transaction_service.iter_transaction_hashes():
tx_hash_binary = tx_hash.binary[::-1]
tx_short_id = compute_short_id(key, tx_hash_binary)
if tx_short_id in short_ids:
tx_content = transaction_service.get_transaction_by_hash(
tx_hash)
if tx_content is None:
logger.debug(
"Hash {} is known by transactions service but content is missing.",
tx_hash)
else:
short_id_to_tx_contents[tx_short_id] = tx_content
if len(short_id_to_tx_contents) == len(short_ids):
break
block_transactions = []
missing_transactions_indices = []
pre_filled_transactions = compact_block.pre_filled_transactions()
total_txs_count = len(pre_filled_transactions) + len(short_ids)
size = 0
block_msg_parts = deque()
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
short_ids_iter = iter(short_ids.keys())
for index in range(total_txs_count):
if index not in pre_filled_transactions:
short_id = next(short_ids_iter)
if short_id in short_id_to_tx_contents:
short_tx = short_id_to_tx_contents[short_id]
block_msg_parts.append(short_tx)
block_transactions.append(short_tx)
size += len(short_tx)
else:
missing_transactions_indices.append(index)
block_transactions.append(None)
else:
pre_filled_transaction = pre_filled_transactions[index]
block_msg_parts.append(pre_filled_transaction)
block_transactions.append(pre_filled_transaction)
size += len(pre_filled_transaction)
recovered_item = CompactBlockRecoveryData(block_transactions,
block_header,
compact_block.magic(),
transaction_service)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if len(missing_transactions_indices) > 0:
recovery_index = self._last_recovery_idx
self._last_recovery_idx += 1
self._recovery_items[
recovery_index] = recovered_item # pyre-ignore
return CompactBlockCompressionResult(False, block_info, None,
recovery_index,
missing_transactions_indices,
[])
result = CompactBlockCompressionResult(False, block_info, None, None,
[], [])
return self._recovered_compact_block_to_bx_block(
result, recovered_item)
def process_compact_block(
self, block_message: CompactBlockBtcMessage, connection: BtcNodeConnection
) -> CompactBlockCompressionResult:
"""
Process compact block for processing on timeout if hold message received.
If no hold exists, compress and broadcast block immediately.
:param block_message: compact block message to process
:param connection: receiving connection (AbstractBlockchainConnection)
"""
block_hash = block_message.block_hash()
parse_result = self._node.message_converter.compact_block_to_bx_block( # pyre-ignore
block_message,
self._node.get_tx_service()
)
block_info = parse_result.block_info
if parse_result.success:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_COMPRESSED_SUCCESS,
network_num=connection.network_num,
start_date_time=block_info.start_datetime,
end_date_time=block_info.end_datetime,
duration=block_info.duration_ms / 1000,
success=parse_result.success,
txs_count=parse_result.block_info.txn_count,
prev_block_hash=parse_result.block_info.prev_block_hash,
original_size=block_info.original_size,
more_info="Compression: {}->{} bytes, {}, {}; Tx count: {}".format(
block_info.original_size,
block_info.compressed_size,
stats_format.percentage(block_info.compression_rate),
stats_format.duration(block_info.duration_ms),
block_info.txn_count)
)
self._node.block_cleanup_service.on_new_block_received(block_hash, block_message.prev_block_hash())
self._process_and_broadcast_compressed_block(
parse_result.bx_block,
connection,
parse_result.block_info,
block_hash
)
else:
missing_indices = parse_result.missing_indices
missing_indices_count = 0 if missing_indices is None else len(missing_indices)
start_datetime = block_info.start_datetime
end_datetime = datetime.utcnow()
duration = (end_datetime - start_datetime).total_seconds()
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_COMPRESSED_FAILED,
network_num=connection.network_num,
start_date_time=start_datetime,
end_date_time=end_datetime,
duration=duration,
success=parse_result.success,
missing_short_id_count=missing_indices_count,
more_info="{:.2f}ms".format(
duration * 1000
)
)
logger.warning(log_messages.UNKNOWN_SHORT_IDS,
missing_indices_count)
return parse_result
def msg_compact_block(self, msg: CompactBlockBtcMessage) -> None:
"""
Handle COMPACT BLOCK message from Bitcoin node
:param msg: COMPACT BLOCK message
"""
block_hash = msg.block_hash()
if not self.node.should_process_block_hash(block_hash):
return
short_ids_count = len(msg.short_ids())
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE,
network_num=self.connection.network_num,
peer=self.connection.peer_desc,
more_info="{} short ids".format(short_ids_count))
if block_hash in self.node.blocks_seen.contents:
self.node.on_block_seen_by_blockchain_node(block_hash)
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.
COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
network_num=self.connection.network_num,
peer=self.connection.peer_desc)
self.connection.log_info(
"Discarding duplicate block {} from local blockchain node.",
block_hash)
return
max_time_offset = self.node.opts.blockchain_block_interval * self.node.opts.blockchain_ignore_block_interval_count
if time.time() - msg.timestamp() >= max_time_offset:
self.connection.log_trace(
"Received block {} more than {} seconds after it was created ({}). Ignoring.",
block_hash, max_time_offset, msg.timestamp())
return
self.node.track_block_from_node_handling_started(block_hash)
if short_ids_count < self.node.opts.compact_block_min_tx_count:
self.connection.log_debug(
"Compact block {} contains {} short transactions, less than limit {}. Requesting full block.",
convert.bytes_to_hex(msg.block_hash().binary), short_ids_count,
btc_constants.BTC_COMPACT_BLOCK_DECOMPRESS_MIN_TX_COUNT)
get_data_msg = GetDataBtcMessage(magic=self.magic,
inv_vects=[
(InventoryType.MSG_BLOCK,
msg.block_hash())
])
self.node.send_msg_to_node(get_data_msg)
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_REQUEST_FULL,
network_num=self.connection.network_num)
return
self.node.block_cleanup_service.on_new_block_received(
msg.block_hash(), msg.prev_block_hash())
self.node.on_block_seen_by_blockchain_node(block_hash)
self.connection.log_info(
"Processing compact block {} from local Bitcoin node.", block_hash)
try:
parse_result = self.node.block_processing_service.process_compact_block(
msg, self.connection)
except MessageConversionError as e:
block_stats.add_block_event_by_block_hash(
e.msg_hash,
BlockStatEventType.BLOCK_CONVERSION_FAILED,
network_num=self.connection.network_num,
conversion_type=e.conversion_type.value)
self.connection.log_warning(log_messages.PROCESS_BLOCK_FAILURE,
e.msg_hash, e)
get_data_msg = GetDataBtcMessage(magic=self.magic,
inv_vects=[
(InventoryType.MSG_BLOCK,
msg.block_hash())
])
self.node.send_msg_to_node(get_data_msg)
return
if not parse_result.success:
self._recovery_compact_blocks.add(block_hash, parse_result)
get_block_txs_msg = GetBlockTransactionsBtcMessage(
magic=self.magic,
block_hash=block_hash,
indices=parse_result.missing_indices)
self.node.send_msg_to_node(get_block_txs_msg)
def get_sample_compact_block():
buf = bytearray(convert.hex_to_bytes(COMPACT_BLOCK_BYTES_HEX))
parsed_block = CompactBlockBtcMessage(buf=buf)
return parsed_block