def msg_block(self, msg: BlockBtcMessage) -> None:
block_stats.add_block_event_by_block_hash(
msg.block_hash(),
BlockStatEventType.REMOTE_BLOCK_RECEIVED_BY_GATEWAY,
network_num=self.connection.network_num,
more_info="Protocol: {}, Network: {}".format(
self.connection.node.opts.blockchain_protocol,
self.connection.node.opts.blockchain_network))
return self.msg_proxy_response(msg)
def clean_block_transactions(
self, block_msg: BlockBtcMessage, transaction_service: TransactionService
) -> None:
start_datetime = datetime.utcnow()
start_time = time.time()
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len()
cleanup_task = self.block_cleanup_tasks.borrow_task()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
init_time = time.time()
task_pool_proxy.run_task(cleanup_task)
task_run_time = time.time()
unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes())
tx_property_fetch_time = time.time()
short_ids = cleanup_task.short_ids()
short_ids_fetch_time = time.time()
short_ids_count = len(short_ids)
tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids)
remove_from_tx_service_time = time.time()
# TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service
block_hash = block_msg.block_hash()
tx_service.on_block_cleaned_up(block_hash)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len()
end_datetime = datetime.utcnow()
end_time = time.time()
logger.statistics(
{
"type": "BlockTransactionsCleanup",
"block_hash": repr(block_hash),
"unknown_tx_hashes_count": unknown_tx_hashes_count,
"short_ids_count": short_ids_count,
"block_transactions_count": cleanup_task.txn_count(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"task_init_time": init_time - start_time,
"task_run_time": task_run_time - init_time,
"tx_property_fetch_time": tx_property_fetch_time - task_run_time,
"short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time,
"remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time,
"duration": end_time - start_time,
"tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup,
"tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup,
}
)
self.block_cleanup_tasks.return_task(cleanup_task)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=short_ids,
unknown_tx_hashes=(
Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string()))
for tx_hash in cleanup_task.unknown_tx_hashes()
)
)
def test_btc_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
timestamp = 1
bits = 2
nonce = 3
btc_block = BlockBtcMessage(
self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns
)
block_hash = btc_block.block_hash()
bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block(
btc_block, self.tx_service, True, 0
)
self.assertEqual(10, block_info.txn_count)
self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3",
block_info.prev_block_hash)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(btc_block.block_hash(), block_info.block_hash)
# TODO: if we convert bloxroute block to a class, add some tests here
parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block(
bloxroute_block,
self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes())
self.assertEqual(self.version, parsed_btc_block.version())
self.assertEqual(self.magic, parsed_btc_block.magic())
self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian())
self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian())
self.assertEqual(timestamp, parsed_btc_block.timestamp())
self.assertEqual(bits, parsed_btc_block.bits())
self.assertEqual(nonce, parsed_btc_block.nonce())
self.assertEqual(len(self.txns), parsed_btc_block.txn_count())
self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum())
self.assertEqual(block_hash, parsed_btc_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(list(block_info.short_ids), self.short_ids)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
start_datetime = datetime.datetime.utcnow()
start_time = time.time()
block_msg = BlockBtcMessage(buf=bx_block_msg)
block_info = BlockInfo(
block_msg.block_hash(),
[],
start_datetime,
datetime.datetime.utcnow(),
(time.time() - start_time) * 1000,
block_msg.txn_count(),
# pyre-fixme[6]: Expected `Optional[str]` for 7th param but got
# `BtcObjectHash`.
block_msg.block_hash(),
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()),
len(block_msg.rawbytes()),
0
)
return BlockDecompressionResult(block_msg, block_info, [], [])
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
