def clean_block_transactions(
self, block_msg: BlockBtcMessage, transaction_service: TransactionService
) -> None:
start_datetime = datetime.utcnow()
start_time = time.time()
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len()
cleanup_task = self.block_cleanup_tasks.borrow_task()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
init_time = time.time()
task_pool_proxy.run_task(cleanup_task)
task_run_time = time.time()
unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes())
tx_property_fetch_time = time.time()
short_ids = cleanup_task.short_ids()
short_ids_fetch_time = time.time()
short_ids_count = len(short_ids)
tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids)
remove_from_tx_service_time = time.time()
# TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service
block_hash = block_msg.block_hash()
tx_service.on_block_cleaned_up(block_hash)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len()
end_datetime = datetime.utcnow()
end_time = time.time()
logger.statistics(
{
"type": "BlockTransactionsCleanup",
"block_hash": repr(block_hash),
"unknown_tx_hashes_count": unknown_tx_hashes_count,
"short_ids_count": short_ids_count,
"block_transactions_count": cleanup_task.txn_count(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"task_init_time": init_time - start_time,
"task_run_time": task_run_time - init_time,
"tx_property_fetch_time": tx_property_fetch_time - task_run_time,
"short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time,
"remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time,
"duration": end_time - start_time,
"tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup,
"tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup,
}
)
self.block_cleanup_tasks.return_task(cleanup_task)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=short_ids,
unknown_tx_hashes=(
Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string()))
for tx_hash in cleanup_task.unknown_tx_hashes()
)
)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def clean_block_transactions_by_block_components(
self, block_hash: Sha256Hash,
transactions_list: Iterable[Sha256Hash],
transaction_service: TransactionService) -> None:
logger.debug("Processing block for cleanup: {}", block_hash)
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
block_short_ids = []
block_unknown_tx_hashes = []
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
start_time = time.time()
for tx_hash in transactions_list:
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, transactions_processed,
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(block_hash, block_short_ids,
block_unknown_tx_hashes)
def clean_block_transactions_by_block_components(
self, block_hash: Sha256Hash,
transactions_list: Iterable[Sha256Hash],
transaction_service: TransactionService) -> None:
logger.debug("Processing block for cleanup: {}", block_hash)
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
start_time = time.time()
if not isinstance(transactions_list, list):
transactions_list = list(transactions_list)
sids = []
confirmation_msg = BlockConfirmationMessage(
block_hash,
transaction_service.network_num,
tx_hashes=transactions_list,
sids=sids)
self.contents_cleanup(transaction_service, confirmation_msg)
self._block_hash_marked_for_cleanup.discard(block_hash)
end_time = time.time()
duration = end_time - start_time
logger.debug(
"Finished cleaning up block {}. Processed {} hashes. Took {:.3f}s.",
block_hash, len(transactions_list), duration)
self.node.post_block_cleanup_tasks(block_hash, sids, transactions_list)
transactions_processed = len(transactions_list)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
transaction_service.log_block_transaction_cleanup_stats(
block_hash, transactions_processed,
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)