def contents_cleanup(transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage,
cleanup_tasks
):
start_datetime = datetime.utcnow()
start_time = time.time()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task = cleanup_tasks.borrow_task()
cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy)
task_pool_proxy.run_task(cleanup_task)
short_ids = cleanup_task.short_ids()
total_content_removed = cleanup_task.total_content_removed()
tx_count = cleanup_task.tx_count()
message_hash = block_confirmation_message.message_hash()
tx_service.update_removed_transactions(total_content_removed, short_ids)
transaction_service.on_block_cleaned_up(message_hash)
end_datetime = datetime.utcnow()
end_time = time.time()
duration = end_time - start_time
logger.statistics(
{
"type": "MemoryCleanup",
"event": "CacheStateAfterBlockCleanup",
"data": transaction_service.get_cache_state_json(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"duration": duration,
"total_content_removed": total_content_removed,
"tx_count": tx_count,
"short_ids_count": len(short_ids),
"message_hash": repr(message_hash),
}
)
cleanup_tasks.return_task(cleanup_task)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def clean_block_transactions_by_block_components(
self, block_hash: Sha256Hash,
transactions_list: Iterable[Sha256Hash],
transaction_service: TransactionService) -> None:
logger.debug("Processing block for cleanup: {}", block_hash)
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
block_short_ids = []
block_unknown_tx_hashes = []
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
start_time = time.time()
for tx_hash in transactions_list:
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, transactions_processed,
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(block_hash, block_short_ids,
block_unknown_tx_hashes)
def contents_cleanup(transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage
):
message_hash = block_confirmation_message.message_hash()
for short_id in block_confirmation_message.short_ids():
transaction_service.remove_transaction_by_short_id(short_id, remove_related_short_ids=True)
for tx_hash in block_confirmation_message.transaction_hashes():
transaction_service.remove_transaction_by_tx_hash(tx_hash)
transaction_service.on_block_cleaned_up(message_hash)
logger.statistics(
{
"type": "MemoryCleanup",
"event": "CacheStateAfterBlockCleanup",
"message_hash": repr(message_hash),
"data": transaction_service.get_cache_state_json()
}
)