def contents_cleanup(transaction_service: TransactionService, block_confirmation_message: AbstractCleanupMessage, cleanup_tasks ): start_datetime = datetime.utcnow() start_time = time.time() tx_service = typing.cast(ExtensionTransactionService, transaction_service) cleanup_task = cleanup_tasks.borrow_task() cleanup_task.init(tpe.InputBytes(block_confirmation_message.buf), tx_service.proxy) task_pool_proxy.run_task(cleanup_task) short_ids = cleanup_task.short_ids() total_content_removed = cleanup_task.total_content_removed() tx_count = cleanup_task.tx_count() message_hash = block_confirmation_message.message_hash() tx_service.update_removed_transactions(total_content_removed, short_ids) transaction_service.on_block_cleaned_up(message_hash) end_datetime = datetime.utcnow() end_time = time.time() duration = end_time - start_time logger.statistics( { "type": "MemoryCleanup", "event": "CacheStateAfterBlockCleanup", "data": transaction_service.get_cache_state_json(), "start_datetime": start_datetime, "end_datetime": end_datetime, "duration": duration, "total_content_removed": total_content_removed, "tx_count": tx_count, "short_ids_count": len(short_ids), "message_hash": repr(message_hash), } ) cleanup_tasks.return_task(cleanup_task)
def clean_block_transactions( self, block_msg: BlockBtcMessage, transaction_service: TransactionService) -> None: block_short_ids = [] block_unknown_tx_hashes = [] start_time = time.time() short_ids_count = 0 unknown_tx_hashes_count = 0 transactions_processed = 0 tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_before_cleanup = transaction_service.get_short_id_count( ) for tx in block_msg.txns(): tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx), length=BTC_SHA_HASH_LEN) short_ids = transaction_service.remove_transaction_by_tx_hash( tx_hash, force=True) if short_ids is None: unknown_tx_hashes_count += 1 block_unknown_tx_hashes.append(tx_hash) else: short_ids_count += len(short_ids) block_short_ids.extend(short_ids) transactions_processed += 1 block_hash = block_msg.block_hash() transaction_service.on_block_cleaned_up(block_hash) end_time = time.time() duration = end_time - start_time tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_after_cleanup = transaction_service.get_short_id_count() logger.debug( "Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} " "short ids. Took {:.3f}s.", block_hash, transactions_processed, unknown_tx_hashes_count, short_ids_count, duration) transaction_service.log_block_transaction_cleanup_stats( block_hash, block_msg.txn_count(), tx_hash_to_contents_len_before_cleanup, tx_hash_to_contents_len_after_cleanup, short_id_count_before_cleanup, short_id_count_after_cleanup) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks( block_hash=block_hash, short_ids=block_short_ids, unknown_tx_hashes=block_unknown_tx_hashes)
def clean_block_transactions_by_block_components( self, block_hash: Sha256Hash, transactions_list: Iterable[Sha256Hash], transaction_service: TransactionService) -> None: logger.debug("Processing block for cleanup: {}", block_hash) tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_before_cleanup = transaction_service.get_short_id_count( ) block_short_ids = [] block_unknown_tx_hashes = [] short_ids_count = 0 unknown_tx_hashes_count = 0 transactions_processed = 0 start_time = time.time() for tx_hash in transactions_list: short_ids = transaction_service.remove_transaction_by_tx_hash( tx_hash, force=True) if short_ids is None: unknown_tx_hashes_count += 1 block_unknown_tx_hashes.append(tx_hash) else: short_ids_count += len(short_ids) block_short_ids.extend(short_ids) transactions_processed += 1 transaction_service.on_block_cleaned_up(block_hash) end_time = time.time() duration = end_time - start_time tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_after_cleanup = transaction_service.get_short_id_count() logger.debug( "Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} " "short ids. Took {:.3f}s.", block_hash, transactions_processed, unknown_tx_hashes_count, short_ids_count, duration) transaction_service.log_block_transaction_cleanup_stats( block_hash, transactions_processed, tx_hash_to_contents_len_before_cleanup, tx_hash_to_contents_len_after_cleanup, short_id_count_before_cleanup, short_id_count_after_cleanup) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks(block_hash, block_short_ids, block_unknown_tx_hashes)
def contents_cleanup(transaction_service: TransactionService, block_confirmation_message: AbstractCleanupMessage ): message_hash = block_confirmation_message.message_hash() for short_id in block_confirmation_message.short_ids(): transaction_service.remove_transaction_by_short_id(short_id, remove_related_short_ids=True) for tx_hash in block_confirmation_message.transaction_hashes(): transaction_service.remove_transaction_by_tx_hash(tx_hash) transaction_service.on_block_cleaned_up(message_hash) logger.statistics( { "type": "MemoryCleanup", "event": "CacheStateAfterBlockCleanup", "message_hash": repr(message_hash), "data": transaction_service.get_cache_state_json() } )