def collect_garbage(self): # type: () -> None """ Runs garbage collection on all of the caches according to the settings. This is a time-consuming process. It is normally kicked-off by drive_garbage_collection :return: """ local_class = CacheData try: if local_class._logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE): if self._is_trailer_cache: local_class._logger.debug_extra_verbose('TRAILER CACHE') else: local_class._logger.debug_extra_verbose('JSON CACHE') if self._is_limit_number_of_cached_files: # # Delete enough of the oldest files to keep the number # within limit number_of_cache_files_to_delete = - self._remaining_allowed_files if number_of_cache_files_to_delete > 0: if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info( 'limit_number_of_cached_files. number_of_files_to_delete:', locale.format("%d", number_of_cache_files_to_delete, grouping=True)) # Order json files by age for cache_file in self._usage_data.get_file_data_by_creation_date(): Monitor.throw_exception_if_abort_requested() self._usage_data.remove_file(cache_file) number_of_cache_files_to_delete -= 1 if number_of_cache_files_to_delete <= 0: break else: if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info( 'limit_number_of_cached_files. Additional allowed files:', locale.format("%d", number_of_cache_files_to_delete, grouping=True)) if self._is_limit_size_of_cache: # # Delete enough of the oldest files to keep the number # within limit max_bytes_in_cache = (self._max_cache_size_mb * 1024 * 1024) bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache() - max_bytes_in_cache) if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info('limit_size_of_cache. max allowed size:', DiskUtils.sizeof_fmt(max_bytes_in_cache)) local_class._logger.debug('actual disk used in cache:', DiskUtils.sizeof_fmt( self._usage_data.get_disk_used_by_cache())) local_class._logger.debug('Amount to delete:', DiskUtils.sizeof_fmt(bytes_of_files_to_delete)) if bytes_of_files_to_delete > 0: # Order json files by age for cache_file in self._usage_data.get_file_data_by_creation_date(): Monitor.throw_exception_if_abort_requested() self._usage_data.remove_file(cache_file) bytes_of_files_to_delete = ( self._usage_data.get_disk_used_by_cache() - max_bytes_in_cache) if bytes_of_files_to_delete <= 0: break if self._is_limit_percent_of_cache_disk: # # Delete enough of the oldest files to keep the number # within limit max_bytes_in_cache = (self._total_size_of_cache_fs * self._max_percent_of_cache_disk / 100.00) bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache() - max_bytes_in_cache) if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info( 'limit_percent of cached files. Calculated max size:', DiskUtils.sizeof_fmt(max_bytes_in_cache)) local_class._logger.info('size to delete:', DiskUtils.sizeof_fmt( bytes_of_files_to_delete)) if bytes_of_files_to_delete > 0: # Order json files by age for cache_file in self._usage_data.get_file_data_by_creation_date(): Monitor.throw_exception_if_abort_requested() self._usage_data.remove_file(cache_file) bytes_of_files_to_delete = ( self._usage_data.get_disk_used_by_cache() - max_bytes_in_cache) if bytes_of_files_to_delete <= 0: break except AbortException: reraise(*sys.exc_info()) except Exception as e: local_class._logger.exception('')
def __init__(self, trailer_cache: bool) -> None: """ Populates this instance with relevant settings. Allows for uniform access to those settings. :param trailer_cache: """ local_class = CacheData if local_class._logger is None: local_class._logger = module_logger.getChild(local_class.__name__) self._usage_data = None self._messages = Messages self._is_trailer_cache = trailer_cache if trailer_cache: self._is_limit_number_of_cached_files = \ Settings.is_limit_number_of_cached_trailers() if self._is_limit_number_of_cached_files: self._max_number_of_files = Settings.get_max_number_of_cached_trailers() self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_trailers() if self._is_limit_size_of_cache: self._max_cache_size_mb = Settings.get_max_size_of_cached_trailers_mb() self._is_limit_percent_of_cache_disk = \ Settings.is_limit_percent_of_cached_trailers() if self._is_limit_percent_of_cache_disk: self._max_percent_of_cache_disk = \ Settings.get_max_percent_of_cached_trailers() # TODO: Delete me # self._max_percent_of_cache_disk = 0.83 else: self._is_limit_number_of_cached_files = \ Settings.is_limit_number_of_cached_json() if self._is_limit_number_of_cached_files: self._max_number_of_files = Settings.get_max_number_of_cached_json() self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_json() if self._is_limit_size_of_cache: self._max_cache_size_mb = Settings.get_max_size_of_cached_json_mb() self._is_limit_percent_of_cache_disk = \ Settings.is_limit_percent_of_cached_json() if self._is_limit_percent_of_cache_disk: self._max_percent_of_cache_disk = \ Settings.get_max_percent_of_cached_json() # TODO_ delete me # self._max_percent_of_cache_disk = 0.0058 if self._is_limit_number_of_cached_files: self._max_number_of_files_str = locale.format("%d", self._max_number_of_files, grouping=True) else: self._max_number_of_files = 0 self._max_number_of_files_str = self._messages.get_msg( Messages.UNLIMITED) if self._is_limit_size_of_cache: self._max_cache_size_mb_str = DiskUtils.sizeof_fmt( self._max_cache_size_mb * 1024 * 1024) else: self._max_cache_size_mb = 0 self._max_cache_size_mb_str = self._messages.get_msg( Messages.UNLIMITED) if self._is_limit_percent_of_cache_disk: self._max_percent_of_cache_disk_str = RATIO_DECIMAL_DIGITS_TO_PRINT.format( self._max_percent_of_cache_disk) + '%' else: self._max_percent_of_cache_disk = 100.0 self._max_percent_of_cache_disk_str = self._messages.get_msg( Messages.UNLIMITED) self._remaining_allowed_files = None self._used_space_in_cache_fs = None self._free_disk_in_cache_fs = None self._total_size_of_cache_fs = None self._disk_used_by_cache = None self._actual_cache_percent = None
def report_status(self): # type: () -> None """ Produces a simple report about the cache using the Settings and UsageData. :return: """ local_class = CacheData try: if self._is_trailer_cache: msg_max_trailers = 'max allowed trailers:' msg_disk_usage = 'max_trailer_cache_disk_usage:' msg_cache_percent = 'max percent of trailer cache disk usage:' msg_files_in_cache = 'trailers_in_cache:' msg_remaining_allowed_files = 'remaining_allowed_trailers:' msg_total_size_of_cache_fs = 'Size of trailer cache fs:' msg_used_space_in_cache_fs = 'Used space in trailer cache fs:' msg_free_space_in_cache_fs = 'free space in trailer cache fs:' msg_actual_fs_cache_percent = 'Actual percent of disk used by trailer ' \ 'cache:' msg_disk_used_by_cache = 'Actual disk used by trailer cache:' else: msg_max_trailers = 'max number of json files:' msg_disk_usage = 'max_json_cache_disk_usage:' msg_cache_percent = 'max_percent of json cache disk usage:' msg_files_in_cache = 'json_files_in_cache' msg_remaining_allowed_files = 'remaining_allowed_json_files' msg_total_size_of_cache_fs = 'Size of json cache fs:' msg_used_space_in_cache_fs = 'Used space in json cache fs:' msg_free_space_in_cache_fs = 'free space in json cache fs:' msg_actual_fs_cache_percent = 'Actual percent of disk used by json ' \ 'cache:' msg_disk_used_by_cache = 'Actual disk used by json cache:' if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info(msg_max_trailers, self._max_number_of_files_str, trace=Trace.STATS_CACHE) local_class._logger.info(msg_disk_usage, self._max_cache_size_mb_str, trace=Trace.STATS_CACHE) local_class._logger.info(msg_cache_percent, self._max_percent_of_cache_disk_str, trace=Trace.STATS_CACHE) files_in_cache = self._usage_data.get_number_of_files() if self._is_limit_number_of_cached_files: self._remaining_allowed_files = (self._max_number_of_files - files_in_cache) remaining_allowed_files_str = locale.format("%d", self._remaining_allowed_files, grouping=True) else: self._remaining_allowed_files = None remaining_allowed_files_str = self._messages.get_msg( Messages.UNLIMITED) self._used_space_in_cache_fs = self._usage_data.get_used_space() self._free_disk_in_cache_fs = self._usage_data.get_free_size() self._total_size_of_cache_fs = self._usage_data.get_total_size() self._disk_used_by_cache = self._usage_data.get_disk_used_by_cache() self._actual_cache_percent = (self._disk_used_by_cache / self._total_size_of_cache_fs) * 100.0 if local_class._logger.isEnabledFor(LazyLogger.INFO): local_class._logger.info(msg_total_size_of_cache_fs, DiskUtils.sizeof_fmt(self._total_size_of_cache_fs)) local_class._logger.info(msg_used_space_in_cache_fs, DiskUtils.sizeof_fmt(self._used_space_in_cache_fs)) local_class._logger.info(msg_free_space_in_cache_fs, DiskUtils.sizeof_fmt(self._free_disk_in_cache_fs)) local_class._logger.info(msg_files_in_cache, locale.format("%d", files_in_cache, grouping=True)) local_class._logger.info(msg_remaining_allowed_files, remaining_allowed_files_str) local_class._logger.info(msg_actual_fs_cache_percent, RATIO_DECIMAL_DIGITS_TO_PRINT.format( self._actual_cache_percent) + '%') local_class._logger.info(msg_disk_used_by_cache, DiskUtils.sizeof_fmt(self._disk_used_by_cache)) except AbortException: reraise(*sys.exc_info()) except Exception as e: local_class._logger.exception('')