def collect_garbage(self):
# type: () -> None
"""
Runs garbage collection on all of the caches according to the
settings.
This is a time-consuming process. It is normally kicked-off by
drive_garbage_collection
:return:
"""
local_class = CacheData
try:
if local_class._logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
if self._is_trailer_cache:
local_class._logger.debug_extra_verbose('TRAILER CACHE')
else:
local_class._logger.debug_extra_verbose('JSON CACHE')
if self._is_limit_number_of_cached_files:
#
# Delete enough of the oldest files to keep the number
# within limit
number_of_cache_files_to_delete = - self._remaining_allowed_files
if number_of_cache_files_to_delete > 0:
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_number_of_cached_files. number_of_files_to_delete:',
locale.format("%d", number_of_cache_files_to_delete,
grouping=True))
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
number_of_cache_files_to_delete -= 1
if number_of_cache_files_to_delete <= 0:
break
else:
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_number_of_cached_files. Additional allowed files:',
locale.format("%d", number_of_cache_files_to_delete,
grouping=True))
if self._is_limit_size_of_cache:
#
# Delete enough of the oldest files to keep the number
# within limit
max_bytes_in_cache = (self._max_cache_size_mb * 1024 * 1024)
bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info('limit_size_of_cache. max allowed size:',
DiskUtils.sizeof_fmt(max_bytes_in_cache))
local_class._logger.debug('actual disk used in cache:',
DiskUtils.sizeof_fmt(
self._usage_data.get_disk_used_by_cache()))
local_class._logger.debug('Amount to delete:',
DiskUtils.sizeof_fmt(bytes_of_files_to_delete))
if bytes_of_files_to_delete > 0:
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
bytes_of_files_to_delete = (
self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if bytes_of_files_to_delete <= 0:
break
if self._is_limit_percent_of_cache_disk:
#
# Delete enough of the oldest files to keep the number
# within limit
max_bytes_in_cache = (self._total_size_of_cache_fs *
self._max_percent_of_cache_disk / 100.00)
bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache() -
max_bytes_in_cache)
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_percent of cached files. Calculated max size:',
DiskUtils.sizeof_fmt(max_bytes_in_cache))
local_class._logger.info('size to delete:',
DiskUtils.sizeof_fmt(
bytes_of_files_to_delete))
if bytes_of_files_to_delete > 0:
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
bytes_of_files_to_delete = (
self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if bytes_of_files_to_delete <= 0:
break
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
local_class._logger.exception('')
def __init__(self, trailer_cache: bool) -> None:
"""
Populates this instance with relevant settings. Allows for uniform
access to those settings.
:param trailer_cache:
"""
local_class = CacheData
if local_class._logger is None:
local_class._logger = module_logger.getChild(local_class.__name__)
self._usage_data = None
self._messages = Messages
self._is_trailer_cache = trailer_cache
if trailer_cache:
self._is_limit_number_of_cached_files = \
Settings.is_limit_number_of_cached_trailers()
if self._is_limit_number_of_cached_files:
self._max_number_of_files = Settings.get_max_number_of_cached_trailers()
self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_trailers()
if self._is_limit_size_of_cache:
self._max_cache_size_mb = Settings.get_max_size_of_cached_trailers_mb()
self._is_limit_percent_of_cache_disk = \
Settings.is_limit_percent_of_cached_trailers()
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk = \
Settings.get_max_percent_of_cached_trailers()
# TODO: Delete me
# self._max_percent_of_cache_disk = 0.83
else:
self._is_limit_number_of_cached_files = \
Settings.is_limit_number_of_cached_json()
if self._is_limit_number_of_cached_files:
self._max_number_of_files = Settings.get_max_number_of_cached_json()
self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_json()
if self._is_limit_size_of_cache:
self._max_cache_size_mb = Settings.get_max_size_of_cached_json_mb()
self._is_limit_percent_of_cache_disk = \
Settings.is_limit_percent_of_cached_json()
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk = \
Settings.get_max_percent_of_cached_json()
# TODO_ delete me
# self._max_percent_of_cache_disk = 0.0058
if self._is_limit_number_of_cached_files:
self._max_number_of_files_str = locale.format("%d",
self._max_number_of_files,
grouping=True)
else:
self._max_number_of_files = 0
self._max_number_of_files_str = self._messages.get_msg(
Messages.UNLIMITED)
if self._is_limit_size_of_cache:
self._max_cache_size_mb_str = DiskUtils.sizeof_fmt(
self._max_cache_size_mb * 1024 * 1024)
else:
self._max_cache_size_mb = 0
self._max_cache_size_mb_str = self._messages.get_msg(
Messages.UNLIMITED)
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk_str = RATIO_DECIMAL_DIGITS_TO_PRINT.format(
self._max_percent_of_cache_disk) + '%'
else:
self._max_percent_of_cache_disk = 100.0
self._max_percent_of_cache_disk_str = self._messages.get_msg(
Messages.UNLIMITED)
self._remaining_allowed_files = None
self._used_space_in_cache_fs = None
self._free_disk_in_cache_fs = None
self._total_size_of_cache_fs = None
self._disk_used_by_cache = None
self._actual_cache_percent = None
def report_status(self):
# type: () -> None
"""
Produces a simple report about the cache using the Settings
and UsageData.
:return:
"""
local_class = CacheData
try:
if self._is_trailer_cache:
msg_max_trailers = 'max allowed trailers:'
msg_disk_usage = 'max_trailer_cache_disk_usage:'
msg_cache_percent = 'max percent of trailer cache disk usage:'
msg_files_in_cache = 'trailers_in_cache:'
msg_remaining_allowed_files = 'remaining_allowed_trailers:'
msg_total_size_of_cache_fs = 'Size of trailer cache fs:'
msg_used_space_in_cache_fs = 'Used space in trailer cache fs:'
msg_free_space_in_cache_fs = 'free space in trailer cache fs:'
msg_actual_fs_cache_percent = 'Actual percent of disk used by trailer ' \
'cache:'
msg_disk_used_by_cache = 'Actual disk used by trailer cache:'
else:
msg_max_trailers = 'max number of json files:'
msg_disk_usage = 'max_json_cache_disk_usage:'
msg_cache_percent = 'max_percent of json cache disk usage:'
msg_files_in_cache = 'json_files_in_cache'
msg_remaining_allowed_files = 'remaining_allowed_json_files'
msg_total_size_of_cache_fs = 'Size of json cache fs:'
msg_used_space_in_cache_fs = 'Used space in json cache fs:'
msg_free_space_in_cache_fs = 'free space in json cache fs:'
msg_actual_fs_cache_percent = 'Actual percent of disk used by json ' \
'cache:'
msg_disk_used_by_cache = 'Actual disk used by json cache:'
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(msg_max_trailers,
self._max_number_of_files_str,
trace=Trace.STATS_CACHE)
local_class._logger.info(msg_disk_usage,
self._max_cache_size_mb_str,
trace=Trace.STATS_CACHE)
local_class._logger.info(msg_cache_percent,
self._max_percent_of_cache_disk_str,
trace=Trace.STATS_CACHE)
files_in_cache = self._usage_data.get_number_of_files()
if self._is_limit_number_of_cached_files:
self._remaining_allowed_files = (self._max_number_of_files -
files_in_cache)
remaining_allowed_files_str = locale.format("%d",
self._remaining_allowed_files,
grouping=True)
else:
self._remaining_allowed_files = None
remaining_allowed_files_str = self._messages.get_msg(
Messages.UNLIMITED)
self._used_space_in_cache_fs = self._usage_data.get_used_space()
self._free_disk_in_cache_fs = self._usage_data.get_free_size()
self._total_size_of_cache_fs = self._usage_data.get_total_size()
self._disk_used_by_cache = self._usage_data.get_disk_used_by_cache()
self._actual_cache_percent = (self._disk_used_by_cache /
self._total_size_of_cache_fs) * 100.0
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(msg_total_size_of_cache_fs,
DiskUtils.sizeof_fmt(self._total_size_of_cache_fs))
local_class._logger.info(msg_used_space_in_cache_fs,
DiskUtils.sizeof_fmt(self._used_space_in_cache_fs))
local_class._logger.info(msg_free_space_in_cache_fs,
DiskUtils.sizeof_fmt(self._free_disk_in_cache_fs))
local_class._logger.info(msg_files_in_cache,
locale.format("%d", files_in_cache, grouping=True))
local_class._logger.info(msg_remaining_allowed_files,
remaining_allowed_files_str)
local_class._logger.info(msg_actual_fs_cache_percent,
RATIO_DECIMAL_DIGITS_TO_PRINT.format(
self._actual_cache_percent) + '%')
local_class._logger.info(msg_disk_used_by_cache,
DiskUtils.sizeof_fmt(self._disk_used_by_cache))
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
local_class._logger.exception('')