def load_unprocessed_movie_cache(cls):
# type: () -> None
"""
:return:
"""
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_unprocessed_movies.json')
path = xbmcvfs.validatePath(path)
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
with CacheIndex.lock:
if os.path.exists(path):
with io.open(path,
mode='rt',
newline=None,
encoding='utf-8') as cacheFile:
cls._unprocessed_movies = json.load(
cacheFile,
encoding='utf-8',
object_hook=CacheIndex.datetime_parser)
cls.last_saved_movie_timestamp = None
cls._unprocessed_movie_changes = 0
else:
cls._unprocessed_movies = {}
Monitor.throw_exception_if_abort_requested()
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
CacheIndex.logger().exception('')
except JSONDecodeError as e:
os.remove(path)
except Exception as e:
CacheIndex.logger().exception('')
def save_cache(cls):
# type: () -> None
"""
:return:
"""
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_discovery_parameters.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
with CacheIndex.lock:
try:
Monitor.throw_exception_if_abort_requested()
with io.open(
path,
mode='wt',
newline=None,
encoding='utf-8',
) as cacheFile:
json_text = cls.to_json()
cacheFile.write(json_text)
cacheFile.flush()
except AbortException:
reraise(*sys.exc_info())
except (IOError) as e:
cls._logger.exception('')
except Exception as e:
cls._logger.exception('')
def load_cache(cls):
# type: () -> None
"""
:return:
"""
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_tmdb_trailers.json')
path = xbmcvfs.validatePath(path)
cls.abort_on_shutdown()
with cls.lock:
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path):
with io.open(path, mode='rt',
newline=None,
encoding='utf-8') as cacheFile:
cls._all_missing_tmdb_trailers = json.load(
cacheFile, encoding='utf-8',
object_hook=TrailerUnavailableCache.datetime_parser)
size = len(cls._all_missing_tmdb_trailers)
Statistics.missing_tmdb_trailers_initial_size(size)
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls._logger.exception('')
except JSONDecodeError as e:
os.remove(path)
except Exception as e:
cls._logger.exception('')
cls.abort_on_shutdown()
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_library_trailers.json')
path = xbmcvfs.validatePath(path)
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path):
with io.open(path, mode='rt',
newline=None,
encoding='utf-8') as cacheFile:
cls._all_missing_library_trailers = json.load(
cacheFile, encoding='utf-8',
object_hook=TrailerUnavailableCache.datetime_parser)
size = len(cls._all_missing_library_trailers)
Statistics.missing_library_trailers_initial_size(size)
except AbortException:
reraise(*sys.exc_info())
except JSONDecodeError as e:
os.remove(path)
except IOError as e:
cls._logger.exception('')
except Exception as e:
cls._logger.exception('')
pass
def save_unprocessed_movie_cache(cls, flush=False):
# type: (bool) -> None
"""
:param flush:
:return:
"""
# TODO: Should use lock here, review locking
with cls.lock:
if cls._unprocessed_movie_changes == 0:
return
if (not flush and
# Constants.TRAILER_CACHE_FLUSH_UPDATES)
(cls._unprocessed_movie_changes < 10)
and
(datetime.datetime.now() - \
cls._last_saved_unprocessed_movie_timestamp)
< datetime.timedelta(minutes=5)):
return
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_unprocessed_movies.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
try:
with io.open(
path,
mode='wt',
newline=None,
encoding='utf-8',
) as cacheFile:
# TODO: Need ability to interrupt when ABORT. Object_handler
# not a valid arg to dumps
json_text = json.dumps(cls.get_unprocessed_movies(),
encoding='utf-8',
ensure_ascii=False,
default=CacheIndex.handler,
indent=3,
sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
cls._last_saved_unprocessed_movie_timestamp = datetime.datetime.now(
)
cls._unprocessed_movie_changes = 0
Monitor.throw_exception_if_abort_requested()
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls.logger().exception('')
except Exception as e:
cls.logger().exception('')
def save_search_pages(self, flush=False):
# type: (bool) -> None
"""
:return:
"""
with CacheIndex.lock:
if (not flush and self.get_number_of_unsaved_changes() <
Constants.TRAILER_CACHE_FLUSH_UPDATES
and self.get_time_since_last_save() <
datetime.timedelta(minutes=5)):
return
saved_pages = len(self._cached_page_by_key.items())
path = xbmcvfs.validatePath(self._path)
temp_path = xbmcvfs.validatePath(self._temp_path)
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
Monitor.throw_exception_if_abort_requested()
with io.open(temp_path,
mode='wt',
newline=None,
encoding='utf-8') as cacheFile:
json_dict = self.to_json()
# TODO: Need ability to interrupt when ABORT. Object_handler
# not a valid arg to dumps
json_text = json.dumps(json_dict,
encoding='utf-8',
ensure_ascii=False,
default=CacheIndex.handler,
indent=3,
sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
self._number_of_unsaved_changes = 0
self._time_of_last_save = datetime.datetime.now()
try:
os.replace(temp_path, path)
except OSError:
self._logger.exception(
f'Failed to replace move information'
f' planned for download: {path}')
Monitor.throw_exception_if_abort_requested()
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
self._logger.exception('')
except JSONDecodeError as e:
os.remove(path)
except Exception as e:
self._logger.exception('')
self._logger.debug_verbose("Entries Saved: ", saved_pages)
def __init__(self, *args, **kwargs):
# type: (*str, **Any) -> None
"""
:param args:
:param kwargs:
"""
self._logger = module_logger.getChild(self.__class__.__name__)
self._file = None
if len(args) == 0:
self._logger.error('Playlist constructor requires an argument')
return
playlist_name = args[0]
self._playlist_name = playlist_name
append = kwargs.get('append', True)
rotate = kwargs.get('rotate', False)
assert append ^ rotate
self.playlist_format = kwargs.get('playlist_format', False)
if self.playlist_format:
self.path = Constants.PLAYLIST_PATH + '/' + \
playlist_name + Playlist.SMART_PLAYLIST_SUFFIX
else:
self.path = Constants.FRONTEND_DATA_PATH + '/' + \
playlist_name # + Playlist.PLAYLIST_SUFFIX
self.path = xbmcvfs.validatePath(self.path)
self.path = xbmcvfs.translatePath(self.path)
DiskUtils.create_path_if_needed(Constants.FRONTEND_DATA_PATH)
if not self.playlist_format:
self.mode = 'wt'
if append:
self.mode = 'at'
else:
self.mode = 'wt'
if rotate:
try:
save_path = Constants.FRONTEND_DATA_PATH + '/' + playlist_name + '.old'
save_path = xbmcvfs.validatePath(save_path)
try:
if os.path.exists(self.path):
os.replace(self.path, save_path)
except Exception as e:
self._logger.exception('')
except Exception as e:
self._logger.exception('')
try:
self._file = io.open(self.path,
mode=self.mode,
buffering=1,
newline=None,
encoding='utf-8')
except Exception as e:
self._logger.exception('')
def read_cached_value_from_disk(cls):
# type: () -> CacheParameters
"""
:return:
"""
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_discovery_parameters.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
saved_preferences = None
with CacheIndex.lock:
try:
if not os.access(path, os.R_OK):
cls._logger.error(
Messages.get_formatted_msg(Messages.CAN_NOT_READ_FILE,
path))
return None
file_mod_time = datetime.datetime.fromtimestamp(
os.path.getmtime(path))
now = datetime.datetime.now()
expiration_time = now - datetime.timedelta(
Settings.get_expire_remote_db_cache_entry_days())
if file_mod_time < expiration_time:
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug('cache file EXPIRED for:', path)
return None
Monitor.throw_exception_if_abort_requested()
with io.open(path, mode='rt', newline=None,
encoding='utf-8') as cacheFile:
saved_preferences = json.load(cacheFile, encoding='utf-8')
saved_preferences = CacheParameters(saved_preferences)
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls._logger.exception('')
exception_occurred = True
except Exception as e:
cls._logger.exception('')
exception_occurred = True
return saved_preferences
def save_found_trailer_ids_cache(cls, flush=False):
# type: (bool) -> None
"""
:param flush:
:return:
"""
with cls.lock:
if cls._unsaved_trailer_changes == 0:
return
if (not flush and (cls._unsaved_trailer_changes <
Constants.TRAILER_CACHE_FLUSH_UPDATES) and
(datetime.datetime.now() - cls._last_saved_trailer_timestamp) <
datetime.timedelta(minutes=5)):
return
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_found_trailers.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
try:
with io.open(
path,
mode='wt',
newline=None,
encoding='utf-8',
) as cacheFile:
found_trailer_id_list = list(
cls._found_tmdb_trailer_ids)
json_text = json.dumps(found_trailer_id_list,
encoding='utf-8',
ensure_ascii=False,
default=CacheIndex.handler,
indent=3,
sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
cls._last_saved_trailer_timestamp = datetime.datetime.now(
)
cls._unsaved_trailer_changes = 0
Monitor.throw_exception_if_abort_requested()
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
CacheIndex.logger().exception('')
except Exception as e:
CacheIndex.logger().exception('')
def onAVStarted(self):
# type: () ->None
"""
Detect when the player is playing something not initiated by this
script. This can be due to a JSON RPC call or similar.Starting the
player via keyboard or remote (that does not use JSON RPC)is
detected by other means (onAction).
Compare the movie that the player is playing versus what we expect
it to play. If they don't match, then assume that something else
launched the movie.
:return:
"""
try:
# All local trailers played by Random Trailers will have a fake genre of
# 'randomtrailers'. However, if a trailer is from a remote source
# such that youtube plugin does the actual playing, then the
# genre will NOT be set to 'randomtrailers'. The use of caching
# of remote trailers will eliminate this issue.
genre = self.getVideoInfoTag().getGenre()
# self._logger.debug('genre:', genre)
if genre != 'randomtrailers':
playing_file = super().getPlayingFile()
playing_file = playing_file
if not (self._is_url and DiskUtils.is_url(playing_file)):
self._is_activated = False
if self._logger.isEnabledFor(LazyLogger.DEBUG):
self._logger.debug(
'Player is playing movie:', playing_file)
self.notify_non_random_trailer_video()
except Exception as e:
pass
def is_more_discovery_needed(cls, movie: MovieType) -> bool:
if movie[Movie.DISCOVERY_STATE] <= Movie.DISCOVERY_COMPLETE:
return False
more_discovery_needed = False
title = movie[Movie.TITLE]
try:
normalized_trailer_path = movie.get(Movie.NORMALIZED_TRAILER)
if normalized_trailer_path is None:
normalized_trailer_path = ''
cached_trailer_path = movie.get(Movie.CACHED_TRAILER)
if cached_trailer_path is None:
cached_trailer_path = ''
if DiskUtils.is_url(movie.get(Movie.TRAILER, '')):
# Remote Trailer
if Settings.is_normalize_volume_of_downloaded_trailers():
try:
if not os.path.exists(normalized_trailer_path):
cls._logger.debug(
f'title: {title} does not exist: '
f'{normalized_trailer_path}')
movie[Movie.NORMALIZED_TRAILER] = None
more_discovery_needed = True
except Exception as e:
cls._logger.log_exception(e)
elif Settings.is_use_trailer_cache():
try:
if not os.path.exists(cached_trailer_path):
cls._logger.debug(
f'title: {title} does not exist: '
f'{cached_trailer_path}')
movie[Movie.CACHED_TRAILER] = None
movie[Movie.NORMALIZED_TRAILER] = None
more_discovery_needed = True
except Exception as e:
cls._logger.log_exception(e)
elif Settings.is_normalize_volume_of_local_trailers():
# Local trailer
try:
if not os.path.exists(normalized_trailer_path):
cls._logger.debug(f'title: {title} does not exist: '
f'{normalized_trailer_path}')
movie[Movie.NORMALIZED_TRAILER] = None
more_discovery_needed = True
except Exception as e:
cls._logger.log_exception(e)
except Exception as e:
cls._logger.log_exception()
if more_discovery_needed:
movie[Movie.DISCOVERY_STATE] = Movie.DISCOVERY_COMPLETE
cls._logger.debug(f'More discovery needed: {title}')
return more_discovery_needed
def write_tmdb_cache_json(cls, movie_id: Union[str, int], source: str,
movie: MovieType) -> None:
"""
Write the given movie information into the cache as JSON
Due to the small size of these files, will not check for
AbortException during write nor save old version of file.
"""
try:
if source is None or source not in Movie.LIB_TMDB_ITUNES_SOURCES:
cls._logger.debug('Invalid source:', source)
movie_id = str(movie_id)
path = Cache.get_json_cache_file_path_for_movie_id(
movie_id, source)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path) and not os.access(path, os.W_OK):
messages = Messages
cls._logger.error(
messages.get_msg(Messages.CAN_NOT_WRITE_FILE) % path)
return None
temp_movie = {}
for key in Movie.TMDB_ENTRY_FIELDS:
temp_movie[key] = movie.get(key)
Monitor.throw_exception_if_abort_requested()
with io.open(
path,
mode='wt',
newline=None,
encoding='utf-8',
) as cacheFile:
json_text = json.dumps(temp_movie,
ensure_ascii=False,
indent=3,
sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
del temp_movie
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
cls._logger.exception('')
def set_playing_file_path(self, file_path):
# type: (str) -> None
"""
:param file_path:
:return:
"""
file_path = file_path
self._is_url = DiskUtils.is_url(file_path)
self._expected_file_path = file_path
def load_search_pages(self):
# type: () -> None
"""
:return:
"""
if self._cached_page_by_key is not None:
return
path = xbmcvfs.validatePath(self._path)
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path):
Monitor.throw_exception_if_abort_requested()
with CacheIndex.lock, io.open(path,
mode='rt',
newline=None,
encoding='utf-8') as cacheFile:
encoded_values = json.load(
cacheFile,
encoding='utf-8',
object_hook=CacheIndex.datetime_parser)
loaded_cached_pages_data = self.from_json(encoded_values)
self._cached_page_by_key = loaded_cached_pages_data._cached_page_by_key
else:
self._cached_page_by_key: Dict[str, CachedPage] = dict()
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
self._logger.exception('')
except JSONDecodeError as e:
os.remove(path)
self._cached_page_by_key = dict()
except Exception as e:
self._logger.exception('')
self._logger.debug_verbose("Loaded entries:",
len(self._cached_page_by_key))
self._time_of_last_save = datetime.datetime.now()
def get_stats_for_caches(self) -> Dict[str, UsageData]:
"""
Get disk usage information for the caches.
Returns a map of UsageData for each cache. Primarily used
by garbage collection and reporting.
:return:
"""
local_class = CacheManager
TRAILER_PATTERN = re.compile(r'^.*-trailer\..*$')
JSON_PATTERN = re.compile(r'^.*\.json$')
TFH_PATTERN = re.compile(r'^.*-movie\..*$')
TRAILER_TYPE = 'trailer'
JSON_TYPE = 'json'
# When the Trailer Cache and Data Cache (.json) are the same
if (Settings.get_downloaded_trailer_cache_path() ==
Settings.get_remote_db_cache_path()):
usage_data_map = DiskUtils.get_stats_for_path(
Settings.get_downloaded_trailer_cache_path(), {
'trailer': (TRAILER_PATTERN, TRAILER_TYPE),
'json': (JSON_PATTERN, JSON_TYPE),
'tfh': (TFH_PATTERN, TRAILER_TYPE)
})
else:
# When Trailer Cache and Data Cache are different directories.
usage_data_map = DiskUtils.get_stats_for_path(
Settings.get_downloaded_trailer_cache_path(), {
'trailer': (TRAILER_PATTERN, TRAILER_TYPE),
'tfh': (TFH_PATTERN, TRAILER_TYPE)
})
json_usage_data = DiskUtils.get_stats_for_path(
Settings.get_remote_db_cache_path(),
{'json': (JSON_PATTERN, JSON_TYPE)})
usage_data_map['json'] = json_usage_data['json']
return usage_data_map
def load_cache(cls) -> None:
"""
:return: True if cache is full and no further discovery needed
"""
with cls.lock:
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tfh_trailers.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path):
with io.open(path, mode='rt', newline=None,
encoding='utf-8') as cacheFile:
cls._cached_trailers = json.load(
cacheFile,
# object_hook=TFHCache.abort_checker,
)
cls.last_saved_movie_timestamp = None
cls._unsaved_trailer_changes = 0
cls.load_creation_date()
else:
cls._cached_trailers = dict()
# Set to an old time so that cache is expired
cls._time_of_index_creation = datetime.datetime(2000, 1, 1)
cls._index_complete = False
except IOError as e:
TFHCache.logger().exception('')
except JSONDecodeError as e:
os.remove(path)
except Exception as e:
TFHCache.logger().exception('')
Monitor.throw_exception_if_abort_requested()
return
def load_found_trailer_cache(cls) -> None:
"""
:return:
"""
path = os.path.join(Settings.get_remote_db_cache_path(), 'index',
'tmdb_found_trailers.json')
path = xbmcvfs.validatePath(path)
try:
parent_dir, file_name = os.path.split(path)
DiskUtils.create_path_if_needed(parent_dir)
if os.path.exists(path):
with CacheIndex.lock, io.open(path,
mode='rt',
newline=None,
encoding='utf-8') as cacheFile:
found_trailers_list = json.load(
cacheFile,
encoding='utf-8',
object_hook=CacheIndex.datetime_parser)
cls._last_saved_trailer_timestamp = datetime.datetime.now()
cls._found_tmdb_trailer_ids: Set[int] = set(
found_trailers_list)
cls._unsaved_trailer_changes = 0
else:
cls._found_tmdb_trailer_ids: Set[int] = set()
Monitor.throw_exception_if_abort_requested()
cls.remove_unprocessed_movies(list(cls._found_tmdb_trailer_ids))
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
CacheIndex.logger().exception('')
except JSONDecodeError as e:
os.remove(path)
except Exception as e:
CacheIndex.logger().exception('')
def __init__(self, trailer_cache: bool) -> None:
"""
Populates this instance with relevant settings. Allows for uniform
access to those settings.
:param trailer_cache:
"""
local_class = CacheData
if local_class._logger is None:
local_class._logger = module_logger.getChild(local_class.__name__)
self._usage_data = None
self._messages = Messages
self._is_trailer_cache = trailer_cache
if trailer_cache:
self._is_limit_number_of_cached_files = \
Settings.is_limit_number_of_cached_trailers()
if self._is_limit_number_of_cached_files:
self._max_number_of_files = Settings.get_max_number_of_cached_trailers()
self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_trailers()
if self._is_limit_size_of_cache:
self._max_cache_size_mb = Settings.get_max_size_of_cached_trailers_mb()
self._is_limit_percent_of_cache_disk = \
Settings.is_limit_percent_of_cached_trailers()
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk = \
Settings.get_max_percent_of_cached_trailers()
# TODO: Delete me
# self._max_percent_of_cache_disk = 0.83
else:
self._is_limit_number_of_cached_files = \
Settings.is_limit_number_of_cached_json()
if self._is_limit_number_of_cached_files:
self._max_number_of_files = Settings.get_max_number_of_cached_json()
self._is_limit_size_of_cache = Settings.is_limit_size_of_cached_json()
if self._is_limit_size_of_cache:
self._max_cache_size_mb = Settings.get_max_size_of_cached_json_mb()
self._is_limit_percent_of_cache_disk = \
Settings.is_limit_percent_of_cached_json()
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk = \
Settings.get_max_percent_of_cached_json()
# TODO_ delete me
# self._max_percent_of_cache_disk = 0.0058
if self._is_limit_number_of_cached_files:
self._max_number_of_files_str = locale.format("%d",
self._max_number_of_files,
grouping=True)
else:
self._max_number_of_files = 0
self._max_number_of_files_str = self._messages.get_msg(
Messages.UNLIMITED)
if self._is_limit_size_of_cache:
self._max_cache_size_mb_str = DiskUtils.sizeof_fmt(
self._max_cache_size_mb * 1024 * 1024)
else:
self._max_cache_size_mb = 0
self._max_cache_size_mb_str = self._messages.get_msg(
Messages.UNLIMITED)
if self._is_limit_percent_of_cache_disk:
self._max_percent_of_cache_disk_str = RATIO_DECIMAL_DIGITS_TO_PRINT.format(
self._max_percent_of_cache_disk) + '%'
else:
self._max_percent_of_cache_disk = 100.0
self._max_percent_of_cache_disk_str = self._messages.get_msg(
Messages.UNLIMITED)
self._remaining_allowed_files = None
self._used_space_in_cache_fs = None
self._free_disk_in_cache_fs = None
self._total_size_of_cache_fs = None
self._disk_used_by_cache = None
self._actual_cache_percent = None
def collect_garbage(self):
# type: () -> None
"""
Runs garbage collection on all of the caches according to the
settings.
This is a time-consuming process. It is normally kicked-off by
drive_garbage_collection
:return:
"""
local_class = CacheData
try:
if local_class._logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
if self._is_trailer_cache:
local_class._logger.debug_extra_verbose('TRAILER CACHE')
else:
local_class._logger.debug_extra_verbose('JSON CACHE')
if self._is_limit_number_of_cached_files:
#
# Delete enough of the oldest files to keep the number
# within limit
number_of_cache_files_to_delete = - self._remaining_allowed_files
if number_of_cache_files_to_delete > 0:
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_number_of_cached_files. number_of_files_to_delete:',
locale.format("%d", number_of_cache_files_to_delete,
grouping=True))
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
number_of_cache_files_to_delete -= 1
if number_of_cache_files_to_delete <= 0:
break
else:
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_number_of_cached_files. Additional allowed files:',
locale.format("%d", number_of_cache_files_to_delete,
grouping=True))
if self._is_limit_size_of_cache:
#
# Delete enough of the oldest files to keep the number
# within limit
max_bytes_in_cache = (self._max_cache_size_mb * 1024 * 1024)
bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info('limit_size_of_cache. max allowed size:',
DiskUtils.sizeof_fmt(max_bytes_in_cache))
local_class._logger.debug('actual disk used in cache:',
DiskUtils.sizeof_fmt(
self._usage_data.get_disk_used_by_cache()))
local_class._logger.debug('Amount to delete:',
DiskUtils.sizeof_fmt(bytes_of_files_to_delete))
if bytes_of_files_to_delete > 0:
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
bytes_of_files_to_delete = (
self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if bytes_of_files_to_delete <= 0:
break
if self._is_limit_percent_of_cache_disk:
#
# Delete enough of the oldest files to keep the number
# within limit
max_bytes_in_cache = (self._total_size_of_cache_fs *
self._max_percent_of_cache_disk / 100.00)
bytes_of_files_to_delete = (self._usage_data.get_disk_used_by_cache() -
max_bytes_in_cache)
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(
'limit_percent of cached files. Calculated max size:',
DiskUtils.sizeof_fmt(max_bytes_in_cache))
local_class._logger.info('size to delete:',
DiskUtils.sizeof_fmt(
bytes_of_files_to_delete))
if bytes_of_files_to_delete > 0:
# Order json files by age
for cache_file in self._usage_data.get_file_data_by_creation_date():
Monitor.throw_exception_if_abort_requested()
self._usage_data.remove_file(cache_file)
bytes_of_files_to_delete = (
self._usage_data.get_disk_used_by_cache()
- max_bytes_in_cache)
if bytes_of_files_to_delete <= 0:
break
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
local_class._logger.exception('')
def report_status(self):
# type: () -> None
"""
Produces a simple report about the cache using the Settings
and UsageData.
:return:
"""
local_class = CacheData
try:
if self._is_trailer_cache:
msg_max_trailers = 'max allowed trailers:'
msg_disk_usage = 'max_trailer_cache_disk_usage:'
msg_cache_percent = 'max percent of trailer cache disk usage:'
msg_files_in_cache = 'trailers_in_cache:'
msg_remaining_allowed_files = 'remaining_allowed_trailers:'
msg_total_size_of_cache_fs = 'Size of trailer cache fs:'
msg_used_space_in_cache_fs = 'Used space in trailer cache fs:'
msg_free_space_in_cache_fs = 'free space in trailer cache fs:'
msg_actual_fs_cache_percent = 'Actual percent of disk used by trailer ' \
'cache:'
msg_disk_used_by_cache = 'Actual disk used by trailer cache:'
else:
msg_max_trailers = 'max number of json files:'
msg_disk_usage = 'max_json_cache_disk_usage:'
msg_cache_percent = 'max_percent of json cache disk usage:'
msg_files_in_cache = 'json_files_in_cache'
msg_remaining_allowed_files = 'remaining_allowed_json_files'
msg_total_size_of_cache_fs = 'Size of json cache fs:'
msg_used_space_in_cache_fs = 'Used space in json cache fs:'
msg_free_space_in_cache_fs = 'free space in json cache fs:'
msg_actual_fs_cache_percent = 'Actual percent of disk used by json ' \
'cache:'
msg_disk_used_by_cache = 'Actual disk used by json cache:'
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(msg_max_trailers,
self._max_number_of_files_str,
trace=Trace.STATS_CACHE)
local_class._logger.info(msg_disk_usage,
self._max_cache_size_mb_str,
trace=Trace.STATS_CACHE)
local_class._logger.info(msg_cache_percent,
self._max_percent_of_cache_disk_str,
trace=Trace.STATS_CACHE)
files_in_cache = self._usage_data.get_number_of_files()
if self._is_limit_number_of_cached_files:
self._remaining_allowed_files = (self._max_number_of_files -
files_in_cache)
remaining_allowed_files_str = locale.format("%d",
self._remaining_allowed_files,
grouping=True)
else:
self._remaining_allowed_files = None
remaining_allowed_files_str = self._messages.get_msg(
Messages.UNLIMITED)
self._used_space_in_cache_fs = self._usage_data.get_used_space()
self._free_disk_in_cache_fs = self._usage_data.get_free_size()
self._total_size_of_cache_fs = self._usage_data.get_total_size()
self._disk_used_by_cache = self._usage_data.get_disk_used_by_cache()
self._actual_cache_percent = (self._disk_used_by_cache /
self._total_size_of_cache_fs) * 100.0
if local_class._logger.isEnabledFor(LazyLogger.INFO):
local_class._logger.info(msg_total_size_of_cache_fs,
DiskUtils.sizeof_fmt(self._total_size_of_cache_fs))
local_class._logger.info(msg_used_space_in_cache_fs,
DiskUtils.sizeof_fmt(self._used_space_in_cache_fs))
local_class._logger.info(msg_free_space_in_cache_fs,
DiskUtils.sizeof_fmt(self._free_disk_in_cache_fs))
local_class._logger.info(msg_files_in_cache,
locale.format("%d", files_in_cache, grouping=True))
local_class._logger.info(msg_remaining_allowed_files,
remaining_allowed_files_str)
local_class._logger.info(msg_actual_fs_cache_percent,
RATIO_DECIMAL_DIGITS_TO_PRINT.format(
self._actual_cache_percent) + '%')
local_class._logger.info(msg_disk_used_by_cache,
DiskUtils.sizeof_fmt(self._disk_used_by_cache))
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
local_class._logger.exception('')
def save_cache(cls, flush: bool = False, complete: bool = False) -> None:
"""
:param flush:
:param complete:
:return:
"""
with cls.lock:
if (not flush and
(cls._unsaved_trailer_changes < 50)
and
(datetime.datetime.now() - cls._last_saved_trailer_timestamp)
< datetime.timedelta(minutes=5)):
return
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tfh_trailers.json')
path = xbmcvfs.validatePath(path)
tmp_path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tfh_trailers.json.tmp')
tmp_path = xbmcvfs.validatePath(tmp_path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
Monitor.throw_exception_if_abort_requested()
with io.open(tmp_path, mode='at', newline=None,
encoding='utf-8', ) as cacheFile:
if complete:
creation_date_str = datetime.datetime.strftime(
cls._time_of_index_creation, '%Y:%m:%d')
else:
creation_date_str = TFHCache.INCOMPLETE_CREATION_DATE_STR
cls._cached_trailers[TFHCache.INDEX_CREATION_DATE] = {
TFHCache.INDEX_CREATION_DATE: creation_date_str,
}
json_text = json.dumps(cls._cached_trailers,
encoding='utf-8',
ensure_ascii=False,
default=TFHCache.abort_checker,
indent=3, sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
# Get rid of dummy entry
del cls._cached_trailers['INDEX_CREATION_DATE']
cls._last_saved_trailer_timestamp = datetime.datetime.now()
cls._unsaved_trailer_changes = 0
try:
os.replace(tmp_path, path)
except OSError:
cls._logger.exception(f'Failed to replace missing trailer'
f' information cache: {path}')
except IOError as e:
TFHCache.logger().exception('')
except Exception as e:
TFHCache.logger().exception('')
Monitor.throw_exception_if_abort_requested()
def save_unprocessed_movie_cache(cls, flush: bool = False) -> None:
"""
:param flush:
:return:
"""
# TODO: Should use lock here, review locking
with cls.lock:
if cls._unprocessed_movie_changes == 0:
return
if (not flush and
# Constants.TRAILER_CACHE_FLUSH_UPDATES)
(cls._unprocessed_movie_changes < 10)
and ((datetime.datetime.now() -
cls._last_saved_unprocessed_movie_timestamp)) <
datetime.timedelta(minutes=5)):
return
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tmdb_unprocessed_movies.json')
path = xbmcvfs.validatePath(path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
# Don't save unneeded fields. Takes up disk and RAM
temp_entries = {}
for tmdb_id, entry in cls.get_unprocessed_movies().items():
temp_entry = {}
for key in Movie.TMDB_PAGE_DATA_FIELDS:
temp_entry[key] = entry[key]
temp_entries[tmdb_id] = temp_entry
json_text = json.dumps(temp_entries,
encoding='utf-8',
ensure_ascii=False,
default=CacheIndex.handler,
indent=3,
sort_keys=True)
with io.open(
path,
mode='wt',
newline=None,
encoding='utf-8',
) as cache_file:
# TODO: Need ability to interrupt when ABORT. Object_handler
# not a valid arg to dumps
cache_file.write(json_text)
cache_file.flush()
cls._last_saved_unprocessed_movie_timestamp = datetime.datetime.now(
)
cls._unprocessed_movie_changes = 0
Monitor.throw_exception_if_abort_requested()
del json_text
del temp_entries
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls.logger().exception('')
except Exception as e:
cls.logger().exception('')
def save_cache(cls, flush: bool = False, complete: bool = False) -> None:
"""
:param flush:
:param complete:
:return:
Typical json entry
Items marked with * are kodi/TMDb artifacts
"BZwDpOQNgpw": {
"adult": false,
"cast": [],
"fanart": "default_fanart",
"genre": [],
"mpaa": "NR",
"normalized_trailer": "/home/fbacher/.kodi/userdata/addon_data/script.video.randomtrailers/cache/hB/tfh_BZwDpOQNgpw_normalized_Larry Karaszewski on SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3) (2017)-movie.mkv",
"original_language": "",
"plot": "But wait! There's more! TFH has a podcast! \n\nIt's THE MOVIES THAT MADE ME, where you can join Oscar-nominated screenwriter Josh Olson and his, ummm, \"co-host\" Joe Dante in conversation with filmmakers, comedians, and all-around interesting people about the movies that made them who they are. Check it out now, and please subscribe wherever podcasts can be found.\n\nBut wait! There's more! TFH has a podcast! \n\nIt's THE MOVIES THAT MADE ME, where you can join Oscar-nominated screenwriter Josh Olson and his, ummm, \"co-host\" Joe Dante in conversation with filmmakers, comedians, and all-around interesting people about the movies that made them who they are. Check it out now, and please subscribe wherever podcasts can be found.\n\niTunes: http://itunes.trailersfromhell.com\nSpotify: http://spotify.trailersfromhell.com\nLibsyn: http://podcast.trailersfromhell.com\nGoogle Play: http://googleplay.trailersfromhell.com\nRSS: http://goo.gl/3faeG7\n\nAs always, you can find more commentary, more reviews, more podcasts, and more deep-dives into the films you don't know you love yet over at the Trailers From Hell mothership: \n\nhttp://www.trailersfromhell.com",
"rating": 4.8974357,
"genre": [],
"rts.actors": "",
"rts.certification": "Unrated",
"rts.certificationImage": "ratings/us/unrated.png",
"rts.directors": "",
"rts.genres": "",
"rts.runtime": "143 [B]Minutes[/B] - ",
"rts.studios": "",
"rts.tfhId": "BZwDpOQNgpw",
"rts.tfh_title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3)",
"rts.title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3) (2017) - TFH ",
"rts.tmdb_id_not_found": true,
"rts.voiced.actors": [],
"rts.voiced.directors": [],
"rts.voiced.studios": [],
"rts.voiced.writers": [],
"rts.writers": "",
"rts.youtube.trailers_in_index": 1449,
"rts.youtube_index": 204,
"runtime": 8580,
"source": "TFH",
"studio": [
[]
],
"tags": [
"smokey and the bandit 3",
"larry karaszewski",
"jackie gleason"
],
"thumbnail": "https://i.ytimg.com/vi_webp/BZwDpOQNgpw/maxresdefault.webp",
"title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3)",
"trailer": "https://youtu.be/BZwDpOQNgpw",
"trailerDiscoveryState": "04_discoveryReadyToDisplay",
"trailerPlayed": true,
"trailerType": "default_trailerType",
"uniqueid": {
"tmdb": "None"
},
"writer": [
[]
],
"year": 2017
}
"""
with cls.lock:
if (not flush and
(cls._unsaved_trailer_changes < 50)
and
(datetime.datetime.now() - cls._last_saved_trailer_timestamp)
< datetime.timedelta(minutes=5)):
return
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tfh_trailers.json')
path = xbmcvfs.validatePath(path)
tmp_path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'tfh_trailers.json.tmp')
tmp_path = xbmcvfs.validatePath(tmp_path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
Monitor.throw_exception_if_abort_requested()
with io.open(tmp_path, mode='at', newline=None,
encoding='utf-8', ) as cacheFile:
if complete:
cls.set_creation_date()
# Set to True when complete, but don't set to False
# when not complete.
cls._cache_complete = True
creation_date_str = datetime.datetime.strftime(
cls._time_of_index_creation, '%Y:%m:%d')
cls._cached_trailers[cls.INDEX_CREATION_DATE] = {
cls.INDEX_CREATION_DATE: creation_date_str,
cls.CACHE_COMPLETE: cls._cache_complete
}
json_text = json.dumps(cls._cached_trailers,
encoding='utf-8',
ensure_ascii=False,
default=TFHCache.abort_checker,
indent=3, sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
# Get rid of dummy entry from local dict
del cls._cached_trailers[cls.INDEX_CREATION_DATE]
cls._last_saved_trailer_timestamp = datetime.datetime.now()
cls._unsaved_trailer_changes = 0
try:
os.replace(tmp_path, path)
except OSError:
cls._logger.exception(f'Failed to replace missing trailer'
f' information cache: {path}')
except IOError as e:
TFHCache.logger().exception('')
except Exception as e:
TFHCache.logger().exception('')
Monitor.throw_exception_if_abort_requested()
def save_cache(cls, ignore_shutdown=False) -> None:
"""
:return:
"""
cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown)
with cls.lock:
if cls.tmdb_unsaved_changes == 0 and cls.library_unsaved_changes == 0:
return
if cls.tmdb_unsaved_changes > 0:
entries_to_delete = []
for key, entry in cls._all_missing_tmdb_trailers.items():
elapsed_time = datetime.date.today() - entry['timestamp']
elapsed_days = elapsed_time.days
if elapsed_days > Settings.get_expire_remote_db_trailer_check_days():
if entry[Movie.UNIQUE_ID_TMDB] in cls._all_missing_tmdb_trailers:
entries_to_delete.append(
entry[Movie.UNIQUE_ID_TMDB])
for entry_to_delete in entries_to_delete:
del cls._all_missing_tmdb_trailers[entry_to_delete]
cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown)
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_tmdb_trailers.json')
temp_path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_tmdb_trailers.json.temp')
# path = path.encode('utf-8')
path = xbmcvfs.validatePath(path)
temp_path = xbmcvfs.validatePath(temp_path)
parent_dir, file_name = os.path.split(temp_path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
with io.open(temp_path, mode='wt', newline=None,
encoding='utf-8', ) as cacheFile:
json_text = \
json.dumps(cls._all_missing_tmdb_trailers,
encoding='utf-8',
ensure_ascii=False,
default=TrailerUnavailableCache.handler,
indent=3, sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
try:
os.replace(temp_path, path)
except OSError:
cls._logger.exception(f'Failed to replace index of movies'
f' missing trailers cache: {path}')
cls.tmdb_last_save = datetime.datetime.now()
cls.tmdb_unsaved_changes = 0
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls._logger.exception('')
except Exception as e:
cls._logger.exception('')
finally:
try:
os.remove(temp_path)
except Exception:
pass
cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown)
if cls.library_unsaved_changes > 0:
entries_to_delete = []
for key, entry in cls._all_missing_library_trailers.items():
elapsed_time = datetime.date.today() - entry['timestamp']
elapsed_days = elapsed_time.days
if elapsed_days > Settings.get_expire_remote_db_trailer_check_days():
if entry[Movie.MOVIEID] in cls._all_missing_library_trailers:
entries_to_delete.append(entry[Movie.MOVIEID])
for entry_to_delete in entries_to_delete:
del cls._all_missing_library_trailers[entry_to_delete]
cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown)
try:
path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_library_trailers.json')
temp_path = os.path.join(Settings.get_remote_db_cache_path(),
'index', 'missing_library_trailers.json.temp')
# path = path.encode('utf-8')
path = xbmcvfs.validatePath(path)
temp_path = xbmcvfs.validatePath(temp_path)
parent_dir, file_name = os.path.split(path)
if not os.path.exists(parent_dir):
DiskUtils.create_path_if_needed(parent_dir)
with io.open(temp_path, mode='wt', newline=None,
encoding='utf-8', ) as cacheFile:
# TODO: Need ability to interrupt when ABORT. Object_handler
# not a valid arg to dumps
json_text = \
json.dumps(cls._all_missing_library_trailers,
encoding='utf-8',
ensure_ascii=False,
default=TrailerUnavailableCache.handler,
indent=3, sort_keys=True)
cacheFile.write(json_text)
cacheFile.flush()
try:
os.replace(temp_path, path)
except OSError:
cls._logger.exception(f'Failed to replace missing trailer'
f' information cache: {path}')
cls.library_last_save = datetime.datetime.now()
cls.library_unsaved_changes = 0
except AbortException:
reraise(*sys.exc_info())
except IOError as e:
cls._logger.exception('')
except Exception as e:
cls._logger.exception('')
finally:
try:
os.remove(temp_path)
except Exception:
pass
