def __init__( self, key='', # type: str total_pages=0, # type: int query_by_year=False # type: bool ): # type: (...) -> None """ :param key: :param total_pages: """ self._logger = module_logger.getChild(type(self).__name__) self._number_of_unsaved_changes = 0 self._time_of_last_save = None self._key = key self._total_pages = total_pages self._total_pages_by_year = {} self._query_by_year = query_by_year self._years_to_query = None self._search_pages_configured = False self._logger.debug('remote_db_cache_path:', Settings.get_remote_db_cache_path()) self._path = os.path.join(Settings.get_remote_db_cache_path(), 'index', f'tmdb_{key}.json') self._temp_path = os.path.join(Settings.get_remote_db_cache_path(), 'index', f'tmdb_{key}.json.tmp') # type: self._cached_page_by_key: Optional[Dict[str, CachedPage]] = None
def load_cache(cls): # type: () -> None """ :return: """ path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_tmdb_trailers.json') path = xbmcvfs.validatePath(path) cls.abort_on_shutdown() with cls.lock: try: parent_dir, file_name = os.path.split(path) DiskUtils.create_path_if_needed(parent_dir) if os.path.exists(path): with io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: cls._all_missing_tmdb_trailers = json.load( cacheFile, encoding='utf-8', object_hook=TrailerUnavailableCache.datetime_parser) size = len(cls._all_missing_tmdb_trailers) Statistics.missing_tmdb_trailers_initial_size(size) except AbortException: reraise(*sys.exc_info()) except IOError as e: cls._logger.exception('') except JSONDecodeError as e: os.remove(path) except Exception as e: cls._logger.exception('') cls.abort_on_shutdown() path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_library_trailers.json') path = xbmcvfs.validatePath(path) try: parent_dir, file_name = os.path.split(path) DiskUtils.create_path_if_needed(parent_dir) if os.path.exists(path): with io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: cls._all_missing_library_trailers = json.load( cacheFile, encoding='utf-8', object_hook=TrailerUnavailableCache.datetime_parser) size = len(cls._all_missing_library_trailers) Statistics.missing_library_trailers_initial_size(size) except AbortException: reraise(*sys.exc_info()) except JSONDecodeError as e: os.remove(path) except IOError as e: cls._logger.exception('') except Exception as e: cls._logger.exception('') pass
def save_cache(cls): # type: () -> None """ :return: """ path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_discovery_parameters.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) with CacheIndex.lock: try: Monitor.throw_exception_if_abort_requested() with io.open( path, mode='wt', newline=None, encoding='utf-8', ) as cacheFile: json_text = cls.to_json() cacheFile.write(json_text) cacheFile.flush() except AbortException: reraise(*sys.exc_info()) except (IOError) as e: cls._logger.exception('') except Exception as e: cls._logger.exception('')
def load_unprocessed_movie_cache(cls): # type: () -> None """ :return: """ path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_unprocessed_movies.json') path = xbmcvfs.validatePath(path) try: parent_dir, file_name = os.path.split(path) DiskUtils.create_path_if_needed(parent_dir) with CacheIndex.lock: if os.path.exists(path): with io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: cls._unprocessed_movies = json.load( cacheFile, encoding='utf-8', object_hook=CacheIndex.datetime_parser) cls.last_saved_movie_timestamp = None cls._unprocessed_movie_changes = 0 else: cls._unprocessed_movies = {} Monitor.throw_exception_if_abort_requested() except AbortException: reraise(*sys.exc_info()) except IOError as e: CacheIndex.logger().exception('') except JSONDecodeError as e: os.remove(path) except Exception as e: CacheIndex.logger().exception('')
def save_unprocessed_movie_cache(cls, flush=False): # type: (bool) -> None """ :param flush: :return: """ # TODO: Should use lock here, review locking with cls.lock: if cls._unprocessed_movie_changes == 0: return if (not flush and # Constants.TRAILER_CACHE_FLUSH_UPDATES) (cls._unprocessed_movie_changes < 10) and (datetime.datetime.now() - \ cls._last_saved_unprocessed_movie_timestamp) < datetime.timedelta(minutes=5)): return path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_unprocessed_movies.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) try: with io.open( path, mode='wt', newline=None, encoding='utf-8', ) as cacheFile: # TODO: Need ability to interrupt when ABORT. Object_handler # not a valid arg to dumps json_text = json.dumps(cls.get_unprocessed_movies(), encoding='utf-8', ensure_ascii=False, default=CacheIndex.handler, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() cls._last_saved_unprocessed_movie_timestamp = datetime.datetime.now( ) cls._unprocessed_movie_changes = 0 Monitor.throw_exception_if_abort_requested() except AbortException: reraise(*sys.exc_info()) except IOError as e: cls.logger().exception('') except Exception as e: cls.logger().exception('')
def get_stats_for_caches(self) -> Dict[str, UsageData]: """ Get disk usage information for the caches. Returns a map of UsageData for each cache. Primarily used by garbage collection and reporting. :return: """ local_class = CacheManager TRAILER_PATTERN = re.compile(r'^.*-trailer\..*$') JSON_PATTERN = re.compile(r'^.*\.json$') TFH_PATTERN = re.compile(r'^.*-movie\..*$') TRAILER_TYPE = 'trailer' JSON_TYPE = 'json' # When the Trailer Cache and Data Cache (.json) are the same if (Settings.get_downloaded_trailer_cache_path() == Settings.get_remote_db_cache_path()): usage_data_map = DiskUtils.get_stats_for_path( Settings.get_downloaded_trailer_cache_path(), { 'trailer': (TRAILER_PATTERN, TRAILER_TYPE), 'json': (JSON_PATTERN, JSON_TYPE), 'tfh': (TFH_PATTERN, TRAILER_TYPE) }) else: # When Trailer Cache and Data Cache are different directories. usage_data_map = DiskUtils.get_stats_for_path( Settings.get_downloaded_trailer_cache_path(), { 'trailer': (TRAILER_PATTERN, TRAILER_TYPE), 'tfh': (TFH_PATTERN, TRAILER_TYPE) }) json_usage_data = DiskUtils.get_stats_for_path( Settings.get_remote_db_cache_path(), {'json': (JSON_PATTERN, JSON_TYPE)}) usage_data_map['json'] = json_usage_data['json'] return usage_data_map
def read_cached_value_from_disk(cls): # type: () -> CacheParameters """ :return: """ path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_discovery_parameters.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) saved_preferences = None with CacheIndex.lock: try: if not os.access(path, os.R_OK): cls._logger.error( Messages.get_formatted_msg(Messages.CAN_NOT_READ_FILE, path)) return None file_mod_time = datetime.datetime.fromtimestamp( os.path.getmtime(path)) now = datetime.datetime.now() expiration_time = now - datetime.timedelta( Settings.get_expire_remote_db_cache_entry_days()) if file_mod_time < expiration_time: if cls._logger.isEnabledFor(LazyLogger.DEBUG): cls._logger.debug('cache file EXPIRED for:', path) return None Monitor.throw_exception_if_abort_requested() with io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: saved_preferences = json.load(cacheFile, encoding='utf-8') saved_preferences = CacheParameters(saved_preferences) except AbortException: reraise(*sys.exc_info()) except IOError as e: cls._logger.exception('') exception_occurred = True except Exception as e: cls._logger.exception('') exception_occurred = True return saved_preferences
def save_found_trailer_ids_cache(cls, flush=False): # type: (bool) -> None """ :param flush: :return: """ with cls.lock: if cls._unsaved_trailer_changes == 0: return if (not flush and (cls._unsaved_trailer_changes < Constants.TRAILER_CACHE_FLUSH_UPDATES) and (datetime.datetime.now() - cls._last_saved_trailer_timestamp) < datetime.timedelta(minutes=5)): return path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_found_trailers.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) try: with io.open( path, mode='wt', newline=None, encoding='utf-8', ) as cacheFile: found_trailer_id_list = list( cls._found_tmdb_trailer_ids) json_text = json.dumps(found_trailer_id_list, encoding='utf-8', ensure_ascii=False, default=CacheIndex.handler, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() cls._last_saved_trailer_timestamp = datetime.datetime.now( ) cls._unsaved_trailer_changes = 0 Monitor.throw_exception_if_abort_requested() except AbortException: reraise(*sys.exc_info()) except IOError as e: CacheIndex.logger().exception('') except Exception as e: CacheIndex.logger().exception('')
def load_cache(cls) -> None: """ :return: True if cache is full and no further discovery needed """ with cls.lock: try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tfh_trailers.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) DiskUtils.create_path_if_needed(parent_dir) if os.path.exists(path): with io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: cls._cached_trailers = json.load( cacheFile, # object_hook=TFHCache.abort_checker, ) cls.last_saved_movie_timestamp = None cls._unsaved_trailer_changes = 0 cls.load_creation_date() else: cls._cached_trailers = dict() # Set to an old time so that cache is expired cls._time_of_index_creation = datetime.datetime(2000, 1, 1) cls._index_complete = False except IOError as e: TFHCache.logger().exception('') except JSONDecodeError as e: os.remove(path) except Exception as e: TFHCache.logger().exception('') Monitor.throw_exception_if_abort_requested() return
def load_found_trailer_cache(cls) -> None: """ :return: """ path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_found_trailers.json') path = xbmcvfs.validatePath(path) try: parent_dir, file_name = os.path.split(path) DiskUtils.create_path_if_needed(parent_dir) if os.path.exists(path): with CacheIndex.lock, io.open(path, mode='rt', newline=None, encoding='utf-8') as cacheFile: found_trailers_list = json.load( cacheFile, encoding='utf-8', object_hook=CacheIndex.datetime_parser) cls._last_saved_trailer_timestamp = datetime.datetime.now() cls._found_tmdb_trailer_ids: Set[int] = set( found_trailers_list) cls._unsaved_trailer_changes = 0 else: cls._found_tmdb_trailer_ids: Set[int] = set() Monitor.throw_exception_if_abort_requested() cls.remove_unprocessed_movies(list(cls._found_tmdb_trailer_ids)) except AbortException: reraise(*sys.exc_info()) except IOError as e: CacheIndex.logger().exception('') except JSONDecodeError as e: os.remove(path) except Exception as e: CacheIndex.logger().exception('')
def save_cache(cls, flush: bool = False, complete: bool = False) -> None: """ :param flush: :param complete: :return: """ with cls.lock: if (not flush and (cls._unsaved_trailer_changes < 50) and (datetime.datetime.now() - cls._last_saved_trailer_timestamp) < datetime.timedelta(minutes=5)): return try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tfh_trailers.json') path = xbmcvfs.validatePath(path) tmp_path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tfh_trailers.json.tmp') tmp_path = xbmcvfs.validatePath(tmp_path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) Monitor.throw_exception_if_abort_requested() with io.open(tmp_path, mode='at', newline=None, encoding='utf-8', ) as cacheFile: if complete: creation_date_str = datetime.datetime.strftime( cls._time_of_index_creation, '%Y:%m:%d') else: creation_date_str = TFHCache.INCOMPLETE_CREATION_DATE_STR cls._cached_trailers[TFHCache.INDEX_CREATION_DATE] = { TFHCache.INDEX_CREATION_DATE: creation_date_str, } json_text = json.dumps(cls._cached_trailers, encoding='utf-8', ensure_ascii=False, default=TFHCache.abort_checker, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() # Get rid of dummy entry del cls._cached_trailers['INDEX_CREATION_DATE'] cls._last_saved_trailer_timestamp = datetime.datetime.now() cls._unsaved_trailer_changes = 0 try: os.replace(tmp_path, path) except OSError: cls._logger.exception(f'Failed to replace missing trailer' f' information cache: {path}') except IOError as e: TFHCache.logger().exception('') except Exception as e: TFHCache.logger().exception('') Monitor.throw_exception_if_abort_requested()
def get_stats_for_path( cls, top: str, patterns: Dict[str, Tuple[Pattern[str], str]]) -> Dict[str, UsageData]: """ Gets disk usage information for a subtree of the filesystem :param top: :param patterns: :return: """ usage_data = None fileMap = {} usage_data_map = {} try: free = 0 total = 0 used = 0 size_on_disk = 0 block_size = None # Available in Python >= 3.3 # shutil.disk_usage(top) # _ntuple_diskusage = collections.namedtuple('usage', 'total used free') # units in bytes disk_usage = cls.disk_usage(top) if disk_usage is not None: free = disk_usage['free'] total = disk_usage['total'] used = disk_usage['used'] block_size = disk_usage['blocksize'] #statvfs = os.statvfs(top) #block_size = statvfs.f_bsize #free = int(statvfs.f_bavail * statvfs.f_frsize / megaByte) #total = int(statvfs.f_blocks * statvfs.f_frsize / megaByte) # used = int((statvfs.f_blocks - statvfs.f_bfree) * # statvfs.f_frsize / megaByte) # st.f_blocks is # blocks in filesystem # f_bavail free blocks for non-super user # f_bsize # preferred block size # f_frsize # fundamental file system block size # f_blocks total blocks in filesystem # f_bfree total # free blocks in filesystem for cache_name, (pattern, cache_type) in patterns.items(): usage_data = UsageData(cache_name, pattern) usage_data.set_free_size(free) usage_data.set_total_size(total) usage_data.set_used_space(used) usage_data.set_block_size(block_size) usage_data_map[cache_name] = usage_data db_cache_file_expiration_days = \ Settings.get_expire_remote_db_cache_entry_days() db_cache_file_expiration_seconds =\ db_cache_file_expiration_days * 24 * 60 * 60 db_cache_path_top = Settings.get_remote_db_cache_path() trailer_cache_file_expiration_days = Settings.get_expire_trailer_cache_days( ) trailer_cache_file_expiration_seconds = \ trailer_cache_file_expiration_days * 24 * 60 * 60 trailer_cache_path_top = Settings.get_downloaded_trailer_cache_path( ) now = datetime.datetime.now() found_directories = set() for root, dirs, files in os.walk(top): for filename in files: for cache_name, (pattern, cache_type) in patterns.items(): Monitor.throw_exception_if_abort_requested() usage_data = usage_data_map[cache_name] if pattern.match(filename): path = os.path.join(root, filename) mod_time = now try: if not os.path.isdir(path): st = os.stat(path) mod_time = datetime.datetime.fromtimestamp( st.st_mtime) size_in_blocks = st.st_size size_on_disk = ( (size_in_blocks - 1) / block_size + 1) * block_size else: found_directories.add(path) except OSError as e: continue # File doesn't exist except Exception as e: cls._logger.exception('') continue deleted = False try: if (top == db_cache_path_top and cache_type == 'json'): if ((now - mod_time).total_seconds() > db_cache_file_expiration_seconds): if cls._logger.isEnabledFor( LazyLogger.INFO): cls._logger.info('deleting:', path) os.remove(path) deleted = True usage_data.add_to_disk_deleted( size_on_disk) break # Next file if (top == trailer_cache_path_top and cache_type == 'trailer'): if ((now - mod_time).total_seconds( ) > trailer_cache_file_expiration_seconds): if cls._logger.isEnabledFor( LazyLogger.INFO): cls._logger.info('deleting:', path) os.remove(path) deleted = True usage_data.add_to_disk_deleted( size_on_disk) break # Next file except AbortException: reraise(*sys.exc_info()) except Exception as e: cls._logger.exception('') if not deleted: file_data = FileData(path, mod_time, size_on_disk) usage_data.add_file_data(file_data) usage_data.add_to_disk_used_by_cache( size_on_disk) for directory in found_directories: try: os.rmdir(directory) except Exception as e: pass except AbortException: reraise(*sys.exc_info()) except Exception as e: cls._logger.exception('') cls._logger.exit() return usage_data_map
def save_cache(cls, ignore_shutdown=False) -> None: """ :return: """ cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown) with cls.lock: if cls.tmdb_unsaved_changes == 0 and cls.library_unsaved_changes == 0: return if cls.tmdb_unsaved_changes > 0: entries_to_delete = [] for key, entry in cls._all_missing_tmdb_trailers.items(): elapsed_time = datetime.date.today() - entry['timestamp'] elapsed_days = elapsed_time.days if elapsed_days > Settings.get_expire_remote_db_trailer_check_days(): if entry[Movie.UNIQUE_ID_TMDB] in cls._all_missing_tmdb_trailers: entries_to_delete.append( entry[Movie.UNIQUE_ID_TMDB]) for entry_to_delete in entries_to_delete: del cls._all_missing_tmdb_trailers[entry_to_delete] cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown) try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_tmdb_trailers.json') temp_path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_tmdb_trailers.json.temp') # path = path.encode('utf-8') path = xbmcvfs.validatePath(path) temp_path = xbmcvfs.validatePath(temp_path) parent_dir, file_name = os.path.split(temp_path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) with io.open(temp_path, mode='wt', newline=None, encoding='utf-8', ) as cacheFile: json_text = \ json.dumps(cls._all_missing_tmdb_trailers, encoding='utf-8', ensure_ascii=False, default=TrailerUnavailableCache.handler, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() try: os.replace(temp_path, path) except OSError: cls._logger.exception(f'Failed to replace index of movies' f' missing trailers cache: {path}') cls.tmdb_last_save = datetime.datetime.now() cls.tmdb_unsaved_changes = 0 except AbortException: reraise(*sys.exc_info()) except IOError as e: cls._logger.exception('') except Exception as e: cls._logger.exception('') finally: try: os.remove(temp_path) except Exception: pass cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown) if cls.library_unsaved_changes > 0: entries_to_delete = [] for key, entry in cls._all_missing_library_trailers.items(): elapsed_time = datetime.date.today() - entry['timestamp'] elapsed_days = elapsed_time.days if elapsed_days > Settings.get_expire_remote_db_trailer_check_days(): if entry[Movie.MOVIEID] in cls._all_missing_library_trailers: entries_to_delete.append(entry[Movie.MOVIEID]) for entry_to_delete in entries_to_delete: del cls._all_missing_library_trailers[entry_to_delete] cls.abort_on_shutdown(ignore_shutdown=ignore_shutdown) try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_library_trailers.json') temp_path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'missing_library_trailers.json.temp') # path = path.encode('utf-8') path = xbmcvfs.validatePath(path) temp_path = xbmcvfs.validatePath(temp_path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) with io.open(temp_path, mode='wt', newline=None, encoding='utf-8', ) as cacheFile: # TODO: Need ability to interrupt when ABORT. Object_handler # not a valid arg to dumps json_text = \ json.dumps(cls._all_missing_library_trailers, encoding='utf-8', ensure_ascii=False, default=TrailerUnavailableCache.handler, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() try: os.replace(temp_path, path) except OSError: cls._logger.exception(f'Failed to replace missing trailer' f' information cache: {path}') cls.library_last_save = datetime.datetime.now() cls.library_unsaved_changes = 0 except AbortException: reraise(*sys.exc_info()) except IOError as e: cls._logger.exception('') except Exception as e: cls._logger.exception('') finally: try: os.remove(temp_path) except Exception: pass
def save_unprocessed_movie_cache(cls, flush: bool = False) -> None: """ :param flush: :return: """ # TODO: Should use lock here, review locking with cls.lock: if cls._unprocessed_movie_changes == 0: return if (not flush and # Constants.TRAILER_CACHE_FLUSH_UPDATES) (cls._unprocessed_movie_changes < 10) and ((datetime.datetime.now() - cls._last_saved_unprocessed_movie_timestamp)) < datetime.timedelta(minutes=5)): return try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tmdb_unprocessed_movies.json') path = xbmcvfs.validatePath(path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) # Don't save unneeded fields. Takes up disk and RAM temp_entries = {} for tmdb_id, entry in cls.get_unprocessed_movies().items(): temp_entry = {} for key in Movie.TMDB_PAGE_DATA_FIELDS: temp_entry[key] = entry[key] temp_entries[tmdb_id] = temp_entry json_text = json.dumps(temp_entries, encoding='utf-8', ensure_ascii=False, default=CacheIndex.handler, indent=3, sort_keys=True) with io.open( path, mode='wt', newline=None, encoding='utf-8', ) as cache_file: # TODO: Need ability to interrupt when ABORT. Object_handler # not a valid arg to dumps cache_file.write(json_text) cache_file.flush() cls._last_saved_unprocessed_movie_timestamp = datetime.datetime.now( ) cls._unprocessed_movie_changes = 0 Monitor.throw_exception_if_abort_requested() del json_text del temp_entries except AbortException: reraise(*sys.exc_info()) except IOError as e: cls.logger().exception('') except Exception as e: cls.logger().exception('')
def get_json_cache_file_path_for_movie_id(cls, movie_id, # type: Union[int, str] source, # type: str error_msg='' # type: str ): # type: (...) -> Union[str, None] """ Returns the path for a cache JSON file for the given movie_id and source. :param movie_id: :param source: :param error_msg: Optional text to add to any error message. Typically a movie title. :return: """ try: prefix = Cache.generate_unique_id_from_source(movie_id, source, error_msg=error_msg) # if cls._logger.isEnabledFor(LazyLogger.DEBUG): # cls._logger.debug('movie_id:', movie_id, 'source:', source, # 'prefix:', prefix) # # To reduce clutter, put cached data into a folder named after the # SOURCE and first character of the id # # For local library entries, just use the first digit from the # numeric id. if source == Movie.LIBRARY_SOURCE: folder = prefix[0] elif source == Movie.TMDB_SOURCE: # # For TMDB entries, the numeric TMDB id is prefaced with: # "tmdb_". Use a folder named "t" + first digit of TMDBID # x = prefix.split('_', 1) folder = 't' + x[1][0] elif source == Movie.TFH_SOURCE: # # For TFH entries, the numeric TFH id is prefaced with: # "tfh_". Use a folder named "h" + first digit of TFH # x = prefix.split('_', 1) folder = 'h' + x[1][0] elif source == Movie.ITUNES_SOURCE: # # For ITunes entries, Apple does not supply an ID, so we # use the TMDB ID instead if we can find it. (A lot of these are # for very new or unreleased movies.) # # The TMDB id here is prefaced with: "appl_". Use a folder named # "a" + first digit of TMDBID. x = prefix.split('_', 1) folder = 'a' + x[1][0] else: cls._logger.debug('Unexpected source:', source, 'movie_id:', movie_id) return None cache_file = prefix + '.json' path = os.path.join(Settings.get_remote_db_cache_path(), folder, cache_file) path = xbmcvfs.validatePath(path) return path except AbortException: reraise(*sys.exc_info()) except Exception as e: cls._logger.exception('') return None
def save_cache(cls, flush: bool = False, complete: bool = False) -> None: """ :param flush: :param complete: :return: Typical json entry Items marked with * are kodi/TMDb artifacts "BZwDpOQNgpw": { "adult": false, "cast": [], "fanart": "default_fanart", "genre": [], "mpaa": "NR", "normalized_trailer": "/home/fbacher/.kodi/userdata/addon_data/script.video.randomtrailers/cache/hB/tfh_BZwDpOQNgpw_normalized_Larry Karaszewski on SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3) (2017)-movie.mkv", "original_language": "", "plot": "But wait! There's more! TFH has a podcast! \n\nIt's THE MOVIES THAT MADE ME, where you can join Oscar-nominated screenwriter Josh Olson and his, ummm, \"co-host\" Joe Dante in conversation with filmmakers, comedians, and all-around interesting people about the movies that made them who they are. Check it out now, and please subscribe wherever podcasts can be found.\n\nBut wait! There's more! TFH has a podcast! \n\nIt's THE MOVIES THAT MADE ME, where you can join Oscar-nominated screenwriter Josh Olson and his, ummm, \"co-host\" Joe Dante in conversation with filmmakers, comedians, and all-around interesting people about the movies that made them who they are. Check it out now, and please subscribe wherever podcasts can be found.\n\niTunes: http://itunes.trailersfromhell.com\nSpotify: http://spotify.trailersfromhell.com\nLibsyn: http://podcast.trailersfromhell.com\nGoogle Play: http://googleplay.trailersfromhell.com\nRSS: http://goo.gl/3faeG7\n\nAs always, you can find more commentary, more reviews, more podcasts, and more deep-dives into the films you don't know you love yet over at the Trailers From Hell mothership: \n\nhttp://www.trailersfromhell.com", "rating": 4.8974357, "genre": [], "rts.actors": "", "rts.certification": "Unrated", "rts.certificationImage": "ratings/us/unrated.png", "rts.directors": "", "rts.genres": "", "rts.runtime": "143 [B]Minutes[/B] - ", "rts.studios": "", "rts.tfhId": "BZwDpOQNgpw", "rts.tfh_title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3)", "rts.title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3) (2017) - TFH ", "rts.tmdb_id_not_found": true, "rts.voiced.actors": [], "rts.voiced.directors": [], "rts.voiced.studios": [], "rts.voiced.writers": [], "rts.writers": "", "rts.youtube.trailers_in_index": 1449, "rts.youtube_index": 204, "runtime": 8580, "source": "TFH", "studio": [ [] ], "tags": [ "smokey and the bandit 3", "larry karaszewski", "jackie gleason" ], "thumbnail": "https://i.ytimg.com/vi_webp/BZwDpOQNgpw/maxresdefault.webp", "title": "SMOKEY IS THE BANDIT (SMOKEY AND THE BANDIT PART 3)", "trailer": "https://youtu.be/BZwDpOQNgpw", "trailerDiscoveryState": "04_discoveryReadyToDisplay", "trailerPlayed": true, "trailerType": "default_trailerType", "uniqueid": { "tmdb": "None" }, "writer": [ [] ], "year": 2017 } """ with cls.lock: if (not flush and (cls._unsaved_trailer_changes < 50) and (datetime.datetime.now() - cls._last_saved_trailer_timestamp) < datetime.timedelta(minutes=5)): return try: path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tfh_trailers.json') path = xbmcvfs.validatePath(path) tmp_path = os.path.join(Settings.get_remote_db_cache_path(), 'index', 'tfh_trailers.json.tmp') tmp_path = xbmcvfs.validatePath(tmp_path) parent_dir, file_name = os.path.split(path) if not os.path.exists(parent_dir): DiskUtils.create_path_if_needed(parent_dir) Monitor.throw_exception_if_abort_requested() with io.open(tmp_path, mode='at', newline=None, encoding='utf-8', ) as cacheFile: if complete: cls.set_creation_date() # Set to True when complete, but don't set to False # when not complete. cls._cache_complete = True creation_date_str = datetime.datetime.strftime( cls._time_of_index_creation, '%Y:%m:%d') cls._cached_trailers[cls.INDEX_CREATION_DATE] = { cls.INDEX_CREATION_DATE: creation_date_str, cls.CACHE_COMPLETE: cls._cache_complete } json_text = json.dumps(cls._cached_trailers, encoding='utf-8', ensure_ascii=False, default=TFHCache.abort_checker, indent=3, sort_keys=True) cacheFile.write(json_text) cacheFile.flush() # Get rid of dummy entry from local dict del cls._cached_trailers[cls.INDEX_CREATION_DATE] cls._last_saved_trailer_timestamp = datetime.datetime.now() cls._unsaved_trailer_changes = 0 try: os.replace(tmp_path, path) except OSError: cls._logger.exception(f'Failed to replace missing trailer' f' information cache: {path}') except IOError as e: TFHCache.logger().exception('') except Exception as e: TFHCache.logger().exception('') Monitor.throw_exception_if_abort_requested()