def get_cached_json(cls, url, # type; str
movie_id=None, # type: Union[str, int, None]
error_msg=None, # type: Union[str, int, None]
source=None, # type: Union[str, None]
dump_results=False, # type: bool
dump_msg='', # type: str
headers=None, # type: Union[dict, None]
params=None, # type: Union[dict, None]
timeout=3.0 # type: int
):
# type: (...) -> (int, str)
"""
Attempt to get cached JSON movie information before using the JSON calls
to get it remotely.
Any information not in the cache will be placed into it after successfully
reading it.
:param url:
:param movie_id:
:param error_msg:
:param source:
:param dump_results:
:param dump_msg:
:param headers:
:param params:
:param timeout:
:return:
"""
if headers is None:
headers = {}
if params is None:
params = {}
trailer_data = None
status = 0
if source is None or source not in Movie.LIB_TMDB_ITUNES_SOURCES:
cls._logger.error('Invalid source:', source)
if Settings.is_use_tmdb_cache():
trailer_data = Cache.read_tmdb_cache_json(
movie_id, source, error_msg=error_msg)
status = 0
if trailer_data is not None:
trailer_data[Movie.CACHED] = True
if trailer_data is None:
status, trailer_data = JsonUtilsBasic.get_json(url, dump_results=dump_results,
dump_msg=dump_msg,
headers=headers,
error_msg=error_msg, params=params,
timeout=timeout)
if (
status == 0 or status == 200) and trailer_data is not None and \
Settings.is_use_tmdb_cache():
Cache.write_tmdb_cache_json(movie_id, source, trailer_data)
return status, trailer_data
def update_database_unique_id(cls, trailer):
# type: ( MovieType) -> None
"""
Update UNIQUE_ID field in database
:param trailer:
:return:
"""
try:
update = True
movie_id = trailer.get(Movie.MOVIEID)
unique_id = trailer.get(Movie.UNIQUE_ID, None)
if unique_id is None:
cls._logger.error('Movie.UNIQUE_ID is None')
return
# "uniqueid":{"imdb": "tt0033517", "unknown": "tt0033517"}
# "uniqueid": {"tmdb": 196862, "imdb": "tt0042784"}
Monitor.throw_exception_if_abort_requested()
json_text = json.dumps(unique_id)
update = '{"jsonrpc": "2.0", "method": "VideoLibrary.SetMovieDetails", \
"params": {\
"movieid": %s, "uniqueid": %s }, "id": 1}' % (movie_id, json_text)
query_result = JsonUtilsBasic.get_kodi_json(
update, dump_results=True)
Monitor.throw_exception_if_abort_requested()
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug('Update TMDBID for:', trailer[Movie.TITLE],
'update json:', update)
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
cls._logger.exception('')
def get_tmdb_id(cls, movie):
# type: (MovieType) -> int
"""
:param movie:
:return:
"""
tmdb_id: Optional[str] = None
imdb_id = None
title = movie[Movie.TITLE]
unique_id = movie.get(Movie.UNIQUE_ID, None)
if unique_id is not None:
# if cls._logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
# for key in unique_id:
# cls._logger.debug_extra_verbose(title, key, unique_id.get(key, ''))
tmdb_id = unique_id.get(Movie.UNIQUE_ID_TMDB, None)
if tmdb_id is not None:
# Make sure we don't have a IMDB id in here by error
if tmdb_id.startswith('tt'):
tmdb_id = None
if tmdb_id is not None:
tmdb_id: int = int(tmdb_id)
else:
imdb_id = unique_id.get(Movie.UNIQUE_ID_IMDB, None)
if imdb_id is None:
imdb_id = unique_id.get(Movie.UNIQUE_ID_UNKNOWN, None)
if imdb_id is not None and not imdb_id.startswith('tt'):
imdb_id = None
if imdb_id is not None:
data = {}
data['external_source'] = 'imdb_id'
# TODO: iso-639-1 gives two char lang. Prefer en-US
data['language'] = Settings.get_lang_iso_639_1()
data['api_key'] = Settings.get_tmdb_api_key()
url = 'http://api.themoviedb.org/3/find/' + str(imdb_id)
try:
Monitor.throw_exception_if_abort_requested()
status_code, tmdb_result = JsonUtilsBasic.get_json(
url, error_msg=title,
params=data, dump_results=True, dump_msg='')
Monitor.throw_exception_if_abort_requested()
if status_code == 0 and tmdb_result is not None:
s_code = tmdb_result.get('status_code', None)
if s_code is not None:
status_code = s_code
if status_code != 0:
pass
elif tmdb_result is not None:
movie_results = tmdb_result.get(
'movie_results', [])
if len(movie_results) == 0:
pass
elif len(movie_results) > 1:
pass
else:
tmdb_id = movie_results[0].get('id', None)
if tmdb_id is None:
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug('Did not find movie for',
'imdb_id:', imdb_id,
'title:', title)
else:
cls.set_tmdb_id(movie, tmdb_id)
cls.update_database_unique_id(movie)
except AbortException:
reraise(*sys.exc_info())
except Exception:
cls._logger.exception('')
if tmdb_id is not None:
tmdb_id = int(tmdb_id)
return tmdb_id
def get_alternate_titles(cls,
movie_title, # type: str
movie_id, # type: Union[int, str]
):
# type: (...) -> MovieType
"""
:param cls:
:param movie_title:
:param movie_id:
:return:
"""
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug('title:', movie_title, 'movie_id:', movie_id)
data = {}
# data['append_to_response'] = 'credits,releases'
data['language'] = Settings.get_lang_iso_639_1()
data['api_key'] = Settings.get_tmdb_api_key()
data['append_to_response'] = 'alternative_titles'
url = 'http://api.themoviedb.org/3/movie/' + str(movie_id)
tmdb_result = None
year = 0
dump_msg = 'movie_id: ' + str(movie_id)
try:
Monitor.throw_exception_if_abort_requested()
status_code, tmdb_result = JsonUtilsBasic.get_json(
url, error_msg=movie_title,
params=data, dump_results=False, dump_msg=dump_msg)
Monitor.throw_exception_if_abort_requested()
if status_code == 0:
s_code = tmdb_result.get('status_code', None)
if s_code is not None:
status_code = s_code
if status_code != 0 and cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug(
'Error getting TMDB data for:', movie_title,
'status:', status_code)
return None
except AbortException:
reraise(*sys.exc_info())
except Exception:
cls._logger.exception('Error processing movie: ', movie_title)
return None
parsed_data = {}
try:
# release_date TMDB key is different from Kodi's
try:
year = tmdb_result['release_date'][:-6]
year = int(year)
except AbortException:
reraise(*sys.exc_info())
except Exception:
year = 0
parsed_data[Movie.YEAR] = year
title = tmdb_result[Movie.TITLE]
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
cls._logger.debug('Processing:', title, 'type:',
type(title).__name__)
parsed_data[Movie.TITLE] = title
studios = tmdb_result['production_companies']
studio = []
for s in studios:
studio.append(s['name'])
parsed_data[Movie.STUDIO] = studio
tmdb_cast_members = tmdb_result['credits']['cast']
cast = []
for cast_member in tmdb_cast_members:
fake_cast_entry = {}
fake_cast_entry['name'] = cast_member['name']
fake_cast_entry['character'] = cast_member['character']
cast.append(fake_cast_entry)
parsed_data[Movie.CAST] = cast
tmdb_crew_members = tmdb_result['credits']['crew']
director = []
writer = []
for crew_member in tmdb_crew_members:
if crew_member['job'] == 'Director':
director.append(crew_member['name'])
if crew_member['department'] == 'Writing':
writer.append(crew_member['name'])
parsed_data[Movie.DIRECTOR] = director
parsed_data[Movie.WRITER] = writer
titles = tmdb_result.get('alternative_titles', {'titles': []})
alt_titles = []
for title in titles['titles']:
alt_title = (title['title'], title['iso_3166_1'])
alt_titles.append(alt_title)
parsed_data['alt_titles'] = alt_titles
original_title = tmdb_result['original_title']
if original_title is not None:
parsed_data[Movie.ORIGINAL_TITLE] = original_title
except AbortException as e:
reraise(*sys.exc_info())
except Exception as e:
cls._logger.exception('%s %s'.format(
'Error getting info for movie_id:', movie_id))
try:
if cls._logger.isEnabledFor(LazyLogger.DEBUG):
json_text = json.dumps(
tmdb_result, indent=3, sort_keys=True)
cls._logger.debug(json_text)
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
cls._logger('failed to get Json data')
parsed_data = None
cls._logger.exit('Finished processing movie: ', movie_title, 'year:',
year)
return parsed_data
def run_worker(self):
# type: () -> None
"""
Initial Discovery of all movies in Kodi.
:return:
"""
# Discovery is done in two parts:
#
# 1- query DB for every movie in library
# 2- Get additional information
#
# There are three types of trailers for these movies:
#
# a- Movies with local trailers
# b- Movies with trailer URLS (typically youtube links from tmdb)
# TMdb will need to be queried for details
# c. Movies with no trailer information, requiring a check with tmdb
# to see if one exists
#
# Because of the above, this manager will query the DB for every movie
# and then only process the ones with local trailers. The others will
# be handed off to their own managers. This is done because of
# the way that this application works:
# Once enough information to identify a movie that matches
# what the user wants, it is added to the pool of movies that
# can be randomly selected for playing. Once a movie has been
# selected, it is placed into a TrailerFetcherQueue. A
# TrailerFetcher then gathers the remaining information so that
# it can be played.
#
# If the lion's share of movies in the pool require significant
# extra processing because they don't have local trailers, then
# the fetcher can get overwhelmed.
clz = DiscoverLibraryMovies
self._selected_keywords = []
self._excluded_keywords = []
self._selected_genres = []
self._excluded_genres = []
if Settings.get_filter_genres():
self._selected_genres = GenreUtils.get_internal_kodi_genre_ids(
GenreUtils.LOCAL_DATABASE, exclude=False)
self._excluded_genres = GenreUtils.get_internal_kodi_genre_ids(
GenreUtils.LOCAL_DATABASE, exclude=True)
self._selected_keywords = GenreUtils.get_internal_kodi_keyword_ids(
GenreUtils.LOCAL_DATABASE, exclude=False)
self._excluded_keywords = GenreUtils.get_internal_kodi_keyword_ids(
GenreUtils.LOCAL_DATABASE, exclude=True)
query = self.create_query(
self._selected_genres, self._excluded_genres, self._selected_keywords,
self._excluded_keywords)
if Monitor.is_abort_requested():
return
start_time = datetime.datetime.now()
Monitor.throw_exception_if_abort_requested() # Expensive operation
query_result = JsonUtilsBasic.get_kodi_json(query, dump_results=False)
Monitor.throw_exception_if_abort_requested()
elapsed_time = datetime.datetime.now() - start_time
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
clz.logger.debug_verbose('Library query seconds:',
elapsed_time.total_seconds())
movies_skipped = 0
movies_found = 0
movies_with_local_trailers = 0
movies_with_trailer_urls = 0
movies_without_trailer_info = 0
self.throw_exception_on_forced_to_stop()
result = query_result.get('result', {})
del query_result
movies = result.get('movies', [])
del result
DiskUtils.RandomGenerator.shuffle(movies)
if self._libraryURLManager is None:
self._libraryURLManager = DiscoverLibraryURLTrailerMovies()
self._libraryNoTrailerInfoManager = DiscoverLibraryNoTrailerMovies()
library_movies = []
library_url_movies = []
library_no_trailer_movies = []
empty_limit = 50
movie_data = None
if Settings.is_enable_movie_stats():
movie_data = LibraryMovieStats()
country_id = Settings.get_country_iso_3166_1().lower()
certifications = WorldCertifications.get_certifications(country_id)
unrated_id = certifications.get_unrated_certification().get_preferred_id()
for movie in movies:
self.throw_exception_on_forced_to_stop()
try:
#clz.logger.debug('movie:', movie)
movies_found += 1
if Settings.get_hide_watched_movies() and Movie.LAST_PLAYED in movie:
if (self.get_days_since_last_played(movie[Movie.LAST_PLAYED],
movie[Movie.TITLE]) >
Settings.get_minimum_days_since_watched()):
movies_skipped += 1
if clz.logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
clz.logger.debug_extra_verbose(movie[Movie.TITLE],
'will not be played due to '
'Hide',
'Watched Movies')
continue
# Normalize certification
# if clz.logger.isEnabledFor(LazyLogger.DEBUG):
# clz.logger.debug('mpaa:', movie[Movie.MPAA],
# 'movie:', movie[Movie.TITLE])
if certifications.is_valid(movie.get(Movie.MPAA, '')):
movie[Movie.MPAA] = unrated_id
certification = certifications.get_certification(
movie.get(Movie.MPAA), movie.get(Movie.ADULT))
movie[Movie.ADULT] = movie.get(Movie.ADULT, False)
if not isinstance(movie[Movie.ADULT], bool):
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
clz.logger.debug_verbose(movie[Movie.TITLE],
'has invalid ADULT field: ',
movie[Movie.ADULT])
movie[Movie.ADULT] = str(movie[Movie.ADULT]).lower == 'true'
movie[Movie.SOURCE] = Movie.LIBRARY_SOURCE
movie.setdefault(Movie.TRAILER, '')
movie[Movie.TYPE] = ''
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
Debug.validate_basic_movie_properties(movie)
if Settings.is_enable_movie_stats():
movie_data.collect_data(movie)
# Basic discovery is complete at this point. Now send
# all of the movies without any trailer information to
# DiscoverLibraryNoTrailerMovies while
# those with trailer URLs to DiscoverLibraryURLTrailerMovies
if certifications.filter(certification):
trailer = movie[Movie.TRAILER]
if trailer == '':
movies_without_trailer_info += 1
library_no_trailer_movies.append(movie)
elif trailer.startswith('plugin://') or trailer.startswith('http'):
movies_with_trailer_urls += 1
library_url_movies.append(movie)
elif Settings.get_include_library_trailers():
movies_with_local_trailers += 1
library_movies.append(movie)
if len(library_movies) >= empty_limit:
self.add_to_discovered_trailers(library_movies)
del library_movies[:]
# Unblock other discovery now that a few movies have been
# found.
if not self._some_movies_discovered_event.isSet():
self._some_movies_discovered_event.set()
if len(library_no_trailer_movies) >= empty_limit:
self._libraryNoTrailerInfoManager.add_to_discovered_trailers(
library_no_trailer_movies)
del library_no_trailer_movies[:]
if len(library_url_movies) >= empty_limit:
self._libraryURLManager.add_to_discovered_trailers(
library_url_movies)
del library_no_trailer_movies[:]
# Unblock other discovery now that a few movies have been
# found.
self._some_movies_discovered_event.set()
except AbortException:
reraise(*sys.exc_info())
except Exception:
clz.logger.exception('')
try:
if len(library_movies) >= 0:
self.add_to_discovered_trailers(library_movies)
if len(library_no_trailer_movies) >= 0:
self._libraryNoTrailerInfoManager.add_to_discovered_trailers(
library_no_trailer_movies)
if len(library_url_movies) >= 0:
self._libraryURLManager.add_to_discovered_trailers(
library_url_movies)
except AbortException:
reraise(*sys.exc_info())
except Exception:
clz.logger.exception('')
if (clz.logger.isEnabledFor(LazyLogger.DEBUG)
and clz.logger.is_trace_enabled(Trace.STATS)):
clz.logger.debug('Local movies found in library:',
movies_found, trace=Trace.STATS)
clz.logger.debug('Local movies filtered out',
movies_skipped, trace=Trace.STATS)
clz.logger.debug('Movies with local trailers:',
movies_with_local_trailers, trace=Trace.STATS)
clz.logger.debug('Movies with trailer URLs:',
movies_with_trailer_urls, trace=Trace.STATS)
clz.logger.debug('Movies with no trailer information:',
movies_without_trailer_info, trace=Trace.STATS)
if Settings.is_enable_movie_stats():
movie_data.report_data()
del movie_data
def __init__(self, title: str, year: Union[str, None],
runtime_seconds: int) -> None:
clz = type(self)
if clz._logger is None:
clz._logger = module_logger.getChild(clz.__name__)
self._title_to_match: str = title
self._year_to_match: Union[str, None] = year
self._runtime_seconds_to_match: int = runtime_seconds
self.candidate_movies: List[clz.CandidateMovie] = []
data = {
'api_key': Settings.get_tmdb_api_key(),
'page': '1',
'query': title,
'language': Settings.get_lang_iso_639_1()
}
if year is not None:
data['primary_release_year'] = year
try:
include_adult = 'false'
country_id = Settings.get_country_iso_3166_1().lower()
certifications = WorldCertifications.get_certifications(country_id)
adult_certification = certifications.get_certification(
'dummy', True)
if certifications.filter(adult_certification):
include_adult = 'true'
data['include_adult'] = include_adult
data['append_to_response'] = 'alternative_titles'
url = 'https://api.themoviedb.org/3/search/movie'
status_code, _info_string = \
JsonUtilsBasic.get_json(url, params=data,
dump_msg='get_tmdb_id_from_title_year',
dump_results=True,
error_msg=title +
f' ({year})')
if clz._logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
clz._logger.debug_extra_verbose(
f'Getting TMDB movie for title: {title} year: {year} '
f'runtime: {runtime_seconds}')
if _info_string is not None:
results = _info_string.get('results', [])
if len(results) > 1:
if clz._logger.isEnabledFor(
LazyLogger.DEBUG_EXTRA_VERBOSE):
clz._logger.debug_extra_verbose(
f'Got multiple matching movies: {title} '
f'year: {year} runtime: {runtime_seconds}')
# TODO: Improve. Create best trailer function from get_tmdb_trailer
# TODO: find best trailer_id
current_language = Settings.get_lang_iso_639_1()
for movie in results:
release_date = movie.get('release_date', '') # 1932-04-22
tmdb_year = release_date[:-6]
movie[Movie.YEAR] = tmdb_year
tmdb_title = movie.get('title', '')
tmdb_id = movie.get('id', None)
tmdb_language = movie.get('original_language')
runtime_minutes = movie.get(Movie.RUNTIME, 0)
runtime_seconds = int(runtime_minutes * 60)
titles = movie.get('alternative_titles', {'titles': []})
alt_titles = []
for title in titles['titles']:
alt_title = (title['title'], title['iso_3166_1'])
alt_titles.append(alt_title)
if clz._logger.isEnabledFor(
LazyLogger.DEBUG_EXTRA_VERBOSE):
clz._logger.debug_extra_verbose(
f'Matching Movie date: {tmdb_year}'
f' tmdb_title: {tmdb_title}'
f' lang: {tmdb_language}'
f' current_lang: {current_language}')
self._add(movie, tmdb_title, tmdb_year, tmdb_language,
tmdb_id, runtime_seconds)
except AbortException:
reraise(*sys.exc_info())
except Exception as e:
clz._logger.exception(e)
def _get_tmdb_id_from_title_year(title: str, year: int) -> Optional[int]:
"""
When we don't have a trailer for a movie, we can
see if TMDB has one.
:param title:
:param year:
:return:
"""
year_str = str(year)
found_movie = None
trailer_id = None
data = {}
data['api_key'] = Settings.get_tmdb_api_key()
data['page'] = '1'
data['query'] = title
data['language'] = Settings.get_lang_iso_639_1()
data['primary_release_year'] = year
try:
include_adult = 'false'
country_id = Settings.get_country_iso_3166_1().lower()
certifications = WorldCertifications.get_certifications(country_id)
adult_certification = certifications.get_certification(
'dummy', True)
if certifications.filter(adult_certification):
include_adult = 'true'
data['include_adult'] = include_adult
url = 'https://api.themoviedb.org/3/search/movie'
status_code, _info_string = \
JsonUtilsBasic.get_json(url, params=data,
dump_msg='get_tmdb_id_from_title_year',
dump_results=True,
error_msg=title +
' (' + year_str + ')')
if _info_string is not None:
results = _info_string.get('results', [])
if len(results) > 1:
if TMDBUtils._logger.isEnabledFor(
LazyLogger.DEBUG_EXTRA_VERBOSE):
TMDBUtils._logger.debug_extra_verbose(
'Got multiple matching movies:', title, 'year:',
year)
# TODO: Improve. Create best trailer function from get_tmdb_trailer
# TODO: find best trailer_id
matches = []
current_language = Settings.get_lang_iso_639_1()
movie = None
for movie in results:
release_date = movie.get('release_date', '') # 1932-04-22
found_year = release_date[:-6]
found_title = movie.get('title', '')
if (found_title.lower() == title.lower()
and found_year == year_str
and movie.get('original_language')
== current_language):
matches.append(movie)
# TODO: Consider close match heuristics.
if len(matches) == 1:
found_movie = matches[0]
elif len(matches) > 1:
if TMDBUtils._logger.isEnabledFor(
LazyLogger.DEBUG_VERBOSE):
TMDBUtils._logger.debug_extra_verbose(
'More than one matching movie in same year choosing first '
'one matching current language.', 'Num choices:',
len(matches))
found_movie = matches[0]
if movie is None:
if TMDBUtils._logger.isEnabledFor(
LazyLogger.DEBUG_EXTRA_VERBOSE):
TMDBUtils._logger.debug_extra_verbose(
'Could not find movie:', title, 'year:', year,
'at TMDB. found', len(results), 'candidates')
for a_movie in results:
release_date = a_movie.get('release_date',
'') # 1932-04-22
found_year = release_date[:-6]
found_title = a_movie.get('title', '')
tmdb_id = a_movie.get('id', None)
if TMDBUtils._logger.isEnabledFor(LazyLogger.DISABLED):
TMDBUtils._logger.debug_extra_verbose(
'found:', found_title, '(', found_year, ')',
'tmdb id:', tmdb_id)
tmdb_data = MovieEntryUtils.get_alternate_titles(
title, tmdb_id)
for alt_title, country in tmdb_data['alt_titles']:
if alt_title.lower() == title.lower():
found_movie = tmdb_data # Not actually in "movie" format
break
else:
if TMDBUtils._logger.isEnabledFor(LazyLogger.INFO):
TMDBUtils._logger.info('Could not find movie:', title,
'year:', year,
'at TMDB. found no candidates')
except AbortException:
reraise(*sys.exc_info())
except Exception:
TMDBUtils._logger.exception('')
tmdb_id = None
if found_movie is not None:
tmdb_id = found_movie.get('id', None)
if tmdb_id is None:
return None
return int(tmdb_id)