def stdout_reader(self):
if Monitor.is_abort_requested():
self.process.kill()
clz = RunCommand
finished = False
while not (finished or self.cmd_finished):
try:
line = self.process.stdout.readline()
if len(line) > 0:
self.stdout_lines.append(line)
except ValueError as e:
rc = self.process.poll()
if rc is not None:
self.rc = rc
# Command complete
finished = True
break
else:
clz.logger.exception(e)
finished = True
except AbortException as e:
finished = True
except Exception as e:
clz.logger.exception(e)
finished = True
def status_hook(self, status: Dict[str, str]) -> None:
clz = BaseInfoHook
if Monitor.is_abort_requested():
self._callback._error = 99
# Kill YoutubeDL
Monitor.throw_exception_if_abort_requested()
status_str = status.get('status', 'missing status')
if status_str is None:
clz._logger.debug('Missing status indication')
elif status_str == 'downloading':
self._download_eta = status.get('eta', 0) # In seconds
elif status_str == 'error':
clz._logger.error('Status:', str(status))
self.error_lines.append('Error downloading')
self._error = VideoDownloader.DOWNLOAD_ERROR
elif status_str == 'finished':
filename = status.get('filename')
tmpfilename = status.get('tmpfilename')
downloaded_bytes = status.get('downloaded_bytes')
total_bytes = status.get('total_bytes')
total_bytes_estimate = status.get('total_bytes_estimate')
elapsed = status.get('elapsed')
eta = status.get('eta')
speed = status.get('speed')
fragment_index = status.get('fragment_index')
fragment_count = status.get('fragment_count')
clz._logger.debug('Finished')
def warning(self, line: str) -> None:
if Monitor.is_abort_requested():
self._callback._error = 99
# Kill YoutubeDL
Monitor.throw_exception_if_abort_requested()
if 'merged' in line:
# str: Requested formats are incompatible for merge and will be merged into
# mkv.
pass
else:
self.warning_lines.append(line)
def log_output(self):
if Monitor.is_abort_requested():
return
clz = RunCommand
if clz.logger.isEnabledFor(LazyLogger.DEBUG):
clz.logger.debug(
f'ffmpeg failed for {self.movie_name} rc: {self.rc}')
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
stdout = '\n'.join(self.stdout_lines)
clz.logger.debug_verbose(f'STDOUT: {stdout}')
stderr = '\n'.join(self.stderr_lines)
clz.logger.debug_verbose(f'STDERR: {stderr}')
def join_dead_threads(cls):
finished = False
while not finished:
with cls._lock:
joined_threads: List[threading.Thread] = []
for thread in cls._threads_to_join:
if not thread.is_alive():
thread.join(timeout=0.0)
if not thread.is_alive():
joined_threads.append(thread)
for thread in joined_threads:
cls._threads_to_join.remove(thread)
if Monitor.is_abort_requested():
finished = True
def error(self, line: str) -> None:
if Monitor.is_abort_requested():
self.set_error(99, force=True)
# Kill YoutubeDL
Monitor.throw_exception_if_abort_requested()
clz = BaseYDLogger
if 'Error Constants.HTTP_TOO_MANY_REQUESTS' in line:
self.set_error(Constants.HTTP_TOO_MANY_REQUESTS, force=True)
type(self).logger.info('Abandoning download. Too Many Requests')
# str: ERROR: (ExtractorError(...), 'wySw1lhMt1s: YouTube said: Unable
# to extract video data')
elif 'Unable to extract' in line:
self.set_error(VideoDownloader.DOWNLOAD_ERROR)
elif 'blocked' in line:
self.set_error(VideoDownloader.BLOCKED_ERROR)
elif 'unavailable' in line:
self.set_error(VideoDownloader.UNAVAILABLE)
else:
self.error_lines.append(line)
def error(self, line: str) -> None:
if Monitor.is_abort_requested():
self._callback._error = 99
# Kill YoutubeDL
Monitor.throw_exception_if_abort_requested()
clz = BaseYDLogger
if 'Error 429' in line:
self._callback._error = 429
clz._initial_tmr_timestamp = datetime.datetime.now()
clz._too_many_requests_timestamp = (datetime.datetime.now() +
RETRY_DELAY)
clz.logger.info('Abandoning download. Too Many Requests')
# str: ERROR: (ExtractorError(...), 'wySw1lhMt1s: YouTube said: Unable
# to extract video data')
elif 'Unable to extract' in line:
self._callback._error = VideoDownloader.DOWNLOAD_ERROR
elif 'blocked' in line:
self._callback._error = VideoDownloader.BLOCKED_ERROR
else:
self.error_lines.append(line)
def debug(self, line: str) -> None:
"""
{"_type": "url", "url": "vYALYAuD5Fw", "ie_key": "Youtube",
"id": "vYALYAuD5Fw", "title": "Larry Karaszewski on ROAD TO SALINA"}
[download] Downloading video 14 of 1448
{"_type": "url", "url":
"dZzFqtlamV4", "ie_key": "Youtube",
"id": "dZzFqtlamV4", "title": "Joe
Dante on HALF HUMAN"}
[download] Downloading video 15 of 1448
{"id": "nrExo_KJROc", "uploader": "Trailers From Hell",
"uploader_id": "trailersfromhell",
"uploader_url": "http://www.youtube.com/user/trailersfromhell",
"channel_id": "UCg7Mllu8AnTjlZ4Vu1FNdjQ",
"channel_url": "http://www.youtube.com/channel/UCg7Mllu8AnTjlZ4Vu1FNdjQ"
"upload_date": "20200908", "license": null, "creator": null,
"title": "Brian Trenchard-Smith on ONCE UPON A TIME IN HOLLYWOOD",
"alt_title": null,
"thumbnails": [{"url":
"https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp
=-oaymwEYCKgBEF5IVfKriqkDCwgBFQAAiEIYAXAB&rs
=AOn4CLCPHEof66nqx4GxE04sOUocr9WywA",
"width": 168, "height": 94, "resolution": "168x94", "id": "0"},
{"url": "https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp
=-oaymwEYCMQBEG5IVfKriqkDCwgBFQAAiEIYAXAB&rs=AOn4CLAm
-CXcCCu0LATG_R347wBxBQj4BQ",
"width": 196, "height": 110, "resolution": "196x110", "id": "1"},
{"url": "https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp
=-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs
=AOn4CLAhW2AcVqdWYiPMZuKENgiCO0gykQ",...
:return:
"""
clz = BaseYDLogger
if Monitor.is_abort_requested():
self._callback._error = 99
# Kill YoutubeDL
Monitor.throw_exception_if_abort_requested()
self.debug_lines.append(line)
if line.startswith('[download] Downloading video'):
try:
_, index_str, total_str = re.split(r'[^0-9]+', line)
self.index = int(index_str)
self.total = int(total_str)
except Exception as e:
clz.logger.exception()
self._callback._error = 1
# if line.startswith('{"_type":'):
# try:
# data = json.loads(line)
# self._trailer_handler(data)
# except Exception as e:
# clz.logger.exception()
if line.startswith('{"id":'):
try:
self.raw_data = json.loads(line)
if self._parse_json_as_youtube:
self._parsed_movie = populate_youtube_movie_info(
self.raw_data, self.url)
# self._trailer_handler(movie_data)
except Exception as e:
clz.logger.exception()
self._callback._error = 2
def run_worker(self):
# type: () -> None
"""
Initial Discovery of all movies in Kodi.
:return:
"""
# Discovery is done in two parts:
#
# 1- query DB for every movie in library
# 2- Get additional information
#
# There are three types of trailers for these movies:
#
# a- Movies with local trailers
# b- Movies with trailer URLS (typically youtube links from tmdb)
# TMdb will need to be queried for details
# c. Movies with no trailer information, requiring a check with tmdb
# to see if one exists
#
# Because of the above, this manager will query the DB for every movie
# and then only process the ones with local trailers. The others will
# be handed off to their own managers. This is done because of
# the way that this application works:
# Once enough information to identify a movie that matches
# what the user wants, it is added to the pool of movies that
# can be randomly selected for playing. Once a movie has been
# selected, it is placed into a TrailerFetcherQueue. A
# TrailerFetcher then gathers the remaining information so that
# it can be played.
#
# If the lion's share of movies in the pool require significant
# extra processing because they don't have local trailers, then
# the fetcher can get overwhelmed.
clz = DiscoverLibraryMovies
self._selected_keywords = []
self._excluded_keywords = []
self._selected_genres = []
self._excluded_genres = []
if Settings.get_filter_genres():
self._selected_genres = GenreUtils.get_internal_kodi_genre_ids(
GenreUtils.LOCAL_DATABASE, exclude=False)
self._excluded_genres = GenreUtils.get_internal_kodi_genre_ids(
GenreUtils.LOCAL_DATABASE, exclude=True)
self._selected_keywords = GenreUtils.get_internal_kodi_keyword_ids(
GenreUtils.LOCAL_DATABASE, exclude=False)
self._excluded_keywords = GenreUtils.get_internal_kodi_keyword_ids(
GenreUtils.LOCAL_DATABASE, exclude=True)
query = self.create_query(
self._selected_genres, self._excluded_genres, self._selected_keywords,
self._excluded_keywords)
if Monitor.is_abort_requested():
return
start_time = datetime.datetime.now()
Monitor.throw_exception_if_abort_requested() # Expensive operation
query_result = JsonUtilsBasic.get_kodi_json(query, dump_results=False)
Monitor.throw_exception_if_abort_requested()
elapsed_time = datetime.datetime.now() - start_time
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
clz.logger.debug_verbose('Library query seconds:',
elapsed_time.total_seconds())
movies_skipped = 0
movies_found = 0
movies_with_local_trailers = 0
movies_with_trailer_urls = 0
movies_without_trailer_info = 0
self.throw_exception_on_forced_to_stop()
result = query_result.get('result', {})
del query_result
movies = result.get('movies', [])
del result
DiskUtils.RandomGenerator.shuffle(movies)
if self._libraryURLManager is None:
self._libraryURLManager = DiscoverLibraryURLTrailerMovies()
self._libraryNoTrailerInfoManager = DiscoverLibraryNoTrailerMovies()
library_movies = []
library_url_movies = []
library_no_trailer_movies = []
empty_limit = 50
movie_data = None
if Settings.is_enable_movie_stats():
movie_data = LibraryMovieStats()
country_id = Settings.get_country_iso_3166_1().lower()
certifications = WorldCertifications.get_certifications(country_id)
unrated_id = certifications.get_unrated_certification().get_preferred_id()
for movie in movies:
self.throw_exception_on_forced_to_stop()
try:
#clz.logger.debug('movie:', movie)
movies_found += 1
if Settings.get_hide_watched_movies() and Movie.LAST_PLAYED in movie:
if (self.get_days_since_last_played(movie[Movie.LAST_PLAYED],
movie[Movie.TITLE]) >
Settings.get_minimum_days_since_watched()):
movies_skipped += 1
if clz.logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE):
clz.logger.debug_extra_verbose(movie[Movie.TITLE],
'will not be played due to '
'Hide',
'Watched Movies')
continue
# Normalize certification
# if clz.logger.isEnabledFor(LazyLogger.DEBUG):
# clz.logger.debug('mpaa:', movie[Movie.MPAA],
# 'movie:', movie[Movie.TITLE])
if certifications.is_valid(movie.get(Movie.MPAA, '')):
movie[Movie.MPAA] = unrated_id
certification = certifications.get_certification(
movie.get(Movie.MPAA), movie.get(Movie.ADULT))
movie[Movie.ADULT] = movie.get(Movie.ADULT, False)
if not isinstance(movie[Movie.ADULT], bool):
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
clz.logger.debug_verbose(movie[Movie.TITLE],
'has invalid ADULT field: ',
movie[Movie.ADULT])
movie[Movie.ADULT] = str(movie[Movie.ADULT]).lower == 'true'
movie[Movie.SOURCE] = Movie.LIBRARY_SOURCE
movie.setdefault(Movie.TRAILER, '')
movie[Movie.TYPE] = ''
if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE):
Debug.validate_basic_movie_properties(movie)
if Settings.is_enable_movie_stats():
movie_data.collect_data(movie)
# Basic discovery is complete at this point. Now send
# all of the movies without any trailer information to
# DiscoverLibraryNoTrailerMovies while
# those with trailer URLs to DiscoverLibraryURLTrailerMovies
if certifications.filter(certification):
trailer = movie[Movie.TRAILER]
if trailer == '':
movies_without_trailer_info += 1
library_no_trailer_movies.append(movie)
elif trailer.startswith('plugin://') or trailer.startswith('http'):
movies_with_trailer_urls += 1
library_url_movies.append(movie)
elif Settings.get_include_library_trailers():
movies_with_local_trailers += 1
library_movies.append(movie)
if len(library_movies) >= empty_limit:
self.add_to_discovered_trailers(library_movies)
del library_movies[:]
# Unblock other discovery now that a few movies have been
# found.
if not self._some_movies_discovered_event.isSet():
self._some_movies_discovered_event.set()
if len(library_no_trailer_movies) >= empty_limit:
self._libraryNoTrailerInfoManager.add_to_discovered_trailers(
library_no_trailer_movies)
del library_no_trailer_movies[:]
if len(library_url_movies) >= empty_limit:
self._libraryURLManager.add_to_discovered_trailers(
library_url_movies)
del library_no_trailer_movies[:]
# Unblock other discovery now that a few movies have been
# found.
self._some_movies_discovered_event.set()
except AbortException:
reraise(*sys.exc_info())
except Exception:
clz.logger.exception('')
try:
if len(library_movies) >= 0:
self.add_to_discovered_trailers(library_movies)
if len(library_no_trailer_movies) >= 0:
self._libraryNoTrailerInfoManager.add_to_discovered_trailers(
library_no_trailer_movies)
if len(library_url_movies) >= 0:
self._libraryURLManager.add_to_discovered_trailers(
library_url_movies)
except AbortException:
reraise(*sys.exc_info())
except Exception:
clz.logger.exception('')
if (clz.logger.isEnabledFor(LazyLogger.DEBUG)
and clz.logger.is_trace_enabled(Trace.STATS)):
clz.logger.debug('Local movies found in library:',
movies_found, trace=Trace.STATS)
clz.logger.debug('Local movies filtered out',
movies_skipped, trace=Trace.STATS)
clz.logger.debug('Movies with local trailers:',
movies_with_local_trailers, trace=Trace.STATS)
clz.logger.debug('Movies with trailer URLs:',
movies_with_trailer_urls, trace=Trace.STATS)
clz.logger.debug('Movies with no trailer information:',
movies_without_trailer_info, trace=Trace.STATS)
if Settings.is_enable_movie_stats():
movie_data.report_data()
del movie_data