def stdout_reader(self): if Monitor.is_abort_requested(): self.process.kill() clz = RunCommand finished = False while not (finished or self.cmd_finished): try: line = self.process.stdout.readline() if len(line) > 0: self.stdout_lines.append(line) except ValueError as e: rc = self.process.poll() if rc is not None: self.rc = rc # Command complete finished = True break else: clz.logger.exception(e) finished = True except AbortException as e: finished = True except Exception as e: clz.logger.exception(e) finished = True
def status_hook(self, status: Dict[str, str]) -> None: clz = BaseInfoHook if Monitor.is_abort_requested(): self._callback._error = 99 # Kill YoutubeDL Monitor.throw_exception_if_abort_requested() status_str = status.get('status', 'missing status') if status_str is None: clz._logger.debug('Missing status indication') elif status_str == 'downloading': self._download_eta = status.get('eta', 0) # In seconds elif status_str == 'error': clz._logger.error('Status:', str(status)) self.error_lines.append('Error downloading') self._error = VideoDownloader.DOWNLOAD_ERROR elif status_str == 'finished': filename = status.get('filename') tmpfilename = status.get('tmpfilename') downloaded_bytes = status.get('downloaded_bytes') total_bytes = status.get('total_bytes') total_bytes_estimate = status.get('total_bytes_estimate') elapsed = status.get('elapsed') eta = status.get('eta') speed = status.get('speed') fragment_index = status.get('fragment_index') fragment_count = status.get('fragment_count') clz._logger.debug('Finished')
def warning(self, line: str) -> None: if Monitor.is_abort_requested(): self._callback._error = 99 # Kill YoutubeDL Monitor.throw_exception_if_abort_requested() if 'merged' in line: # str: Requested formats are incompatible for merge and will be merged into # mkv. pass else: self.warning_lines.append(line)
def log_output(self): if Monitor.is_abort_requested(): return clz = RunCommand if clz.logger.isEnabledFor(LazyLogger.DEBUG): clz.logger.debug( f'ffmpeg failed for {self.movie_name} rc: {self.rc}') if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE): stdout = '\n'.join(self.stdout_lines) clz.logger.debug_verbose(f'STDOUT: {stdout}') stderr = '\n'.join(self.stderr_lines) clz.logger.debug_verbose(f'STDERR: {stderr}')
def join_dead_threads(cls): finished = False while not finished: with cls._lock: joined_threads: List[threading.Thread] = [] for thread in cls._threads_to_join: if not thread.is_alive(): thread.join(timeout=0.0) if not thread.is_alive(): joined_threads.append(thread) for thread in joined_threads: cls._threads_to_join.remove(thread) if Monitor.is_abort_requested(): finished = True
def error(self, line: str) -> None: if Monitor.is_abort_requested(): self.set_error(99, force=True) # Kill YoutubeDL Monitor.throw_exception_if_abort_requested() clz = BaseYDLogger if 'Error Constants.HTTP_TOO_MANY_REQUESTS' in line: self.set_error(Constants.HTTP_TOO_MANY_REQUESTS, force=True) type(self).logger.info('Abandoning download. Too Many Requests') # str: ERROR: (ExtractorError(...), 'wySw1lhMt1s: YouTube said: Unable # to extract video data') elif 'Unable to extract' in line: self.set_error(VideoDownloader.DOWNLOAD_ERROR) elif 'blocked' in line: self.set_error(VideoDownloader.BLOCKED_ERROR) elif 'unavailable' in line: self.set_error(VideoDownloader.UNAVAILABLE) else: self.error_lines.append(line)
def error(self, line: str) -> None: if Monitor.is_abort_requested(): self._callback._error = 99 # Kill YoutubeDL Monitor.throw_exception_if_abort_requested() clz = BaseYDLogger if 'Error 429' in line: self._callback._error = 429 clz._initial_tmr_timestamp = datetime.datetime.now() clz._too_many_requests_timestamp = (datetime.datetime.now() + RETRY_DELAY) clz.logger.info('Abandoning download. Too Many Requests') # str: ERROR: (ExtractorError(...), 'wySw1lhMt1s: YouTube said: Unable # to extract video data') elif 'Unable to extract' in line: self._callback._error = VideoDownloader.DOWNLOAD_ERROR elif 'blocked' in line: self._callback._error = VideoDownloader.BLOCKED_ERROR else: self.error_lines.append(line)
def debug(self, line: str) -> None: """ {"_type": "url", "url": "vYALYAuD5Fw", "ie_key": "Youtube", "id": "vYALYAuD5Fw", "title": "Larry Karaszewski on ROAD TO SALINA"} [download] Downloading video 14 of 1448 {"_type": "url", "url": "dZzFqtlamV4", "ie_key": "Youtube", "id": "dZzFqtlamV4", "title": "Joe Dante on HALF HUMAN"} [download] Downloading video 15 of 1448 {"id": "nrExo_KJROc", "uploader": "Trailers From Hell", "uploader_id": "trailersfromhell", "uploader_url": "http://www.youtube.com/user/trailersfromhell", "channel_id": "UCg7Mllu8AnTjlZ4Vu1FNdjQ", "channel_url": "http://www.youtube.com/channel/UCg7Mllu8AnTjlZ4Vu1FNdjQ" "upload_date": "20200908", "license": null, "creator": null, "title": "Brian Trenchard-Smith on ONCE UPON A TIME IN HOLLYWOOD", "alt_title": null, "thumbnails": [{"url": "https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp =-oaymwEYCKgBEF5IVfKriqkDCwgBFQAAiEIYAXAB&rs =AOn4CLCPHEof66nqx4GxE04sOUocr9WywA", "width": 168, "height": 94, "resolution": "168x94", "id": "0"}, {"url": "https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp =-oaymwEYCMQBEG5IVfKriqkDCwgBFQAAiEIYAXAB&rs=AOn4CLAm -CXcCCu0LATG_R347wBxBQj4BQ", "width": 196, "height": 110, "resolution": "196x110", "id": "1"}, {"url": "https://i.ytimg.com/vi/nrExo_KJROc/hqdefault.jpg?sqp =-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs =AOn4CLAhW2AcVqdWYiPMZuKENgiCO0gykQ",... :return: """ clz = BaseYDLogger if Monitor.is_abort_requested(): self._callback._error = 99 # Kill YoutubeDL Monitor.throw_exception_if_abort_requested() self.debug_lines.append(line) if line.startswith('[download] Downloading video'): try: _, index_str, total_str = re.split(r'[^0-9]+', line) self.index = int(index_str) self.total = int(total_str) except Exception as e: clz.logger.exception() self._callback._error = 1 # if line.startswith('{"_type":'): # try: # data = json.loads(line) # self._trailer_handler(data) # except Exception as e: # clz.logger.exception() if line.startswith('{"id":'): try: self.raw_data = json.loads(line) if self._parse_json_as_youtube: self._parsed_movie = populate_youtube_movie_info( self.raw_data, self.url) # self._trailer_handler(movie_data) except Exception as e: clz.logger.exception() self._callback._error = 2
def run_worker(self): # type: () -> None """ Initial Discovery of all movies in Kodi. :return: """ # Discovery is done in two parts: # # 1- query DB for every movie in library # 2- Get additional information # # There are three types of trailers for these movies: # # a- Movies with local trailers # b- Movies with trailer URLS (typically youtube links from tmdb) # TMdb will need to be queried for details # c. Movies with no trailer information, requiring a check with tmdb # to see if one exists # # Because of the above, this manager will query the DB for every movie # and then only process the ones with local trailers. The others will # be handed off to their own managers. This is done because of # the way that this application works: # Once enough information to identify a movie that matches # what the user wants, it is added to the pool of movies that # can be randomly selected for playing. Once a movie has been # selected, it is placed into a TrailerFetcherQueue. A # TrailerFetcher then gathers the remaining information so that # it can be played. # # If the lion's share of movies in the pool require significant # extra processing because they don't have local trailers, then # the fetcher can get overwhelmed. clz = DiscoverLibraryMovies self._selected_keywords = [] self._excluded_keywords = [] self._selected_genres = [] self._excluded_genres = [] if Settings.get_filter_genres(): self._selected_genres = GenreUtils.get_internal_kodi_genre_ids( GenreUtils.LOCAL_DATABASE, exclude=False) self._excluded_genres = GenreUtils.get_internal_kodi_genre_ids( GenreUtils.LOCAL_DATABASE, exclude=True) self._selected_keywords = GenreUtils.get_internal_kodi_keyword_ids( GenreUtils.LOCAL_DATABASE, exclude=False) self._excluded_keywords = GenreUtils.get_internal_kodi_keyword_ids( GenreUtils.LOCAL_DATABASE, exclude=True) query = self.create_query( self._selected_genres, self._excluded_genres, self._selected_keywords, self._excluded_keywords) if Monitor.is_abort_requested(): return start_time = datetime.datetime.now() Monitor.throw_exception_if_abort_requested() # Expensive operation query_result = JsonUtilsBasic.get_kodi_json(query, dump_results=False) Monitor.throw_exception_if_abort_requested() elapsed_time = datetime.datetime.now() - start_time if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE): clz.logger.debug_verbose('Library query seconds:', elapsed_time.total_seconds()) movies_skipped = 0 movies_found = 0 movies_with_local_trailers = 0 movies_with_trailer_urls = 0 movies_without_trailer_info = 0 self.throw_exception_on_forced_to_stop() result = query_result.get('result', {}) del query_result movies = result.get('movies', []) del result DiskUtils.RandomGenerator.shuffle(movies) if self._libraryURLManager is None: self._libraryURLManager = DiscoverLibraryURLTrailerMovies() self._libraryNoTrailerInfoManager = DiscoverLibraryNoTrailerMovies() library_movies = [] library_url_movies = [] library_no_trailer_movies = [] empty_limit = 50 movie_data = None if Settings.is_enable_movie_stats(): movie_data = LibraryMovieStats() country_id = Settings.get_country_iso_3166_1().lower() certifications = WorldCertifications.get_certifications(country_id) unrated_id = certifications.get_unrated_certification().get_preferred_id() for movie in movies: self.throw_exception_on_forced_to_stop() try: #clz.logger.debug('movie:', movie) movies_found += 1 if Settings.get_hide_watched_movies() and Movie.LAST_PLAYED in movie: if (self.get_days_since_last_played(movie[Movie.LAST_PLAYED], movie[Movie.TITLE]) > Settings.get_minimum_days_since_watched()): movies_skipped += 1 if clz.logger.isEnabledFor(LazyLogger.DEBUG_EXTRA_VERBOSE): clz.logger.debug_extra_verbose(movie[Movie.TITLE], 'will not be played due to ' 'Hide', 'Watched Movies') continue # Normalize certification # if clz.logger.isEnabledFor(LazyLogger.DEBUG): # clz.logger.debug('mpaa:', movie[Movie.MPAA], # 'movie:', movie[Movie.TITLE]) if certifications.is_valid(movie.get(Movie.MPAA, '')): movie[Movie.MPAA] = unrated_id certification = certifications.get_certification( movie.get(Movie.MPAA), movie.get(Movie.ADULT)) movie[Movie.ADULT] = movie.get(Movie.ADULT, False) if not isinstance(movie[Movie.ADULT], bool): if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE): clz.logger.debug_verbose(movie[Movie.TITLE], 'has invalid ADULT field: ', movie[Movie.ADULT]) movie[Movie.ADULT] = str(movie[Movie.ADULT]).lower == 'true' movie[Movie.SOURCE] = Movie.LIBRARY_SOURCE movie.setdefault(Movie.TRAILER, '') movie[Movie.TYPE] = '' if clz.logger.isEnabledFor(LazyLogger.DEBUG_VERBOSE): Debug.validate_basic_movie_properties(movie) if Settings.is_enable_movie_stats(): movie_data.collect_data(movie) # Basic discovery is complete at this point. Now send # all of the movies without any trailer information to # DiscoverLibraryNoTrailerMovies while # those with trailer URLs to DiscoverLibraryURLTrailerMovies if certifications.filter(certification): trailer = movie[Movie.TRAILER] if trailer == '': movies_without_trailer_info += 1 library_no_trailer_movies.append(movie) elif trailer.startswith('plugin://') or trailer.startswith('http'): movies_with_trailer_urls += 1 library_url_movies.append(movie) elif Settings.get_include_library_trailers(): movies_with_local_trailers += 1 library_movies.append(movie) if len(library_movies) >= empty_limit: self.add_to_discovered_trailers(library_movies) del library_movies[:] # Unblock other discovery now that a few movies have been # found. if not self._some_movies_discovered_event.isSet(): self._some_movies_discovered_event.set() if len(library_no_trailer_movies) >= empty_limit: self._libraryNoTrailerInfoManager.add_to_discovered_trailers( library_no_trailer_movies) del library_no_trailer_movies[:] if len(library_url_movies) >= empty_limit: self._libraryURLManager.add_to_discovered_trailers( library_url_movies) del library_no_trailer_movies[:] # Unblock other discovery now that a few movies have been # found. self._some_movies_discovered_event.set() except AbortException: reraise(*sys.exc_info()) except Exception: clz.logger.exception('') try: if len(library_movies) >= 0: self.add_to_discovered_trailers(library_movies) if len(library_no_trailer_movies) >= 0: self._libraryNoTrailerInfoManager.add_to_discovered_trailers( library_no_trailer_movies) if len(library_url_movies) >= 0: self._libraryURLManager.add_to_discovered_trailers( library_url_movies) except AbortException: reraise(*sys.exc_info()) except Exception: clz.logger.exception('') if (clz.logger.isEnabledFor(LazyLogger.DEBUG) and clz.logger.is_trace_enabled(Trace.STATS)): clz.logger.debug('Local movies found in library:', movies_found, trace=Trace.STATS) clz.logger.debug('Local movies filtered out', movies_skipped, trace=Trace.STATS) clz.logger.debug('Movies with local trailers:', movies_with_local_trailers, trace=Trace.STATS) clz.logger.debug('Movies with trailer URLs:', movies_with_trailer_urls, trace=Trace.STATS) clz.logger.debug('Movies with no trailer information:', movies_without_trailer_info, trace=Trace.STATS) if Settings.is_enable_movie_stats(): movie_data.report_data() del movie_data