def test_progress_track() -> None: console = Console( file=io.StringIO(), force_terminal=True, width=60, color_system="truecolor", legacy_windows=False, _environ={}, ) progress = Progress(console=console, auto_refresh=False, get_time=MockClock(auto=True)) test = ["foo", "bar", "baz"] expected_values = iter(test) with progress: for value in progress.track(test, description="test"): assert value == next(expected_values) result = console.file.getvalue() print(repr(result)) expected = "\x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h" print(expected) print(repr(expected)) print(result) print(repr(result)) assert result == expected with pytest.raises(ValueError): for n in progress.track(5): pass
def track(iterable, total=None, description="Working...", transient=False): """ Spawns a progress bar to monitor the progress of a for loop over an iterable sequence with detailed information. Arguments: iterable: list or other iterable object total: int. Total length of iterable description: str. Text to preprend to the progress bar. Returs: elements of iterable """ description = f"[{orange}]" + description columns = [description] + COLUMNS if total is None: try: total = len(iterable) except Exception: raise ValueError( "Could not get total from iterable, pass a total value." ) track_progress = Progress(*columns, transient=transient) with track_progress: yield from track_progress.track( iterable, total=total, description=description, )
def update_images(image_list: List[str]): from rich.markup import escape from rich.progress import MofNCompleteColumn, Progress from localstack.utils.container_utils.container_client import ContainerException from localstack.utils.docker_utils import DOCKER_CLIENT updated_count = 0 failed_count = 0 progress = Progress(*Progress.get_default_columns(), MofNCompleteColumn(), transient=True, console=console) with progress: for image in progress.track(image_list, description="Processing image..."): try: updated = False hash_before_pull = DOCKER_CLIENT.inspect_image( image_name=image, pull=False)["Id"] DOCKER_CLIENT.pull_image(image) if (hash_before_pull != DOCKER_CLIENT.inspect_image( image_name=image, pull=False)["Id"]): updated = True updated_count += 1 console.print( f":heavy_check_mark: Image {escape(image)} {'updated' if updated else 'up-to-date'}.", style="bold" if updated else None, highlight=False, ) except ContainerException as e: console.print( f":heavy_multiplication_x: Image {escape(image)} pull failed: {e.message}", style="bold red", highlight=False, ) failed_count += 1 console.rule() console.print( f"Images updated: {updated_count}, Images failed: {failed_count}, total images processed: {len(image_list)}." )
def test_no_output_if_progress_is_disabled() -> None: console = Console( file=io.StringIO(), force_terminal=True, width=60, color_system="truecolor", legacy_windows=False, _environ={}, ) progress = Progress( console=console, disable=True, ) test = ["foo", "bar", "baz"] expected_values = iter(test) with progress: for value in progress.track(test, description="test"): assert value == next(expected_values) result = console.file.getvalue() print(repr(result)) expected = "" assert result == expected
def get_movies_to_exclude(self, providers, fast=True, disable_progress=False): exclude_movies = {} # Get all movies listed in Radarr logger.debug("Getting all the movies from Radarr") radarr_movies = self.radarr_client.movie.get_all_movies() # Get the providers listed for the configured locale from JustWatch # and filter it with the given providers. This will ensure only the correct # providers are in the dictionary. raw_jw_providers = self.justwatch_client.get_providers() jw_providers = filters.get_providers(raw_jw_providers, providers) logger.debug( f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}" ) progress = Progress(disable=disable_progress) with progress: for movie in progress.track(radarr_movies): # Set the minimal base variables radarr_id = movie["id"] title = movie["title"] tmdb_id = movie["tmdbId"] filesize = movie["sizeOnDisk"] release_date = filters.get_release_date(movie) # Log the title and Radarr ID logger.debug( f"Processing title: {title} with Radarr ID: {radarr_id} and TMDB ID: {tmdb_id}" ) # Find the movie jw_id, jw_movie_data = self._find_movie(movie, jw_providers, fast, exclude=True) if jw_movie_data: # Get all the providers the movie is streaming on movie_providers = filters.get_jw_providers(jw_movie_data) # Loop over the configured providers and check if the provider # matches the providers advertised at the movie. If a match is found # update the exclude_movies dict matched_providers = list( set(movie_providers.keys()) & set(jw_providers.keys())) if matched_providers: clear_names = [ provider_details["clear_name"] for provider_id, provider_details in jw_providers.items() if provider_id in matched_providers ] exclude_movies.update({ radarr_id: { "title": title, "filesize": filesize, "release_date": release_date, "radarr_object": movie, "tmdb_id": tmdb_id, "jw_id": jw_id, "providers": clear_names, } }) logger.debug( f"{title} is streaming on {', '.join(clear_names)}" ) return exclude_movies
def get_movies_to_re_add(self, providers, fast=True, disable_progress=False): re_add_movies = {} # Get all movies listed in Radarr and filter it to only include not monitored movies logger.debug("Getting all the movies from Radarr") radarr_movies = self.radarr_client.movie.get_all_movies() radarr_not_monitored_movies = [ movie for movie in radarr_movies if not movie["monitored"] ] # Get the providers listed for the configured locale from JustWatch # and filter it with the given providers. This will ensure only the correct # providers are in the dictionary. raw_jw_providers = self.justwatch_client.get_providers() jw_providers = filters.get_providers(raw_jw_providers, providers) logger.debug( f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}" ) progress = Progress(disable=disable_progress) with progress: for movie in progress.track(radarr_not_monitored_movies): # Set the minimal base variables radarr_id = movie["id"] title = movie["title"] tmdb_id = movie["tmdbId"] release_date = filters.get_release_date(movie) # Log the title and Radarr ID logger.debug( f"Processing title: {title} with Radarr ID: {radarr_id} and TMDB ID: {tmdb_id}" ) # Find the movie jw_id, jw_movie_data = self._find_movie(movie, jw_providers, fast, exclude=False) if jw_movie_data: # Get all the providers the movie is streaming on movie_providers = filters.get_jw_providers(jw_movie_data) # Check if the JustWatch providers matching the movie providers matched_providers = list( set(movie_providers.keys()) & set(jw_providers.keys())) if not matched_providers: re_add_movies.update({ radarr_id: { "title": title, "release_date": release_date, "radarr_object": movie, "tmdb_id": tmdb_id, "jw_id": jw_id, } }) logger.debug( f"{title} is not streaming on a configured provider" ) return re_add_movies
from time import sleep from rich.progress import Progress p = Progress() for i in p.track(range(1, 31)): p.log(f":thumbs_up: we are making progress : {i}") sleep(0.5)
def parsingYa(login): # Запись в txt if Ya == '4': file_txt = open(dirresults + "/results/Yandex_parser/" + str(hvostfile) + '_' + time.strftime("%d_%m_%Y_%H_%M_%S", time_data) + ".txt", "w", encoding="utf-8") # raise Exception("") else: file_txt = open(dirresults + "/results/Yandex_parser/" + str(login) + ".txt", "w", encoding="utf-8") progressYa = Progress("[progress.percentage]{task.percentage:>3.0f}%", auto_refresh=False) # Парсинг for login in progressYa.track(listlogin, description=""): urlYa = f'https://yandex.ru/collections/api/users/{login}/' try: r = my_session.get(urlYa, headers = head0, timeout=3) except: print(Fore.RED + "\nОшибка\n" + Style.RESET_ALL) continue try: rdict = json.loads(r.text) except: rdict = {} rdict.update(public_id="Увы", display_name="-No-") pub = rdict.get("public_id") name = rdict.get("display_name") email=str(login)+"@yandex.ru" if rdict.get("display_name") == "-No-": if Ya != '4': print(Style.BRIGHT + Fore.RED + "\nНе сработало") else: wZ1bad.append(str(login)) continue continue else: table1 = Table(title = "\n" + Style.BRIGHT + Fore.RED + str(login) + Style.RESET_ALL, style="green") table1.add_column("Имя", style="magenta") table1.add_column("Логин", style="cyan") table1.add_column("E-mail", style="cyan") if Ya == '3': table1.add_row(name,"Пропуск","Пропуск") else: table1.add_row(name,login,email) console = Console() console.print(table1) market=f"https://market.yandex.ru/user/{pub}/reviews" collections=f"https://yandex.ru/collections/user/{login}/" if Ya == '3': music=f"\033[33;1mПропуск\033[0m" else: music=f"https://music.yandex.ru/users/{login}/tracks" dzen=f"https://zen.yandex.ru/user/{pub}" qu=f"https://yandex.ru/q/profile/{pub}/" raion=f"https://local.yandex.ru/users/{pub}/" print("\033[32;1mЯ.Маркет:\033[0m", market) print("\033[32;1mЯ.Картинки:\033[0m", collections) print("\033[32;1mЯ.Музыка:\033[0m", music) print("\033[32;1mЯ.Дзен:\033[0m", dzen) print("\033[32;1mЯ.Кью:\033[0m", qu) print("\033[32;1mЯ.Район:\033[0m", raion) yalist=[market, collections, music, dzen, qu, raion] file_txt.write(f"{login} | {email} | {name}\n{market}\n{collections}\n{music}\n{dzen}\n{qu}\n{raion}\n\n\n",) progressYa.refresh() for webopen in yalist: if webopen == music and Ya == '3': continue else: webbrowser.open(webopen) if Ya == '4': # запись в txt концовка file_txt.write(f"\nНеобработанные данные из файла '{hvostfile}':\n") for badsites in wZ1bad: file_txt.write(f"{badsites}\n") file_txt.write(f"\nОбновлено: " + time.strftime("%d/%m/%Y_%H:%M:%S", time_data) + ".") file_txt.close()
from rich.panel import Panel from rich.progress import Progress JOBS = [100, 150, 25, 70, 110, 90] progress = Progress(auto_refresh=False) master_task = progress.add_task("overall", total=sum(JOBS)) jobs_task = progress.add_task("jobs") progress.console.print( Panel( "[bold blue]A demonstration of progress with a current task and overall progress.", padding=1, ) ) with progress: for job_no, job in enumerate(JOBS): progress.log(f"Starting job #{job_no}") sleep(0.2) progress.reset(jobs_task, total=job, description=f"job [bold yellow]#{job_no}") progress.start_task(jobs_task) for wait in progress.track(range(job), task_id=jobs_task): sleep(0.01) progress.advance(master_task, job) progress.log(f"Job #{job_no} is complete") progress.log( Panel(":sparkle: All done! :sparkle:", border_style="green", padding=1) )
def get_series_to_re_add(self, providers, fast=True, disable_progress=False, tmdb_api_key=None): re_add_series = {} # Setup TMDB if there is an API key provided if tmdb_api_key: self.tmdb = pytmdb.TMDB(tmdb_api_key) # Get all series listed in Sonarr logger.debug("Getting all the series from Sonarr") sonarr_series = self.sonarr_client.serie.get_all_series() # Get the providers listed for the configured locale from JustWatch # and filter it with the given providers. This will ensure only the correct # providers are in the dictionary. raw_jw_providers = self.justwatch_client.get_providers() jw_providers = filters.get_providers(raw_jw_providers, providers) logger.debug( f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}" ) progress = Progress(disable=disable_progress) with progress: for serie in progress.track(sonarr_series): # Set the minimal base variables sonarr_id = serie["id"] title = serie["title"] release_year = serie["year"] ended = serie["ended"] # Get episodes of the serie episodes = self.sonarr_client.episode.get_episodes_of_serie( sonarr_id) # Get JustWatch serie data jw_id, jw_serie_data = self._find_serie(serie, jw_providers, tmdb_api_key, fast, exclude=False) # Continue if the proper JustWatch ID is found if jw_serie_data: logger.debug(f"Look up season data for {title}") jw_seasons = jw_serie_data["seasons"] # Loop over the seasons for jw_season in jw_seasons: jw_season_title = jw_season.get( "title", f"Season {jw_season['season_number']}") jw_season_id = jw_season["id"] jw_season_data = self.justwatch_client.get_season( jw_season_id) jw_episodes = jw_season_data.get("episodes", []) logger.debug( f"Processing season {jw_season_title} of {title}") # Loop over the episodes and check if there are providers for episode in jw_episodes: season_number = episode["season_number"] episode_number = episode["episode_number"] # Get episode providers episode_providers = filters.get_jw_providers( episode) # Check if the providers of the episodes matches the configured providers providers_match = [ provider_details["clear_name"] for provider_id, provider_details in jw_providers.items() if provider_id in episode_providers.keys() ] if not providers_match: # Get the episode data from the information in Sonarr sonarr_episode_data = filters.get_episode_data( episodes, season_number, episode_number) re_add_series.update({ sonarr_id: { "title": title, "release_year": release_year, "ended": ended, "jw_id": jw_id, "sonarr_object": serie, "episodes": re_add_series[sonarr_id]["episodes"] + [{ "season": season_number, "episode": episode_number, **sonarr_episode_data, }] if re_add_series.get(sonarr_id) else [{ "season": season_number, "episode": episode_number, **sonarr_episode_data, }], } }) logger.debug( f"{title} S{season_number}E{episode_number} is not streaming on a configured provider" ) # Check if the full season could be excluded rather than seperate episodes for re_add_id, re_add_entry in re_add_series.items(): sonarr_object = re_add_entry["sonarr_object"] sonarr_seasons = sonarr_object["seasons"] re_add_episodes = re_add_entry["episodes"] seasons_to_re_add = [] season_numbers = [] # Loop over the seasons registerd in Sonarr for season in sonarr_seasons: sonarr_total_episodes = season["statistics"][ "totalEpisodeCount"] sonarr_season_monitored = season["monitored"] sonarr_season_number = int(season["seasonNumber"]) # Get the total amount of episodes re_add_total_episodes = [ episode for episode in re_add_episodes if episode["season"] == sonarr_season_number ] # Check if the amount of episodes to exclude is greater or equals the total episodes in Sonarr if len(re_add_total_episodes) >= sonarr_total_episodes: season_numbers.append(sonarr_season_number) seasons_to_re_add.append({ "season": sonarr_season_number, "monitored": sonarr_season_monitored, }) # Re order the re_add_series dict to strip the episodes if the whole season can be excluded # or if the episode is not monitored but the season is updated_re_add_episodes = [] for episode in re_add_episodes: episode_season = episode["season"] episode_monitored = episode.get("monitored", True) season_monitored = [ season["monitored"] for season in seasons_to_re_add if season["season"] == episode_season ] if all(season_monitored) and not episode_monitored: updated_re_add_episodes.append(episode) elif episode_season not in season_numbers: updated_re_add_episodes.append(episode) # Save all episode IDs in case we need to re add the whole serie all_episode_ids = [ episode["episode_id"] for episode in re_add_episodes if episode.get("episode_id", False) ] re_add_series[re_add_id]["all_episode_ids"] = all_episode_ids re_add_series[re_add_id]["episodes"] = updated_re_add_episodes re_add_series[re_add_id]["seasons"] = seasons_to_re_add return re_add_series
def get_series_to_exclude(self, providers, fast=True, disable_progress=False, tmdb_api_key=None): exclude_series = {} # Get all series listed in Sonarr logger.debug("Getting all the series from Sonarr") sonarr_series = self.sonarr_client.serie.get_all_series() # Get the providers listed for the configured locale from JustWatch # and filter it with the given providers. This will ensure only the correct # providers are in the dictionary. raw_jw_providers = self.justwatch_client.get_providers() jw_providers = filters.get_providers(raw_jw_providers, providers) logger.debug( f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}" ) progress = Progress(disable=disable_progress) with progress: for serie in progress.track(sonarr_series): # Set the minimal base variables sonarr_id = serie["id"] title = serie["title"] filesize = serie.get("statistics", {}).get("sizeOnDisk", 0) release_year = serie["year"] ended = serie["ended"] # Get episodes of the serie episodes = self.sonarr_client.episode.get_episodes_of_serie( sonarr_id) # Get JustWatch serie data jw_id, jw_serie_data = self._find_serie(serie, jw_providers, tmdb_api_key, fast, exclude=True) # Continue if the proper JustWatch ID is found if jw_serie_data: logger.debug(f"Look up season data for {title}") jw_seasons = jw_serie_data["seasons"] # Loop over the seasons for jw_season in jw_seasons: jw_season_title = jw_season.get( "title", f"Season {jw_season['season_number']}") jw_season_id = jw_season["id"] jw_season_data = self.justwatch_client.get_season( jw_season_id) jw_episodes = jw_season_data.get("episodes", []) logger.debug( f"Processing season {jw_season_title} of {title}") # Loop over the episodes and check if there are providers for episode in jw_episodes: season_number = episode["season_number"] episode_number = episode["episode_number"] # Get episode providers episode_providers = filters.get_jw_providers( episode) # Check if the providers of the episodes matches the configured providers providers_match = [ provider_details["clear_name"] for provider_id, provider_details in jw_providers.items() if provider_id in episode_providers.keys() ] if providers_match: # Get the episode data from the information in Sonarr sonarr_episode_data = filters.get_episode_data( episodes, season_number, episode_number) sonarr_episode_id = filters.get_episode_file_id( episodes, season_number, episode_number) exclude_series.update({ sonarr_id: { "title": title, "filesize": filesize, "release_year": release_year, "ended": ended, "jw_id": jw_id, "sonarr_object": serie, "sonarr_file_ids": exclude_series[sonarr_id] ["sonarr_file_ids"] + sonarr_episode_id if exclude_series.get(sonarr_id) else sonarr_episode_id, "episodes": exclude_series[sonarr_id]["episodes"] + [{ "season": season_number, "episode": episode_number, "providers": providers_match, **sonarr_episode_data, }] if exclude_series.get(sonarr_id) else [{ "season": season_number, "episode": episode_number, "providers": providers_match, **sonarr_episode_data, }], } }) logger.debug( f"{title} S{season_number}E{episode_number} is streaming on {', '.join(providers_match)}" ) # Check if the full season could be excluded rather than seperate episodes for exclude_id, exclude_entry in exclude_series.items(): sonarr_object = exclude_entry["sonarr_object"] sonarr_seasons = sonarr_object["seasons"] exclude_episodes = exclude_entry["episodes"] seasons_to_exclude = [] season_numbers = [] # Loop over the seasons registerd in Sonarr for season in sonarr_seasons: sonarr_total_episodes = season["statistics"][ "totalEpisodeCount"] sonarr_season_monitored = season["monitored"] sonarr_season_has_file = bool( season["statistics"]["episodeFileCount"]) sonarr_season_number = int(season["seasonNumber"]) # Get the total amount of episodes exclude_total_episodes = [ episode for episode in exclude_episodes if episode["season"] == sonarr_season_number ] # Get a list of providers of the season season_providers = [ episode["providers"] for episode in exclude_episodes if episode["season"] == sonarr_season_number ] season_providers = filters.flatten(season_providers) # Check if the amount of episodes to exclude is greater or equal the total episodes in Sonarr if len(exclude_total_episodes) >= sonarr_total_episodes: season_numbers.append(sonarr_season_number) seasons_to_exclude.append({ "season": sonarr_season_number, "providers": season_providers, "monitored": sonarr_season_monitored, "has_file": sonarr_season_has_file, }) # Re order the exclude_series dict to strip the episodes if the whole season can be excluded updated_exclude_episodes = [ episode for episode in exclude_episodes if episode["season"] not in season_numbers ] exclude_series[exclude_id]["episodes"] = updated_exclude_episodes exclude_series[exclude_id]["seasons"] = seasons_to_exclude exclude_series[exclude_id][ "providers"] = filters.get_providers_from_seasons_episodes( exclude_entry["seasons"], exclude_entry["episodes"]) return exclude_series