def test_progress_track() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=60,
color_system="truecolor",
legacy_windows=False,
_environ={},
)
progress = Progress(console=console,
auto_refresh=False,
get_time=MockClock(auto=True))
test = ["foo", "bar", "baz"]
expected_values = iter(test)
with progress:
for value in progress.track(test, description="test"):
assert value == next(expected_values)
result = console.file.getvalue()
print(repr(result))
expected = "\x1b[?25l\r\x1b[2Ktest \x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 0%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━\x1b[0m\x1b[38;5;237m╺\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m 33%\x1b[0m \x1b[36m-:--:--\x1b[0m\r\x1b[2Ktest \x1b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\x1b[38;2;249;38;114m╸\x1b[0m\x1b[38;5;237m━━━━━━━━━━━━━\x1b[0m \x1b[35m 67%\x1b[0m \x1b[36m0:00:06\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\r\x1b[2Ktest \x1b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m \x1b[35m100%\x1b[0m \x1b[36m0:00:00\x1b[0m\n\x1b[?25h"
print(expected)
print(repr(expected))
print(result)
print(repr(result))
assert result == expected
with pytest.raises(ValueError):
for n in progress.track(5):
pass
def track(iterable, total=None, description="Working...", transient=False):
"""
Spawns a progress bar to monitor the progress of a for loop over
an iterable sequence with detailed information.
Arguments:
iterable: list or other iterable object
total: int. Total length of iterable
description: str. Text to preprend to the progress bar.
Returs:
elements of iterable
"""
description = f"[{orange}]" + description
columns = [description] + COLUMNS
if total is None:
try:
total = len(iterable)
except Exception:
raise ValueError(
"Could not get total from iterable, pass a total value."
)
track_progress = Progress(*columns, transient=transient)
with track_progress:
yield from track_progress.track(
iterable, total=total, description=description,
)
def update_images(image_list: List[str]):
from rich.markup import escape
from rich.progress import MofNCompleteColumn, Progress
from localstack.utils.container_utils.container_client import ContainerException
from localstack.utils.docker_utils import DOCKER_CLIENT
updated_count = 0
failed_count = 0
progress = Progress(*Progress.get_default_columns(),
MofNCompleteColumn(),
transient=True,
console=console)
with progress:
for image in progress.track(image_list,
description="Processing image..."):
try:
updated = False
hash_before_pull = DOCKER_CLIENT.inspect_image(
image_name=image, pull=False)["Id"]
DOCKER_CLIENT.pull_image(image)
if (hash_before_pull != DOCKER_CLIENT.inspect_image(
image_name=image, pull=False)["Id"]):
updated = True
updated_count += 1
console.print(
f":heavy_check_mark: Image {escape(image)} {'updated' if updated else 'up-to-date'}.",
style="bold" if updated else None,
highlight=False,
)
except ContainerException as e:
console.print(
f":heavy_multiplication_x: Image {escape(image)} pull failed: {e.message}",
style="bold red",
highlight=False,
)
failed_count += 1
console.rule()
console.print(
f"Images updated: {updated_count}, Images failed: {failed_count}, total images processed: {len(image_list)}."
)
def test_no_output_if_progress_is_disabled() -> None:
console = Console(
file=io.StringIO(),
force_terminal=True,
width=60,
color_system="truecolor",
legacy_windows=False,
_environ={},
)
progress = Progress(
console=console,
disable=True,
)
test = ["foo", "bar", "baz"]
expected_values = iter(test)
with progress:
for value in progress.track(test, description="test"):
assert value == next(expected_values)
result = console.file.getvalue()
print(repr(result))
expected = ""
assert result == expected
def get_movies_to_exclude(self,
providers,
fast=True,
disable_progress=False):
exclude_movies = {}
# Get all movies listed in Radarr
logger.debug("Getting all the movies from Radarr")
radarr_movies = self.radarr_client.movie.get_all_movies()
# Get the providers listed for the configured locale from JustWatch
# and filter it with the given providers. This will ensure only the correct
# providers are in the dictionary.
raw_jw_providers = self.justwatch_client.get_providers()
jw_providers = filters.get_providers(raw_jw_providers, providers)
logger.debug(
f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}"
)
progress = Progress(disable=disable_progress)
with progress:
for movie in progress.track(radarr_movies):
# Set the minimal base variables
radarr_id = movie["id"]
title = movie["title"]
tmdb_id = movie["tmdbId"]
filesize = movie["sizeOnDisk"]
release_date = filters.get_release_date(movie)
# Log the title and Radarr ID
logger.debug(
f"Processing title: {title} with Radarr ID: {radarr_id} and TMDB ID: {tmdb_id}"
)
# Find the movie
jw_id, jw_movie_data = self._find_movie(movie,
jw_providers,
fast,
exclude=True)
if jw_movie_data:
# Get all the providers the movie is streaming on
movie_providers = filters.get_jw_providers(jw_movie_data)
# Loop over the configured providers and check if the provider
# matches the providers advertised at the movie. If a match is found
# update the exclude_movies dict
matched_providers = list(
set(movie_providers.keys()) & set(jw_providers.keys()))
if matched_providers:
clear_names = [
provider_details["clear_name"] for provider_id,
provider_details in jw_providers.items()
if provider_id in matched_providers
]
exclude_movies.update({
radarr_id: {
"title": title,
"filesize": filesize,
"release_date": release_date,
"radarr_object": movie,
"tmdb_id": tmdb_id,
"jw_id": jw_id,
"providers": clear_names,
}
})
logger.debug(
f"{title} is streaming on {', '.join(clear_names)}"
)
return exclude_movies
def get_movies_to_re_add(self,
providers,
fast=True,
disable_progress=False):
re_add_movies = {}
# Get all movies listed in Radarr and filter it to only include not monitored movies
logger.debug("Getting all the movies from Radarr")
radarr_movies = self.radarr_client.movie.get_all_movies()
radarr_not_monitored_movies = [
movie for movie in radarr_movies if not movie["monitored"]
]
# Get the providers listed for the configured locale from JustWatch
# and filter it with the given providers. This will ensure only the correct
# providers are in the dictionary.
raw_jw_providers = self.justwatch_client.get_providers()
jw_providers = filters.get_providers(raw_jw_providers, providers)
logger.debug(
f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}"
)
progress = Progress(disable=disable_progress)
with progress:
for movie in progress.track(radarr_not_monitored_movies):
# Set the minimal base variables
radarr_id = movie["id"]
title = movie["title"]
tmdb_id = movie["tmdbId"]
release_date = filters.get_release_date(movie)
# Log the title and Radarr ID
logger.debug(
f"Processing title: {title} with Radarr ID: {radarr_id} and TMDB ID: {tmdb_id}"
)
# Find the movie
jw_id, jw_movie_data = self._find_movie(movie,
jw_providers,
fast,
exclude=False)
if jw_movie_data:
# Get all the providers the movie is streaming on
movie_providers = filters.get_jw_providers(jw_movie_data)
# Check if the JustWatch providers matching the movie providers
matched_providers = list(
set(movie_providers.keys()) & set(jw_providers.keys()))
if not matched_providers:
re_add_movies.update({
radarr_id: {
"title": title,
"release_date": release_date,
"radarr_object": movie,
"tmdb_id": tmdb_id,
"jw_id": jw_id,
}
})
logger.debug(
f"{title} is not streaming on a configured provider"
)
return re_add_movies
from time import sleep
from rich.progress import Progress
p = Progress()
for i in p.track(range(1, 31)):
p.log(f":thumbs_up: we are making progress : {i}")
sleep(0.5)
def parsingYa(login):
# Запись в txt
if Ya == '4':
file_txt = open(dirresults + "/results/Yandex_parser/" + str(hvostfile) + '_' + time.strftime("%d_%m_%Y_%H_%M_%S", time_data) + ".txt", "w", encoding="utf-8")
# raise Exception("")
else:
file_txt = open(dirresults + "/results/Yandex_parser/" + str(login) + ".txt", "w", encoding="utf-8")
progressYa = Progress("[progress.percentage]{task.percentage:>3.0f}%", auto_refresh=False)
# Парсинг
for login in progressYa.track(listlogin, description=""):
urlYa = f'https://yandex.ru/collections/api/users/{login}/'
try:
r = my_session.get(urlYa, headers = head0, timeout=3)
except:
print(Fore.RED + "\nОшибка\n" + Style.RESET_ALL)
continue
try:
rdict = json.loads(r.text)
except:
rdict = {}
rdict.update(public_id="Увы", display_name="-No-")
pub = rdict.get("public_id")
name = rdict.get("display_name")
email=str(login)+"@yandex.ru"
if rdict.get("display_name") == "-No-":
if Ya != '4':
print(Style.BRIGHT + Fore.RED + "\nНе сработало")
else:
wZ1bad.append(str(login))
continue
continue
else:
table1 = Table(title = "\n" + Style.BRIGHT + Fore.RED + str(login) + Style.RESET_ALL, style="green")
table1.add_column("Имя", style="magenta")
table1.add_column("Логин", style="cyan")
table1.add_column("E-mail", style="cyan")
if Ya == '3':
table1.add_row(name,"Пропуск","Пропуск")
else:
table1.add_row(name,login,email)
console = Console()
console.print(table1)
market=f"https://market.yandex.ru/user/{pub}/reviews"
collections=f"https://yandex.ru/collections/user/{login}/"
if Ya == '3':
music=f"\033[33;1mПропуск\033[0m"
else:
music=f"https://music.yandex.ru/users/{login}/tracks"
dzen=f"https://zen.yandex.ru/user/{pub}"
qu=f"https://yandex.ru/q/profile/{pub}/"
raion=f"https://local.yandex.ru/users/{pub}/"
print("\033[32;1mЯ.Маркет:\033[0m", market)
print("\033[32;1mЯ.Картинки:\033[0m", collections)
print("\033[32;1mЯ.Музыка:\033[0m", music)
print("\033[32;1mЯ.Дзен:\033[0m", dzen)
print("\033[32;1mЯ.Кью:\033[0m", qu)
print("\033[32;1mЯ.Район:\033[0m", raion)
yalist=[market, collections, music, dzen, qu, raion]
file_txt.write(f"{login} | {email} | {name}\n{market}\n{collections}\n{music}\n{dzen}\n{qu}\n{raion}\n\n\n",)
progressYa.refresh()
for webopen in yalist:
if webopen == music and Ya == '3':
continue
else:
webbrowser.open(webopen)
if Ya == '4':
# запись в txt концовка
file_txt.write(f"\nНеобработанные данные из файла '{hvostfile}':\n")
for badsites in wZ1bad:
file_txt.write(f"{badsites}\n")
file_txt.write(f"\nОбновлено: " + time.strftime("%d/%m/%Y_%H:%M:%S", time_data) + ".")
file_txt.close()
from rich.panel import Panel
from rich.progress import Progress
JOBS = [100, 150, 25, 70, 110, 90]
progress = Progress(auto_refresh=False)
master_task = progress.add_task("overall", total=sum(JOBS))
jobs_task = progress.add_task("jobs")
progress.console.print(
Panel(
"[bold blue]A demonstration of progress with a current task and overall progress.",
padding=1,
)
)
with progress:
for job_no, job in enumerate(JOBS):
progress.log(f"Starting job #{job_no}")
sleep(0.2)
progress.reset(jobs_task, total=job, description=f"job [bold yellow]#{job_no}")
progress.start_task(jobs_task)
for wait in progress.track(range(job), task_id=jobs_task):
sleep(0.01)
progress.advance(master_task, job)
progress.log(f"Job #{job_no} is complete")
progress.log(
Panel(":sparkle: All done! :sparkle:", border_style="green", padding=1)
)
def get_series_to_re_add(self,
providers,
fast=True,
disable_progress=False,
tmdb_api_key=None):
re_add_series = {}
# Setup TMDB if there is an API key provided
if tmdb_api_key:
self.tmdb = pytmdb.TMDB(tmdb_api_key)
# Get all series listed in Sonarr
logger.debug("Getting all the series from Sonarr")
sonarr_series = self.sonarr_client.serie.get_all_series()
# Get the providers listed for the configured locale from JustWatch
# and filter it with the given providers. This will ensure only the correct
# providers are in the dictionary.
raw_jw_providers = self.justwatch_client.get_providers()
jw_providers = filters.get_providers(raw_jw_providers, providers)
logger.debug(
f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}"
)
progress = Progress(disable=disable_progress)
with progress:
for serie in progress.track(sonarr_series):
# Set the minimal base variables
sonarr_id = serie["id"]
title = serie["title"]
release_year = serie["year"]
ended = serie["ended"]
# Get episodes of the serie
episodes = self.sonarr_client.episode.get_episodes_of_serie(
sonarr_id)
# Get JustWatch serie data
jw_id, jw_serie_data = self._find_serie(serie,
jw_providers,
tmdb_api_key,
fast,
exclude=False)
# Continue if the proper JustWatch ID is found
if jw_serie_data:
logger.debug(f"Look up season data for {title}")
jw_seasons = jw_serie_data["seasons"]
# Loop over the seasons
for jw_season in jw_seasons:
jw_season_title = jw_season.get(
"title", f"Season {jw_season['season_number']}")
jw_season_id = jw_season["id"]
jw_season_data = self.justwatch_client.get_season(
jw_season_id)
jw_episodes = jw_season_data.get("episodes", [])
logger.debug(
f"Processing season {jw_season_title} of {title}")
# Loop over the episodes and check if there are providers
for episode in jw_episodes:
season_number = episode["season_number"]
episode_number = episode["episode_number"]
# Get episode providers
episode_providers = filters.get_jw_providers(
episode)
# Check if the providers of the episodes matches the configured providers
providers_match = [
provider_details["clear_name"] for provider_id,
provider_details in jw_providers.items()
if provider_id in episode_providers.keys()
]
if not providers_match:
# Get the episode data from the information in Sonarr
sonarr_episode_data = filters.get_episode_data(
episodes, season_number, episode_number)
re_add_series.update({
sonarr_id: {
"title":
title,
"release_year":
release_year,
"ended":
ended,
"jw_id":
jw_id,
"sonarr_object":
serie,
"episodes":
re_add_series[sonarr_id]["episodes"] +
[{
"season": season_number,
"episode": episode_number,
**sonarr_episode_data,
}] if re_add_series.get(sonarr_id) else
[{
"season": season_number,
"episode": episode_number,
**sonarr_episode_data,
}],
}
})
logger.debug(
f"{title} S{season_number}E{episode_number} is not streaming on a configured provider"
)
# Check if the full season could be excluded rather than seperate episodes
for re_add_id, re_add_entry in re_add_series.items():
sonarr_object = re_add_entry["sonarr_object"]
sonarr_seasons = sonarr_object["seasons"]
re_add_episodes = re_add_entry["episodes"]
seasons_to_re_add = []
season_numbers = []
# Loop over the seasons registerd in Sonarr
for season in sonarr_seasons:
sonarr_total_episodes = season["statistics"][
"totalEpisodeCount"]
sonarr_season_monitored = season["monitored"]
sonarr_season_number = int(season["seasonNumber"])
# Get the total amount of episodes
re_add_total_episodes = [
episode for episode in re_add_episodes
if episode["season"] == sonarr_season_number
]
# Check if the amount of episodes to exclude is greater or equals the total episodes in Sonarr
if len(re_add_total_episodes) >= sonarr_total_episodes:
season_numbers.append(sonarr_season_number)
seasons_to_re_add.append({
"season":
sonarr_season_number,
"monitored":
sonarr_season_monitored,
})
# Re order the re_add_series dict to strip the episodes if the whole season can be excluded
# or if the episode is not monitored but the season is
updated_re_add_episodes = []
for episode in re_add_episodes:
episode_season = episode["season"]
episode_monitored = episode.get("monitored", True)
season_monitored = [
season["monitored"] for season in seasons_to_re_add
if season["season"] == episode_season
]
if all(season_monitored) and not episode_monitored:
updated_re_add_episodes.append(episode)
elif episode_season not in season_numbers:
updated_re_add_episodes.append(episode)
# Save all episode IDs in case we need to re add the whole serie
all_episode_ids = [
episode["episode_id"] for episode in re_add_episodes
if episode.get("episode_id", False)
]
re_add_series[re_add_id]["all_episode_ids"] = all_episode_ids
re_add_series[re_add_id]["episodes"] = updated_re_add_episodes
re_add_series[re_add_id]["seasons"] = seasons_to_re_add
return re_add_series
def get_series_to_exclude(self,
providers,
fast=True,
disable_progress=False,
tmdb_api_key=None):
exclude_series = {}
# Get all series listed in Sonarr
logger.debug("Getting all the series from Sonarr")
sonarr_series = self.sonarr_client.serie.get_all_series()
# Get the providers listed for the configured locale from JustWatch
# and filter it with the given providers. This will ensure only the correct
# providers are in the dictionary.
raw_jw_providers = self.justwatch_client.get_providers()
jw_providers = filters.get_providers(raw_jw_providers, providers)
logger.debug(
f"Got the following providers: {', '.join([v['clear_name'] for _, v in jw_providers.items()])}"
)
progress = Progress(disable=disable_progress)
with progress:
for serie in progress.track(sonarr_series):
# Set the minimal base variables
sonarr_id = serie["id"]
title = serie["title"]
filesize = serie.get("statistics", {}).get("sizeOnDisk", 0)
release_year = serie["year"]
ended = serie["ended"]
# Get episodes of the serie
episodes = self.sonarr_client.episode.get_episodes_of_serie(
sonarr_id)
# Get JustWatch serie data
jw_id, jw_serie_data = self._find_serie(serie,
jw_providers,
tmdb_api_key,
fast,
exclude=True)
# Continue if the proper JustWatch ID is found
if jw_serie_data:
logger.debug(f"Look up season data for {title}")
jw_seasons = jw_serie_data["seasons"]
# Loop over the seasons
for jw_season in jw_seasons:
jw_season_title = jw_season.get(
"title", f"Season {jw_season['season_number']}")
jw_season_id = jw_season["id"]
jw_season_data = self.justwatch_client.get_season(
jw_season_id)
jw_episodes = jw_season_data.get("episodes", [])
logger.debug(
f"Processing season {jw_season_title} of {title}")
# Loop over the episodes and check if there are providers
for episode in jw_episodes:
season_number = episode["season_number"]
episode_number = episode["episode_number"]
# Get episode providers
episode_providers = filters.get_jw_providers(
episode)
# Check if the providers of the episodes matches the configured providers
providers_match = [
provider_details["clear_name"] for provider_id,
provider_details in jw_providers.items()
if provider_id in episode_providers.keys()
]
if providers_match:
# Get the episode data from the information in Sonarr
sonarr_episode_data = filters.get_episode_data(
episodes, season_number, episode_number)
sonarr_episode_id = filters.get_episode_file_id(
episodes, season_number, episode_number)
exclude_series.update({
sonarr_id: {
"title":
title,
"filesize":
filesize,
"release_year":
release_year,
"ended":
ended,
"jw_id":
jw_id,
"sonarr_object":
serie,
"sonarr_file_ids":
exclude_series[sonarr_id]
["sonarr_file_ids"] + sonarr_episode_id
if exclude_series.get(sonarr_id) else
sonarr_episode_id,
"episodes":
exclude_series[sonarr_id]["episodes"] +
[{
"season": season_number,
"episode": episode_number,
"providers": providers_match,
**sonarr_episode_data,
}] if exclude_series.get(sonarr_id)
else [{
"season": season_number,
"episode": episode_number,
"providers": providers_match,
**sonarr_episode_data,
}],
}
})
logger.debug(
f"{title} S{season_number}E{episode_number} is streaming on {', '.join(providers_match)}"
)
# Check if the full season could be excluded rather than seperate episodes
for exclude_id, exclude_entry in exclude_series.items():
sonarr_object = exclude_entry["sonarr_object"]
sonarr_seasons = sonarr_object["seasons"]
exclude_episodes = exclude_entry["episodes"]
seasons_to_exclude = []
season_numbers = []
# Loop over the seasons registerd in Sonarr
for season in sonarr_seasons:
sonarr_total_episodes = season["statistics"][
"totalEpisodeCount"]
sonarr_season_monitored = season["monitored"]
sonarr_season_has_file = bool(
season["statistics"]["episodeFileCount"])
sonarr_season_number = int(season["seasonNumber"])
# Get the total amount of episodes
exclude_total_episodes = [
episode for episode in exclude_episodes
if episode["season"] == sonarr_season_number
]
# Get a list of providers of the season
season_providers = [
episode["providers"] for episode in exclude_episodes
if episode["season"] == sonarr_season_number
]
season_providers = filters.flatten(season_providers)
# Check if the amount of episodes to exclude is greater or equal the total episodes in Sonarr
if len(exclude_total_episodes) >= sonarr_total_episodes:
season_numbers.append(sonarr_season_number)
seasons_to_exclude.append({
"season":
sonarr_season_number,
"providers":
season_providers,
"monitored":
sonarr_season_monitored,
"has_file":
sonarr_season_has_file,
})
# Re order the exclude_series dict to strip the episodes if the whole season can be excluded
updated_exclude_episodes = [
episode for episode in exclude_episodes
if episode["season"] not in season_numbers
]
exclude_series[exclude_id]["episodes"] = updated_exclude_episodes
exclude_series[exclude_id]["seasons"] = seasons_to_exclude
exclude_series[exclude_id][
"providers"] = filters.get_providers_from_seasons_episodes(
exclude_entry["seasons"], exclude_entry["episodes"])
return exclude_series
