def retrieve_info(self):
self.log("attempting to retrieve video info using youtube-dl...")
formats = [f for f in YoutubeDLFormats]
preferred = YoutubeDLFormats.preferred_format(self.url)
if preferred is not None:
self.log(
fcs(f"trying preferred format first: p<{preferred.value}>",
format_chars=["<", ">"]))
formats.remove(preferred)
formats.insert(0, preferred)
for vid_format in formats:
self.log(f"trying format: {vid_format.value}")
self.options["format"] = vid_format.value
self.format = vid_format
try:
with youtube_dl.YoutubeDL(self.options) as ydl:
self.info = ydl.extract_info(self.url, download=False)
self.filename = self.generate_filename()
self.log(
fcs(f"i[success!] generated filename: p[{self.filename}]"))
self.log(
fcs(f"i<success!> using format: p<{self.format.value}>",
format_chars=["<", ">"]))
return # succeeded
except youtube_dl.utils.DownloadError as error:
self.log(error)
pass
except AttributeError:
pass
# did not succeed
pfcs("e[error] could not retrieve info using youtube-dl!")
def thread_downloader(cli_args):
SharedData().set_status(DownloaderStatus.Init)
show_list = ScheduledShowList(cli_args)
SharedData().set_info(SharedDataKey.ShowList, show_list)
if show_list.empty():
print("no shows to process.. exiting.")
return
if cli_args.simulate:
log("running simulation..")
if cli_args.set_all_dl:
for show in show_list:
if show.should_download(print_to_log=False):
show.downloaded_today = True
log(fcs(f"setting i[{show.name}] as downloaded today"))
if cli_args.force_download:
for show in show_list:
show.download(force=True, simulate=cli_args.simulate)
weekday = today_weekday()
log(fcs(f"today is b[{Day(weekday).name}]"))
while True:
if weekday != today_weekday():
log("new day, resetting all show \"downloaded\" flags")
for show in show_list:
show.reset_downloaded_today()
weekday = today_weekday()
log(fcs(f"today is b[{Day(weekday).name}]"))
log("checking shows...")
sleep_to_next_airdate = True
for show in show_list:
force = show.force_download
show.download(simulate=cli_args.simulate, force=force)
SharedData().set_info(SharedDataKey.FileName, None)
if show.should_download(print_to_log=False):
sleep_to_next_airdate = False
print_separator()
if sleep_to_next_airdate:
show = show_list.next_show()
sleep_time = show.shortest_airtime()
wake_date = get_now() + timedelta(seconds=sleep_time)
wake_date_str = date_to_time_str(wake_date)
if wake_date.weekday() != weekday:
wake_date_str = date_to_full_str(wake_date)
log(fcs(f"sleeping p[{show.shortest_airtime()}] (to {wake_date_str}) - "
f"next show is b[{show.name}]"))
SharedData().set_info(SharedDataKey.NextShow, show.name)
else:
sleep_time = 60 * 5 # try again in 5 minutes, show has failed dl
sleep_time_delta = timedelta(seconds=sleep_time)
log(fcs(f"sleeping p[{sleep_time_delta}]"))
SharedData().set_status(DownloaderStatus.Sleeping)
while sleep_time > 0:
if show_list.reloaded:
log("breaking sleep, show list was reloaded")
break
SharedData().set_info(SharedDataKey.NextShowSeconds, sleep_time)
sleep_time -= 10
sleep(10)
if not SharedData().run:
log("stopping downloader")
return
def _setup_discovery(self):
file_path = ConfigurationManager().path("cookies_txt")
if not Path(file_path).exists():
file_path = Path(__file__).resolve().parent / "cookies.txt"
if not Path(file_path).exists():
self.log(fcs("e[error]: could not load cookies for discoveryplus"))
else:
self.log(fcs("loading o[cookie.txt] for discoveryplus"))
self.options["cookiefile"] = str(file_path)
def __init__(self, file_path, args):
self.path = file_path
self.args = args
if args.highlight:
self.highlighted = str(file_path).replace(
args.highlight, fcs(f"b[{args.highlight}]"))
else:
self.highlighted = ""
self.filename = file_path.name
self.colorized_filename = fcs(f"i[{file_path.name}]")
def print(self):
_name = self.parent_name or self.path.stem
_type_str = fcs("o[UNKN]")
if self.is_movie:
_type_str = fcs("b[MOVI]")
elif self.is_tvshow:
_type_str = fcs("p[SHOW]")
if self.parent_is_season_dir:
_name = self.path.stem
pfcs(f"i<[{self.index:04d}]> [{_type_str}] {_name}", format_chars=("<", ">"))
def subtitle_url(self) -> list:
if self.sub_url_list:
self.log("already retrieved/gotten subtitle url")
return self.sub_url_list
if self.id == 0:
self.log("show id is 0, cannot retrive subtitle url")
return None
if self.sub_m3u_url:
self.log("have already retrieved subtitle m3u url",
info_str_line2=fcs(f"p[{self.sub_m3u_url}]"))
else:
self.log("attempting to retrieve subtitle url...")
data_url = f"https://playback-api.b17g.net/media/{self.id}?"\
f"service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine"
res = SessionSingleton().get(data_url)
try:
hls_url = res.json()["playbackItem"]["manifestUrl"]
self.log("got manifest url:",
info_str_line2=fcs(f"p[{hls_url}]"))
except KeyError as key_error:
self.log("failed to retrieve manifest url from",
fcs(f"o[{data_url}]"))
return []
hls_data = SessionSingleton().get(hls_url)
last_path = hls_url.split("/")[-1]
hls_url_prefix = hls_url.replace(last_path, "")
self.log("using hls url prefix", hls_url_prefix)
for line in hls_data.text.splitlines():
if all([
x in line
for x in ["TYPE=SUBTITLES", "URI=", 'LANGUAGE="sv"']
]):
sub_url_suffix = line.split('URI="')[1]
sub_url_suffix = sub_url_suffix[:-1]
self.sub_m3u_url = hls_url_prefix + "/" + sub_url_suffix
break
if not self.sub_m3u_url:
self.log("could not retrieve subtitle m3u url")
return []
self.log("using subtitle m3u url:",
info_str_line2=fcs(f"p[{self.sub_m3u_url}]"))
res = SessionSingleton().get(self.sub_m3u_url)
vtt_files = []
last_path = hls_url.split("/")[-1]
hls_url_prefix = hls_url.replace(last_path, "")
for line in res.text.splitlines():
if ".webvtt" in line:
vtt_files.append(hls_url_prefix + "/" + line)
self.sub_url_list = vtt_files
if self.sub_url_list:
self.log(f"found {len(self.sub_url_list)} vtt files")
return self.sub_url_list
self.log(fcs("w[WARNING] could not find vtt link in subtitle m3u!"))
self.sub_m3u_url = ""
return []
def print(self):
tot = 0
for bank_str in self.accounts:
tot_bank = 0
for _, account in self.accounts[bank_str].items():
account.print()
val = account.get_latest_balance().value
tot_bank += val
tot += val
print(fcs(f" o[- {bank_str} total]: {to_kr_str(tot_bank)}"))
print_line()
print(fcs(f"o[ - total]: {to_kr_str(tot)}"))
def subtitle_url(self):
if self.sub_url:
self.log("already retrieved/gotten subtitle url")
return self.sub_url
content_url = f"{self.CONT_URL_PREFIX}{self.episode_path}"
res = SessionSingleton().get(content_url)
try:
stream_url = res.json()['_embedded']['viafreeBlocks'][0][
'_embedded']['program']['_links']['streamLink']['href']
self.log(fcs("got stream url"), fcs(f"i[{stream_url}]"))
except KeyError as _:
self.log(fcs("failed to retrieve stream url from:"),
fcs(f"o[{content_url}]"))
return ""
res = SessionSingleton().get(stream_url)
try:
sub_url = self._find_sv_sub(res.json())
if not sub_url:
raise ValueError()
self.log(fcs("got sv subtitle url"), fcs(f"i[{sub_url}]"))
self.sub_url = sub_url
except (KeyError, IndexError, ValueError) as _:
self.log(fcs("failed to retrieve subtitle url from:"),
fcs(f"o[{stream_url}]"))
return ""
return self.sub_url
def subtitle_url(self):
if self.sub_url:
self.log("already retrieved/gotten subtitle url")
return self.sub_url
if self.id == 0:
self.log("show id is 0, cannot retrive subtitle url")
return ""
SessionSingleton().load_cookies_txt()
if self.sub_m3u_url:
self.log("have already retrieved subtitle m3u url",
info_str_line2=fcs(f"p[{self.sub_m3u_url}]"))
else:
res = SessionSingleton().get(
f"{self.API_URL}/playback/videoPlaybackInfo/{self.id}")
try:
hls_url = res.json(
)["data"]["attributes"]["streaming"]["hls"]["url"]
self.log("got HLS url:", info_str_line2=fcs(f"p[{hls_url}]"))
except KeyError as key_error:
self.log("failed to retrieve HLS url from videoPlaybackInfo")
return ""
hls_data = SessionSingleton().get(hls_url)
hls_url_prefix = hls_url.split("playlist.m3u8")[0]
self.log("using hls url prefix", hls_url_prefix)
for line in hls_data.text.splitlines():
if all([
x in line
for x in ["TYPE=SUBTITLES", "URI=", 'LANGUAGE="sv"']
]):
sub_url_suffix = line.split('URI="')[1]
sub_url_suffix = sub_url_suffix[:-1]
self.sub_m3u_url = hls_url_prefix + "/" + sub_url_suffix
break
if not self.sub_m3u_url:
self.log("could not retrieve subtitle m3u url")
return ""
self.log("using subtitle m3u url:",
info_str_line2=fcs(f"p[{self.sub_m3u_url}]"))
res = SessionSingleton().get(self.sub_m3u_url)
for line in res.text.splitlines():
if ".vtt" in line:
hls_url_prefix = hls_url.split("playlist.m3u8")[0]
sub_url = hls_url_prefix + "/" + line
self.sub_url = sub_url
return self.sub_url
self.log(fcs("w[WARNING] could not find vtt link in subtitle m3u!"))
self.sub_m3u_url = ""
return ""
def __init__(self, data: dict, cli_args):
super().__init__(cli_args.verbose)
self.raw_data = data
self.name = data["name"]
self.dest_path = self._parse_dest_path(data["dest"])
self.log(f"set dest path: {self.dest_path}")
self.filter_dict = data.get("filter", {})
self.url = data["url"]
self.use_title = data.get("use_title", False)
self._force_dl = False
if cli_args.simulate:
self.disabled = random.randint(0, 100) > 90
else:
if data.get("process", False):
self.disabled = False
else:
self.disabled = True
self.skip_sub = data.get("skip_sub", False)
self.downloaded_today = False
self.airtimes: List[Airtime] = []
self.set_log_prefix(f"show_{self.name}".replace(" ", "_").upper())
self.log("init")
pfcs(f"added show i[{self.name}]")
for day, time in data["airtime"].items():
self.airtimes.append(Airtime(day, time))
if not self.disabled:
self._validate()
else:
self.log(fcs("o[is disabled] will not process!"))
def log(info_str, info_str_line2=""):
log_prefix = "SCHEDULER"
print(fcs(f"i[({log_prefix})]"),
f"{get_now_color_str()}:", info_str)
if info_str_line2:
spaces = " " * len(f"({log_prefix}) ")
print(f"{spaces}{info_str_line2}")
def _parse_json_schedule(self):
file_path = self._cli_args.json_file
if file_path is None:
self.error("no ripper schedule file set!")
return None
if not file_path.exists():
self.error(fcs(f"could not find file: e[{file_path}]"))
return None
try:
with open(file_path) as json_file:
self.log(fcs(f"loaded: i[{file_path}]"))
return json.load(json_file)
except Exception as error:
print(error)
self.error(fcs(f"could parse json file: e[{file_path}]"))
return None
def get_episodes(self, revered_order=False, limit=None):
if self.ep_list:
return super().get_episodes(revered_order, limit)
res = SessionSingleton().get(f"{self.url}")
match = re.search(r"__svtplay_apollo'] = ({.*});", res.text)
if not match:
print("could not parse data!")
return []
json_data = json.loads(match.group(1))
season_slug = ""
show_name = ""
self.log("processing json data...")
for key in json_data.keys():
slug = json_data[key].get("slug", "")
if slug and slug in self.url:
season_slug = slug
self.log(f"got slug: {season_slug}")
show_name = json_data[key].get("name", "")
if show_name:
self.log(f"got show name: \"{show_name}\"")
else:
self.log(fcs(f"w[warning] failed to get show name!"))
break
episode_keys = self.find_episode_keys(json_data, season_slug)
for ep_key in episode_keys:
obj = self.key_to_obj(json_data, ep_key)
if obj is not None:
obj.set_data(show=show_name)
self.ep_list.append(obj)
return super().get_episodes(revered_order, limit)
def get_episodes(self, revered_order=False, limit=None):
if self.ep_list:
return super().get_episodes(revered_order, limit)
res = self.session.get(self.url)
splits = res.text.split("\"programs\":")
candidates = []
try:
for index, string in enumerate(splits, 0):
if index == 0:
continue
if not string.startswith("["):
continue
index_of_list_end = string.rfind("]")
candidates.append(string[:index_of_list_end + 1])
except Exception as error:
self.log(fcs("e[error]"), error)
return []
if not candidates:
self.log("failed to retrieve episodes!")
return []
json_data = self.candidates_to_json(candidates)
if not json_data:
self.log("failed to retrieve episodes!")
return []
for episode_data in json_data:
self.ep_list.append(
ViafreeEpisodeData(episode_data, verbose=self.print_log))
return super().get_episodes(revered_order, limit)
def show_list(self, one_line: bool = False):
_missing_str = fcs(f" o[{self.name} >> MISSING <<]")
if not one_line:
_missing_str += "\n "
id_str = "id: " + cstr(f"{self.data.get('id', 0)}", Color.LightBlue)
aired_str = "aired: " + cstr(self.data.get("airdate"), Color.LightBlue)
name_str = "\"" + cstr(self.data.get("name"), Color.LightBlue) + "\""
print(f"{_missing_str} {name_str} {id_str} {aired_str}")
def should_download(self, print_to_log=True):
if self.downloaded_today:
return False
sec_to = self.shortest_airtime()
if sec_to > 0:
delta = timedelta(seconds=sec_to)
if print_to_log:
self.log(fcs(f"b[{self.name}] airs in i[{delta}]..."))
return sec_to < 0
def show_list(self, one_line: bool = False):
se_str = f"S{self.season:02d}E{self.episode:02d}"
subs_str = "subs: "
for sub in SubtitleLang:
has = sub in self.subs
if sub == SubtitleLang.Unknown:
if has:
subs_str += cstr(sub.value + " ", Color.LightYellow)
continue
subs_str += cstr(sub.value + " ",
Color.LightGreen if has else Color.DarkGrey)
_output = fcs(f" i[{self.filename: <{self.row_len}}]")
if not one_line:
_output += "\n "
_output += fcs(f" b[{se_str}] {subs_str}")
print(_output, end=" " if one_line and self.extras else "\n")
if self.extras:
self.print_extras(only_airdate=one_line)
def key_to_obj(self, json_data: dict, key: str):
re_ix = r"\d{3}$"
re_part = r"[dD]el+\s(?P<ep_num>\d+)\sav\s\d+"
re_url = r"\/(?P<ep_id>\d+).+sasong\-(?P<season_num>\d+)"
ep_data = json_data.get(key, {})
if not ep_data:
return None
determined_ep_number = None
determined_season_number = None
match = re.search(re_ix, ep_data.get("id", ""))
if match:
determined_ep_number = int(match.group(0))
match = re.search(re_part, ep_data.get("longDescription", ""))
if match:
part_num = int(match.groupdict().get("ep_num", None))
if determined_ep_number is not None:
if part_num != determined_ep_number:
print(f"found several ep numbers for{key}!")
determined_ep_number = part_num
if determined_ep_number is None:
self.log(fcs(f"w[warning] failed to get ep_num for key {key}!"))
url_key = ep_data["urls"]["id"]
url_str = json_data.get(url_key, {}).get("svtplay", "")
if not url_str:
self.log(fcs(f"w[warning] failed to get url for ep key {key}!"))
return None
match = re.search(re_url, url_str)
ep_id = None
if match:
determined_season_number = int(match.groupdict().get(
"season_num", None))
ep_id = int(match.groupdict().get("ep_id", None))
if determined_season_number is None:
self.log(
fcs(f"w[warning] failed to get season_num for url w[{url_str}]!"
), "--> setting to \"S01\"")
determined_season_number = 1
obj = SVTPlayEpisodeData()
obj.set_data(id=ep_id,
title=ep_data.get("name", "N/A"),
url=url_str,
season_num=determined_season_number,
episode_num=determined_ep_number)
return obj
def to_kr_str(value: int, color=True, show_prefix=False):
ret = f"{value:,}".replace(",", " ") + " kr"
if show_prefix:
prefix = "+" if value >= 0 else ""
else:
prefix = ""
if color:
format_char = "i" if value >= 0 else "r"
return fcs(f"{format_char}[{prefix}{ret}]")
return prefix + ret
def download(self, destination_path=None):
if not self.info:
self.warn("no video info available, skipping download!",
force=True)
return None
if destination_path:
self.dest_path = destination_path
if self.file_already_exists():
self.log(f"file already exists: o[{self.filename}], skipping",
force=True)
return None
if not self.simulate:
self.log(fcs(f"downloading to: p[{self.dest_path}]"))
retries = 0
while retries < 4:
if retries:
self.log(fcs(f"retry attempt w[{retries}]"))
print("retrying...")
time.sleep(1)
try:
with youtube_dl.YoutubeDL(self.options) as ydl:
ydl.params["outtmpl"] = str(self.get_dest_path())
ydl.download([self.url])
if self.file_already_exists():
self.download_succeeded = True
return str(self.get_dest_path())
except youtube_dl.utils.DownloadError as error:
print("\ngot error during download attempt:\n", error)
except KeyboardInterrupt:
print("user cancelled...")
return None
retries += 1
self.log(fcs("e[failed download!]"))
return None
else:
self.log("downloading")
self.log(f"dest: {self.get_dest_path()}")
self.download_succeeded = True
self.log(f"setting download succeeded: {self.download_succeeded}")
return str(self.get_dest_path())
def search(self) -> bool:
if not self._query:
self.error("search: query missing")
return False
self.log(fcs(f"search: using query i[{self._query}]"))
resp = requests.get(self.URL, params={})
if resp.status_code != 200 or not resp.json().get("success", False):
self.error("search: get request failed")
return False
_data = resp.json().get("data", {})
if not _data:
self.error("search: no data in result")
return False
return self._parse_response(_data)
def __init__(self, sub_url, video_file_path, sim=False, verbose=False):
super().__init__(verbose=verbose)
self.set_log_prefix("SUB_DOWNLOAD")
if sim:
self.set_log_prefix_2("SIMULATED")
self.url = sub_url
self.print_log = verbose
self.video_file_path = video_file_path
self.simulate = sim
self.filename = self.determine_file_name()
self.dest_path = Path(self.video_file_path).parent
self.download_succeeded = False
self.data_obj = None
self.log(fcs(f"using filename p[{self.filename}]"))
self.srt_index = 1
def search(self) -> bool:
if not self._query:
self.error("search: query missing")
return False
self.log(fcs(f"search: using query i[{self._query}]"))
_url = self.URL
resp = requests.get(_url,
params={
"searchstr": self._query,
"searchcat": "all"
})
if resp.status_code != 200:
self.error("search: get request failed")
return False
if not resp.text:
return False
return self._parse_response(resp.text)
def _find_sv_sub(self, data: dict):
try:
_list = data["embedded"]["subtitles"]
except IndexError as error:
self.warn("could not find subtitle list for episode!")
return None
if not isinstance(_list, list):
self.warn('data["embedded"]["subtitles"] was not a list!')
return None
sv_url = None
for _sub_data in _list:
data = _sub_data.get("data", {})
lang = data.get("language", None)
if lang:
self.log(fcs(f"found sub for language: i[{lang}]"))
if lang == "sv":
sv_url = _sub_data.get("link", {}).get("href", None)
return sv_url
def _validate(self):
if not self.dest_path.exists():
self.warn(fcs(f"no such destination dir: w[{self.dest_path}] : ({self.name})"))
def log_error(self, err_str, format_string=False, force=False):
self._print_with_tag(err_str,
tag=fcs("e[error]"),
format_string=format_string,
force=force)
def log_warn(self, warn_str, format_string=False, force=False):
self._print_with_tag(warn_str,
tag=fcs("w[warning]"),
format_string=format_string,
force=force)
def download(self, force=False, simulate=False):
if not force and not self.should_download():
return False
if force:
self.log("forced download...")
self.log(fcs(f"trying to download episodes for i[{self.name}]"))
objects = self.get_url_objects()
if not objects:
self.warn(fcs(f"failed to retrieve urls for i[{self.name}], setting as downloaded..."))
self.downloaded_today = True
return False
for obj in objects:
SharedData().set_status(DownloaderStatus.ProcessingShow)
ripper = PlayRipperYoutubeDl(obj.url(),
sim=simulate,
dest=self.dest_path,
ep_data=obj,
verbose=True,
use_title=self.use_title)
if not ripper.file_already_exists():
try:
SharedData().set_status(DownloaderStatus.Downloading)
SharedData().set_info(SharedDataKey.FileName, ripper.filename or "None")
file_path = ripper.download()
SharedData().set_status(DownloaderStatus.ProcessingShow)
except Exception as error:
self.error(fcs(f"got exception when trying to download {self.name}"))
SharedData().add_error(str(error))
return False
if file_path and ripper.download_succeeded:
log_entry = DownloadedShowLogEntry(timestamp=get_now_str(),
show_name=self.name,
file_path=file_path)
SharedData().add_downloaded_item(log_entry)
self.log(fcs(f"downloaded: i[{str(file_path)}]"))
self.downloaded_today = True
if not simulate:
write_to_log(log_entry)
else:
file_path = ripper.get_dest_path()
self.log(fcs(f"i[{file_path.name}] already exists, skipping dl..."))
if file_path and not self.skip_sub and not simulate:
existing = SubRipper.vid_file_has_subs(file_path)
if existing is not False:
self.log(fcs(f"i[{existing.name}] already exists, skipping sub dl..."))
print_separator()
continue
sub_rip = SubRipper(retrive_sub_url(obj), str(file_path), verbose=True)
if not sub_rip.file_already_exists():
self.log(fcs(f"trying to download subtitles: i[{sub_rip.filename}]"))
try:
SharedData().set_status(DownloaderStatus.DownloadingSubtitles)
SharedData().set_info(SharedDataKey.FileName, sub_rip.filename)
sub_rip.download()
SharedData().set_status(DownloaderStatus.ProcessingShow)
except Exception as error:
self.error(fcs(f"got exception when trying to download subs for {self.name}"))
SharedData().add_error(str(error))
else:
self.log(fcs(f"i[{sub_rip.filename}] already exists, skipping sub dl..."))
elif self.skip_sub:
self.log(fcs(f"skipping subtitle download for i[{self.name}]"))
print_separator()
return True
def print(self, show_date=True, end="\n"):
if show_date:
suffix = fcs(f" d[({self.date.strftime(DATE_FMT)})]")
else:
suffix = ""
print(to_kr_str(self.value) + suffix, end=end)
def print_change(self, other):
change = self.value - other.value
percentage = round(self.value / other.value * 100 - 100, 2)
print("->", to_kr_str(change, show_prefix=True), "since",
other.date.strftime(DATE_FMT), fcs(f"d[({percentage:.2f} %)]"))
