def retrieve_info(self): self.log("attempting to retrieve video info using youtube-dl...") formats = [f for f in YoutubeDLFormats] preferred = YoutubeDLFormats.preferred_format(self.url) if preferred is not None: self.log( fcs(f"trying preferred format first: p<{preferred.value}>", format_chars=["<", ">"])) formats.remove(preferred) formats.insert(0, preferred) for vid_format in formats: self.log(f"trying format: {vid_format.value}") self.options["format"] = vid_format.value self.format = vid_format try: with youtube_dl.YoutubeDL(self.options) as ydl: self.info = ydl.extract_info(self.url, download=False) self.filename = self.generate_filename() self.log( fcs(f"i[success!] generated filename: p[{self.filename}]")) self.log( fcs(f"i<success!> using format: p<{self.format.value}>", format_chars=["<", ">"])) return # succeeded except youtube_dl.utils.DownloadError as error: self.log(error) pass except AttributeError: pass # did not succeed pfcs("e[error] could not retrieve info using youtube-dl!")
def thread_downloader(cli_args): SharedData().set_status(DownloaderStatus.Init) show_list = ScheduledShowList(cli_args) SharedData().set_info(SharedDataKey.ShowList, show_list) if show_list.empty(): print("no shows to process.. exiting.") return if cli_args.simulate: log("running simulation..") if cli_args.set_all_dl: for show in show_list: if show.should_download(print_to_log=False): show.downloaded_today = True log(fcs(f"setting i[{show.name}] as downloaded today")) if cli_args.force_download: for show in show_list: show.download(force=True, simulate=cli_args.simulate) weekday = today_weekday() log(fcs(f"today is b[{Day(weekday).name}]")) while True: if weekday != today_weekday(): log("new day, resetting all show \"downloaded\" flags") for show in show_list: show.reset_downloaded_today() weekday = today_weekday() log(fcs(f"today is b[{Day(weekday).name}]")) log("checking shows...") sleep_to_next_airdate = True for show in show_list: force = show.force_download show.download(simulate=cli_args.simulate, force=force) SharedData().set_info(SharedDataKey.FileName, None) if show.should_download(print_to_log=False): sleep_to_next_airdate = False print_separator() if sleep_to_next_airdate: show = show_list.next_show() sleep_time = show.shortest_airtime() wake_date = get_now() + timedelta(seconds=sleep_time) wake_date_str = date_to_time_str(wake_date) if wake_date.weekday() != weekday: wake_date_str = date_to_full_str(wake_date) log(fcs(f"sleeping p[{show.shortest_airtime()}] (to {wake_date_str}) - " f"next show is b[{show.name}]")) SharedData().set_info(SharedDataKey.NextShow, show.name) else: sleep_time = 60 * 5 # try again in 5 minutes, show has failed dl sleep_time_delta = timedelta(seconds=sleep_time) log(fcs(f"sleeping p[{sleep_time_delta}]")) SharedData().set_status(DownloaderStatus.Sleeping) while sleep_time > 0: if show_list.reloaded: log("breaking sleep, show list was reloaded") break SharedData().set_info(SharedDataKey.NextShowSeconds, sleep_time) sleep_time -= 10 sleep(10) if not SharedData().run: log("stopping downloader") return
def _setup_discovery(self): file_path = ConfigurationManager().path("cookies_txt") if not Path(file_path).exists(): file_path = Path(__file__).resolve().parent / "cookies.txt" if not Path(file_path).exists(): self.log(fcs("e[error]: could not load cookies for discoveryplus")) else: self.log(fcs("loading o[cookie.txt] for discoveryplus")) self.options["cookiefile"] = str(file_path)
def __init__(self, file_path, args): self.path = file_path self.args = args if args.highlight: self.highlighted = str(file_path).replace( args.highlight, fcs(f"b[{args.highlight}]")) else: self.highlighted = "" self.filename = file_path.name self.colorized_filename = fcs(f"i[{file_path.name}]")
def print(self): _name = self.parent_name or self.path.stem _type_str = fcs("o[UNKN]") if self.is_movie: _type_str = fcs("b[MOVI]") elif self.is_tvshow: _type_str = fcs("p[SHOW]") if self.parent_is_season_dir: _name = self.path.stem pfcs(f"i<[{self.index:04d}]> [{_type_str}] {_name}", format_chars=("<", ">"))
def subtitle_url(self) -> list: if self.sub_url_list: self.log("already retrieved/gotten subtitle url") return self.sub_url_list if self.id == 0: self.log("show id is 0, cannot retrive subtitle url") return None if self.sub_m3u_url: self.log("have already retrieved subtitle m3u url", info_str_line2=fcs(f"p[{self.sub_m3u_url}]")) else: self.log("attempting to retrieve subtitle url...") data_url = f"https://playback-api.b17g.net/media/{self.id}?"\ f"service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine" res = SessionSingleton().get(data_url) try: hls_url = res.json()["playbackItem"]["manifestUrl"] self.log("got manifest url:", info_str_line2=fcs(f"p[{hls_url}]")) except KeyError as key_error: self.log("failed to retrieve manifest url from", fcs(f"o[{data_url}]")) return [] hls_data = SessionSingleton().get(hls_url) last_path = hls_url.split("/")[-1] hls_url_prefix = hls_url.replace(last_path, "") self.log("using hls url prefix", hls_url_prefix) for line in hls_data.text.splitlines(): if all([ x in line for x in ["TYPE=SUBTITLES", "URI=", 'LANGUAGE="sv"'] ]): sub_url_suffix = line.split('URI="')[1] sub_url_suffix = sub_url_suffix[:-1] self.sub_m3u_url = hls_url_prefix + "/" + sub_url_suffix break if not self.sub_m3u_url: self.log("could not retrieve subtitle m3u url") return [] self.log("using subtitle m3u url:", info_str_line2=fcs(f"p[{self.sub_m3u_url}]")) res = SessionSingleton().get(self.sub_m3u_url) vtt_files = [] last_path = hls_url.split("/")[-1] hls_url_prefix = hls_url.replace(last_path, "") for line in res.text.splitlines(): if ".webvtt" in line: vtt_files.append(hls_url_prefix + "/" + line) self.sub_url_list = vtt_files if self.sub_url_list: self.log(f"found {len(self.sub_url_list)} vtt files") return self.sub_url_list self.log(fcs("w[WARNING] could not find vtt link in subtitle m3u!")) self.sub_m3u_url = "" return []
def print(self): tot = 0 for bank_str in self.accounts: tot_bank = 0 for _, account in self.accounts[bank_str].items(): account.print() val = account.get_latest_balance().value tot_bank += val tot += val print(fcs(f" o[- {bank_str} total]: {to_kr_str(tot_bank)}")) print_line() print(fcs(f"o[ - total]: {to_kr_str(tot)}"))
def subtitle_url(self): if self.sub_url: self.log("already retrieved/gotten subtitle url") return self.sub_url content_url = f"{self.CONT_URL_PREFIX}{self.episode_path}" res = SessionSingleton().get(content_url) try: stream_url = res.json()['_embedded']['viafreeBlocks'][0][ '_embedded']['program']['_links']['streamLink']['href'] self.log(fcs("got stream url"), fcs(f"i[{stream_url}]")) except KeyError as _: self.log(fcs("failed to retrieve stream url from:"), fcs(f"o[{content_url}]")) return "" res = SessionSingleton().get(stream_url) try: sub_url = self._find_sv_sub(res.json()) if not sub_url: raise ValueError() self.log(fcs("got sv subtitle url"), fcs(f"i[{sub_url}]")) self.sub_url = sub_url except (KeyError, IndexError, ValueError) as _: self.log(fcs("failed to retrieve subtitle url from:"), fcs(f"o[{stream_url}]")) return "" return self.sub_url
def subtitle_url(self): if self.sub_url: self.log("already retrieved/gotten subtitle url") return self.sub_url if self.id == 0: self.log("show id is 0, cannot retrive subtitle url") return "" SessionSingleton().load_cookies_txt() if self.sub_m3u_url: self.log("have already retrieved subtitle m3u url", info_str_line2=fcs(f"p[{self.sub_m3u_url}]")) else: res = SessionSingleton().get( f"{self.API_URL}/playback/videoPlaybackInfo/{self.id}") try: hls_url = res.json( )["data"]["attributes"]["streaming"]["hls"]["url"] self.log("got HLS url:", info_str_line2=fcs(f"p[{hls_url}]")) except KeyError as key_error: self.log("failed to retrieve HLS url from videoPlaybackInfo") return "" hls_data = SessionSingleton().get(hls_url) hls_url_prefix = hls_url.split("playlist.m3u8")[0] self.log("using hls url prefix", hls_url_prefix) for line in hls_data.text.splitlines(): if all([ x in line for x in ["TYPE=SUBTITLES", "URI=", 'LANGUAGE="sv"'] ]): sub_url_suffix = line.split('URI="')[1] sub_url_suffix = sub_url_suffix[:-1] self.sub_m3u_url = hls_url_prefix + "/" + sub_url_suffix break if not self.sub_m3u_url: self.log("could not retrieve subtitle m3u url") return "" self.log("using subtitle m3u url:", info_str_line2=fcs(f"p[{self.sub_m3u_url}]")) res = SessionSingleton().get(self.sub_m3u_url) for line in res.text.splitlines(): if ".vtt" in line: hls_url_prefix = hls_url.split("playlist.m3u8")[0] sub_url = hls_url_prefix + "/" + line self.sub_url = sub_url return self.sub_url self.log(fcs("w[WARNING] could not find vtt link in subtitle m3u!")) self.sub_m3u_url = "" return ""
def __init__(self, data: dict, cli_args): super().__init__(cli_args.verbose) self.raw_data = data self.name = data["name"] self.dest_path = self._parse_dest_path(data["dest"]) self.log(f"set dest path: {self.dest_path}") self.filter_dict = data.get("filter", {}) self.url = data["url"] self.use_title = data.get("use_title", False) self._force_dl = False if cli_args.simulate: self.disabled = random.randint(0, 100) > 90 else: if data.get("process", False): self.disabled = False else: self.disabled = True self.skip_sub = data.get("skip_sub", False) self.downloaded_today = False self.airtimes: List[Airtime] = [] self.set_log_prefix(f"show_{self.name}".replace(" ", "_").upper()) self.log("init") pfcs(f"added show i[{self.name}]") for day, time in data["airtime"].items(): self.airtimes.append(Airtime(day, time)) if not self.disabled: self._validate() else: self.log(fcs("o[is disabled] will not process!"))
def log(info_str, info_str_line2=""): log_prefix = "SCHEDULER" print(fcs(f"i[({log_prefix})]"), f"{get_now_color_str()}:", info_str) if info_str_line2: spaces = " " * len(f"({log_prefix}) ") print(f"{spaces}{info_str_line2}")
def _parse_json_schedule(self): file_path = self._cli_args.json_file if file_path is None: self.error("no ripper schedule file set!") return None if not file_path.exists(): self.error(fcs(f"could not find file: e[{file_path}]")) return None try: with open(file_path) as json_file: self.log(fcs(f"loaded: i[{file_path}]")) return json.load(json_file) except Exception as error: print(error) self.error(fcs(f"could parse json file: e[{file_path}]")) return None
def get_episodes(self, revered_order=False, limit=None): if self.ep_list: return super().get_episodes(revered_order, limit) res = SessionSingleton().get(f"{self.url}") match = re.search(r"__svtplay_apollo'] = ({.*});", res.text) if not match: print("could not parse data!") return [] json_data = json.loads(match.group(1)) season_slug = "" show_name = "" self.log("processing json data...") for key in json_data.keys(): slug = json_data[key].get("slug", "") if slug and slug in self.url: season_slug = slug self.log(f"got slug: {season_slug}") show_name = json_data[key].get("name", "") if show_name: self.log(f"got show name: \"{show_name}\"") else: self.log(fcs(f"w[warning] failed to get show name!")) break episode_keys = self.find_episode_keys(json_data, season_slug) for ep_key in episode_keys: obj = self.key_to_obj(json_data, ep_key) if obj is not None: obj.set_data(show=show_name) self.ep_list.append(obj) return super().get_episodes(revered_order, limit)
def get_episodes(self, revered_order=False, limit=None): if self.ep_list: return super().get_episodes(revered_order, limit) res = self.session.get(self.url) splits = res.text.split("\"programs\":") candidates = [] try: for index, string in enumerate(splits, 0): if index == 0: continue if not string.startswith("["): continue index_of_list_end = string.rfind("]") candidates.append(string[:index_of_list_end + 1]) except Exception as error: self.log(fcs("e[error]"), error) return [] if not candidates: self.log("failed to retrieve episodes!") return [] json_data = self.candidates_to_json(candidates) if not json_data: self.log("failed to retrieve episodes!") return [] for episode_data in json_data: self.ep_list.append( ViafreeEpisodeData(episode_data, verbose=self.print_log)) return super().get_episodes(revered_order, limit)
def show_list(self, one_line: bool = False): _missing_str = fcs(f" o[{self.name} >> MISSING <<]") if not one_line: _missing_str += "\n " id_str = "id: " + cstr(f"{self.data.get('id', 0)}", Color.LightBlue) aired_str = "aired: " + cstr(self.data.get("airdate"), Color.LightBlue) name_str = "\"" + cstr(self.data.get("name"), Color.LightBlue) + "\"" print(f"{_missing_str} {name_str} {id_str} {aired_str}")
def should_download(self, print_to_log=True): if self.downloaded_today: return False sec_to = self.shortest_airtime() if sec_to > 0: delta = timedelta(seconds=sec_to) if print_to_log: self.log(fcs(f"b[{self.name}] airs in i[{delta}]...")) return sec_to < 0
def show_list(self, one_line: bool = False): se_str = f"S{self.season:02d}E{self.episode:02d}" subs_str = "subs: " for sub in SubtitleLang: has = sub in self.subs if sub == SubtitleLang.Unknown: if has: subs_str += cstr(sub.value + " ", Color.LightYellow) continue subs_str += cstr(sub.value + " ", Color.LightGreen if has else Color.DarkGrey) _output = fcs(f" i[{self.filename: <{self.row_len}}]") if not one_line: _output += "\n " _output += fcs(f" b[{se_str}] {subs_str}") print(_output, end=" " if one_line and self.extras else "\n") if self.extras: self.print_extras(only_airdate=one_line)
def key_to_obj(self, json_data: dict, key: str): re_ix = r"\d{3}$" re_part = r"[dD]el+\s(?P<ep_num>\d+)\sav\s\d+" re_url = r"\/(?P<ep_id>\d+).+sasong\-(?P<season_num>\d+)" ep_data = json_data.get(key, {}) if not ep_data: return None determined_ep_number = None determined_season_number = None match = re.search(re_ix, ep_data.get("id", "")) if match: determined_ep_number = int(match.group(0)) match = re.search(re_part, ep_data.get("longDescription", "")) if match: part_num = int(match.groupdict().get("ep_num", None)) if determined_ep_number is not None: if part_num != determined_ep_number: print(f"found several ep numbers for{key}!") determined_ep_number = part_num if determined_ep_number is None: self.log(fcs(f"w[warning] failed to get ep_num for key {key}!")) url_key = ep_data["urls"]["id"] url_str = json_data.get(url_key, {}).get("svtplay", "") if not url_str: self.log(fcs(f"w[warning] failed to get url for ep key {key}!")) return None match = re.search(re_url, url_str) ep_id = None if match: determined_season_number = int(match.groupdict().get( "season_num", None)) ep_id = int(match.groupdict().get("ep_id", None)) if determined_season_number is None: self.log( fcs(f"w[warning] failed to get season_num for url w[{url_str}]!" ), "--> setting to \"S01\"") determined_season_number = 1 obj = SVTPlayEpisodeData() obj.set_data(id=ep_id, title=ep_data.get("name", "N/A"), url=url_str, season_num=determined_season_number, episode_num=determined_ep_number) return obj
def to_kr_str(value: int, color=True, show_prefix=False): ret = f"{value:,}".replace(",", " ") + " kr" if show_prefix: prefix = "+" if value >= 0 else "" else: prefix = "" if color: format_char = "i" if value >= 0 else "r" return fcs(f"{format_char}[{prefix}{ret}]") return prefix + ret
def download(self, destination_path=None): if not self.info: self.warn("no video info available, skipping download!", force=True) return None if destination_path: self.dest_path = destination_path if self.file_already_exists(): self.log(f"file already exists: o[{self.filename}], skipping", force=True) return None if not self.simulate: self.log(fcs(f"downloading to: p[{self.dest_path}]")) retries = 0 while retries < 4: if retries: self.log(fcs(f"retry attempt w[{retries}]")) print("retrying...") time.sleep(1) try: with youtube_dl.YoutubeDL(self.options) as ydl: ydl.params["outtmpl"] = str(self.get_dest_path()) ydl.download([self.url]) if self.file_already_exists(): self.download_succeeded = True return str(self.get_dest_path()) except youtube_dl.utils.DownloadError as error: print("\ngot error during download attempt:\n", error) except KeyboardInterrupt: print("user cancelled...") return None retries += 1 self.log(fcs("e[failed download!]")) return None else: self.log("downloading") self.log(f"dest: {self.get_dest_path()}") self.download_succeeded = True self.log(f"setting download succeeded: {self.download_succeeded}") return str(self.get_dest_path())
def search(self) -> bool: if not self._query: self.error("search: query missing") return False self.log(fcs(f"search: using query i[{self._query}]")) resp = requests.get(self.URL, params={}) if resp.status_code != 200 or not resp.json().get("success", False): self.error("search: get request failed") return False _data = resp.json().get("data", {}) if not _data: self.error("search: no data in result") return False return self._parse_response(_data)
def __init__(self, sub_url, video_file_path, sim=False, verbose=False): super().__init__(verbose=verbose) self.set_log_prefix("SUB_DOWNLOAD") if sim: self.set_log_prefix_2("SIMULATED") self.url = sub_url self.print_log = verbose self.video_file_path = video_file_path self.simulate = sim self.filename = self.determine_file_name() self.dest_path = Path(self.video_file_path).parent self.download_succeeded = False self.data_obj = None self.log(fcs(f"using filename p[{self.filename}]")) self.srt_index = 1
def search(self) -> bool: if not self._query: self.error("search: query missing") return False self.log(fcs(f"search: using query i[{self._query}]")) _url = self.URL resp = requests.get(_url, params={ "searchstr": self._query, "searchcat": "all" }) if resp.status_code != 200: self.error("search: get request failed") return False if not resp.text: return False return self._parse_response(resp.text)
def _find_sv_sub(self, data: dict): try: _list = data["embedded"]["subtitles"] except IndexError as error: self.warn("could not find subtitle list for episode!") return None if not isinstance(_list, list): self.warn('data["embedded"]["subtitles"] was not a list!') return None sv_url = None for _sub_data in _list: data = _sub_data.get("data", {}) lang = data.get("language", None) if lang: self.log(fcs(f"found sub for language: i[{lang}]")) if lang == "sv": sv_url = _sub_data.get("link", {}).get("href", None) return sv_url
def _validate(self): if not self.dest_path.exists(): self.warn(fcs(f"no such destination dir: w[{self.dest_path}] : ({self.name})"))
def log_error(self, err_str, format_string=False, force=False): self._print_with_tag(err_str, tag=fcs("e[error]"), format_string=format_string, force=force)
def log_warn(self, warn_str, format_string=False, force=False): self._print_with_tag(warn_str, tag=fcs("w[warning]"), format_string=format_string, force=force)
def download(self, force=False, simulate=False): if not force and not self.should_download(): return False if force: self.log("forced download...") self.log(fcs(f"trying to download episodes for i[{self.name}]")) objects = self.get_url_objects() if not objects: self.warn(fcs(f"failed to retrieve urls for i[{self.name}], setting as downloaded...")) self.downloaded_today = True return False for obj in objects: SharedData().set_status(DownloaderStatus.ProcessingShow) ripper = PlayRipperYoutubeDl(obj.url(), sim=simulate, dest=self.dest_path, ep_data=obj, verbose=True, use_title=self.use_title) if not ripper.file_already_exists(): try: SharedData().set_status(DownloaderStatus.Downloading) SharedData().set_info(SharedDataKey.FileName, ripper.filename or "None") file_path = ripper.download() SharedData().set_status(DownloaderStatus.ProcessingShow) except Exception as error: self.error(fcs(f"got exception when trying to download {self.name}")) SharedData().add_error(str(error)) return False if file_path and ripper.download_succeeded: log_entry = DownloadedShowLogEntry(timestamp=get_now_str(), show_name=self.name, file_path=file_path) SharedData().add_downloaded_item(log_entry) self.log(fcs(f"downloaded: i[{str(file_path)}]")) self.downloaded_today = True if not simulate: write_to_log(log_entry) else: file_path = ripper.get_dest_path() self.log(fcs(f"i[{file_path.name}] already exists, skipping dl...")) if file_path and not self.skip_sub and not simulate: existing = SubRipper.vid_file_has_subs(file_path) if existing is not False: self.log(fcs(f"i[{existing.name}] already exists, skipping sub dl...")) print_separator() continue sub_rip = SubRipper(retrive_sub_url(obj), str(file_path), verbose=True) if not sub_rip.file_already_exists(): self.log(fcs(f"trying to download subtitles: i[{sub_rip.filename}]")) try: SharedData().set_status(DownloaderStatus.DownloadingSubtitles) SharedData().set_info(SharedDataKey.FileName, sub_rip.filename) sub_rip.download() SharedData().set_status(DownloaderStatus.ProcessingShow) except Exception as error: self.error(fcs(f"got exception when trying to download subs for {self.name}")) SharedData().add_error(str(error)) else: self.log(fcs(f"i[{sub_rip.filename}] already exists, skipping sub dl...")) elif self.skip_sub: self.log(fcs(f"skipping subtitle download for i[{self.name}]")) print_separator() return True
def print(self, show_date=True, end="\n"): if show_date: suffix = fcs(f" d[({self.date.strftime(DATE_FMT)})]") else: suffix = "" print(to_kr_str(self.value) + suffix, end=end)
def print_change(self, other): change = self.value - other.value percentage = round(self.value / other.value * 100 - 100, 2) print("->", to_kr_str(change, show_prefix=True), "since", other.date.strftime(DATE_FMT), fcs(f"d[({percentage:.2f} %)]"))