def test_youtube_download_parallel(tmp_path): def download_and_assert(url, outtmpl, yt_downloader): options = BestMp4.get_options( filepath=outtmpl, ) yt_downloader.download(url, options) assert outtmpl.with_suffix(".mp4").exists() yt_downloader = YoutubeDownloader(threads=2) videos = { "Bc5QSUhL6co": tmp_path / "video1.%(ext)s", "a3HZ8S2H-GQ": tmp_path / "video2.%(ext)s", "3HFBR0UQPes": tmp_path / "video3.%(ext)s", "oiWWKumrLH8": tmp_path / "video4.%(ext)s", } with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: fs = [ executor.submit(download_and_assert, key, val, yt_downloader) for key, val in videos.items() ] done, not_done = concurrent.futures.wait( fs, return_when=concurrent.futures.ALL_COMPLETED ) assert len(done) == 4 for future in done: assert future.exception() is None yt_downloader.shutdown()
def download_video_files_parallel(self): """download videos and images parallely""" self.yt_downloader = YoutubeDownloader(threads=1) with concurrent.futures.ThreadPoolExecutor( max_workers=self.threads ) as executor: fs = [ executor.submit(self.download_video_files, video) for video in self.videos ] concurrent.futures.wait(fs, return_when=concurrent.futures.ALL_COMPLETED) self.yt_downloader.shutdown()
def test_youtube_download_contextmanager(nb_workers, videos, tmp_path): with YoutubeDownloader(threads=nb_workers) as yt_downloader: assert yt_downloader.executor._max_workers == nb_workers yt_downloader.download("Bc5QSUhL6co", BestMp4.get_options(target_dir=tmp_path)) fs = [ yt_downloader.download(video, BestMp4.get_options(target_dir=tmp_path), wait=False) for video in videos ] done, not_done = concurrent.futures.wait( fs, return_when=concurrent.futures.ALL_COMPLETED) assert len(done) == len(videos) and len(not_done) == 0
def test_youtube_download_nowait(tmp_path): with YoutubeDownloader(threads=1) as yt_downloader: future = yt_downloader.download( "https://vimeo.com/619427082", BestMp4.get_options(target_dir=tmp_path), wait=False, ) assert future.running() assert not yt_downloader.executor._shutdown done, not_done = concurrent.futures.wait( [future], return_when=concurrent.futures.ALL_COMPLETED) assert future.exception() is None assert len(done) == 1 assert len(not_done) == 0
def test_youtube_download_serial(url, video_id, tmp_path): yt_downloader = YoutubeDownloader(threads=1) options = BestWebm.get_options( target_dir=tmp_path, filepath=pathlib.Path("%(id)s/video.%(ext)s"), ) yt_downloader.download(url, options) assert tmp_path.joinpath(video_id).joinpath("video.webm").exists() yt_downloader.shutdown()
class Ted2Zim: def __init__( self, topics, debug, name, video_format, low_quality, output_dir, no_zim, fname, languages, locale_name, title, description, creator, publisher, tags, keep_build_dir, autoplay, use_any_optimized_version, s3_url_with_credentials, playlist, subtitles_enough, subtitles_setting, tmp_dir, threads, ): # video-encoding info self.video_format = video_format self.low_quality = low_quality # zim params self.fname = fname self.languages = ( [] if languages is None else [l.strip() for l in languages.split(",")] ) self.tags = [] if tags is None else [t.strip() for t in tags.split(",")] self.title = title self.description = description self.creator = creator self.publisher = publisher self.name = name # directory setup self.output_dir = pathlib.Path(output_dir).expanduser().resolve() if tmp_dir: pathlib.Path(tmp_dir).mkdir(parents=True, exist_ok=True) self.build_dir = pathlib.Path(tempfile.mkdtemp(dir=tmp_dir)) # scraper options self.topics = ( [] if not topics else [c.strip().replace(" ", "+") for c in topics.split(",")] ) self.autoplay = autoplay self.playlist = playlist self.subtitles_enough = subtitles_enough self.subtitles_setting = ( subtitles_setting if subtitles_setting == ALL or subtitles_setting == MATCHING or subtitles_setting == NONE else self.to_ted_langcodes( [lang.strip() for lang in subtitles_setting.split(",")] ) ) self.threads = threads self.yt_downloader = None # optimization cache self.s3_url_with_credentials = s3_url_with_credentials self.use_any_optimized_version = use_any_optimized_version self.s3_storage = None self.video_quality = "low" if self.low_quality else "high" # debug/developer options self.no_zim = no_zim self.keep_build_dir = keep_build_dir self.debug = debug # class members self.videos = [] self.playlist_title = None self.playlist_description = None self.source_languages = ( [] if not self.languages else self.to_ted_langcodes(self.languages) ) self.zim_lang = None self.already_visited = [] # set and record locale for translations locale_details = get_language_details(locale_name) if locale_details["querytype"] != "locale": locale_name = locale_details["iso-639-1"] try: self.locale = setlocale(ROOT_DIR, locale_name) except locale.Error: logger.error( f"No locale for {locale_name}. Use --locale to specify it. " "defaulting to en_US" ) self.locale = setlocale(ROOT_DIR, "en") # locale's language code self.locale_name = self.to_ted_langcodes(locale_name) @property def templates_dir(self): return ROOT_DIR.joinpath("templates") @property def videos_dir(self): return self.build_dir.joinpath("videos") @property def ted_videos_json(self): return self.build_dir.joinpath("ted_videos.json") @property def ted_topics_json(self): return self.build_dir.joinpath("ted_topics.json") @property def talks_base_url(self): return BASE_URL + "talks" @property def playlists_base_url(self): return BASE_URL + "playlists" def append_part1_or_part3(self, lang_code_list, lang_info): """Fills missing ISO languages codes for all in list lang_code_list: list og lang codes lang_info: see zimscraperlib.i18n""" # ignore extra language mappings if supplied query was an iso-639-1 code if "part1" in lang_info["iso_types"]: lang_code_list.append(lang_info["iso-639-1"]) # supplied query was not iso-639-1 else: if lang_info["iso-639-1"]: lang_code_list.append(lang_info["iso-639-1"]) # check for extra language codes to include if lang_info["iso-639-1"] in TEDLANGS["mappings"]: for code in TEDLANGS["mappings"][lang_info["iso-639-1"]]: lang_code_list.append(code) elif lang_info["iso-639-3"]: lang_code_list.append(lang_info["iso-639-3"]) else: supplied_lang = lang_info["query"] logger.error(f"Language {supplied_lang} is not supported by TED") def to_ted_langcodes(self, languages): """Converts languages queries into TED language codes Examples: ["English", "fr", "hin"] => ["en", "fr", "hi"] ["chi", "fake"] => ["zh", "zh-cn", "zh-tw"] """ lang_code_list = [] for lang in languages: lang_info = get_language_details(lang, failsafe=True) if lang_info: if lang_info["querytype"] == "purecode": self.append_part1_or_part3(lang_code_list, lang_info) elif lang_info["querytype"] == "locale": query = lang_info["query"].replace("_", "-") if query in TEDLANGS["locales"]: lang_code_list.append(query) else: self.append_part1_or_part3(lang_code_list, lang_info) else: self.append_part1_or_part3(lang_code_list, lang_info) return list(set(lang_code_list)) def extract_videos_from_playlist(self, playlist): """extracts metadata for all videos in the given playlist calls extract_video_info on all links to get this data """ playlist_url = f"{self.playlists_base_url}/{playlist}" logger.debug(f"extract_videos_from_playlist: {playlist_url}") soup = BeautifulSoup(download_link(playlist_url).text, features="html.parser") video_elements = soup.find_all("a", attrs={"class": "group"}) self.playlist_title = soup.find("h1").string self.playlist_description = soup.find("p", attrs={"class": "text-base"}).string for element in video_elements: relative_path = element.get("href") url = urllib.parse.urljoin(self.talks_base_url, relative_path) if self.extract_info_from_video_page(url): if self.source_languages and len(self.source_languages) > 1: other_lang_urls = self.generate_urls_for_other_languages(url) logger.debug( f"Searching info for the video in other {len(other_lang_urls)} language(s)" ) for lang_url in other_lang_urls: self.extract_info_from_video_page(lang_url) self.already_visited.append(urllib.parse.urlparse(url)[2]) logger.debug(f"Seen {relative_path}") logger.debug(f"Total videos found on playlist: {len(video_elements)}") if not video_elements: raise ValueError("Wrong playlist ID supplied. No videos found") def generate_search_result_and_scrape(self, topic_url, total_videos_scraped): """generates a search result and returns the total number of videos scraped""" page = 1 while True: logger.debug(f"generate_search_result_and_scrape: {topic_url}&page={page}") html = download_link(f"{topic_url}&page={page}").text nb_videos_extracted, nb_videos_on_page = self.extract_videos_on_topic_page( html ) if nb_videos_on_page == 0: break total_videos_scraped += nb_videos_extracted page += 1 return total_videos_scraped def extract_videos_from_topics(self, topic): """extracts metadata for required number of videos on different topics""" logger.debug(f"Fetching video links for topic: {topic}") topic_url = f"{self.talks_base_url}?topics%5B%5D={topic}" total_videos_scraped = 0 if self.source_languages: for lang in self.source_languages: topic_url = topic_url + f"&language={lang}" total_videos_scraped = self.generate_search_result_and_scrape( topic_url, total_videos_scraped ) else: total_videos_scraped = self.generate_search_result_and_scrape( topic_url, total_videos_scraped ) logger.info(f"Total video links found in {topic}: {total_videos_scraped}") if total_videos_scraped == 0: return False return True def update_zim_metadata(self): if not self.languages: self.zim_lang = "eng" else: if len(self.source_languages) > 1: self.zim_lang = "mul" else: lang_info = get_language_details( self.source_languages[0], failsafe=True ) if lang_info: self.zim_lang = lang_info["iso-639-3"] else: self.zim_lang = "eng" if self.playlist: if not self.title: self.title = self.playlist_title.strip() if not self.description: self.description = self.playlist_description.strip() else: if len(self.topics) > 1: if not self.title: self.title = "TED Collection" if not self.description: self.description = "A selection of TED videos from several topics" else: topic_str = self.topics[0].replace("+", " ") if not self.title: self.title = f"{topic_str.capitalize()} from TED" if not self.description: self.description = f"A selection of {topic_str} videos from TED" def get_display_name(self, lang_code, lang_name): """Display name for language""" lang_info = get_language_details(lang_code, failsafe=True) if lang_code != "en" and lang_info: return lang_info["native"] + " - " + lang_name return lang_name def get_subtitle_dict(self, lang): """dict of language name and code from a larger dict lang Example: { 'languageCode': 'en', 'languageName': 'English' } """ return { "languageName": self.get_display_name( lang["languageCode"], lang["languageName"] ), "languageCode": lang["languageCode"], } def generate_subtitle_list(self, video_id, langs, page_lang, audio_lang): """List of all subtitle languages with link to their pages""" subtitles = [] if self.subtitles_setting == ALL or ( not self.source_languages and self.topics and self.subtitles_setting != NONE ): subtitles = [self.get_subtitle_dict(lang) for lang in langs] elif self.subtitles_setting == MATCHING or ( self.subtitles_enough and self.subtitles_setting == NONE and page_lang != audio_lang ): subtitles = [ self.get_subtitle_dict(lang) for lang in langs if lang["languageCode"] == page_lang ] elif self.subtitles_setting and self.subtitles_setting != NONE: if not self.subtitles_enough and self.topics: subtitles = [ self.get_subtitle_dict(lang) for lang in langs if lang["languageCode"] in self.subtitles_setting ] else: subtitles = [ self.get_subtitle_dict(lang) for lang in langs if lang["languageCode"] in self.subtitles_setting or lang["languageCode"] in self.source_languages ] return update_subtitles_list(video_id, subtitles) def generate_urls_for_other_languages(self, url): """Possible URLs for other requested languages based on a video url""" urls = [] page_lang, query = self.get_lang_code_from_url(url, with_full_query=True) url_parts = list(urllib.parse.urlparse(url)) # update the language query field value with other languages and form URLs for language in self.source_languages: if language != page_lang: query.update({"language": language}) url_parts[4] = urllib.parse.urlencode(query) urls.append(urllib.parse.urlunparse(url_parts)) return urls def extract_videos_on_topic_page(self, page_html): # all videos are embedded in a <div> with the class name 'row'. # we are searching for the div inside this div, that has an <a>-tag # with the class name 'media__image', because this is the relative # link to the representative TED talk. It turns this relative link to # an absolute link and calls extract_video_info for them soup = BeautifulSoup(page_html, features="html.parser") video_links = soup.select("div.row div.media__image a") nb_extracted = 0 nb_listed = len(video_links) logger.debug(f"{nb_listed} video(s) found on current page") for video_link in video_links: url = urllib.parse.urljoin(self.talks_base_url, video_link["href"]) if self.extract_info_from_video_page(url): nb_extracted += 1 if self.source_languages and len(self.source_languages) > 1: other_lang_urls = self.generate_urls_for_other_languages(url) logger.debug( f"Searching info for video in other {len(other_lang_urls)} language(s)" ) for lang_url in other_lang_urls: self.extract_info_from_video_page(lang_url) self.already_visited.append(urllib.parse.urlparse(url)[2]) logger.debug(f"Seen {video_link['href']}") return nb_extracted, nb_listed def get_lang_code_from_url(self, url, with_full_query=False): """gets the queried language code from a ted talk url""" # sample - https://www.ted.com/talks/alex_rosenthal_the_gauntlet_think_like_a_coder_ep_8?language=ja url_parts = list(urllib.parse.urlparse(url)) # explode url to extract `language` query field value query = dict(urllib.parse.parse_qsl(url_parts[4])) current_lang = query.get("language") if with_full_query: return current_lang, query return current_lang def extract_download_link(self, talk_data): """Returns download link / youtube video ID for a TED video""" if ( isinstance(talk_data.get("resources", {}).get("h264"), list) and len(talk_data["resources"]["h264"]) and talk_data["resources"]["h264"][0].get("file") ): logger.debug( "Using h264 resource link for bitrate=" f"{talk_data['resources']['h264'][0].get('bitrate')}" ) return talk_data["resources"]["h264"][0]["file"] logger.error("No download link found for the video") return None def update_videos_list( self, video_id, lang_code, lang_name, title, description, speaker, speaker_profession, speaker_bio, speaker_picture, date, thumbnail, video_link, length, subtitles, ): # append to self.videos and return if not present if not [video for video in self.videos if video.get("id", None) == video_id]: self.videos.append( { "id": video_id, "languages": [ { "languageCode": lang_code, "languageName": self.get_display_name(lang_code, lang_name), } ], "title": [{"lang": lang_code, "text": title}], "description": [{"lang": lang_code, "text": description}], "speaker": speaker, "speaker_profession": speaker_profession, "speaker_bio": speaker_bio, "speaker_picture": speaker_picture, "date": date, "thumbnail": thumbnail, "video_link": video_link, "length": length, "subtitles": subtitles, } ) logger.debug(f"Successfully inserted video {video_id} into video list") return True # update localized meta for video if already in self.videos # based on --subtitles=matching logger.debug(f"Video {video_id} already present in video list") for index, video in enumerate(self.videos): if video.get("id", None) == video_id: if {"lang": lang_code, "text": title} not in video["title"]: self.videos[index]["title"].append( {"lang": lang_code, "text": title} ) self.videos[index]["description"].append( {"lang": lang_code, "text": description} ) self.videos[index]["languages"].append( { "languageCode": lang_code, "languageName": self.get_display_name(lang_code, lang_name), } ) if self.subtitles_setting == MATCHING or self.subtitles_setting == NONE: self.videos[index]["subtitles"] += subtitles return False def extract_video_info_from_json(self, json_data): player_data = json.loads(json_data["playerData"]) lang_code = json_data["language"] lang_name = [ lang["languageName"] for lang in player_data["languages"] if lang["languageCode"] == lang_code ][-1] # talk_info = json_data["talks"][0] native_talk_language = player_data["nativeLanguage"] if ( not self.subtitles_enough and self.source_languages and native_talk_language != lang_code and self.topics ): return False # Extract the speaker of the TED talk if len(json_data["speakers"]): speaker_info = json_data["speakers"][0] speaker = " ".join( [ speaker_info.get("firstName"), speaker_info.get("middleName"), speaker_info.get("lastName"), ] ) else: speaker_info = { "description": "None", "whotheyare": "None", "photo_url": "", } if "presenterDisplayName" in json_data: speaker = json_data["presenterDisplayName"] else: speaker = "None" # Extract the ted talk details from json video_id = json_data["id"] speaker_profession = speaker_info["description"] speaker_bio = speaker_info.get("whoTheyAre", "-") speaker_picture = speaker_info.get("avatar", "-") title = json_data["title"] description = json_data["description"] date = dateutil.parser.parse(json_data["recordedOn"]).strftime("%d %B %Y") length = int(json_data["duration"]) // 60 thumbnail = player_data["thumb"] video_link = self.extract_download_link(player_data) if not video_link: logger.error("No suitable download link found. Skipping video") return False langs = player_data["languages"] subtitles = self.generate_subtitle_list( video_id, langs, lang_code, native_talk_language ) return self.update_videos_list( video_id=video_id, lang_code=lang_code, lang_name=lang_name, title=title, description=description, speaker=speaker, speaker_profession=speaker_profession, speaker_bio=speaker_bio, speaker_picture=speaker_picture, date=date, thumbnail=thumbnail, video_link=video_link, length=length, subtitles=subtitles, ) def extract_info_from_video_page(self, url, retry_count=0): """extract all info from a TED video page url and update self.videos""" # Every TED video page has a <script>-tag with a Javascript # object with JSON in it. We will just stip away the object # signature and load the json to extract meta-data out of it. # returns True if successfully scraped new video # don't scrape if URL already visited if urllib.parse.urlparse(url)[2] in self.already_visited: return False # don't scrape if maximum retry count is reached if retry_count > 5: logger.error("Max retries exceeded. Skipping video") return False logger.debug(f"extract_info_from_video_page: {url}") soup = BeautifulSoup(download_link(url).text, features="html.parser") json_data = json.loads( soup.find("script", attrs={"id": "__NEXT_DATA__"}).string )["props"]["pageProps"]["videoData"] requested_lang_code = self.get_lang_code_from_url(url) if requested_lang_code and json_data["language"] != requested_lang_code: logger.error( f"Video has not yet been translated into {requested_lang_code}" ) return False return self.extract_video_info_from_json(json_data) def add_default_language(self): """add metatada in default language (english or first avail) on all videos""" for video in self.videos: en_found = False for index, lang in enumerate(video["languages"]): if lang["languageCode"] == "en": en_found = True video["title"] = [ {"lang": "default", "text": video["title"][index]["text"]} ] + video["title"] video["description"] = [ {"lang": "default", "text": video["description"][index]["text"]} ] + video["description"] if not en_found: video["title"] = [ {"lang": "default", "text": video["title"][0]["text"]} ] + video["title"] video["description"] = [ {"lang": "default", "text": video["description"][0]["text"]} ] + video["description"] # update video slug video["slug"] = slugify(video["title"][0]["text"], separator="-") def render_video_pages(self): # Render static html pages from the scraped video data and # save the pages in build_dir/<video-id>/index.html env = jinja2.Environment( loader=jinja2.FileSystemLoader(str(self.templates_dir)), autoescape=True ) for video in self.videos: titles = video["title"] html = env.get_template("article.html").render( speaker=video["speaker"], languages=video["subtitles"], speaker_bio=video["speaker_bio"].replace("Full bio", ""), speaker_img=video["speaker_picture"], date=video["date"], profession=video["speaker_profession"], video_format=self.video_format, autoplay=self.autoplay, video_id=str(video["id"]), title=get_main_title(titles, self.locale_name), titles=titles, descriptions=video["description"], back_to_list=_("Back to the list"), ) html_path = self.build_dir.joinpath(video["slug"]) with open(html_path, "w", encoding="utf-8") as html_page: html_page.write(html) def render_home_page(self): # Render the homepage env = jinja2.Environment( loader=jinja2.FileSystemLoader(str(self.templates_dir)), autoescape=True ) all_langs = { language["languageCode"]: language["languageName"] for video in self.videos for language in video["subtitles"] + video["languages"] } languages = [ {"languageName": value, "languageCode": key} for key, value in all_langs.items() ] languages = sorted(languages, key=lambda x: x["languageName"]) html = env.get_template("home.html").render( languages=languages, page_title=_("TED Talks"), language_filter_text=_("Filter by language"), back_to_top=_("Back to the top"), pagination_text=_("Page"), ) home_page_path = self.build_dir.joinpath("index") with open(home_page_path, "w", encoding="utf-8") as html_page: html_page.write(html) def copy_files_to_build_directory(self): # Copy files from template_dir to build_dir assets_dir = self.templates_dir.joinpath("assets") if assets_dir.exists(): shutil.copytree( assets_dir, self.build_dir.joinpath("assets"), dirs_exist_ok=True ) shutil.copy( self.templates_dir.joinpath("favicon.png"), self.build_dir.joinpath("favicon.png"), ) def generate_datafile(self): """Generate data.js inside assets folder""" video_list = [] for video in self.videos: lang_codes = [lang["languageCode"] for lang in video["subtitles"]] + [ lang["languageCode"] for lang in video["languages"] ] json_data = { "languages": [lang_code for lang_code in set(lang_codes)], "id": video["id"], "description": video["description"], "title": video["title"], "speaker": video["speaker"], "slug": video["slug"], } video_list.append(json_data) assets_path = self.build_dir.joinpath("assets") if not assets_path.exists(): assets_path.mkdir(parents=True) with open(assets_path.joinpath("data.js"), "w") as data_file: data_file.write("json_data = " + json.dumps(video_list, indent=4)) def download_jpeg_image_and_convert( self, url, fpath, preset_options={}, resize=None ): """downloads a JPEG image and converts and optimizes it into desired format detected from fpath""" org_jpeg_path = pathlib.Path( tempfile.NamedTemporaryFile(delete=False, suffix=".jpg").name ) save_large_file(url, org_jpeg_path) if resize is not None: resize_image( org_jpeg_path, width=resize[0], height=resize[1], method="cover", ) optimize_image( org_jpeg_path, fpath, convert=True, delete_src=True, **preset_options ) logger.debug(f"Converted {org_jpeg_path} to {fpath} and optimized ") def download_speaker_image( self, video_id, video_title, video_speaker, speaker_path ): """downloads the speaker image""" downloaded_from_cache = False preset = WebpMedium() if self.s3_storage: s3_key = f"speaker_image/{video_id}" downloaded_from_cache = self.download_from_cache( s3_key, speaker_path, preset.VERSION ) if not downloaded_from_cache: try: # download an image of the speaker if not video_speaker: logger.debug("Speaker doesn't have an image") else: logger.debug(f"Downloading Speaker image for {video_title}") self.download_jpeg_image_and_convert( video_speaker, speaker_path, preset_options=preset.options ) except Exception: logger.error(f"Could not download speaker image for {video_title}") else: if self.s3_storage and video_speaker: self.upload_to_cache(s3_key, speaker_path, preset.VERSION) def download_thumbnail( self, video_id, video_title, video_thumbnail, thumbnail_path ): """download the thumbnail""" downloaded_from_cache = False preset = WebpMedium() if self.s3_storage: s3_key = f"thumbnail/{video_id}" downloaded_from_cache = self.download_from_cache( s3_key, thumbnail_path, preset.VERSION ) if not downloaded_from_cache: try: # download the thumbnail of the video logger.debug(f"Downloading thumbnail for {video_title}") self.download_jpeg_image_and_convert( video_thumbnail, thumbnail_path, preset_options=preset.options, resize=(248, 187), ) except Exception: logger.error(f"Could not download thumbnail for {video_title}") else: if self.s3_storage: self.upload_to_cache(s3_key, thumbnail_path, preset.VERSION) def download_video_files(self, video): """download all video files (video, thumbnail, speaker)""" # Download all the TED talk videos and the meta-data for it. # Save the videos in build_dir/{video id}/video.mp4. # Save the thumbnail for the video in build_dir/{video id}/thumbnail.jpg. # Save the image of the speaker in build_dir/{video id}/speaker.jpg. # set up variables video_id = str(video["id"]) # Take the english version of title or else whatever language it's available in video_title = video["title"][0]["text"] video_link = video["video_link"] video_speaker = video["speaker_picture"] video_thumbnail = video["thumbnail"] video_dir = self.videos_dir.joinpath(video_id) org_video_file_path = video_dir.joinpath("video.mp4") req_video_file_path = video_dir.joinpath(f"video.{self.video_format}") speaker_path = video_dir.joinpath("speaker.webp") thumbnail_path = video_dir.joinpath("thumbnail.webp") # ensure that video directory exists if not video_dir.exists(): video_dir.mkdir(parents=True) # set preset preset = {"mp4": VideoMp4Low}.get(self.video_format, VideoWebmLow)() # download video downloaded_from_cache = False logger.debug(f"Downloading {video_title}") if self.s3_storage: s3_key = f"{self.video_format}/{self.video_quality}/{video_id}" downloaded_from_cache = self.download_from_cache( s3_key, req_video_file_path, preset.VERSION ) if not downloaded_from_cache: try: if "https://" not in video_link: options = ( BestWebm if self.video_format == "webm" else BestMp4 ).get_options( target_dir=video_dir, filepath=pathlib.Path("video.%(ext)s") ) self.yt_downloader.download(video_link, options) else: save_large_file(video_link, org_video_file_path) except Exception: logger.error(f"Could not download {org_video_file_path}") # download speaker and thumbnail images self.download_speaker_image(video_id, video_title, video_speaker, speaker_path) self.download_thumbnail(video_id, video_title, video_thumbnail, thumbnail_path) # recompress if necessary try: if not downloaded_from_cache: post_process_video( video_dir, video_id, preset, self.video_format, self.low_quality, ) except Exception as e: logger.error(f"Failed to post process video {video_id}") logger.debug(e) else: # upload to cache only if recompress was successful if self.s3_storage and not downloaded_from_cache: self.upload_to_cache(s3_key, req_video_file_path, preset.VERSION) def download_video_files_parallel(self): """download videos and images parallely""" self.yt_downloader = YoutubeDownloader(threads=1) with concurrent.futures.ThreadPoolExecutor( max_workers=self.threads ) as executor: fs = [ executor.submit(self.download_video_files, video) for video in self.videos ] concurrent.futures.wait(fs, return_when=concurrent.futures.ALL_COMPLETED) self.yt_downloader.shutdown() def download_subtitles(self, index, video): """download, converts and writes VTT subtitles for a video at a specific index in self.videos""" # Download the subtitle files, generate a WebVTT file # and save the subtitles in # build_dir/{video id}/subs/subs_{language code}.vtt if not video["subtitles"]: return video_dir = self.videos_dir.joinpath(video["id"]) subs_dir = video_dir.joinpath("subs") if not subs_dir.exists(): subs_dir.mkdir(parents=True) else: logger.debug(f"Subs dir exists already") # download subtitles logger.debug(f"Downloading subtitles for {video['title'][0]['text']}") valid_subs = [] for subtitle in video["subtitles"]: time.sleep(0.5) # throttling vtt_subtitle = WebVTT(subtitle["link"]).convert() if not vtt_subtitle: logger.error( f"Subtitle file for {subtitle['languageCode']} could not be created" ) continue valid_subs.append(subtitle) vtt_path = subs_dir.joinpath(f"subs_{subtitle['languageCode']}.vtt") with open(vtt_path, "w", encoding="utf-8") as sub_file: sub_file.write(vtt_subtitle) self.videos[index]["subtitles"] = valid_subs def download_subtitles_parallel(self): """download subtitles for all videos parallely""" with concurrent.futures.ThreadPoolExecutor( max_workers=self.threads ) as executor: fs = [ executor.submit(self.download_subtitles, index, video) for index, video in enumerate(self.videos) ] concurrent.futures.wait(fs, return_when=concurrent.futures.ALL_COMPLETED) def s3_credentials_ok(self): logger.info("Testing S3 Optimization Cache credentials") self.s3_storage = KiwixStorage(self.s3_url_with_credentials) if not self.s3_storage.check_credentials( list_buckets=True, bucket=True, write=True, read=True, failsafe=True ): logger.error("S3 cache connection error testing permissions.") logger.error(f" Server: {self.s3_storage.url.netloc}") logger.error(f" Bucket: {self.s3_storage.bucket_name}") logger.error(f" Key ID: {self.s3_storage.params.get('keyid')}") logger.error(f" Public IP: {get_public_ip()}") return False return True def download_from_cache(self, key, object_path, encoder_version): """whether it downloaded from S3 cache""" if self.use_any_optimized_version: if not self.s3_storage.has_object(key, self.s3_storage.bucket_name): return False else: if not self.s3_storage.has_object_matching_meta( key, tag="encoder_version", value=f"v{encoder_version}" ): return False object_path.parent.mkdir(parents=True, exist_ok=True) try: self.s3_storage.download_file(key, object_path) except Exception as exc: logger.error(f"{key} failed to download from cache: {exc}") return False logger.info(f"downloaded {object_path} from cache at {key}") return True def upload_to_cache(self, key, object_path, encoder_version): """whether it uploaded from S3 cache""" try: self.s3_storage.upload_file( object_path, key, meta={"encoder_version": f"v{encoder_version}"} ) except Exception as exc: logger.error(f"{key} failed to upload to cache: {exc}") return False logger.info(f"uploaded {object_path} to cache at {key}") return True def remove_failed_topics_and_check_extraction(self, failed_topics): """removes failed topics from topics list and raises error if scraper cannot continue""" for topic in failed_topics: self.topics.remove(topic) if not self.topics: if self.source_languages: raise ValueError( "No videos found for any topic in the language(s) requested. Check topic(s) and/or language code supplied to --languages" ) raise ValueError("Wrong topic(s) were supplied. No videos found") def run(self): logger.info( f"Starting scraper with:\n" f" langs: {', '.join(self.source_languages)}\n" f" subtitles : {', '.join(self.subtitles_setting) if isinstance(self.subtitles_setting, list) else self.subtitles_setting}\n" f" video format : {self.video_format}" ) if self.s3_url_with_credentials and not self.s3_credentials_ok(): raise ValueError("Unable to connect to Optimization Cache. Check its URL.") if self.s3_storage: logger.info( f"Using cache: {self.s3_storage.url.netloc} with bucket: {self.s3_storage.bucket_name}" ) # playlist mode requested if self.playlist: self.extract_videos_from_playlist(self.playlist) # topic(s) mode requested else: failed = [] for topic in self.topics: if not self.extract_videos_from_topics(topic): failed.append(topic) else: logger.debug(f"Successfully scraped {topic}") self.remove_failed_topics_and_check_extraction(failed) self.add_default_language() self.update_zim_metadata() self.download_video_files_parallel() self.download_subtitles_parallel() self.render_home_page() self.render_video_pages() self.copy_files_to_build_directory() self.generate_datafile() # zim creation and cleanup if not self.no_zim: self.fname = ( self.fname or f"{self.name.replace(' ', '-')}_{{period}}.zim" ).format(period=datetime.datetime.now().strftime("%Y-%m")) logger.info("building ZIM file") if not self.output_dir.exists(): self.output_dir.mkdir(parents=True) make_zim_file( build_dir=self.build_dir, fpath=self.output_dir.joinpath(self.fname), name=self.name, main_page="index", favicon="favicon.png", title=self.title, description=self.description, language=self.zim_lang, creator=self.creator, publisher=self.publisher, tags=self.tags + ["_category:ted", "ted", "_videos:yes"], scraper=SCRAPER, ) if not self.keep_build_dir: logger.info("removing temp folder") shutil.rmtree(self.build_dir, ignore_errors=True) logger.info("Done Everything")
def test_youtube_download_error(tmp_path): yt_downloader = YoutubeDownloader(threads=1) with pytest.raises(Exception): yt_downloader.download("11", BestMp4.get_options()) yt_downloader.shutdown()
def test_youtube_download_contextmanager(tmp_path): with YoutubeDownloader(threads=1) as yt_downloader: yt_downloader.download("https://vimeo.com/619427082", BestWebm.get_options(target_dir=tmp_path)) assert yt_downloader.executor._shutdown assert tmp_path.joinpath("video.mp4").exists() # videmo doesn't offer webm