def clear_subtitles_cache(lang_codes=None, locale_root=LOCALE_ROOT): """ Language codes will be converted to django format (e.g. en_US) """ lang_codes = lang_codes or os.listdir(locale_root) for lang_code in lang_codes: lang_code = lcode_to_ietf(lang_code) # Clear the status file lm_file = get_lang_map_filepath(lang_code) with open(lm_file, "r") as fp: download_status = json.load(fp) for key in download_status: download_status[key] = { u"downloaded": False, u"last_success": u"", u"last_attempt": u"", u"api_response": u"", } with open(lm_file, "w") as fp: json.dump(download_status, fp) # Delete all srt files srt_path = get_srt_path(lang_code) if os.path.exists(srt_path): shutil.rmtree(srt_path)
def zip_language_packs(lang_codes=None): """Zip up and expose all language packs converts all into ietf """ lang_codes = lang_codes or os.listdir(LOCALE_ROOT) lang_codes = [lcode_to_ietf(lc) for lc in lang_codes] logging.info("Zipping up %d language pack(s)" % len(lang_codes)) for lang_code_ietf in lang_codes: lang_code_django = lcode_to_django_dir(lang_code_ietf) lang_locale_path = os.path.join(LOCALE_ROOT, lang_code_django) if not os.path.exists(lang_locale_path): logging.warn("Unexpectedly skipping missing directory: %s" % lang_code_django) elif not os.path.isdir(lang_locale_path): logging.error("Skipping language where a file exists where a directory was expected: %s" % lang_code_django) # Create a zipfile for this language zip_filepath = get_language_pack_filepath(lang_code_ietf) ensure_dir(os.path.dirname(zip_filepath)) logging.info("Creating zip file in %s" % zip_filepath) z = zipfile.ZipFile(zip_filepath, 'w', zipfile.ZIP_DEFLATED) # Get every single file in the directory and zip it up for metadata_file in glob.glob('%s/*.json' % lang_locale_path): z.write(os.path.join(lang_locale_path, metadata_file), arcname=os.path.basename(metadata_file)) srt_dirpath = get_srt_path(lang_code_django) for srt_file in glob.glob(os.path.join(srt_dirpath, "*.srt")): z.write(srt_file, arcname=os.path.join("subtitles", os.path.basename(srt_file))) z.close() logging.info("Done.")
def move_srts(lang_code): """ Srts live in the locale directory, but that's not exposed at any URL. So instead, we have to move the srts out to /static/subtitles/[lang_code]/ """ lang_code_ietf = lcode_to_ietf(lang_code) lang_code_django = lcode_to_django_dir(lang_code) subtitles_static_dir = os.path.join(settings.STATIC_ROOT, "subtitles") src_dir = os.path.join(LOCALE_ROOT, lang_code_django, "subtitles") dest_dir = get_srt_path(lang_code_django) ensure_dir(dest_dir) lang_subtitles = glob.glob(os.path.join(src_dir, "*.srt")) logging.info("Moving %d subtitles from %s to %s" % (len(lang_subtitles), src_dir, dest_dir)) for fil in lang_subtitles: srt_dest_path = os.path.join(dest_dir, os.path.basename(fil)) if os.path.exists(srt_dest_path): os.remove(srt_dest_path) shutil.move(fil, srt_dest_path) if os.listdir(src_dir): logging.warn("%s is not empty; will not remove. Please check that all subtitles were moved." % src_dir) else: logging.info("Removing empty source directory (%s)." % src_dir) shutil.rmtree(src_dir)
def store_new_counts(lang_code, data_path=SUBTITLES_DATA_ROOT, locale_root=LOCALE_ROOT): """Write a new dictionary of srt file counts in respective download folders""" language_subtitle_count = {} subtitles_path = get_srt_path(lang_code) lang_name = get_language_name(lang_code) try: count = len(glob.glob("%s/*.srt" % subtitles_path)) language_subtitle_count[lang_name] = {} language_subtitle_count[lang_name]["count"] = count language_subtitle_count[lang_name]["code"] = lang_code except LanguageNameDoesNotExist as ldne: count = 0 logging.debug(ldne) except: count = 0 logging.info("%-4s subtitles for %-20s" % ("No", lang_name)) # Always write to disk. write_count_to_json(language_subtitle_count, data_path) return count
def download_if_criteria_met( videos, lang_code, force, response_code, date_since_attempt, frequency_to_save, *args, **kwargs ): """Execute download of subtitle if it meets the criteria specified by the command line args Note: videos are a dict; keys=youtube_id, values=data Note: lang_code is in IETF format. """ date_specified = convert_date_input(date_since_attempt) # Filter up front, for efficiency (& reporting's sake) n_videos = len(videos) logging.info( "There are (up to) %s total videos with subtitles for language '%s'. Let's go get them!" % (n_videos, lang_code) ) # Filter based on response code if response_code and response_code != "all": logging.info("Filtering based on response code (%s)..." % response_code) response_code_filter = partial(lambda vid, rcode: rcode == vid["api_response"], rcode=response_code) videos = dict([(k, v) for k, v in videos.iteritems() if response_code_filter(v)]) logging.info( "%4d of %4d videos match your specified response code (%s)" % (len(videos), n_videos, response_code) ) if date_specified: logging.info("Filtering based on date...") videos_copy = copy.deepcopy(videos) for k, v in videos.items(): if not v["last_attempt"] or datetime.datetime.strptime(v["last_attempt"], "%Y-%m-%d") < date_specified: continue else: del videos_copy[k] videos = videos_copy logging.info( "%4d of %4d videos need refreshing (last refresh more recent than %s)" % (len(videos), n_videos, date_specified) ) # Loop over videos needing refreshing n_loops = 0 srt_count = None for youtube_id, entry in videos.items(): previously_downloaded = entry.get("downloaded") if previously_downloaded and not force: logging.info("Already downloaded %s/%s. To redownload, run again with -f." % (lang_code, youtube_id)) continue logging.debug("Attempting to download subtitle for lang: %s and YouTube ID: %s" % (lang_code, youtube_id)) response = download_subtitle(youtube_id, lang_code, format="srt") time_of_attempt = unicode(datetime.datetime.now().date()) if response == "client-error" or response == "server-error": # Couldn't download logging.info("%s/%s.srt: Updating JSON file to record error (%s)." % (lang_code, youtube_id, response)) update_json(youtube_id, lang_code, previously_downloaded, response, time_of_attempt) else: dirpath = get_srt_path(lang_code) fullpath = os.path.join(dirpath, youtube_id + ".srt") ensure_dir(dirpath) logging.debug("Writing file to %s" % fullpath) with open(fullpath, "w") as fp: fp.write(response.encode("UTF-8")) logging.info("%s/%s.srt: Updating JSON file to record success." % (lang_code, youtube_id)) update_json(youtube_id, lang_code, True, "success", time_of_attempt) # Update srt availability mapping n_loops += 1 if n_loops % frequency_to_save == 0 or n_loops == len(videos.keys()): srt_count = store_new_counts(lang_code=lang_code) logging.info( "%s: On loop %d / %d, stored: subtitle count = %d." % (lang_code, n_loops, len(videos), srt_count) ) # Summarize output if srt_count is None: # only none if nothing was done. logging.info("Nothing was done.") else: logging.info( "We now have %d subtitles (amara thought they had %d) for language '%s'!" % (srt_count, n_videos, lang_code) )
def stamp_availability_on_video(video, format="mp4", force=False, stamp_urls=True, videos_path=settings.CONTENT_ROOT): """ Stamp all relevant urls and availability onto a video object (if necessary), including: * whether the video is available (on disk or online) """ def compute_video_availability(youtube_id, format, videos_path=settings.CONTENT_ROOT): return {"on_disk": is_video_on_disk(youtube_id, format, videos_path=videos_path)} def compute_video_metadata(youtube_id, format): return {"stream_type": "video/%s" % format} def compute_video_urls(youtube_id, format, lang_code, on_disk=None, thumb_format="png", videos_path=settings.CONTENT_ROOT): if on_disk is None: on_disk = is_video_on_disk(youtube_id, format, videos_path=videos_path) if on_disk: video_base_url = settings.CONTENT_URL + youtube_id stream_url = video_base_url + ".%s" % format thumbnail_url = video_base_url + ".png" elif settings.BACKUP_VIDEO_SOURCE and lang_code == "en": dict_vals = {"youtube_id": youtube_id, "video_format": format, "thumb_format": thumb_format } stream_url = settings.BACKUP_VIDEO_SOURCE % dict_vals thumbnail_url = settings.BACKUP_THUMBNAIL_SOURCE % dict_vals if settings.BACKUP_THUMBNAIL_SOURCE else None else: return {} # no URLs return {"stream": stream_url, "thumbnail": thumbnail_url} video_availability = video.get("availability", {}) if not force else {} en_youtube_id = get_youtube_id(video["id"], None) video_map = get_id2oklang_map(video["id"]) or {} if not "on_disk" in video_availability: for lang_code, youtube_id in video_map.iteritems(): video_availability[lang_code] = compute_video_availability(youtube_id, format=format, videos_path=videos_path) video_availability["en"] = video_availability.get("en", {"on_disk": False}) # en should always be defined # Summarize status any_on_disk = any([lang_avail["on_disk"] for lang_avail in video_availability.values()]) any_available = any_on_disk or bool(settings.BACKUP_VIDEO_SOURCE) if stamp_urls: # Loop over all known dubbed videos for lang_code, youtube_id in video_map.iteritems(): urls = compute_video_urls(youtube_id, format, lang_code, on_disk=video_availability[lang_code]["on_disk"], videos_path=videos_path) if urls: # Only add properties if anything is available. video_availability[lang_code].update(urls) video_availability[lang_code].update(compute_video_metadata(youtube_id, format)) # Get the (english) subtitle urls subtitle_lang_codes = get_langs_with_subtitle(en_youtube_id) subtitles_tuple = [(lc, get_srt_url(en_youtube_id, lc)) for lc in subtitle_lang_codes if os.path.exists(get_srt_path(lc, en_youtube_id))] subtitles_urls = dict(subtitles_tuple) video_availability["en"]["subtitles"] = subtitles_urls # now scrub any values that don't actually exist for lang_code in video_availability.keys(): if not video_availability[lang_code]["on_disk"] and len(video_availability[lang_code]) == 1: del video_availability[lang_code] # Now summarize some availability onto the video itself video["availability"] = video_availability video["on_disk"] = any_on_disk video["available"] = any_available return video