def zip_language_packs(lang_codes=None):
"""Zip up and expose all language packs"""
lang_codes = lang_codes or listdir(LOCALE_ROOT)
logging.info("Zipping up %d language pack(s)" % len(lang_codes))
ensure_dir(settings.LANGUAGE_PACK_ROOT)
for lang in lang_codes:
lang_locale_path = os.path.join(LOCALE_ROOT, lang)
if not os.path.exists(lang_locale_path):
logging.warn("Unexpectedly skipping missing directory: %s" % lang)
elif not os.path.isdir(lang_locale_path):
logging.error("Skipping language where a file exists: %s" % lang)
# Create a zipfile for this language
zip_path = os.path.join(settings.LANGUAGE_PACK_ROOT, version.VERSION)
ensure_dir(zip_path)
z = zipfile.ZipFile(os.path.join(zip_path, "%s.zip" % convert_language_code_format(lang)), 'w')
# Get every single file in the directory and zip it up
for metadata_file in glob.glob('%s/*.json' % lang_locale_path):
z.write(os.path.join(lang_locale_path, metadata_file), arcname=os.path.basename(metadata_file))
for mo_file in glob.glob('%s/LC_MESSAGES/*.mo' % lang_locale_path):
z.write(os.path.join(lang_locale_path, mo_file), arcname=os.path.join("LC_MESSAGES", os.path.basename(mo_file)))
for srt_file in glob.glob('%s/subtitles/*.srt' % lang_locale_path):
z.write(os.path.join(lang_locale_path, srt_file), arcname=os.path.join("subtitles", os.path.basename(srt_file)))
z.close()
logging.info("Done.")
def process_request(self, request):
"""
Process requests to set language, redirect to the same URL to continue processing
without leaving the "set" in the browser history.
"""
# Set the set of available languages
if "language_choices" not in request.session:
request.session["language_choices"] = list(LanguagePack.objects.all())
request.session["language_codes"] = [convert_language_code_format(lp.code) for lp in request.session["language_choices"]]
# Set the current language, and redirect (to clean browser history)
if request.is_admin and request.GET.get("set_default_language"):
Settings.set("default_language", request.GET["set_default_language"])
request.session["default_language"] = request.GET["set_default_language"]
logging.debug("setting default language to %s" % request.GET["set_default_language"])
request.session["django_language"] = request.GET["set_default_language"]
logging.debug("setting session language to %s" % request.session["django_language"])
return HttpResponseRedirect(request.path)
elif request.GET.get("set_language"):
request.session["django_language"] = request.GET["set_language"]
logging.debug("setting session language to %s" % request.session["django_language"])
return HttpResponseRedirect(request.path)
# Process the current language
if "default_language" not in request.session:
request.session["default_language"] = Settings.get("default_language") or settings.LANGUAGE_CODE
if "django_language" not in request.session:
request.session["django_language"] = request.session["default_language"]
logging.debug("setting session language to %s" % request.session["django_language"])
request.language = request.session["django_language"]
def handle(self, **options):
if not settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the central server.")
obliterate_old_schema()
# Raw language code for srts
update_srts(days=options["days"], lang_code=options["lang_code"])
# Converted language code for language packs
update_language_packs(lang_codes=[convert_language_code_format(options["lang_code"])] if options["lang_code"] != "all" else None)
def process_request(self, request):
"""
Process requests to set language, redirect to the same URL to continue processing
without leaving the "set" in the browser history.
"""
# Set the set of available languages
if "language_choices" not in request.session:
request.session["language_choices"] = list(
LanguagePack.objects.all())
request.session["language_codes"] = [
convert_language_code_format(lp.code)
for lp in request.session["language_choices"]
]
# Set the current language, and redirect (to clean browser history)
if request.is_admin and request.GET.get("set_default_language"):
Settings.set("default_language",
request.GET["set_default_language"])
request.session["default_language"] = request.GET[
"set_default_language"]
logging.debug("setting default language to %s" %
request.GET["set_default_language"])
request.session["django_language"] = request.GET[
"set_default_language"]
logging.debug("setting session language to %s" %
request.session["django_language"])
return HttpResponseRedirect(request.path)
elif request.GET.get("set_language"):
request.session["django_language"] = request.GET["set_language"]
logging.debug("setting session language to %s" %
request.session["django_language"])
return HttpResponseRedirect(request.path)
# Process the current language
if "default_language" not in request.session:
request.session["default_language"] = Settings.get(
"default_language") or settings.LANGUAGE_CODE
if "django_language" not in request.session:
request.session["django_language"] = request.session[
"default_language"]
logging.debug("setting session language to %s" %
request.session["django_language"])
request.language = request.session["django_language"]
def download_srt_from_3rd_party(*args, **kwargs):
"""Download subtitles specified by command line args"""
lang_code = kwargs.get("lang_code", None)
# if language specified, do those, if not do all
if not lang_code:
raise CommandError("You must specify a language code or 'all' with -l")
elif lang_code == "all":
bad_languages = {}
for filename in get_all_download_status_files():
try:
videos = json.loads(open(filename).read())
except Exception as e:
logging.error(e)
raise CommandError(
"Unable to open %s. The file might be corrupted. Please re-run the generate_subtitle_map command to regenerate it."
% filename)
try:
lang_code = os.path.basename(filename).split("_")[0]
kwargs["lang_code"] = lang_code
download_if_criteria_met(videos, *args, **kwargs)
except Exception as e:
logging.error("Error downloading subtitles for %s: %s" %
(lang_code, e))
bad_languages[lang_code] = e
continue
# now report final results
if bad_languages:
raise CommandError(
"Failed to download subtitles for the following languages: %s"
% bad_languages.keys())
else:
srt_list_path = get_lang_map_filepath(
convert_language_code_format(lang_code))
try:
videos = json.loads(open(srt_list_path).read())
except:
logging.warning(
"No subtitles available for download for language code %s. Skipping."
% lang_code)
else:
download_if_criteria_met(videos, *args, **kwargs)
def extract_new_po(tmp_dir_path=os.path.join(LOCALE_ROOT, "tmp"), language_codes=[]):
"""Move newly downloaded po files to correct location in locale direction"""
logging.info("Unpacking new translations")
update_languages = os.listdir(tmp_dir_path)
if language_codes: # limit based on passed in limitations
update_languages = set(update_languages).intersect(set(language_codes))
for lang in update_languages:
converted_code = convert_language_code_format(lang)
# ensure directory exists in locale folder, and then overwrite local po files with new ones
ensure_dir(os.path.join(LOCALE_ROOT, converted_code, "LC_MESSAGES"))
for po_file in glob.glob(os.path.join(tmp_dir_path, lang, "*/*.po")):
if "js" in os.path.basename(po_file):
shutil.copy(po_file, os.path.join(LOCALE_ROOT, converted_code, "LC_MESSAGES", "djangojs.po"))
else:
shutil.copy(po_file, os.path.join(LOCALE_ROOT, converted_code, "LC_MESSAGES", "django.po"))
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError(
"This must only be run on distributed servers server.")
code = convert_language_code_format(options["lang_code"])
software_version = options["software_version"]
if code == settings.LANGUAGE_CODE:
logging.info(
"Note: language code set to default language. This is fine (and may be intentional), but you may specify a language other than '%s' with -l"
% code)
if software_version == version.VERSION:
logging.info(
"Note: software version set to default version. This is fine (and may be intentional), but you may specify a software version other than '%s' with -s"
% version.VERSION)
# Download the language pack
try:
self.start("Downloading language pack '%s'" % code)
zip_file = get_language_pack(code, software_version)
except CommandError as e: # 404
sys.exit(
'404 Not found: Could not download language pack file %s ' %
_language_pack_url(code, software_version))
# Unpack into locale directory
self.next_stage("Unpacking language pack '%s'" % code)
unpack_language(code, zip_file)
# Update database with meta info
self.next_stage("Updating database for language pack '%s'" % code)
update_database(code)
#
self.next_stage("Creating static files for language pack '%s'" % code)
update_jsi18n_file(code)
#
move_srts(code)
self.complete("Finished processing language pack %s" % code)
def download_srt_from_3rd_party(*args, **kwargs):
"""Download subtitles specified by command line args"""
lang_code = kwargs.get("lang_code", None)
# if language specified, do those, if not do all
if not lang_code:
raise CommandError("You must specify a language code or 'all' with -l")
elif lang_code == "all":
bad_languages = {}
for filename in get_all_download_status_files():
try:
videos = json.loads(open(filename).read())
except Exception as e:
logging.error(e)
raise CommandError("Unable to open %s. The file might be corrupted. Please re-run the generate_subtitle_map command to regenerate it." % filename)
try:
lang_code = os.path.basename(filename).split("_")[0]
kwargs["lang_code"] = lang_code
download_if_criteria_met(videos, *args, **kwargs)
except Exception as e:
logging.error("Error downloading subtitles for %s: %s" % (lang_code, e))
bad_languages[lang_code] = e
continue
# now report final results
if bad_languages:
raise CommandError("Failed to download subtitles for the following languages: %s" % bad_languages.keys())
else:
srt_list_path = get_lang_map_filepath(convert_language_code_format(lang_code))
try:
videos = json.loads(open(srt_list_path).read())
except:
logging.warning("No subtitles available for download for language code %s. Skipping." % lang_code)
else:
download_if_criteria_met(videos, *args, **kwargs)
def obliterate_old_schema():
"""Move srt files from static/srt to locale directory and file them by language code, delete any old locale directories"""
srt_root = os.path.join(settings.STATIC_ROOT, "srt")
for locale_root in settings.LOCALE_PATHS:
if not os.path.exists(locale_root):
continue
for lang in os.listdir(locale_root):
# Skips if not a directory
if not os.path.isdir(os.path.join(locale_root, lang)):
continue
# If it isn't crowdin/django format, keeeeeeellllllll
if lang != convert_language_code_format(lang):
logging.info("Deleting %s directory because it does not fit our language code format standards" % lang)
shutil.rmtree(os.path.join(locale_root, lang))
if os.path.exists(os.path.join(settings.STATIC_ROOT, "srt")):
logging.info("Outdated schema detected for storing srt files. Hang tight, the moving crew is on it.")
for lang in os.listdir(srt_root):
# Skips if not a directory
if not os.path.isdir(os.path.join(srt_root, lang)):
continue
lang_srt_path = os.path.join(srt_root, lang, "subtitles/")
lang_locale_path = os.path.join(locale_root, lang)
ensure_dir(lang_locale_path)
dst = os.path.join(lang_locale_path, "subtitles")
for srt_file_path in glob.glob(os.path.join(lang_srt_path, "*.srt")):
base_path, srt_filename = os.path.split(srt_file_path)
if not os.path.exists(os.path.join(dst, srt_filename)):
ensure_dir(dst)
shutil.move(srt_file_path, os.path.join(dst, srt_filename))
shutil.rmtree(srt_root)
logging.info("Move completed.")
def generate_metadata(lang_codes=None, broken_langs=None):
"""Loop through locale folder, create or update language specific meta and create or update master file, skipping broken languages"""
logging.info("Generating new po file metadata")
master_file = []
# loop through all languages in locale, update master file
crowdin_meta_dict = get_crowdin_meta()
subtitle_counts = json.loads(open(settings.SUBTITLES_DATA_ROOT + "subtitle_counts.json").read())
for lang in os.listdir(LOCALE_ROOT):
# skips anything not a directory
if not os.path.isdir(os.path.join(LOCALE_ROOT, lang)):
logging.info("Skipping %s because it is not a directory" % lang)
continue
elif lang in broken_langs:
logging.info("Skipping %s because it triggered an error during compilemessages. The admins should have received a report about this and must fix it before this pack will be updateed." % lang)
continue
crowdin_meta = next((meta for meta in crowdin_meta_dict if meta["code"] == convert_language_code_format(lang_code=lang, for_crowdin=True)), {})
try:
local_meta = json.loads(open(os.path.join(LOCALE_ROOT, lang, "%s_metadata.json" % lang)).read())
except:
local_meta = {}
try:
# update metadata
updated_meta = {
"code": crowdin_meta.get("code") or convert_language_code_format(lang),
"name": crowdin_meta.get("name") or get_language_name(convert_language_code_format(lang)),
"percent_translated": int(crowdin_meta.get("approved_progress", 0)),
"phrases": int(crowdin_meta.get("phrases", 0)),
"approved_translations": int(crowdin_meta.get("approved", 0)),
}
# Obtain current number of subtitles
entry = subtitle_counts.get(get_language_name(lang), {})
srt_count = entry.get("count", 0)
updated_meta.update({
"software_version": version.VERSION,
"subtitle_count": srt_count,
})
except LanguageNotFoundError:
logging.error("Unrecognized language; must skip: %s" % lang)
continue
language_pack_version = increment_language_pack_version(local_meta, updated_meta)
updated_meta["language_pack_version"] = language_pack_version
local_meta.update(updated_meta)
# Write locally (this is used on download by distributed server to update it's database)
with open(os.path.join(LOCALE_ROOT, lang, "%s_metadata.json" % lang), 'w') as output:
json.dump(local_meta, output)
# Update master (this is used for central server to handle API requests for data)
master_file.append(local_meta)
# Save updated master
ensure_dir(settings.LANGUAGE_PACK_ROOT)
with open(os.path.join(settings.LANGUAGE_PACK_ROOT, LANGUAGE_PACK_AVAILABILITY_FILENAME), 'w') as output:
json.dump(master_file, output)
logging.info("Local record of translations updated")