def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None):
"""
Return list of languages to not rezip due to errors in compile process.
Then email admins errors.
"""
broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)', err) or []
if lang_codes:
# Only show the errors relevant to the list of language codes passed in.
lang_codes = set([lcode_to_django_dir(lc) for lc in lang_codes])
broken_codes = list(set(broken_codes).intersection(lang_codes))
if broken_codes:
logging.warning("Found %d errors while compiling in codes %s. Mailing admins report now." % (len(broken_codes), ', '.join(broken_codes)))
subject = "Error while compiling po files"
commands = "\n".join(["python manage.py compilemessages -l %s" % lc for lc in broken_codes])
message = """The following codes had errors when compiling their po files: %s.
Please rerun the following commands to see specific line numbers
that need to be corrected on CrowdIn, before we can update the language packs.
%s""" % (
', '.join([lcode_to_ietf(lc) for lc in broken_codes]),
commands,
)
if not settings.DEBUG:
mail_admins(subject=subject, message=message)
logging.info("Report sent.")
else:
logging.info("DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s" % (subject, message))
return broken_codes
def download_srt_from_3rd_party(*args, **kwargs):
"""Download subtitles specified by command line args"""
lang_code = kwargs.get("lang_code", None)
# if language specified, do those, if not do all
if not lang_code:
raise CommandError("You must specify a language code or 'all' with -l")
elif lang_code == "all":
bad_languages = {}
for filename in get_all_download_status_files():
try:
videos = json.loads(open(filename).read())
except Exception as e:
logging.error(e)
raise CommandError(
"Unable to open %s. The file might be corrupted. Please re-run the generate_subtitle_map command to regenerate it."
% filename)
try:
lang_code = os.path.basename(filename).split("_")[0]
kwargs["lang_code"] = lang_code
download_if_criteria_met(videos, *args, **kwargs)
except Exception as e:
logging.error("Error downloading subtitles for %s: %s" %
(lang_code, e))
bad_languages[lang_code] = e
continue
# now report final results
if bad_languages:
raise CommandError(
"Failed to download subtitles for the following languages: %s"
% bad_languages.keys())
else:
srt_list_path = get_lang_map_filepath(
convert_language_code_format(lang_code))
try:
videos = json.loads(open(srt_list_path).read())
except:
logging.warning(
"No subtitles available for download for language code %s. Skipping."
% lang_code)
else:
download_if_criteria_met(videos, *args, **kwargs)
def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None):
"""
Return list of languages to not rezip due to errors in compile process.
Then email admins errors.
"""
broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)',
err) or []
if lang_codes:
# Only show the errors relevant to the list of language codes passed in.
lang_codes = set([lcode_to_django_dir(lc) for lc in lang_codes])
broken_codes = list(set(broken_codes).intersection(lang_codes))
if broken_codes:
logging.warning(
"Found %d errors while compiling in codes %s. Mailing admins report now."
% (len(broken_codes), ', '.join(broken_codes)))
subject = "Error while compiling po files"
commands = "\n".join([
"python manage.py compilemessages -l %s" % lc
for lc in broken_codes
])
message = """The following codes had errors when compiling their po files: %s.
Please rerun the following commands to see specific line numbers
that need to be corrected on CrowdIn, before we can update the language packs.
%s""" % (
', '.join([lcode_to_ietf(lc) for lc in broken_codes]),
commands,
)
if not settings.DEBUG:
mail_admins(subject=subject, message=message)
logging.info("Report sent.")
else:
logging.info(
"DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s"
% (subject, message))
return broken_codes
def download_srt_from_3rd_party(*args, **kwargs):
"""Download subtitles specified by command line args"""
lang_code = kwargs.get("lang_code", None)
# if language specified, do those, if not do all
if not lang_code:
raise CommandError("You must specify a language code or 'all' with -l")
elif lang_code == "all":
bad_languages = {}
for filename in get_all_download_status_files():
try:
videos = json.loads(open(filename).read())
except Exception as e:
logging.error(e)
raise CommandError("Unable to open %s. The file might be corrupted. Please re-run the generate_subtitle_map command to regenerate it." % filename)
try:
lang_code = os.path.basename(filename).split("_")[0]
kwargs["lang_code"] = lang_code
download_if_criteria_met(videos, *args, **kwargs)
except Exception as e:
logging.error("Error downloading subtitles for %s: %s" % (lang_code, e))
bad_languages[lang_code] = e
continue
# now report final results
if bad_languages:
raise CommandError("Failed to download subtitles for the following languages: %s" % bad_languages.keys())
else:
srt_list_path = get_lang_map_filepath(convert_language_code_format(lang_code))
try:
videos = json.loads(open(srt_list_path).read())
except:
logging.warning("No subtitles available for download for language code %s. Skipping." % lang_code)
else:
download_if_criteria_met(videos, *args, **kwargs)
def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None):
"""Return list of languages to not rezip due to errors in compile process. Email admins errors"""
broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)', err)
if lang_codes:
broken_codes = list(set(broken_codes) - set(lang_codes))
if broken_codes:
logging.warning("Found %d errors while compiling in codes %s. Mailing admins report now." % (len(broken_codes), ', '.join(broken_codes)))
subject = "Error while compiling po files"
commands = ""
for code in broken_codes:
commands += "\npython manage.py compilemessages -l %s" % code
message = """The following codes had errors when compiling their po files: %s.
Please rerun the following commands to see specific line numbers
that need to be corrected on CrowdIn, before we can update the language packs.
%s""" % (', '.join(broken_codes), commands)
if not settings.DEBUG:
mail_admins(subject=subject, message=message)
logging.info("Report sent.")
else:
logging.info("DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s" % (subject, message))
return broken_codes
def generate_metadata(package_metadata=None, version=VERSION, force_version_update=False):
"""Loop through locale folder, create or update language specific meta
and create or update master file, skipping broken languages
"""
logging.info("Generating new language pack metadata")
lang_codes = package_metadata.keys() if package_metadata else os.listdir(LOCALE_ROOT)
broken_langs = [lc for lc, md in package_metadata.iteritems() if md.get("broken")] if package_metadata else []
master_filepath = get_language_pack_availability_filepath(version=version)
master_metadata = softload_json(master_filepath, logger=logging.warn, errmsg="Error opening master language pack metadata")
# loop through all languages in locale, update master file
crowdin_meta_dict = download_crowdin_metadata()
for lc in lang_codes:
lang_code_django = lcode_to_django_dir(lc)
lang_code_ietf = lcode_to_ietf(lc)
lang_name = get_language_name(lang_code_ietf)
metadata_filepath = get_language_pack_metadata_filepath(lang_code_ietf, version=version)
ensure_dir(os.path.dirname(metadata_filepath))
if broken_langs and lang_code_django in broken_langs: # broken_langs is django format
logging.info("Skipping directory %s because it did not compile." % lang_code_django)
continue
# Gather existing metadata
crowdin_meta = next((meta for meta in crowdin_meta_dict if meta["code"] == lang_code_ietf), {})
stored_meta = softload_json(metadata_filepath, logger=logging.info, errmsg="Could not open %s language pack metadata" % lc)
updated_meta = package_metadata.get(lang_code_ietf) or {}
updated_meta.update({
"code": lang_code_ietf, # user-facing code
"name": lang_name,
"software_version": version,
})
try:
# Augment the metadata
updated_meta.update(get_language_names(lang_code_django))
except LanguageNotFoundError:
logging.warning("Unrecognized language; unable to add extra naming metadata %s" % lang_code_django)
continue
if force_version_update:
language_pack_version = 1 + stored_meta.get("language_pack_version", 0) # will increment to one
else:
language_pack_version = increment_language_pack_version(stored_meta, updated_meta)
updated_meta["language_pack_version"] = language_pack_version
stored_meta.update(updated_meta)
# Write locally (this is used on download by distributed server to update it's database)
with open(metadata_filepath, 'w') as output:
json.dump(stored_meta, output)
# Update master (this is used for central server to handle API requests for data)
master_metadata[lang_code_ietf] = stored_meta
# Save updated master
ensure_dir(os.path.dirname(master_filepath))
with open(master_filepath, 'w') as fp:
json.dump(master_metadata, fp)
logging.info("Local record of translations updated")
def update_translations(lang_codes=None,
download_kalite_translations=True,
download_ka_translations=True,
zip_file=None,
ka_zip_file=None,
use_local=False,
version=VERSION):
"""
Download translations (if necessary), repurpose them into needed files,
then move the resulting files to the versioned storage directory.
"""
package_metadata = {}
if use_local:
for lang_code in lang_codes:
lang_code = lcode_to_ietf(lang_code)
package_metadata[lang_code] = {}
combined_po_file = get_po_build_path(lang_code, version=version)
combined_metadata = get_po_metadata(combined_po_file)
package_metadata[lang_code]["approved_translations"] = combined_metadata["approved_translations"]
package_metadata[lang_code]["phrases"] = combined_metadata["phrases"]
else:
logging.info("Downloading %s language(s)" % lang_codes)
# Download latest UI translations from CrowdIn
for lang_code in (lang_codes or [None]):
lang_code = lcode_to_ietf(lang_code)
lang_code_crowdin = get_supported_language_map(lang_code)['crowdin']
if not lang_code_crowdin:
logging.warning('Interface translations for %s are disabled for now' % lang_code)
raise SkipTranslations
# we make it a defaultdict so that if no value is present it's automatically 0
package_metadata[lang_code] = defaultdict(
lambda: 0,
{
'approved_translations': 0,
'phrases': 0,
'kalite_ntranslations': 0,
'kalite_nphrases': 0,
}) # these values will likely yield the wrong values when download_kalite_translations == False.
if not download_kalite_translations:
logging.info("Skipping KA Lite translations")
kalite_po_file = None
else:
logging.info("Downloading KA Lite translations...")
kalite_po_file = download_latest_translations(
lang_code=lang_code_crowdin,
project_id=settings.CROWDIN_PROJECT_ID,
project_key=settings.CROWDIN_PROJECT_KEY,
zip_file=zip_file or (os.path.join(CROWDIN_CACHE_DIR, "kalite-%s.zip" % lang_code_crowdin) if settings.DEBUG else None),
)
# We have the po file, now get metadata.
kalite_metadata = get_po_metadata(kalite_po_file)
package_metadata[lang_code]["approved_translations"] = kalite_metadata["approved_translations"]
package_metadata[lang_code]["phrases"] = kalite_metadata["phrases"]
package_metadata[lang_code]["kalite_ntranslations"] = kalite_metadata["approved_translations"]
package_metadata[lang_code]["kalite_nphrases"] = kalite_metadata["phrases"]
# Download Khan Academy translations too
if not download_ka_translations:
logging.info("Skipping KA translations")
combined_po_file = None
else:
logging.info("Downloading Khan Academy translations...")
combined_po_file = download_latest_translations(
lang_code=lang_code_crowdin,
project_id=settings.KA_CROWDIN_PROJECT_ID,
project_key=settings.KA_CROWDIN_PROJECT_KEY,
zip_file=ka_zip_file or (os.path.join(CROWDIN_CACHE_DIR, "ka-%s.zip" % lang_code_crowdin) if settings.DEBUG else None),
combine_with_po_file=kalite_po_file,
rebuild=False, # just to be friendly to KA--we shouldn't force a rebuild
download_type="ka",
)
# we have the po file; now
ka_metadata = get_po_metadata(combined_po_file)
package_metadata[lang_code]["approved_translations"] = ka_metadata["approved_translations"]
package_metadata[lang_code]["phrases"] = ka_metadata["phrases"]
package_metadata[lang_code]["ka_ntranslations"] = ka_metadata["approved_translations"] - package_metadata[lang_code]["kalite_ntranslations"]
package_metadata[lang_code]["ka_nphrases"] = ka_metadata["phrases"] - package_metadata[lang_code]["kalite_nphrases"]
# here we compute the percent translated
if download_ka_translations or download_kalite_translations:
pmlc = package_metadata[lang_code] # shorter name, less characters
if pmlc['kalite_nphrases'] == pmlc['ka_nphrases'] == 0:
pmlc['percent_translated'] = 0
else:
pmlc["percent_translated"] = 100. * (pmlc['kalite_ntranslations'] + pmlc['ka_ntranslations']) / float(pmlc['kalite_nphrases'] + pmlc['ka_nphrases'])
# english is always 100% translated
if lang_code == 'en':
pmlc['percent_translated'] = 100
return package_metadata
def generate_metadata(package_metadata=None,
version=VERSION,
force_version_update=False):
"""Loop through locale folder, create or update language specific meta
and create or update master file, skipping broken languages
"""
logging.info("Generating new language pack metadata")
lang_codes = package_metadata.keys() if package_metadata else os.listdir(
LOCALE_ROOT)
broken_langs = [
lc for lc, md in package_metadata.iteritems() if md.get("broken")
] if package_metadata else []
master_filepath = get_language_pack_availability_filepath(version=version)
master_metadata = softload_json(
master_filepath,
logger=logging.warn,
errmsg="Error opening master language pack metadata")
# loop through all languages in locale, update master file
crowdin_meta_dict = download_crowdin_metadata()
for lc in lang_codes:
lang_code_django = lcode_to_django_dir(lc)
lang_code_ietf = lcode_to_ietf(lc)
lang_name = get_language_name(lang_code_ietf)
metadata_filepath = get_language_pack_metadata_filepath(
lang_code_ietf, version=version)
ensure_dir(os.path.dirname(metadata_filepath))
if broken_langs and lang_code_django in broken_langs: # broken_langs is django format
logging.info("Skipping directory %s because it did not compile." %
lang_code_django)
continue
# Gather existing metadata
crowdin_meta = next(
(meta
for meta in crowdin_meta_dict if meta["code"] == lang_code_ietf),
{})
stored_meta = softload_json(
metadata_filepath,
logger=logging.info,
errmsg="Could not open %s language pack metadata" % lc)
updated_meta = package_metadata.get(lang_code_ietf) or {}
updated_meta.update({
"code": lang_code_ietf, # user-facing code
"name": lang_name,
"software_version": version,
})
try:
# Augment the metadata
updated_meta.update(get_language_names(lang_code_django))
except LanguageNotFoundError:
logging.warning(
"Unrecognized language; unable to add extra naming metadata %s"
% lang_code_django)
continue
if force_version_update:
language_pack_version = 1 + stored_meta.get(
"language_pack_version", 0) # will increment to one
else:
language_pack_version = increment_language_pack_version(
stored_meta, updated_meta)
updated_meta["language_pack_version"] = language_pack_version
stored_meta.update(updated_meta)
# Write locally (this is used on download by distributed server to update it's database)
with open(metadata_filepath, 'w') as output:
json.dump(stored_meta, output)
# Update master (this is used for central server to handle API requests for data)
master_metadata[lang_code_ietf] = stored_meta
# Save updated master
ensure_dir(os.path.dirname(master_filepath))
with open(master_filepath, 'w') as fp:
json.dump(master_metadata, fp)
logging.info("Local record of translations updated")
def update_translations(lang_codes=None,
download_kalite_translations=True,
download_ka_translations=True,
zip_file=None,
ka_zip_file=None,
use_local=False,
version=VERSION):
"""
Download translations (if necessary), repurpose them into needed files,
then move the resulting files to the versioned storage directory.
"""
package_metadata = {}
if use_local:
for lang_code in lang_codes:
lang_code = lcode_to_ietf(lang_code)
package_metadata[lang_code] = {}
combined_po_file = get_po_build_path(lang_code, version=version)
combined_metadata = get_po_metadata(combined_po_file)
package_metadata[lang_code][
"approved_translations"] = combined_metadata[
"approved_translations"]
package_metadata[lang_code]["phrases"] = combined_metadata[
"phrases"]
else:
logging.info("Downloading %s language(s)" % lang_codes)
# Download latest UI translations from CrowdIn
for lang_code in (lang_codes or [None]):
lang_code = lcode_to_ietf(lang_code)
lang_code_crowdin = get_supported_language_map(
lang_code)['crowdin']
if not lang_code_crowdin:
logging.warning(
'Interface translations for %s are disabled for now' %
lang_code)
raise SkipTranslations
# we make it a defaultdict so that if no value is present it's automatically 0
package_metadata[lang_code] = defaultdict(
lambda: 0, {
'approved_translations': 0,
'phrases': 0,
'kalite_ntranslations': 0,
'kalite_nphrases': 0,
}
) # these values will likely yield the wrong values when download_kalite_translations == False.
if not download_kalite_translations:
logging.info("Skipping KA Lite translations")
kalite_po_file = None
else:
logging.info("Downloading KA Lite translations...")
kalite_po_file = download_latest_translations(
lang_code=lang_code_crowdin,
project_id=settings.CROWDIN_PROJECT_ID,
project_key=settings.CROWDIN_PROJECT_KEY,
zip_file=zip_file or (os.path.join(
CROWDIN_CACHE_DIR, "kalite-%s.zip" %
lang_code_crowdin) if settings.DEBUG else None),
)
# We have the po file, now get metadata.
kalite_metadata = get_po_metadata(kalite_po_file)
package_metadata[lang_code][
"approved_translations"] = kalite_metadata[
"approved_translations"]
package_metadata[lang_code]["phrases"] = kalite_metadata["phrases"]
package_metadata[lang_code][
"kalite_ntranslations"] = kalite_metadata[
"approved_translations"]
package_metadata[lang_code]["kalite_nphrases"] = kalite_metadata[
"phrases"]
# Download Khan Academy translations too
if not download_ka_translations:
logging.info("Skipping KA translations")
combined_po_file = None
else:
logging.info("Downloading Khan Academy translations...")
combined_po_file = download_latest_translations(
lang_code=lang_code_crowdin,
project_id=settings.KA_CROWDIN_PROJECT_ID,
project_key=settings.KA_CROWDIN_PROJECT_KEY,
zip_file=ka_zip_file or (os.path.join(
CROWDIN_CACHE_DIR, "ka-%s.zip" %
lang_code_crowdin) if settings.DEBUG else None),
combine_with_po_file=kalite_po_file,
rebuild=
False, # just to be friendly to KA--we shouldn't force a rebuild
download_type="ka",
)
# we have the po file; now
ka_metadata = get_po_metadata(combined_po_file)
package_metadata[lang_code]["approved_translations"] = ka_metadata[
"approved_translations"]
package_metadata[lang_code]["phrases"] = ka_metadata["phrases"]
package_metadata[lang_code]["ka_ntranslations"] = ka_metadata[
"approved_translations"] - package_metadata[lang_code][
"kalite_ntranslations"]
package_metadata[lang_code]["ka_nphrases"] = ka_metadata[
"phrases"] - package_metadata[lang_code]["kalite_nphrases"]
# here we compute the percent translated
if download_ka_translations or download_kalite_translations:
pmlc = package_metadata[
lang_code] # shorter name, less characters
if pmlc['kalite_nphrases'] == pmlc['ka_nphrases'] == 0:
pmlc['percent_translated'] = 0
else:
pmlc["percent_translated"] = 100. * (
pmlc['kalite_ntranslations'] + pmlc['ka_ntranslations']
) / float(pmlc['kalite_nphrases'] + pmlc['ka_nphrases'])
# english is always 100% translated
if lang_code == 'en':
pmlc['percent_translated'] = 100
return package_metadata
