def zip_language_packs(lang_codes=None):
"""Zip up and expose all language packs
converts all into ietf
"""
lang_codes = lang_codes or os.listdir(LOCALE_ROOT)
lang_codes = [lcode_to_ietf(lc) for lc in lang_codes]
logging.info("Zipping up %d language pack(s)" % len(lang_codes))
for lang_code_ietf in lang_codes:
lang_code_django = lcode_to_django_dir(lang_code_ietf)
lang_locale_path = os.path.join(LOCALE_ROOT, lang_code_django)
if not os.path.exists(lang_locale_path):
logging.warn("Unexpectedly skipping missing directory: %s" % lang_code_django)
elif not os.path.isdir(lang_locale_path):
logging.error("Skipping language where a file exists where a directory was expected: %s" % lang_code_django)
# Create a zipfile for this language
zip_filepath = get_language_pack_filepath(lang_code_ietf)
ensure_dir(os.path.dirname(zip_filepath))
logging.info("Creating zip file in %s" % zip_filepath)
z = zipfile.ZipFile(zip_filepath, 'w', zipfile.ZIP_DEFLATED)
# Get every single file in the directory and zip it up
for metadata_file in glob.glob('%s/*.json' % lang_locale_path):
z.write(os.path.join(lang_locale_path, metadata_file), arcname=os.path.basename(metadata_file))
srt_dirpath = get_srt_path(lang_code_django)
for srt_file in glob.glob(os.path.join(srt_dirpath, "*.srt")):
z.write(srt_file, arcname=os.path.join("subtitles", os.path.basename(srt_file)))
z.close()
logging.info("Done.")
def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None):
"""
Return list of languages to not rezip due to errors in compile process.
Then email admins errors.
"""
broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)', err) or []
if lang_codes:
# Only show the errors relevant to the list of language codes passed in.
lang_codes = set([lcode_to_django_dir(lc) for lc in lang_codes])
broken_codes = list(set(broken_codes).intersection(lang_codes))
if broken_codes:
logging.warning("Found %d errors while compiling in codes %s. Mailing admins report now." % (len(broken_codes), ', '.join(broken_codes)))
subject = "Error while compiling po files"
commands = "\n".join(["python manage.py compilemessages -l %s" % lc for lc in broken_codes])
message = """The following codes had errors when compiling their po files: %s.
Please rerun the following commands to see specific line numbers
that need to be corrected on CrowdIn, before we can update the language packs.
%s""" % (
', '.join([lcode_to_ietf(lc) for lc in broken_codes]),
commands,
)
if not settings.DEBUG:
mail_admins(subject=subject, message=message)
logging.info("Report sent.")
else:
logging.info("DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s" % (subject, message))
return broken_codes
def move_srts(lang_code):
"""
Srts live in the locale directory, but that's not exposed at any URL. So instead,
we have to move the srts out to /static/subtitles/[lang_code]/
"""
lang_code_ietf = lcode_to_ietf(lang_code)
lang_code_django = lcode_to_django_dir(lang_code)
subtitles_static_dir = os.path.join(settings.STATIC_ROOT, "subtitles")
src_dir = os.path.join(LOCALE_ROOT, lang_code_django, "subtitles")
dest_dir = get_srt_path(lang_code_django)
ensure_dir(dest_dir)
lang_subtitles = glob.glob(os.path.join(src_dir, "*.srt"))
logging.info("Moving %d subtitles from %s to %s" % (len(lang_subtitles), src_dir, dest_dir))
for fil in lang_subtitles:
srt_dest_path = os.path.join(dest_dir, os.path.basename(fil))
if os.path.exists(srt_dest_path):
os.remove(srt_dest_path)
shutil.move(fil, srt_dest_path)
if os.listdir(src_dir):
logging.warn("%s is not empty; will not remove. Please check that all subtitles were moved." % src_dir)
else:
logging.info("Removing empty source directory (%s)." % src_dir)
shutil.rmtree(src_dir)
def unpack_language(lang_code, zip_file):
"""Unpack zipped language pack into locale directory"""
lang_code = lcode_to_django_dir(lang_code)
logging.info("Unpacking new translations")
ensure_dir(os.path.join(LOCALE_ROOT, lang_code, "LC_MESSAGES"))
## Unpack into temp dir
z = zipfile.ZipFile(StringIO(zip_file))
z.extractall(os.path.join(LOCALE_ROOT, lang_code))
def extract_new_po(extract_path, combine_with_po_file=None, lang="all"):
"""Move newly downloaded po files to correct location in locale
direction. Returns the location of the po file if a single
language is given, or a list of locations if language is
'all'.
"""
if combine_with_po_file:
assert lang != 'all', "You can only combine a po file with only one other po file. Please select a specific language, not 'all'."
assert os.path.basename(combine_with_po_file) in ["django.po", "djangojs.po"], "File %s does not seem to be either django.po or djangojs.po."
if lang == 'all':
languages = os.listdir(extract_path)
return [extract_new_po(os.path.join(extract_path, l), lang=l) for l in languages]
else:
converted_code = lcode_to_django_dir(lang)
# ensure directory exists in locale folder, and then overwrite local po files with new ones
dest_path = os.path.join(LOCALE_ROOT, converted_code, "LC_MESSAGES")
ensure_dir(dest_path)
dest_file = os.path.join(dest_path, 'django.po')
build_file = os.path.join(dest_path, 'djangobuild.po') # so we dont clobber previous django.po that we build
src_po_files = all_po_files(extract_path)
concat_command = ['msgcat', '-o', build_file, '--no-location']
# filter out po files that are giving me problems
src_po_files = filter(lambda po_file: not ('learn.math.trigonometry.exercises' in po_file or 'learn.math.algebra.exercises' in po_file),
src_po_files)
concat_command += src_po_files
if combine_with_po_file and os.path.exists(combine_with_po_file):
concat_command += [combine_with_po_file]
backups = [sys.stdout, sys.stderr]
try:
sys.stdout = StringIO.StringIO() # capture output
sys.stderr = StringIO.StringIO()
p = subprocess.call(concat_command)
out = sys.stdout.getvalue() # release output
err = sys.stderr.getvalue() # release err
finally:
sys.stdout = backups[0]
sys.stderr = backups[1]
shutil.move(build_file, dest_file)
return dest_file
def handle(self, *args, **options):
if not settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the central server.")
if not options["lang_code"] or options["lang_code"].lower() == "all":
lang_codes = ['all']
else:
lang_codes = [lcode_to_django_dir(lc) for lc in options["lang_code"].split(",")]
# If no_update is set, then disable all update options.
for key in options:
if key.startswith("update_"):
options[key] = options[key] and not options["no_update"]
upgrade_old_schema()
package_metadata = dict([(lang_code, {}) for lang_code in lang_codes])
if options['low_mem']:
logging.info('Making the GC more aggressive...')
gc.set_threshold(36, 2, 2)
# Update all the latest srts, using raw language code
if options['update_srts']:
update_srts(days=options["days"], lang_codes=lang_codes)
for lang_code in lang_codes:
package_metadata[lang_code]["subtitle_count"] = get_subtitle_count(lang_code)
# Update the dubbed video mappings
if options['update_dubbed']:
get_dubbed_video_map(force=True)
for lang_code in lang_codes:
dv_map = get_dubbed_video_map(lang_code)
package_metadata[lang_code]["num_dubbed_videos"] = len(dv_map) if dv_map else 0
# Update the exercises
for lang_code in lang_codes:
if options['update_exercises']:
call_command("scrape_exercises", lang_code=lang_code)
package_metadata[lang_code]["num_exercises"] = get_localized_exercise_count(lang_code)
# Loop through new UI translations & subtitles, create/update unified meta data
generate_metadata(lang_codes=lang_codes, broken_langs=broken_langs, package_metadata=package_metadata)
# Zip
package_sizes = zip_language_packs(lang_codes=lang_codes)
logging.debug("Package sizes: %s" % package_sizes)
# Loop through new UI translations & subtitles, create/update unified meta data
update_metadata(package_sizes)
def compile_po_files(lang_codes=None, failure_ok=True):
"""
Compile all po files in locale directory.
First argument (lang_codes) can be None (means all), a list/tuple, or even a string (shh...)
"""
# before running compilemessages, ensure in correct directory
change_dir_to_project_root()
if not lang_codes or len(lang_codes) > 1:
(out, err, rc) = call_command_with_output('compilemessages')
else:
lang_code = lang_codes if isinstance(lang_codes, basestring) else lang_codes[0]
(out, err, rc) = call_command_with_output('compilemessages', locale=lcode_to_django_dir(lang_code))
if err and not failure_ok:
raise CommandError("Failure compiling po files: %s" % err)
return out, err, rc
def update_language_packs(lang_codes=None, download_ka_translations=True, zip_file=None, ka_zip_file=None, use_local=False):
# Loop through new UI translations & subtitles, create/update unified meta data
generate_metadata(lang_codes=lang_codes)
if use_local:
for lang_code in lang_codes:
lang_code = lcode_to_ietf(lang_code)
package_metadata[lang_code] = {}
combined_po_file = os.path.join(LOCALE_ROOT, lcode_to_django_dir(lang_code), "LC_MESSAGES", "django.po")
combined_metadata = get_po_metadata(combined_po_file)
package_metadata[lang_code]["approved_translations"] = combined_metadata["approved_translations"]
package_metadata[lang_code]["phrases"] = combined_metadata["phrases"]
else:
logging.info("Downloading %s language(s)" % lang_codes)
# Zip
zip_language_packs(lang_codes=lang_codes)
def generate_metadata(lang_codes=None, broken_langs=None, added_ka=False):
"""Loop through locale folder, create or update language specific meta
and create or update master file, skipping broken languages
note: broken_langs must be in django format.
"""
logging.info("Generating new language pack metadata")
if broken_langs is None:
broken_langs = tuple()
lang_codes = lang_codes or os.listdir(LOCALE_ROOT)
try:
with open(get_language_pack_availability_filepath(), "r") as fp:
master_metadata = json.load(fp)
if isinstance(master_metadata, list):
logging.info("Code switched from list to dict to support single language LanguagePack updates; converting your old list storage for dictionary storage.")
master_list = master_metadata
master_metadata = {}
for lang_meta in master_list:
master_metadata[lang_meta["code"]] = lang_meta
except Exception as e:
logging.warn("Error opening language pack metadata: %s; resetting" % e)
master_metadata = {}
# loop through all languages in locale, update master file
crowdin_meta_dict = download_crowdin_metadata()
with open(SUBTITLE_COUNTS_FILEPATH, "r") as fp:
subtitle_counts = json.load(fp)
for lc in lang_codes:
lang_code_django = lcode_to_django_dir(lc)
lang_code_ietf = lcode_to_ietf(lc)
lang_name = get_language_name(lang_code_ietf)
# skips anything not a directory, or with errors
if not os.path.isdir(os.path.join(LOCALE_ROOT, lang_code_django)):
logging.info("Skipping item %s because it is not a directory" % lang_code_django)
continue
elif lang_code_django in broken_langs: # broken_langs is django format
logging.info("Skipping directory %s because it triggered an error during compilemessages. The admins should have received a report about this and must fix it before this pack will be updateed." % lang_code_django)
continue
# Gather existing metadata
crowdin_meta = next((meta for meta in crowdin_meta_dict if meta["code"] == lang_code_ietf), {})
metadata_filepath = get_language_pack_metadata_filepath(lang_code_ietf)
try:
with open(metadata_filepath) as fp:
local_meta = json.load(fp)
except Exception as e:
logging.warn("Error opening language pack metadata (%s): %s; resetting" % (metadata_filepath, e))
local_meta = {}
try:
# update metadata
updated_meta = {
"code": lcode_to_ietf(crowdin_meta.get("code") or lang_code_django), # user-facing code
"name": (crowdin_meta.get("name") or lang_name),
"percent_translated": int(crowdin_meta.get("approved_progress", 0)),
"phrases": int(crowdin_meta.get("phrases", 0)),
"approved_translations": int(crowdin_meta.get("approved", 0)),
}
# Obtain current number of subtitles
entry = subtitle_counts.get(lang_name, {})
srt_count = entry.get("count", 0)
updated_meta.update({
"software_version": version.VERSION,
"subtitle_count": srt_count,
})
except LanguageNotFoundError:
logging.error("Unrecognized language; must skip item %s" % lang_code_django)
continue
language_pack_version = increment_language_pack_version(local_meta, updated_meta)
updated_meta["language_pack_version"] = language_pack_version + int(added_ka)
local_meta.update(updated_meta)
# Write locally (this is used on download by distributed server to update it's database)
with open(metadata_filepath, 'w') as output:
json.dump(local_meta, output)
# Update master (this is used for central server to handle API requests for data)
master_metadata[lang_code_ietf] = local_meta
# Save updated master
ensure_dir(os.path.dirname(get_language_pack_availability_filepath()))
with open(get_language_pack_availability_filepath(), 'w') as output:
json.dump(master_metadata, output)
logging.info("Local record of translations updated")
