def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None): """ Return list of languages to not rezip due to errors in compile process. Then email admins errors. """ broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)', err) or [] if lang_codes: # Only show the errors relevant to the list of language codes passed in. lang_codes = set([lcode_to_django_dir(lc) for lc in lang_codes]) broken_codes = list(set(broken_codes).intersection(lang_codes)) if broken_codes: logging.warning("Found %d errors while compiling in codes %s. Mailing admins report now." % (len(broken_codes), ', '.join(broken_codes))) subject = "Error while compiling po files" commands = "\n".join(["python manage.py compilemessages -l %s" % lc for lc in broken_codes]) message = """The following codes had errors when compiling their po files: %s. Please rerun the following commands to see specific line numbers that need to be corrected on CrowdIn, before we can update the language packs. %s""" % ( ', '.join([lcode_to_ietf(lc) for lc in broken_codes]), commands, ) if not settings.DEBUG: mail_admins(subject=subject, message=message) logging.info("Report sent.") else: logging.info("DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s" % (subject, message)) return broken_codes
def generate_metadata(package_metadata=None, version=VERSION, force_version_update=False): """Loop through locale folder, create or update language specific meta and create or update master file, skipping broken languages """ logging.info("Generating new language pack metadata") lang_codes = package_metadata.keys() if package_metadata else os.listdir(LOCALE_ROOT) broken_langs = [lc for lc, md in package_metadata.iteritems() if md.get("broken")] if package_metadata else [] master_filepath = get_language_pack_availability_filepath(version=version) master_metadata = softload_json(master_filepath, logger=logging.warn, errmsg="Error opening master language pack metadata") # loop through all languages in locale, update master file crowdin_meta_dict = download_crowdin_metadata() for lc in lang_codes: lang_code_django = lcode_to_django_dir(lc) lang_code_ietf = lcode_to_ietf(lc) lang_name = get_language_name(lang_code_ietf) metadata_filepath = get_language_pack_metadata_filepath(lang_code_ietf, version=version) ensure_dir(os.path.dirname(metadata_filepath)) if broken_langs and lang_code_django in broken_langs: # broken_langs is django format logging.info("Skipping directory %s because it did not compile." % lang_code_django) continue # Gather existing metadata crowdin_meta = next((meta for meta in crowdin_meta_dict if meta["code"] == lang_code_ietf), {}) stored_meta = softload_json(metadata_filepath, logger=logging.info, errmsg="Could not open %s language pack metadata" % lc) updated_meta = package_metadata.get(lang_code_ietf) or {} updated_meta.update({ "code": lang_code_ietf, # user-facing code "name": lang_name, "software_version": version, }) try: # Augment the metadata updated_meta.update(get_language_names(lang_code_django)) except LanguageNotFoundError: logging.warning("Unrecognized language; unable to add extra naming metadata %s" % lang_code_django) continue if force_version_update: language_pack_version = 1 + stored_meta.get("language_pack_version", 0) # will increment to one else: language_pack_version = increment_language_pack_version(stored_meta, updated_meta) updated_meta["language_pack_version"] = language_pack_version stored_meta.update(updated_meta) # Write locally (this is used on download by distributed server to update it's database) with open(metadata_filepath, 'w') as output: json.dump(stored_meta, output) # Update master (this is used for central server to handle API requests for data) master_metadata[lang_code_ietf] = stored_meta # Save updated master ensure_dir(os.path.dirname(master_filepath)) with open(master_filepath, 'w') as fp: json.dump(master_metadata, fp) logging.info("Local record of translations updated")
def update_translations(lang_codes=None, download_kalite_translations=True, download_ka_translations=True, zip_file=None, ka_zip_file=None, use_local=False, version=VERSION): """ Download translations (if necessary), repurpose them into needed files, then move the resulting files to the versioned storage directory. """ package_metadata = {} if use_local: for lang_code in lang_codes: lang_code = lcode_to_ietf(lang_code) package_metadata[lang_code] = {} combined_po_file = get_po_build_path(lang_code, version=version) combined_metadata = get_po_metadata(combined_po_file) package_metadata[lang_code]["approved_translations"] = combined_metadata["approved_translations"] package_metadata[lang_code]["phrases"] = combined_metadata["phrases"] else: logging.info("Downloading %s language(s)" % lang_codes) # Download latest UI translations from CrowdIn for lang_code in (lang_codes or [None]): lang_code = lcode_to_ietf(lang_code) lang_code_crowdin = get_supported_language_map(lang_code)['crowdin'] if not lang_code_crowdin: logging.warning('Interface translations for %s are disabled for now' % lang_code) raise SkipTranslations # we make it a defaultdict so that if no value is present it's automatically 0 package_metadata[lang_code] = defaultdict( lambda: 0, { 'approved_translations': 0, 'phrases': 0, 'kalite_ntranslations': 0, 'kalite_nphrases': 0, }) # these values will likely yield the wrong values when download_kalite_translations == False. if not download_kalite_translations: logging.info("Skipping KA Lite translations") kalite_po_file = None else: logging.info("Downloading KA Lite translations...") kalite_po_file = download_latest_translations( lang_code=lang_code_crowdin, project_id=settings.CROWDIN_PROJECT_ID, project_key=settings.CROWDIN_PROJECT_KEY, zip_file=zip_file or (os.path.join(CROWDIN_CACHE_DIR, "kalite-%s.zip" % lang_code_crowdin) if settings.DEBUG else None), ) # We have the po file, now get metadata. kalite_metadata = get_po_metadata(kalite_po_file) package_metadata[lang_code]["approved_translations"] = kalite_metadata["approved_translations"] package_metadata[lang_code]["phrases"] = kalite_metadata["phrases"] package_metadata[lang_code]["kalite_ntranslations"] = kalite_metadata["approved_translations"] package_metadata[lang_code]["kalite_nphrases"] = kalite_metadata["phrases"] # Download Khan Academy translations too if not download_ka_translations: logging.info("Skipping KA translations") combined_po_file = None else: logging.info("Downloading Khan Academy translations...") combined_po_file = download_latest_translations( lang_code=lang_code_crowdin, project_id=settings.KA_CROWDIN_PROJECT_ID, project_key=settings.KA_CROWDIN_PROJECT_KEY, zip_file=ka_zip_file or (os.path.join(CROWDIN_CACHE_DIR, "ka-%s.zip" % lang_code_crowdin) if settings.DEBUG else None), combine_with_po_file=kalite_po_file, rebuild=False, # just to be friendly to KA--we shouldn't force a rebuild download_type="ka", ) # we have the po file; now ka_metadata = get_po_metadata(combined_po_file) package_metadata[lang_code]["approved_translations"] = ka_metadata["approved_translations"] package_metadata[lang_code]["phrases"] = ka_metadata["phrases"] package_metadata[lang_code]["ka_ntranslations"] = ka_metadata["approved_translations"] - package_metadata[lang_code]["kalite_ntranslations"] package_metadata[lang_code]["ka_nphrases"] = ka_metadata["phrases"] - package_metadata[lang_code]["kalite_nphrases"] # here we compute the percent translated if download_ka_translations or download_kalite_translations: pmlc = package_metadata[lang_code] # shorter name, less characters if pmlc['kalite_nphrases'] == pmlc['ka_nphrases'] == 0: pmlc['percent_translated'] = 0 else: pmlc["percent_translated"] = 100. * (pmlc['kalite_ntranslations'] + pmlc['ka_ntranslations']) / float(pmlc['kalite_nphrases'] + pmlc['ka_nphrases']) # english is always 100% translated if lang_code == 'en': pmlc['percent_translated'] = 100 return package_metadata