def pproc_newcontext_po(po, pot_messages, pot_stats): print("Adding new contexts to {}...".format(po)) messages, state, stats = utils.parse_messages(po) known_ctxt = stats["contexts"] print("Already known (present) context(s): {}".format(str(known_ctxt))) new_ctxt = set() added = 0 # Only use valid already translated messages! allowed_keys = state["trans_msg"] - state["fuzzy_msg"] - state["comm_msg"] for key in pot_messages.keys(): ctxt, msgid = key if ctxt in known_ctxt: continue new_ctxt.add(ctxt) for t_ctxt in known_ctxt: # XXX The first match will win, this might not be optimal... t_key = (t_ctxt, msgid) if t_key in allowed_keys: # Wrong comments (sources) will be removed by msgmerge... messages[key] = messages[t_key] messages[key]["msgctxt_lines"] = [ctxt] added += 1 utils.write_messages(po, messages, state["comm_msg"], state["fuzzy_msg"]) print("Finished!\n {} new context(s) was/were added {}, adding {} new " "messages.\n".format(len(new_ctxt), str(new_ctxt), added)) return 0
def do_clean(po, strict): print("Cleaning {}...".format(po)) messages, states, u1 = utils.parse_messages(po) if strict and states["is_broken"]: print("ERROR! This .po file is broken!") return 1 for msgkey in states["comm_msg"]: del messages[msgkey] utils.write_messages(po, messages, states["comm_msg"], states["fuzzy_msg"]) print("Removed {} commented messages.".format(len(states["comm_msg"]))) return 0
def main(): import argparse parser = argparse.ArgumentParser(description="" \ "Preprocesses right-to-left languages.\n" \ "You can use it either standalone, or through " \ "import_po_from_branches or update_trunk.\n\n" \ "Note: This has been tested on Linux, not 100% it will " \ "work nicely on Windows or OsX.\n" \ "Note: This uses ctypes, as there is no py3 binding for " \ "fribidi currently. This implies you only need the " \ "compiled C library to run it.\n" \ "Note: It handles some formating/escape codes (like " \ "\\\", %s, %x12, %.4f, etc.), protecting them from ugly " \ "(evil) fribidi, which seems completely unaware of such " \ "things (as unicode is...).") parser.add_argument('dst', metavar='dst.po', help="The dest po into which write the " \ "pre-processed messages.") parser.add_argument('src', metavar='src.po', help="The po's to pre-process messages.") args = parser.parse_args() msgs, state, u1 = utils.parse_messages(args.src) if state["is_broken"]: print("Source po is BROKEN, aborting.") return 1 keys = [] trans = [] for key, val in msgs.items(): keys.append(key) trans.append("".join(val["msgstr_lines"])) trans = log2vis(trans) for key, trn in zip(keys, trans): # Mono-line for now... msgs[key]["msgstr_lines"] = [trn] utils.write_messages(args.dst, msgs, state["comm_msg"], state["fuzzy_msg"]) print("RTL pre-process completed.") return 0
def main(): import argparse parser = argparse.ArgumentParser(description="Import advanced enough po’s " \ "from branches to trunk.") parser.add_argument('-t', '--threshold', type=int, help="Import threshold, as a percentage.") parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.") parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.") args = parser.parse_args() ret = 0 threshold = float(settings.IMPORT_MIN_LEVEL) / 100.0 if args.threshold is not None: threshold = float(args.threshold) / 100.0 for lang in os.listdir(BRANCHES_DIR): if args.langs and lang not in args.langs: continue po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po"))) if os.path.exists(po): po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE) msgs, state, stats = utils.parse_messages(po) tot_msgs = stats["tot_msg"] trans_msgs = stats["trans_msg"] lvl = 0.0 if tot_msgs: lvl = float(trans_msgs) / float(tot_msgs) if lvl > threshold: if state["is_broken"] and args.strict: print("{:<10}: {:>6.1%} done, but BROKEN, skipped." \ "".format(lang, lvl)) ret = 1 else: if os.path.exists(po_is_rtl): out_po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po"))) out_raw_po = os.path.join(TRUNK_PO_DIR, "_".join((lang, "raw.po"))) keys = [] trans = [] for k, m in msgs.items(): keys.append(k) trans.append("".join(m["msgstr_lines"])) trans = rtl_preprocess.log2vis(trans) for k, t in zip(keys, trans): # Mono-line for now... msgs[k]["msgstr_lines"] = [t] utils.write_messages(out_po, msgs, state["comm_msg"], state["fuzzy_msg"]) # Also copies org po! shutil.copy(po, out_raw_po) print("{:<10}: {:>6.1%} done, enough translated " \ "messages, processed and copied to trunk." \ "".format(lang, lvl)) else: shutil.copy(po, TRUNK_PO_DIR) print("{:<10}: {:>6.1%} done, enough translated " \ "messages, copied to trunk.".format(lang, lvl)) else: if state["is_broken"] and args.strict: print("{:<10}: {:>6.1%} done, BROKEN and not enough " \ "translated messages, skipped".format(lang, lvl)) ret = 1 else: print("{:<10}: {:>6.1%} done, not enough translated " \ "messages, skipped.".format(lang, lvl)) return ret
def main(): parser = argparse.ArgumentParser(description="Update blender.pot file " \ "from messages.txt") parser.add_argument('-w', '--warning', action="store_true", help="Show warnings.") parser.add_argument('-i', '--input', metavar="File", help="Input messages file path.") parser.add_argument('-o', '--output', metavar="File", help="Output pot file path.") args = parser.parse_args() if args.input: global FILE_NAME_MESSAGES FILE_NAME_MESSAGES = args.input if args.output: global FILE_NAME_POT FILE_NAME_POT = args.output print("Running fake py gettext…") # Not using any more xgettext, simpler to do it ourself! messages = {} py_xgettext(messages) print("Finished, found {} messages.".format(len(messages))) if SPELL_CACHE and os.path.exists(SPELL_CACHE): with open(SPELL_CACHE, 'rb') as f: spell_cache = pickle.load(f) else: spell_cache = set() print(len(spell_cache)) print("Generating POT file {}…".format(FILE_NAME_POT)) msgs, states = gen_empty_pot() tot_messages, _a = merge_messages(msgs, states, messages, True, spell_cache) # add messages collected automatically from RNA print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES)) messages = {} with open(FILE_NAME_MESSAGES, encoding="utf-8") as f: srcs = [] context = "" for line in f: line = utils.stripeol(line) if line.startswith(COMMENT_PREFIX): srcs.append(line[len(COMMENT_PREFIX):].strip()) elif line.startswith(CONTEXT_PREFIX): context = line[len(CONTEXT_PREFIX):].strip() else: key = (context, line) messages[key] = srcs srcs = [] context = "" num_added, num_present = merge_messages(msgs, states, messages, True, spell_cache) tot_messages += num_added print("\tMerged {} messages ({} were already present)." "".format(num_added, num_present)) # Write back all messages into blender.pot. utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"], states["fuzzy_msg"]) print(len(spell_cache)) if SPELL_CACHE and spell_cache: with open(SPELL_CACHE, 'wb') as f: pickle.dump(spell_cache, f) print("Finished, total: {} messages!".format(tot_messages - 1)) return 0
def main(): parser = argparse.ArgumentParser(description="Update blender.pot file from messages.txt and source code parsing, " "and performs some checks over msgids.") parser.add_argument('-w', '--warning', action="store_true", help="Show warnings.") parser.add_argument('-i', '--input', metavar="File", help="Input messages file path.") parser.add_argument('-o', '--output', metavar="File", help="Output pot file path.") args = parser.parse_args() if args.input: global FILE_NAME_MESSAGES FILE_NAME_MESSAGES = args.input if args.output: global FILE_NAME_POT FILE_NAME_POT = args.output print("Running fake py gettext…") # Not using any more xgettext, simpler to do it ourself! messages = utils.new_messages() py_xgettext(messages) print("Finished, found {} messages.".format(len(messages))) if SPELL_CACHE and os.path.exists(SPELL_CACHE): with open(SPELL_CACHE, 'rb') as f: spell_cache = pickle.load(f) else: spell_cache = set() print("Generating POT file {}…".format(FILE_NAME_POT)) msgs, states = gen_empty_pot() tot_messages, _a = merge_messages(msgs, states, messages, True, spell_cache) # add messages collected automatically from RNA print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES)) messages = utils.new_messages() with open(FILE_NAME_MESSAGES, encoding="utf-8") as f: srcs = [] context = "" for line in f: line = utils.stripeol(line) if line.startswith(COMMENT_PREFIX): srcs.append(line[len(COMMENT_PREFIX):].strip()) elif line.startswith(CONTEXT_PREFIX): context = line[len(CONTEXT_PREFIX):].strip() else: key = (context, line) messages[key] = srcs srcs = [] context = "" num_added, num_present = merge_messages(msgs, states, messages, True, spell_cache) tot_messages += num_added print("\tMerged {} messages ({} were already present)." "".format(num_added, num_present)) print("\tAdding languages labels...") messages = {(CONTEXT_DEFAULT, lng[1]): ("Languages’ labels from bl_i18n_utils/settings.py",) for lng in LANGUAGES} messages.update({(CONTEXT_DEFAULT, cat[1]): ("Language categories’ labels from bl_i18n_utils/settings.py",) for cat in LANGUAGES_CATEGORIES}) num_added, num_present = merge_messages(msgs, states, messages, True, spell_cache) tot_messages += num_added print("\tAdded {} language messages.".format(num_added)) # Write back all messages into blender.pot. utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"], states["fuzzy_msg"]) if SPELL_CACHE and spell_cache: with open(SPELL_CACHE, 'wb') as f: pickle.dump(spell_cache, f) print("Finished, total: {} messages!".format(tot_messages - 1)) return 0
def main(): import argparse parser = argparse.ArgumentParser(description="" \ "Merge one or more .po files into the first dest one.\n" \ "If a msgkey (msgctxt, msgid) is present in more than " \ "one merged po, the one in the first file wins, unless " \ "it’s marked as fuzzy and one later is not.\n" \ "The fuzzy flag is removed if necessary.\n" \ "All other comments are never modified.\n" \ "Commented messages in dst will always remain " \ "commented, and commented messages are never merged " \ "from sources.") parser.add_argument('-s', '--stats', action="store_true", help="Show statistics info.") parser.add_argument('-r', '--replace', action="store_true", help="Replace existing messages of same \"level\" already in dest po.") parser.add_argument('dst', metavar='dst.po', help="The dest po into which merge the others.") parser.add_argument('src', metavar='src.po', nargs='+', help="The po's to merge into the dst.po one.") args = parser.parse_args() ret = 0 done_msgkeys = set() done_fuzzy_msgkeys = set() nbr_merged = 0 nbr_replaced = 0 nbr_added = 0 nbr_unfuzzied = 0 dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst) if dst_states["is_broken"]: print("Dest po is BROKEN, aborting.") return 1 if args.stats: print("Dest po, before merging:") utils.print_stats(dst_stats, prefix="\t") # If we don’t want to replace existing valid translations, pre-populate # done_msgkeys and done_fuzzy_msgkeys. if not args.replace: done_msgkeys = dst_states["trans_msg"].copy() done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy() for po in args.src: messages, states, stats = utils.parse_messages(po) if states["is_broken"]: print("\tSrc po {} is BROKEN, skipping.".format(po)) ret = 1 continue print("\tMerging {}...".format(po)) if args.stats: print("\t\tMerged po stats:") utils.print_stats(stats, prefix="\t\t\t") for msgkey, val in messages.items(): msgctxt, msgid = msgkey # This msgkey has already been completely merged, or is a commented one, # or the new message is commented, skip it. if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]): continue is_ttip = utils.is_tooltip(msgid) # New messages does not yet exists in dest. if msgkey not in dst_messages: dst_messages[msgkey] = messages[msgkey] if msgkey in states["fuzzy_msg"]: done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) elif msgkey in states["trans_msg"]: done_msgkeys.add(msgkey) dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 nbr_added += 1 dst_stats["tot_msg"] += 1 if is_ttip: dst_stats["tot_ttips"] += 1 # From now on, the new messages is already in dst. # New message is neither translated nor fuzzy, skip it. elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]): continue # From now on, the new message is either translated or fuzzy! # The new message is translated. elif msgkey in states["trans_msg"]: dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"] done_msgkeys.add(msgkey) done_fuzzy_msgkeys.discard(msgkey) if msgkey in dst_states["fuzzy_msg"]: dst_states["fuzzy_msg"].remove(msgkey) nbr_unfuzzied += 1 if msgkey not in dst_states["trans_msg"]: dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 else: nbr_replaced += 1 nbr_merged += 1 # The new message is fuzzy, org one is fuzzy too, # and this msgkey has not yet been merged. elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys): dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"] done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) nbr_merged += 1 nbr_replaced += 1 utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"]) print("Merged completed. {} messages were merged (among which {} were replaced), " \ "{} were added, {} were \"un-fuzzied\"." \ "".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied)) if args.stats: print("Final merged po stats:") utils.print_stats(dst_stats, prefix="\t") return ret
def main(): import argparse parser = argparse.ArgumentParser(description="" \ "Merge one or more .po files into the first dest one.\n" \ "If a msgkey (msgctxt, msgid) is present in more than " \ "one merged po, the one in the first file wins, unless " \ "it’s marked as fuzzy and one later is not.\n" \ "The fuzzy flag is removed if necessary.\n" \ "All other comments are never modified.\n" \ "Commented messages in dst will always remain " \ "commented, and commented messages are never merged " \ "from sources.") parser.add_argument('-s', '--stats', action="store_true", help="Show statistics info.") parser.add_argument( '-r', '--replace', action="store_true", help="Replace existing messages of same \"level\" already in dest po.") parser.add_argument('dst', metavar='dst.po', help="The dest po into which merge the others.") parser.add_argument('src', metavar='src.po', nargs='+', help="The po's to merge into the dst.po one.") args = parser.parse_args() ret = 0 done_msgkeys = set() done_fuzzy_msgkeys = set() nbr_merged = 0 nbr_replaced = 0 nbr_added = 0 nbr_unfuzzied = 0 dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst) if dst_states["is_broken"]: print("Dest po is BROKEN, aborting.") return 1 if args.stats: print("Dest po, before merging:") utils.print_stats(dst_stats, prefix="\t") # If we don’t want to replace existing valid translations, pre-populate # done_msgkeys and done_fuzzy_msgkeys. if not args.replace: done_msgkeys = dst_states["trans_msg"].copy() done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy() for po in args.src: messages, states, stats = utils.parse_messages(po) if states["is_broken"]: print("\tSrc po {} is BROKEN, skipping.".format(po)) ret = 1 continue print("\tMerging {}...".format(po)) if args.stats: print("\t\tMerged po stats:") utils.print_stats(stats, prefix="\t\t\t") for msgkey, val in messages.items(): msgctxt, msgid = msgkey # This msgkey has already been completely merged, or is a commented one, # or the new message is commented, skip it. if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]): continue is_ttip = utils.is_tooltip(msgid) # New messages does not yet exists in dest. if msgkey not in dst_messages: dst_messages[msgkey] = messages[msgkey] if msgkey in states["fuzzy_msg"]: done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) elif msgkey in states["trans_msg"]: done_msgkeys.add(msgkey) dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 nbr_added += 1 dst_stats["tot_msg"] += 1 if is_ttip: dst_stats["tot_ttips"] += 1 # From now on, the new messages is already in dst. # New message is neither translated nor fuzzy, skip it. elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]): continue # From now on, the new message is either translated or fuzzy! # The new message is translated. elif msgkey in states["trans_msg"]: dst_messages[msgkey]["msgstr_lines"] = messages[msgkey][ "msgstr_lines"] done_msgkeys.add(msgkey) done_fuzzy_msgkeys.discard(msgkey) if msgkey in dst_states["fuzzy_msg"]: dst_states["fuzzy_msg"].remove(msgkey) nbr_unfuzzied += 1 if msgkey not in dst_states["trans_msg"]: dst_states["trans_msg"].add(msgkey) dst_stats["trans_msg"] += 1 if is_ttip: dst_stats["trans_ttips"] += 1 else: nbr_replaced += 1 nbr_merged += 1 # The new message is fuzzy, org one is fuzzy too, # and this msgkey has not yet been merged. elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys): dst_messages[msgkey]["msgstr_lines"] = messages[msgkey][ "msgstr_lines"] done_fuzzy_msgkeys.add(msgkey) dst_states["fuzzy_msg"].add(msgkey) nbr_merged += 1 nbr_replaced += 1 utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"]) print("Merged completed. {} messages were merged (among which {} were replaced), " \ "{} were added, {} were \"un-fuzzied\"." \ "".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied)) if args.stats: print("Final merged po stats:") utils.print_stats(dst_stats, prefix="\t") return ret