def pproc_newcontext_po(po, pot_messages, pot_stats):
print("Adding new contexts to {}...".format(po))
messages, state, stats = utils.parse_messages(po)
known_ctxt = stats["contexts"]
print("Already known (present) context(s): {}".format(str(known_ctxt)))
new_ctxt = set()
added = 0
# Only use valid already translated messages!
allowed_keys = state["trans_msg"] - state["fuzzy_msg"] - state["comm_msg"]
for key in pot_messages.keys():
ctxt, msgid = key
if ctxt in known_ctxt:
continue
new_ctxt.add(ctxt)
for t_ctxt in known_ctxt:
# XXX The first match will win, this might not be optimal...
t_key = (t_ctxt, msgid)
if t_key in allowed_keys:
# Wrong comments (sources) will be removed by msgmerge...
messages[key] = messages[t_key]
messages[key]["msgctxt_lines"] = [ctxt]
added += 1
utils.write_messages(po, messages, state["comm_msg"], state["fuzzy_msg"])
print("Finished!\n {} new context(s) was/were added {}, adding {} new "
"messages.\n".format(len(new_ctxt), str(new_ctxt), added))
return 0
def pproc_newcontext_po(po, pot_messages, pot_stats):
print("Adding new contexts to {}...".format(po))
messages, state, stats = utils.parse_messages(po)
known_ctxt = stats["contexts"]
print("Already known (present) context(s): {}".format(str(known_ctxt)))
new_ctxt = set()
added = 0
# Only use valid already translated messages!
allowed_keys = state["trans_msg"] - state["fuzzy_msg"] - state["comm_msg"]
for key in pot_messages.keys():
ctxt, msgid = key
if ctxt in known_ctxt:
continue
new_ctxt.add(ctxt)
for t_ctxt in known_ctxt:
# XXX The first match will win, this might not be optimal...
t_key = (t_ctxt, msgid)
if t_key in allowed_keys:
# Wrong comments (sources) will be removed by msgmerge...
messages[key] = messages[t_key]
messages[key]["msgctxt_lines"] = [ctxt]
added += 1
utils.write_messages(po, messages, state["comm_msg"], state["fuzzy_msg"])
print("Finished!\n {} new context(s) was/were added {}, adding {} new "
"messages.\n".format(len(new_ctxt), str(new_ctxt), added))
return 0
def do_clean(po, strict):
print("Cleaning {}...".format(po))
messages, states, u1 = utils.parse_messages(po)
if strict and states["is_broken"]:
print("ERROR! This .po file is broken!")
return 1
for msgkey in states["comm_msg"]:
del messages[msgkey]
utils.write_messages(po, messages, states["comm_msg"], states["fuzzy_msg"])
print("Removed {} commented messages.".format(len(states["comm_msg"])))
return 0
def do_clean(po, strict):
print("Cleaning {}...".format(po))
messages, states, u1 = utils.parse_messages(po)
if strict and states["is_broken"]:
print("ERROR! This .po file is broken!")
return 1
for msgkey in states["comm_msg"]:
del messages[msgkey]
utils.write_messages(po, messages, states["comm_msg"], states["fuzzy_msg"])
print("Removed {} commented messages.".format(len(states["comm_msg"])))
return 0
def main():
import argparse
parser = argparse.ArgumentParser(description="" \
"Preprocesses right-to-left languages.\n" \
"You can use it either standalone, or through " \
"import_po_from_branches or update_trunk.\n\n" \
"Note: This has been tested on Linux, not 100% it will " \
"work nicely on Windows or OsX.\n" \
"Note: This uses ctypes, as there is no py3 binding for " \
"fribidi currently. This implies you only need the " \
"compiled C library to run it.\n" \
"Note: It handles some formating/escape codes (like " \
"\\\", %s, %x12, %.4f, etc.), protecting them from ugly " \
"(evil) fribidi, which seems completely unaware of such " \
"things (as unicode is...).")
parser.add_argument('dst', metavar='dst.po',
help="The dest po into which write the " \
"pre-processed messages.")
parser.add_argument('src',
metavar='src.po',
help="The po's to pre-process messages.")
args = parser.parse_args()
msgs, state, u1 = utils.parse_messages(args.src)
if state["is_broken"]:
print("Source po is BROKEN, aborting.")
return 1
keys = []
trans = []
for key, val in msgs.items():
keys.append(key)
trans.append("".join(val["msgstr_lines"]))
trans = log2vis(trans)
for key, trn in zip(keys, trans):
# Mono-line for now...
msgs[key]["msgstr_lines"] = [trn]
utils.write_messages(args.dst, msgs, state["comm_msg"], state["fuzzy_msg"])
print("RTL pre-process completed.")
return 0
def main():
import argparse
parser = argparse.ArgumentParser(description="" \
"Preprocesses right-to-left languages.\n" \
"You can use it either standalone, or through " \
"import_po_from_branches or update_trunk.\n\n" \
"Note: This has been tested on Linux, not 100% it will " \
"work nicely on Windows or OsX.\n" \
"Note: This uses ctypes, as there is no py3 binding for " \
"fribidi currently. This implies you only need the " \
"compiled C library to run it.\n" \
"Note: It handles some formating/escape codes (like " \
"\\\", %s, %x12, %.4f, etc.), protecting them from ugly " \
"(evil) fribidi, which seems completely unaware of such " \
"things (as unicode is...).")
parser.add_argument('dst', metavar='dst.po',
help="The dest po into which write the " \
"pre-processed messages.")
parser.add_argument('src', metavar='src.po',
help="The po's to pre-process messages.")
args = parser.parse_args()
msgs, state, u1 = utils.parse_messages(args.src)
if state["is_broken"]:
print("Source po is BROKEN, aborting.")
return 1
keys = []
trans = []
for key, val in msgs.items():
keys.append(key)
trans.append("".join(val["msgstr_lines"]))
trans = log2vis(trans)
for key, trn in zip(keys, trans):
# Mono-line for now...
msgs[key]["msgstr_lines"] = [trn]
utils.write_messages(args.dst, msgs, state["comm_msg"], state["fuzzy_msg"])
print("RTL pre-process completed.")
return 0
def main():
import argparse
parser = argparse.ArgumentParser(description="Import advanced enough po’s " \
"from branches to trunk.")
parser.add_argument('-t', '--threshold', type=int,
help="Import threshold, as a percentage.")
parser.add_argument('-s', '--strict', action="store_true",
help="Raise an error if a po is broken.")
parser.add_argument('langs', metavar='ISO_code', nargs='*',
help="Restrict processed languages to those.")
args = parser.parse_args()
ret = 0
threshold = float(settings.IMPORT_MIN_LEVEL) / 100.0
if args.threshold is not None:
threshold = float(args.threshold) / 100.0
for lang in os.listdir(BRANCHES_DIR):
if args.langs and lang not in args.langs:
continue
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
if os.path.exists(po):
po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE)
msgs, state, stats = utils.parse_messages(po)
tot_msgs = stats["tot_msg"]
trans_msgs = stats["trans_msg"]
lvl = 0.0
if tot_msgs:
lvl = float(trans_msgs) / float(tot_msgs)
if lvl > threshold:
if state["is_broken"] and args.strict:
print("{:<10}: {:>6.1%} done, but BROKEN, skipped." \
"".format(lang, lvl))
ret = 1
else:
if os.path.exists(po_is_rtl):
out_po = os.path.join(TRUNK_PO_DIR,
".".join((lang, "po")))
out_raw_po = os.path.join(TRUNK_PO_DIR,
"_".join((lang, "raw.po")))
keys = []
trans = []
for k, m in msgs.items():
keys.append(k)
trans.append("".join(m["msgstr_lines"]))
trans = rtl_preprocess.log2vis(trans)
for k, t in zip(keys, trans):
# Mono-line for now...
msgs[k]["msgstr_lines"] = [t]
utils.write_messages(out_po, msgs, state["comm_msg"],
state["fuzzy_msg"])
# Also copies org po!
shutil.copy(po, out_raw_po)
print("{:<10}: {:>6.1%} done, enough translated " \
"messages, processed and copied to trunk." \
"".format(lang, lvl))
else:
shutil.copy(po, TRUNK_PO_DIR)
print("{:<10}: {:>6.1%} done, enough translated " \
"messages, copied to trunk.".format(lang, lvl))
else:
if state["is_broken"] and args.strict:
print("{:<10}: {:>6.1%} done, BROKEN and not enough " \
"translated messages, skipped".format(lang, lvl))
ret = 1
else:
print("{:<10}: {:>6.1%} done, not enough translated " \
"messages, skipped.".format(lang, lvl))
return ret
def main():
parser = argparse.ArgumentParser(description="Update blender.pot file " \
"from messages.txt")
parser.add_argument('-w', '--warning', action="store_true",
help="Show warnings.")
parser.add_argument('-i', '--input', metavar="File",
help="Input messages file path.")
parser.add_argument('-o', '--output', metavar="File",
help="Output pot file path.")
args = parser.parse_args()
if args.input:
global FILE_NAME_MESSAGES
FILE_NAME_MESSAGES = args.input
if args.output:
global FILE_NAME_POT
FILE_NAME_POT = args.output
print("Running fake py gettext…")
# Not using any more xgettext, simpler to do it ourself!
messages = {}
py_xgettext(messages)
print("Finished, found {} messages.".format(len(messages)))
if SPELL_CACHE and os.path.exists(SPELL_CACHE):
with open(SPELL_CACHE, 'rb') as f:
spell_cache = pickle.load(f)
else:
spell_cache = set()
print(len(spell_cache))
print("Generating POT file {}…".format(FILE_NAME_POT))
msgs, states = gen_empty_pot()
tot_messages, _a = merge_messages(msgs, states, messages,
True, spell_cache)
# add messages collected automatically from RNA
print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES))
messages = {}
with open(FILE_NAME_MESSAGES, encoding="utf-8") as f:
srcs = []
context = ""
for line in f:
line = utils.stripeol(line)
if line.startswith(COMMENT_PREFIX):
srcs.append(line[len(COMMENT_PREFIX):].strip())
elif line.startswith(CONTEXT_PREFIX):
context = line[len(CONTEXT_PREFIX):].strip()
else:
key = (context, line)
messages[key] = srcs
srcs = []
context = ""
num_added, num_present = merge_messages(msgs, states, messages,
True, spell_cache)
tot_messages += num_added
print("\tMerged {} messages ({} were already present)."
"".format(num_added, num_present))
# Write back all messages into blender.pot.
utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
states["fuzzy_msg"])
print(len(spell_cache))
if SPELL_CACHE and spell_cache:
with open(SPELL_CACHE, 'wb') as f:
pickle.dump(spell_cache, f)
print("Finished, total: {} messages!".format(tot_messages - 1))
return 0
def main():
parser = argparse.ArgumentParser(description="Update blender.pot file from messages.txt and source code parsing, "
"and performs some checks over msgids.")
parser.add_argument('-w', '--warning', action="store_true",
help="Show warnings.")
parser.add_argument('-i', '--input', metavar="File",
help="Input messages file path.")
parser.add_argument('-o', '--output', metavar="File",
help="Output pot file path.")
args = parser.parse_args()
if args.input:
global FILE_NAME_MESSAGES
FILE_NAME_MESSAGES = args.input
if args.output:
global FILE_NAME_POT
FILE_NAME_POT = args.output
print("Running fake py gettext…")
# Not using any more xgettext, simpler to do it ourself!
messages = utils.new_messages()
py_xgettext(messages)
print("Finished, found {} messages.".format(len(messages)))
if SPELL_CACHE and os.path.exists(SPELL_CACHE):
with open(SPELL_CACHE, 'rb') as f:
spell_cache = pickle.load(f)
else:
spell_cache = set()
print("Generating POT file {}…".format(FILE_NAME_POT))
msgs, states = gen_empty_pot()
tot_messages, _a = merge_messages(msgs, states, messages,
True, spell_cache)
# add messages collected automatically from RNA
print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES))
messages = utils.new_messages()
with open(FILE_NAME_MESSAGES, encoding="utf-8") as f:
srcs = []
context = ""
for line in f:
line = utils.stripeol(line)
if line.startswith(COMMENT_PREFIX):
srcs.append(line[len(COMMENT_PREFIX):].strip())
elif line.startswith(CONTEXT_PREFIX):
context = line[len(CONTEXT_PREFIX):].strip()
else:
key = (context, line)
messages[key] = srcs
srcs = []
context = ""
num_added, num_present = merge_messages(msgs, states, messages,
True, spell_cache)
tot_messages += num_added
print("\tMerged {} messages ({} were already present)."
"".format(num_added, num_present))
print("\tAdding languages labels...")
messages = {(CONTEXT_DEFAULT, lng[1]):
("Languages’ labels from bl_i18n_utils/settings.py",)
for lng in LANGUAGES}
messages.update({(CONTEXT_DEFAULT, cat[1]):
("Language categories’ labels from bl_i18n_utils/settings.py",)
for cat in LANGUAGES_CATEGORIES})
num_added, num_present = merge_messages(msgs, states, messages,
True, spell_cache)
tot_messages += num_added
print("\tAdded {} language messages.".format(num_added))
# Write back all messages into blender.pot.
utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
states["fuzzy_msg"])
if SPELL_CACHE and spell_cache:
with open(SPELL_CACHE, 'wb') as f:
pickle.dump(spell_cache, f)
print("Finished, total: {} messages!".format(tot_messages - 1))
return 0
def main():
import argparse
parser = argparse.ArgumentParser(description="" \
"Merge one or more .po files into the first dest one.\n" \
"If a msgkey (msgctxt, msgid) is present in more than " \
"one merged po, the one in the first file wins, unless " \
"it’s marked as fuzzy and one later is not.\n" \
"The fuzzy flag is removed if necessary.\n" \
"All other comments are never modified.\n" \
"Commented messages in dst will always remain " \
"commented, and commented messages are never merged " \
"from sources.")
parser.add_argument('-s', '--stats', action="store_true",
help="Show statistics info.")
parser.add_argument('-r', '--replace', action="store_true",
help="Replace existing messages of same \"level\" already in dest po.")
parser.add_argument('dst', metavar='dst.po',
help="The dest po into which merge the others.")
parser.add_argument('src', metavar='src.po', nargs='+',
help="The po's to merge into the dst.po one.")
args = parser.parse_args()
ret = 0
done_msgkeys = set()
done_fuzzy_msgkeys = set()
nbr_merged = 0
nbr_replaced = 0
nbr_added = 0
nbr_unfuzzied = 0
dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst)
if dst_states["is_broken"]:
print("Dest po is BROKEN, aborting.")
return 1
if args.stats:
print("Dest po, before merging:")
utils.print_stats(dst_stats, prefix="\t")
# If we don’t want to replace existing valid translations, pre-populate
# done_msgkeys and done_fuzzy_msgkeys.
if not args.replace:
done_msgkeys = dst_states["trans_msg"].copy()
done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy()
for po in args.src:
messages, states, stats = utils.parse_messages(po)
if states["is_broken"]:
print("\tSrc po {} is BROKEN, skipping.".format(po))
ret = 1
continue
print("\tMerging {}...".format(po))
if args.stats:
print("\t\tMerged po stats:")
utils.print_stats(stats, prefix="\t\t\t")
for msgkey, val in messages.items():
msgctxt, msgid = msgkey
# This msgkey has already been completely merged, or is a commented one,
# or the new message is commented, skip it.
if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]):
continue
is_ttip = utils.is_tooltip(msgid)
# New messages does not yet exists in dest.
if msgkey not in dst_messages:
dst_messages[msgkey] = messages[msgkey]
if msgkey in states["fuzzy_msg"]:
done_fuzzy_msgkeys.add(msgkey)
dst_states["fuzzy_msg"].add(msgkey)
elif msgkey in states["trans_msg"]:
done_msgkeys.add(msgkey)
dst_states["trans_msg"].add(msgkey)
dst_stats["trans_msg"] += 1
if is_ttip:
dst_stats["trans_ttips"] += 1
nbr_added += 1
dst_stats["tot_msg"] += 1
if is_ttip:
dst_stats["tot_ttips"] += 1
# From now on, the new messages is already in dst.
# New message is neither translated nor fuzzy, skip it.
elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]):
continue
# From now on, the new message is either translated or fuzzy!
# The new message is translated.
elif msgkey in states["trans_msg"]:
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
done_msgkeys.add(msgkey)
done_fuzzy_msgkeys.discard(msgkey)
if msgkey in dst_states["fuzzy_msg"]:
dst_states["fuzzy_msg"].remove(msgkey)
nbr_unfuzzied += 1
if msgkey not in dst_states["trans_msg"]:
dst_states["trans_msg"].add(msgkey)
dst_stats["trans_msg"] += 1
if is_ttip:
dst_stats["trans_ttips"] += 1
else:
nbr_replaced += 1
nbr_merged += 1
# The new message is fuzzy, org one is fuzzy too,
# and this msgkey has not yet been merged.
elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys):
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
done_fuzzy_msgkeys.add(msgkey)
dst_states["fuzzy_msg"].add(msgkey)
nbr_merged += 1
nbr_replaced += 1
utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"])
print("Merged completed. {} messages were merged (among which {} were replaced), " \
"{} were added, {} were \"un-fuzzied\"." \
"".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
if args.stats:
print("Final merged po stats:")
utils.print_stats(dst_stats, prefix="\t")
return ret
def main():
import argparse
parser = argparse.ArgumentParser(description="" \
"Merge one or more .po files into the first dest one.\n" \
"If a msgkey (msgctxt, msgid) is present in more than " \
"one merged po, the one in the first file wins, unless " \
"it’s marked as fuzzy and one later is not.\n" \
"The fuzzy flag is removed if necessary.\n" \
"All other comments are never modified.\n" \
"Commented messages in dst will always remain " \
"commented, and commented messages are never merged " \
"from sources.")
parser.add_argument('-s',
'--stats',
action="store_true",
help="Show statistics info.")
parser.add_argument(
'-r',
'--replace',
action="store_true",
help="Replace existing messages of same \"level\" already in dest po.")
parser.add_argument('dst',
metavar='dst.po',
help="The dest po into which merge the others.")
parser.add_argument('src',
metavar='src.po',
nargs='+',
help="The po's to merge into the dst.po one.")
args = parser.parse_args()
ret = 0
done_msgkeys = set()
done_fuzzy_msgkeys = set()
nbr_merged = 0
nbr_replaced = 0
nbr_added = 0
nbr_unfuzzied = 0
dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst)
if dst_states["is_broken"]:
print("Dest po is BROKEN, aborting.")
return 1
if args.stats:
print("Dest po, before merging:")
utils.print_stats(dst_stats, prefix="\t")
# If we don’t want to replace existing valid translations, pre-populate
# done_msgkeys and done_fuzzy_msgkeys.
if not args.replace:
done_msgkeys = dst_states["trans_msg"].copy()
done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy()
for po in args.src:
messages, states, stats = utils.parse_messages(po)
if states["is_broken"]:
print("\tSrc po {} is BROKEN, skipping.".format(po))
ret = 1
continue
print("\tMerging {}...".format(po))
if args.stats:
print("\t\tMerged po stats:")
utils.print_stats(stats, prefix="\t\t\t")
for msgkey, val in messages.items():
msgctxt, msgid = msgkey
# This msgkey has already been completely merged, or is a commented one,
# or the new message is commented, skip it.
if msgkey in (done_msgkeys | dst_states["comm_msg"]
| states["comm_msg"]):
continue
is_ttip = utils.is_tooltip(msgid)
# New messages does not yet exists in dest.
if msgkey not in dst_messages:
dst_messages[msgkey] = messages[msgkey]
if msgkey in states["fuzzy_msg"]:
done_fuzzy_msgkeys.add(msgkey)
dst_states["fuzzy_msg"].add(msgkey)
elif msgkey in states["trans_msg"]:
done_msgkeys.add(msgkey)
dst_states["trans_msg"].add(msgkey)
dst_stats["trans_msg"] += 1
if is_ttip:
dst_stats["trans_ttips"] += 1
nbr_added += 1
dst_stats["tot_msg"] += 1
if is_ttip:
dst_stats["tot_ttips"] += 1
# From now on, the new messages is already in dst.
# New message is neither translated nor fuzzy, skip it.
elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]):
continue
# From now on, the new message is either translated or fuzzy!
# The new message is translated.
elif msgkey in states["trans_msg"]:
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey][
"msgstr_lines"]
done_msgkeys.add(msgkey)
done_fuzzy_msgkeys.discard(msgkey)
if msgkey in dst_states["fuzzy_msg"]:
dst_states["fuzzy_msg"].remove(msgkey)
nbr_unfuzzied += 1
if msgkey not in dst_states["trans_msg"]:
dst_states["trans_msg"].add(msgkey)
dst_stats["trans_msg"] += 1
if is_ttip:
dst_stats["trans_ttips"] += 1
else:
nbr_replaced += 1
nbr_merged += 1
# The new message is fuzzy, org one is fuzzy too,
# and this msgkey has not yet been merged.
elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys):
dst_messages[msgkey]["msgstr_lines"] = messages[msgkey][
"msgstr_lines"]
done_fuzzy_msgkeys.add(msgkey)
dst_states["fuzzy_msg"].add(msgkey)
nbr_merged += 1
nbr_replaced += 1
utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"],
dst_states["fuzzy_msg"])
print("Merged completed. {} messages were merged (among which {} were replaced), " \
"{} were added, {} were \"un-fuzzied\"." \
"".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
if args.stats:
print("Final merged po stats:")
utils.print_stats(dst_stats, prefix="\t")
return ret
