def test_write_and_read_po_file_with_non_ascii_string(temp):
from sphinx_intl import catalog
cat = Catalog(locale='ja', domain='domain', fuzzy=False)
msg = Message('Hello World', u'こんにちは世界')
cat[msg.id] = msg
po_file = (temp / 'domain.po')
catalog.dump_po(po_file, cat)
cat2 = catalog.load_po(po_file)
assert cat2[msg.id].string == msg.string
def run(self):
#exit(0)
try:
self.po_file_list = self.getPOFileList()
for po_file in self.po_file_list:
po_doc = c.load_po(po_file)
if self.dry_run:
print("File to be flatted:", po_file)
else:
#self.dump_po(po_file, po_doc)
c.dump_po(po_file, po_doc)
except Exception as e:
print(e)
print(po_file)
def updatePOtoPO(self):
po_cat_dict = self.poCatToDic(self.po_cat)
po_pot_dict = self.poCatToDic(self.pot_cat)
# print("po_pot_dict")
# pp(po_pot_dict)
#pp(self.sorted_po_cat)
#exit(0)
changed = False
for k, v in po_pot_dict.items():
print("run:", k, v)
is_in_old_dict = (k in po_cat_dict)
if not is_in_old_dict:
print("NEW ENTRY k:[{}], v:[{}]".format(k, v))
continue
old_dict_entry = po_cat_dict[k]
old_dict_entry_msgid = old_dict_entry.id
old_dict_entry_msgstr = old_dict_entry.string
is_fuzzy = old_dict_entry.fuzzy
has_translation = (old_dict_entry_msgstr is not None) and (
len(old_dict_entry_msgstr) > 0)
if not has_translation:
print("IGNORED No translation k:[{}], v:[{}]".format(k, v))
v.string = old_dict_entry_msgstr
if is_fuzzy:
v.flags.add('fuzzy')
print("Updating k[{}], v:[{}], fuzzy:[{}]".format(k, v, is_fuzzy))
changed = True
if (changed):
if self.dry_run:
print(
f'DRY_RUN: changed content IS NOT SAVED to:{self.out_file}'
)
else:
#new_po_file = "/Users/hoangduytran/new_vi.po"
print(f'Saving content of new pot_cat to:{self.out_file}')
c.dump_po(self.out_file, self.pot_cat, line_width=99)
def doctree_resolved(app, doctree, docname):
def abbreviating():
remove_items = []
new_items = {}
for k, v in trans_finder.master_dic.items():
ref_list = RefList(msg=v)
new_v = ref_list.quotedToAbbrev(k)
has_new_v = (new_v is not None) and (len(new_v) > 0)
if has_new_v:
new_entry = {k: new_v}
new_items.update(new_entry)
has_remove_items = (len(remove_items) > 0)
if has_remove_items:
for k in remove_items:
dd(f'Delete from dictionary:[{k}]')
del trans_finder.master_dic[k]
is_writing_changes = (len(new_items) > 0)
if is_writing_changes:
trans_finder.master_dic.update(new_items)
dic_file = '/Users/hoangduytran/blender_manual/test_dic.json'
print(
f'Writing changes to: {dic_file}, number of records:{len(new_items)}'
)
trans_finder.writeJSONDic(dict_list=trans_finder.master_dic,
file_name=dic_file)
def checkDictForMultipleMeaningsInTrans():
pattern = re.compile(r'(\w+(/\w+)+)')
k: str = None
v: str = None
for k, v in trans_finder.master_dic.items():
k_found_list = pattern.findall(k)
v_found_list = pattern.findall(v)
is_same = (len(k_found_list) == len(v_found_list))
if is_same:
continue
k_is_single_word = (len(k.split(' ')) == 1)
if k_is_single_word:
continue
PP(k_found_list)
print('-' * 3)
PP(v_found_list)
print('-' * 3)
print(f'{k}\n\n{v}')
print('-' * 40)
def trimmingText(text):
txt_has_trimmable_ending = (cm.TRIMMABLE_ENDING.search(text)
is not None)
txt_has_trimmable_beginning = (cm.TRIMMABLE_BEGINNING.search(text)
is not None)
is_trim = (txt_has_trimmable_ending or txt_has_trimmable_beginning)
if not is_trim:
return text, False
text = cm.TRIMMABLE_BEGINNING.sub('', text)
text = cm.TRIMMABLE_ENDING.sub('', text)
return text, True
def removeDictBeginAndEndingPuncts():
remove_set = {}
add_set = {}
for k, v in trans_finder.master_dic.items():
# is_debug = ('Cut to' in k)
# if is_debug:
# dd('DEBUG')
trimmed_k, is_trimmed_k = trimmingText(k)
trimmed_v, is_trimmed_v = trimmingText(v)
changed = (is_trimmed_k or is_trimmed_v)
if changed:
remove_entry = {k: v}
remove_set.update(remove_entry)
add_entry = {trimmed_k: trimmed_v}
add_set.update(add_entry)
print(f'[{k}]')
print(f'[{trimmed_k}]')
print('-' * 3)
print(f'[{v}]')
print(f'[{trimmed_v}]')
print('-' * 40)
changed = False
for k, v in remove_set.items():
remove_entry = {k: v}
print(f'remove:{remove_entry}')
del trans_finder.master_dic[k]
changed = True
for k, v in add_set.items():
add_entry = {k: v}
trans_finder.master_dic.update(add_entry)
print(f'added: {add_entry}')
changed = True
if changed:
new_dict = cleanupLeadingTrailingPunct(trans_finder.master_dic)
test_to_file = '/Users/hoangduytran/blender_manual/ref_dict_0005.json'
trans_finder.writeJSONDic(dict_list=new_dict,
file_name=test_to_file)
def removeDuplication(txt_with_punct):
# is_debug = (txt_with_punct.endswith('::'))
# if is_debug:
# dd('DEBUG')
cropped_txt, begin_with_punctuations, ending_with_punctuations = cm.beginAndEndPunctuation(
txt_with_punct, is_single=True)
trans = trans_finder.isInList(cropped_txt)
is_repeat = (trans is not None)
if not is_repeat:
cropped_txt, begin_with_punctuations, ending_with_punctuations = cm.beginAndEndPunctuation(
txt_with_punct, is_single=False)
trans = trans_finder.isInList(cropped_txt)
is_repeat = (trans is not None)
return is_repeat
def cleanupLeadingTrailingPunct(d_dict):
return_dict = {}
for k, v in d_dict.items():
trimmed_k = str(k)
trimmed_v = str(v)
found_k = cm.WORD_WITHOUT_QUOTE.search(k)
found_v = cm.WORD_WITHOUT_QUOTE.search(v)
if found_k:
trimmed_k = found_k.group(1)
if found_v:
trimmed_v = found_v.group(1)
entry = {trimmed_k: trimmed_v}
return_dict.update(entry)
return return_dict
def refToDictItems(ref_list):
ref_dict = {}
ref: RefRecord = None
interest_ref = [
RefType.REF,
RefType.DOC,
RefType.GA,
RefType.TERM,
]
for ref in ref_list:
# print(ref)
type = ref.getOrigin().getRefType()
first_ref = ref.getRefItemByIndex(0)
ref_text = first_ref.getText()
is_debug = ('Poor mans steadycam' in ref_text)
if is_debug:
dd('DEBUG')
en_part = None
vn_part = None
d_dict = {}
if type == RefType.MENUSELECTION:
print(f'MENUSELECTION:{type}')
text_list = cm.MENU_TEXT_REVERSE.findall(ref_text)
length = len(text_list)
i_index = 0
for i in range(length):
tran = text_list[i_index]
if i_index + 1 < length:
orig = text_list[i_index + 1]
else:
print('ERROR: Orig is NOT THERE, use original')
orig = ref.getOrigin().getText()
entry = {orig: tran}
print(f'menu:{entry}')
d_dict.update(entry)
i_index += 2
if i_index >= length:
break
elif type == RefType.ABBR:
print(f'ABBR:{type}')
text_list = cm.ABBREV_TEXT_REVERSE.findall(ref_text)
abbr = text_list[0]
defin = text_list[1]
has_further_explanation = (': ' in defin)
if has_further_explanation:
exp_list = defin.split(': ')
orig_part = exp_list[0]
further_exp = exp_list[1]
print(
f'abbr:{abbr}; orig_part:{orig_part}; further_exp:{further_exp}'
)
if abbr.isascii():
entry = {abbr: f'{orig_part}, {further_exp}'}
elif orig_part.isascii():
entry = {orig_part: f'{further_exp}, {abbr}'}
else:
entry = {further_exp: f'{orig_part}, {abbr}'}
d_dict.update(entry)
else:
print(f'abbr:{abbr}; defin:{defin}')
if defin.isascii():
entry = {defin: abbr}
else:
entry = {abbr: defin}
d_dict.update(entry)
elif type in interest_ref:
print(f'GENERIC_REF:{type}')
text_list = cm.REF_TEXT_REVERSE.findall(ref_text)
has_text = (len(text_list) > 0)
if not has_text:
origin_text = ref.getOrigin().getText()
print(f'ERROR: origin_text:{origin_text}')
# print(f'{text_list}, appeared to be empty!!!')
else:
vn_part, en_part = text_list[0]
print(f'en_part:{en_part} vn_part:{vn_part}')
entry = {en_part: vn_part}
d_dict.update(entry)
else:
dd(f'{type} is not the type we are looking for.')
ref_dict.update(d_dict)
return_dict = cleanupLeadingTrailingPunct(d_dict)
return return_dict
def listDictRefsToDict():
interest_ref_list = [
RefType.MENUSELECTION,
RefType.REF,
RefType.DOC,
RefType.GA,
RefType.TERM,
RefType.ABBR,
]
ref_dict = {}
ref_dict_filename = '/Users/hoangduytran/blender_manual/ref_dict_refsonly.json'
for k, v in trans_finder.master_dic.items():
ref_list = RefList(msg=v, keep_orig=False, tf=trans_finder)
ref_list.parseMessage()
inter_ref_list = ref_list.getListOfRefType(interest_ref_list)
has_ref = (len(inter_ref_list) > 0)
if not has_ref:
continue
current_ref_dict = refToDictItems(inter_ref_list)
ref_dict.update(current_ref_dict)
has_dict_content = (len(ref_dict) > 0)
if has_dict_content:
trans_finder.writeJSONDic(dict_list=ref_dict,
file_name=ref_dict_filename)
def tranRef(msg, is_keep_original):
ref_list = RefList(msg=msg,
keep_orig=is_keep_original,
tf=trans_finder)
ref_list.parseMessage()
ref_list.translateRefList()
tran = ref_list.getTranslation()
# trans_finder.addDictEntry((msg, tran))
# print("Got translation from REF_LIST")
return tran
# def fuzzyTextSimilar(txt1 : str, txt2 : str, accept_ratio):
# try:
# similar_ratio = LE.ratio(txt1, txt2)
# is_similar = (similar_ratio >= accept_ratio)
# return is_similar
# except Exception as e:
# print(e)
# return False
def getTimeNow(self):
local_time = timezone('Europe/London')
fmt = '%Y-%m-%d %H:%M%z'
loc_dt = local_time.localize(datetime.datetime.now())
formatted_dt = loc_dt.strftime(fmt)
return formatted_dt
# is_running = runAppOrNot()
# if not is_running:
# return
# correctingDictionary()
# checkDictKeyboard()
# checkDictRef()
# checkNonTranslatedDictWords()
# checkDictForMultipleMeaningsInTrans()
# removeDictBeginAndEndingPuncts()
# listDictRefsToDict()
# trans_finder.saveMasterDict()
# exit(0)
try:
is_debug = ('vr_scene_inspection' in docname)
if is_debug:
dd('DEBUG')
ex_env_key = 'EX_PO_TRANS'
is_ex_env_set = (ex_env_key in os.environ)
if not is_ex_env_set:
return
ex_env_key_value = os.environ[ex_env_key]
is_ex_set_true = (ex_env_key_value.lower() == 'true')
if not is_ex_set_true:
return
debug_file = cm.debug_file
if debug_file:
is_debug_file = (debug_file in docname)
if not is_debug_file:
return
build_dir = "build/rstdoc"
po_vi_dir = "locale/vi/LC_MESSAGES"
po_file_path = "{}.po".format(docname)
local_path = os.path.dirname(os.path.abspath(__file__))
blender_docs_path = cm.BLENDER_DOCS # os.path.dirname(local_path)
locale_vi_path = "locale/vi/LC_MESSAGES"
po_path = os.path.join(blender_docs_path,
os.path.join(locale_vi_path, po_file_path))
if not os.path.isfile(po_path):
msg = f'po_path: {po_path} NOT FOUND!'
print(msg)
raise Exception(msg)
exit(0)
# #loading local po file to get translation if any
po_dic, current_po_cat = trans_finder.loadPOAsDic(po_path)
trans_finder.flatPOFile(po_path)
rst_output_location = os.path.join(blender_docs_path, build_dir)
output_path = os.path.join(rst_output_location, po_file_path)
local_time = timezone(TIME_ZONE)
time_now = local_time.localize(datetime.datetime.now())
local_locale = locale.getlocale()[0]
current_header = current_po_cat._get_header_comment()
new_po_cat = Catalog(locale="vi",
header_comment=current_header,
project=current_po_cat.project,
version=current_po_cat.version,
copyright_holder=YOUR_ID,
creation_date=current_po_cat.creation_date,
revision_date=time_now,
last_translator=YOUR_ID,
language_team=YOUR_TRANSLATION_TEAM)
dd("#" * 80)
dd("filename: {}".format(output_path))
# msgid = "Lines should be less than 120 characters long."
# msgstr = "Số chữ trong các dòng phải ít hơn 120 ký tự de lam gi."
# trans_finder.addDictEntry((msgid, msgstr), False)
# exit(0)
for node, msg in extract_messages(doctree):
msg = msg.strip()
dd("=" * 80)
dd("msgid:[{}]".format(msg))
# clean up po file
is_inline = isinstance(node, nodes.inline)
is_emphasis = isinstance(node, nodes.emphasis)
is_title = isinstance(node, nodes.title)
is_term = isinstance(node, nodes.term)
is_rubric = isinstance(node, nodes.rubric)
is_field_name = isinstance(node, nodes.field_name)
is_reference = isinstance(node, nodes.reference)
is_strong = isinstance(node, nodes.strong)
is_keep_original = (is_inline or is_emphasis or is_title or is_term
or is_rubric or is_field_name or is_reference
or is_strong)
tran = None
# is_debug = ('Get involved in discussions' in msg)
# if is_debug:
# dd('DEBUG')
is_ignore = ig.isIgnored(msg)
if is_ignore:
print(f'IGNORED: {msg}')
continue
# is_added = False
tran, is_ignore = trans_finder.findTranslation(msg)
if is_ignore:
continue
has_translation = (tran is not None)
if not has_translation:
is_debug = ('is based on the OpenXR specification' in msg)
if is_debug:
dd('Debug')
ref_list = RefList(msg=msg,
keep_orig=is_keep_original,
tf=trans_finder)
ref_list.parseMessage()
ref_list.translateRefList()
tran = ref_list.getTranslation()
# tran = tranRef(msg, is_keep_original)
has_translation = (tran is not None)
if not has_translation:
tran = po_dic[msg]
has_translation = (tran is not None)
if has_translation:
has_month = ('Tháng ' in tran)
has_original = (msg.lower() in tran.lower())
has_link = (cm.REF_LINK.search(tran) is not None)
can_ignore = (has_month or has_original or has_link)
is_repeat = is_keep_original and not can_ignore
if is_repeat:
print('Repeating MSG')
tran = cm.matchCase(msg, tran)
tran = f'{tran} -- {msg}'
print(f'Repeating MSG:{tran}')
if tran is not None:
new_po_cat.add(msg, string=tran)
else:
new_po_cat.add(msg, string="")
print(f'msgid \"{msg}\"')
if tran is not None:
print(f'msgstr \"{tran}\"')
else:
print('msgstr \"\"')
print("Output to the path:", new_po_cat, output_path)
c.dump_po(output_path, new_po_cat)
# dd('DEBUG')
except Exception as e:
df.LOG(f'{e}', error=True)
def replaceAndReport(self, changed_file):
# these lines are taken from /Library/Python/3.7/site-packages/babel/messages/catalog.py
# put here for references
#
# self.domain = domain
# self.locale = locale
# self._header_comment = header_comment
# self._messages = OrderedDict()
# self.project = project or 'PROJECT'
# self.version = version or 'VERSION'
# self.copyright_holder = copyright_holder or 'ORGANIZATION'
# self.msgid_bugs_address = msgid_bugs_address or 'EMAIL@ADDRESS'
# self.last_translator = last_translator or 'FULL NAME <EMAIL@ADDRESS>'
# """Name and email address of the last translator."""
# self.language_team = language_team or 'LANGUAGE <*****@*****.**>'
# """Name and email address of the language team."""
# self.charset = charset or 'utf-8'
# if creation_date is None:
# creation_date = datetime.now(LOCALTZ)
# elif isinstance(creation_date, datetime) and not creation_date.tzinfo:
# creation_date = creation_date.replace(tzinfo=LOCALTZ)
# self.creation_date = creation_date
# if revision_date is None:
# revision_date = 'YEAR-MO-DA HO:MI+ZONE'
# elif isinstance(revision_date, datetime) and not revision_date.tzinfo:
# revision_date = revision_date.replace(tzinfo=LOCALTZ)
# self.revision_date = revision_date
# self.fuzzy = fuzzy
# self.obsolete = OrderedDict() # Dictionary of obsolete messages
# self._num_plurals = None
# self._plural_expr = None
cat_changed = False
header_changed = False
in_po_cat = c.load_po(changed_file)
local_time = timezone(TIME_ZONE)
time_now = local_time.localize(datetime.datetime.now())
if self.modi_time:
change_time = time_now.strftime('%Y-%m-%d %H:%M%z')
print("PO-Revision-Date:", change_time)
in_po_cat.revision_date = time_now
cat_changed = True
is_you_last_translator = (YOUR_ID in in_po_cat.last_translator)
if not is_you_last_translator:
print("Last-Translator:", YOUR_ID)
in_po_cat.last_translator = YOUR_ID
cat_changed = True
is_language_team_already_set = (YOUR_TRANSLATION_TEAM
in in_po_cat.language_team)
if not is_language_team_already_set:
print("Language-Team:", YOUR_TRANSLATION_TEAM)
in_po_cat.language_team = YOUR_TRANSLATION_TEAM
cat_changed = True
header = in_po_cat._get_header_comment()
is_you_already_first_author = (YOUR_NAME
in header) and (default_first_author
not in header)
if not is_you_already_first_author:
print("FIRST AUTHOR <EMAIL@ADDRESS>", YOUR_ID)
header = header.replace(default_first_author, YOUR_NAME)
header = header.replace(default_mail_address, YOUR_EMAIL)
is_year_set = (header.find(default_year) < 0)
if not is_year_set:
year = time_now.strftime('%Y')
print(default_year, year)
actual_year = ", {}.".format(year)
header = header.replace(default_year, actual_year)
header_changed = True
in_po_cat.header_comment = header
changed = (cat_changed or header_changed)
if changed:
if self.dry_run:
print("Processing file:", changed_file)
print("-" * 80)
else:
#self.basic_io.writeTextFile(changed_file, data)
c.dump_po(changed_file, in_po_cat)
print("Wrote changes to:", changed_file)
def run(self):
self.loadFiles()
po_cat_dict = self.poCatToDic(self.po_cat)
print("po_cat_dict")
pp(po_cat_dict)
po_pot_dict = self.poCatToDic(self.pot_cat)
print("po_pot_dict")
pp(po_pot_dict)
ignore_list = {
("Volume", "Sound"): "Âm Lượng",
("volume", "sound"): "Âm Lượng",
("Volume", ""): "Thể Tích",
("volume", ""): "Thể Tích",
#("",""):"",
}
changed = False
for index, item in enumerate(po_pot_dict.items()):
if index == 0:
continue
k, v = item
is_ignore = (k in ignore_list)
if is_ignore:
ignore_value = ignore_list[k]
v.string = ignore_value
print("Ignoring item:", k, v.string)
continue
print("run:", k, v)
is_in_old_dict = (k in po_cat_dict)
if not is_in_old_dict:
print("NEW ENTRY k:[{}], v:[{}]".format(k, v))
continue
old_dict_entry = po_cat_dict[k]
old_dict_entry_msgid = old_dict_entry.id
old_dict_entry_msgstr = old_dict_entry.string
is_fuzzy = old_dict_entry.fuzzy
has_translation = (old_dict_entry_msgstr is not None) and (
len(old_dict_entry_msgstr) > 0)
if not has_translation:
print("IGNORED No translation k:[{}], v:[{}]".format(k, v))
v.string = old_dict_entry_msgstr
if is_fuzzy:
v.flags.add('fuzzy')
print("Updating k[{}], v:[{}], fuzzy:[{}]".format(k, v, is_fuzzy))
changed = True
if (changed and not self.is_dry_run):
new_po_file = "/Users/hoangduytran/new_vi_0001.po"
print("Saving content of new pot_cat to:{}".format(new_po_file))
#self.dump_po(new_po_file, self.pot_cat)
c.dump_po(new_po_file, self.pot_cat)
print("Transferred translations from:", self.from_vipo, "to:",
self.to_vipo, "is_dry_run:", self.is_dry_run)