def test_simple(self):
file_list = get_file_list(os.path.join(TEST_PATH, "test_files", "test_dir"), ['txt', 'yaml'])
file_list.sort()
self.assertEqual(file_list, [os.path.join(TEST_PATH, "test_files", "test_dir", "d1", "f4.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f2.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f5.yaml")])
def test_simple(self):
file_list = get_file_list(
os.path.join(TEST_PATH, "test_files", "test_dir"), ['txt', 'yaml'])
file_list.sort()
self.assertEqual(file_list, [
os.path.join(TEST_PATH, "test_files", "test_dir", "d1", "f4.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f2.txt"),
os.path.join(TEST_PATH, "test_files", "test_dir", "f5.yaml")
])
def test_aggregation(self):
po_file_list = get_file_list(
os.path.join(TEST_PATH, "test_files", "docs"), ["po", "pot"])
po_file_list.sort()
extract_all_untranslated_po_entries(
po_file_list, os.path.join(TEST_PATH, "temp_files"))
with open(os.path.join(TEST_PATH, "temp_files", "source")) as f:
self.assertEqual(
f.read().strip(),
"Administration\nAggregation\nA high-level introduction to aggregation.\nIntroduces the use and operation of the data aggregation modalities available in MongoDB."
)
def test_files(self):
po_file_list = get_file_list(os.path.join(TEST_PATH, "temp_files", "docs"),["po","pot"])
po_file_list.sort()
write_po_files(po_file_list,os.path.join(TEST_PATH, "test_files", "doc_filler.txt"))
f1 = polib.pofile(os.path.join(TEST_PATH, "temp_files", "docs", "aggregation.po"))
f2 = polib.pofile(os.path.join(TEST_PATH, "test_files", "filled_docs", "aggregation.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
f1 = polib.pofile(os.path.join(TEST_PATH, "temp_files", "docs", "administration.po"))
f2 = polib.pofile(os.path.join(TEST_PATH, "test_files", "filled_docs", "administration.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
def auto_approve_po_entries(po_path):
''' This function automatically approves any untranslated sentence in a
po file that should be identical in the target language. These sentences
are of the form ``:word:\`link\```
:param string po_path: the path to the top level directory of the po_files
'''
po_file_list = get_file_list(po_path, ["po", "pot"])
reg = re.compile('^:[a-zA-Z0-9]+:`(?!.*<.*>.*)[^`]*`$')
for fn in po_file_list:
po = polib.pofile(fn)
for entry in po.untranslated_entries():
match = re.match(reg, entry.msgid.encode('utf-8'))
if match is not None and match.group() == entry.msgid.encode('utf-8'):
entry.msgstr = entry.msgid
po.save()
def create_corpus_from_po(po_path, source_doc_fn, target_doc_fn):
'''This function opens up the output files and then goes through the files
in the file list and writes them all to two corpus files.
:param string po_path: Path to po file or directory of po files
:param string source_doc_fn: Name of file to put source lanaguge text in.
:param string target_doc_fn: Name of file to put target lanaguge text in.
'''
# path is a directory now
logger.info("walking path "+po_path)
with open(source_doc_fn, "w", 1) as source_doc:
with open(target_doc_fn, "w", 1) as target_doc:
file_list = get_file_list(po_path, ["po", "pot"])
for fn in file_list:
write_from_po_file(source_doc, target_doc, fn)
def create_corpus_from_po(po_path, source_doc_fn, target_doc_fn):
'''This function opens up the output files and then goes through the files
in the file list and writes them all to two corpus files.
:param string po_path: Path to po file or directory of po files
:param string source_doc_fn: Name of file to put source lanaguge text in.
:param string target_doc_fn: Name of file to put target lanaguge text in.
'''
# path is a directory now
logger.info("walking path " + po_path)
with open(source_doc_fn, "w", 1) as source_doc:
with open(target_doc_fn, "w", 1) as target_doc:
file_list = get_file_list(po_path, ["po", "pot"])
for fn in file_list:
write_from_po_file(source_doc, target_doc, fn)
def test_files(self):
po_file_list = get_file_list(
os.path.join(TEST_PATH, "temp_files", "docs"), ["po", "pot"])
po_file_list.sort()
write_po_files(po_file_list,
os.path.join(TEST_PATH, "test_files", "doc_filler.txt"))
f1 = polib.pofile(
os.path.join(TEST_PATH, "temp_files", "docs", "aggregation.po"))
f2 = polib.pofile(
os.path.join(TEST_PATH, "test_files", "filled_docs",
"aggregation.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
f1 = polib.pofile(
os.path.join(TEST_PATH, "temp_files", "docs", "administration.po"))
f2 = polib.pofile(
os.path.join(TEST_PATH, "test_files", "filled_docs",
"administration.po"))
for l1, l2 in zip(f1, f2):
self.assertEqual(l1, l2)
def translate_po_files(po_path, tconf, protected_file=None):
''' This function translates a directory of po files in three steps:
First it extracts the untranslated entries from every po file into one
big file. Then it translates that file. Lastly it fills in all of the
po files in the same order the entries were extracted, removing the text
from any translated entries.
:param string po_path: the path to the top level directory of the po_files
:param config tconf: translation config object
:param string protected_file: path to file with regexes to protect
'''
with TempDir() as temp_dir:
po_file_list = get_file_list(po_path, ["po", "pot"])
temp_file = extract_all_untranslated_po_entries(po_file_list, temp_dir)
trans_file = temp_file + '.translated'
translate_file(temp_file, trans_file, tconf, protected_file, temp_dir)
# flips the file if the language is right to left
if tconf.settings.foreign in ['he', 'ar']:
flipped_file = trans_file + '.flip'
flip_text_direction(trans_file, flipped_file)
trans_file = flipped_file
write_po_files(po_file_list, trans_file)
def test_bad_file(self):
self.assertEqual(
get_file_list(
os.path.join(TEST_PATH, "temp_files", "mergdfadafadfaed.txt"),
['txt']), [])
def test_single(self):
self.assertEqual(
get_file_list(
os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt"),
['txt']),
[os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt")])
def test_aggregation(self):
po_file_list = get_file_list(os.path.join(TEST_PATH, "test_files", "docs"), ["po","pot"])
po_file_list.sort()
extract_all_untranslated_po_entries(po_file_list, os.path.join(TEST_PATH, "temp_files"))
with open(os.path.join(TEST_PATH, "temp_files", "source")) as f:
self.assertEqual(f.read().strip(), "Administration\nAggregation\nA high-level introduction to aggregation.\nIntroduces the use and operation of the data aggregation modalities available in MongoDB.")
def test_bad_file(self):
self.assertEqual(get_file_list(os.path.join(TEST_PATH, "temp_files", "mergdfadafadfaed.txt"),['txt']),[])
def test_single(self):
self.assertEqual(get_file_list(os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt"), ['txt']),
[os.path.join(TEST_PATH, "test_files", "test_dir", "f1.txt")])