def load_mo(mo_file, encoding="utf-8") -> Iterator[TranslationItem]:
mo_file.seek(0)
magic_number = mo_file.read(4)
if magic_number != MO_MAGIC:
raise ValueError("Wrong mo-file format")
mo_file.seek(8)
number_of_strings = read_uint(mo_file)
original_string_table_offset = read_uint(mo_file)
translation_string_table_offset = read_uint(mo_file)
for i in range(number_of_strings):
context, _, original_string = load_string(
mo_file, original_string_table_offset + i * 8).rpartition(b"\x04")
translation_string = load_string(
mo_file, translation_string_table_offset + i * 8)
original_string = original_string.decode(encoding)
translation_string = translation_string.decode(encoding)
if context:
context = context.decode(encoding)
yield TranslationItem(context=context,
text=original_string,
translation=translation_string)
else:
yield TranslationItem(text=original_string,
translation=translation_string)
def load_po(po_file: Iterator[str]) -> Iterator[TranslationItem]:
item = defaultdict(str)
prev = None
for line in po_file:
line = line.strip()
if not line:
if item:
yield TranslationItem(text=item["msgid"],
translation=item.get("msgstr"),
context=item.get("msgctxt"))
item = defaultdict(str)
prev = None
elif line.startswith("#"):
# key, _, value = line.partition(" ")
# item[key] += value
# if key == "#":
# item[key] += "\n"
pass # ignore comments
elif line.startswith('"'):
assert prev is not None
item[prev] += unescape_string(strip_once(line, '"'))
else:
# msgid, msgstr, msgctxt etc.
key, value = line.split(maxsplit=1)
item[key] = unescape_string(strip_once(value, '"'))
prev = key
if item:
yield TranslationItem(text=item["msgid"],
translation=item.get("msgstr"),
context=item.get("msgctxt"))
def test_load_mo():
d = [
TranslationItem("Word1", "Translation1"),
TranslationItem("Word2", "Translation2"),
TranslationItem("LongWord", "LongTranslation", context="context"),
]
mo_file = create_mo(d)
assert list(load_mo(mo_file)) == d
def test_save_load_po():
entries: List[TranslationItem] = [
TranslationItem("asddf", "qwert"),
TranslationItem("xcvf", "fghrth"),
TranslationItem("cvbeb", "jtyjkty"),
]
template = (item.text for item in entries)
file = StringIO()
save_po(file, template, entries)
file.seek(0)
assert list(load_po(file))[1:] == entries
def extract_translatables_from_file(file, file_path, join_paragraphs, keys):
for text_block, is_translatable, line_number in parse_plain_text_file(
file, join_paragraphs):
if is_translatable:
if text_block in keys:
print("Key already exists:", repr(text_block), file=sys.stderr)
else:
keys.add(text_block)
yield TranslationItem(text=text_block.rstrip("\n"),
source_file=file_path.name,
line_number=line_number)
def extract_translatables(files: Iterable[Path]) -> Iterator[TranslationItem]:
for file_path in files:
if file_path.is_file():
print("File:", file_path.name, file=sys.stderr)
with open(file_path) as file:
for i, line in enumerate(file, 1):
text = line.rstrip("\n")
if text:
yield TranslationItem(text=text,
source_file=file_path.name,
line_number=i)
def main(
source_file: typer.FileText,
destination_file: typer.FileTextWrite,
no_ignore: bool = False,
):
template = (line.rstrip("\n") for line in source_file)
ignore_rules = dont_ignore if no_ignore else all_ignore_rules
file_name = Path(source_file.name).name
filtered_lines = (TranslationItem(text=line,
source_file=file_name,
line_number=line_number)
for line_number, line in enumerate(template, 1)
if not ignore_rules(line))
save_pot(destination_file, filtered_lines)
def extract_translatables_from_raws(file: Iterable[str]) -> Iterator[TranslationItem]:
translation_keys: Set[Tuple[str, Tuple[str, ...]]] = set()
for item in parse_raw_file(file):
if item.translatable:
tag_parts = item.tag_parts
if (
"TILE" not in tag_parts[0]
and any(is_translatable(s) for s in tag_parts[1:])
and (item.context, tuple(tag_parts)) not in translation_keys # Don't add duplicate items to translate
):
if not is_translatable(tag_parts[-1]):
last = last_suitable(tag_parts, is_translatable)
tag_parts = tag_parts[:last]
tag_parts.append("") # Add an empty element to the tag to mark the tag as not completed
translation_keys.add((item.context, tuple(tag_parts)))
yield TranslationItem(context=item.context, text=join_tag(tag_parts), line_number=item.line_number)
def test_load_po():
data = strip_margin(
"""
|# Some comment
|#: body_default.txt:7
|msgctxt "BODY:BASIC_1PARTBODY"
|msgid "[BP:UB:body:bodies]"
|msgstr "[BP:UB:тело:тела]"
"""
)
expected = TranslationItem(
context="BODY:BASIC_1PARTBODY",
text="[BP:UB:body:bodies]",
translation="[BP:UB:тело:тела]",
# translator_comment="Some comment\n",
# source_file="body_default.txt",
# line_number=7,
)
result = next(load_po(StringIO(data)))
assert result == expected
def test_po_reader():
po_content = strip_margin(
r"""
|msgid ""
|msgstr ""
|"Project-Id-Version: Dwarf Fortress\n"
|"PO-Revision-Date: 2019-11-20 10:25+0000\n"
|"Content-Type: text/plain; charset=UTF-8\n"
|"Content-Transfer-Encoding: 8bit\n"
|"Language: ru\n"
|
|# Some comment
|#: body_default.txt:7
|msgctxt "BODY:BASIC_1PARTBODY"
|msgid "[BP:UB:body:bodies]"
|msgstr "[BP:UB:тело:тела]"
"""
)
po = PoReader(StringIO(po_content))
assert po.meta == {
"Project-Id-Version": "Dwarf Fortress",
"PO-Revision-Date": "2019-11-20 10:25+0000",
"Content-Type": "text/plain; charset=UTF-8",
"Content-Transfer-Encoding": "8bit",
"Language": "ru",
}
assert next(po) == TranslationItem(
context="BODY:BASIC_1PARTBODY",
text="[BP:UB:body:bodies]",
translation="[BP:UB:тело:тела]",
# translator_comment="Some comment\n",
# source_file="body_default.txt",
# line_number=7,
)
|[BIOME:GRASSLAND_TEMPERATE]
|
|[CREATURE:BIRD_CARDINAL] - context will be changed to CREATURE:BIRD_CARDINAL
|[DESCRIPTION:A small bright red bird with a distinctive crest, found in temperate forests.]
|[NAME:cardinal:cardinals:cardinal]
|[CASTE_NAME:cardinal:cardinals:cardinal]
|[GENERAL_CHILD_NAME:cardinal hatchling:cardinal hatchlings]
|[CREATURE_TILE:144][COLOR:4:0:1]
|[PETVALUE:30][NATURAL][PET]
"""
).strip(),
[
TranslationItem(
context="CREATURE:BIRD_BLUEJAY",
text=(
"[DESCRIPTION:"
"A small blue-crested bird living in temperate woodlands, known for its harsh chirps.]"
),
),
TranslationItem(context="CREATURE:BIRD_BLUEJAY", text="[NAME:blue jay:blue jays:blue jay]"),
TranslationItem(context="CREATURE:BIRD_BLUEJAY", text="[CASTE_NAME:blue jay:blue jays:blue jay]"),
TranslationItem(
context="CREATURE:BIRD_BLUEJAY",
text="[GENERAL_CHILD_NAME:blue jay hatchling:blue jay hatchlings]",
),
TranslationItem(
context="CREATURE:BIRD_CARDINAL",
text="[DESCRIPTION:A small bright red bird with a distinctive crest, found in temperate forests.]",
),
TranslationItem(context="CREATURE:BIRD_CARDINAL", text="[NAME:cardinal:cardinals:cardinal]"),
TranslationItem(context="CREATURE:BIRD_CARDINAL", text="[CASTE_NAME:cardinal:cardinals:cardinal]"),