def apply(self, **kwargs: Any) -> None:
source = self.document["source"]
msgstr = ""
# XXX check if this is reliable
assert source.startswith(self.env.srcdir)
docname = path.splitext(
relative_path(path.join(self.env.srcdir, "dummy"), source))[0]
textdomain = docname_to_domain(docname, self.config.gettext_compact)
# fetch translations
dirs = [
path.join(self.env.srcdir, directory)
for directory in self.config.locale_dirs
]
# sphinx.locale.init changes its args type by version
# so, ignore mypy check for the following call
catalog, has_catalog = init_locale(
dirs,
self.config.language,
textdomain # type: ignore
)
if not has_catalog:
return
# for translated text keep original as ORIGINAL_TEXT_ATTR attribute
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# self.logger.info("add %s attr to node." % ORIGINAL_TEXT_ATTR)
node[ORIGINAL_TEXT_ATTR] = msg
def test_extract_messages_without_rawsource():
"""
Check node.rawsource is fall-backed by using node.astext() value.
`extract_message` which is used from Sphinx i18n feature drop ``not node.rawsource``
nodes. So, all nodes which want to translate must have ``rawsource`` value.
However, sometimes node.rawsource is not set.
For example: recommonmark-0.2.0 doesn't set rawsource to `paragraph` node.
refs #1994: Fall back to node's astext() during i18n message extraction.
"""
p = nodes.paragraph()
p.append(nodes.Text('test'))
p.append(nodes.Text('sentence'))
assert not p.rawsource # target node must not have rawsource value
document = create_new_document()
document.append(p)
_transform(document)
assert_node_count(extract_messages(document), nodes.TextElement, 1)
assert [m for n, m in extract_messages(document)][0], 'text sentence'
def test_extract_messages_without_rawsource():
"""
Check node.rawsource is fall-backed by using node.astext() value.
`extract_message` which is used from Sphinx i18n feature drop ``not node.rawsource``
nodes. So, all nodes which want to translate must have ``rawsource`` value.
However, sometimes node.rawsource is not set.
For example: recommonmark-0.2.0 doesn't set rawsource to `paragraph` node.
refs #1994: Fall back to node's astext() during i18n message extraction.
"""
p = nodes.paragraph()
p.append(nodes.Text('test'))
p.append(nodes.Text('sentence'))
assert not p.rawsource # target node must not have rawsource value
document = create_new_document()
document.append(p)
_transform(document)
assert_node_count(extract_messages(document), nodes.TextElement, 1)
assert [m for n, m in extract_messages(document)][0], 'text sentence'
def write_doc(self, docname: str, doctree: nodes.document) -> None:
catalog = self.catalogs[docname_to_domain(docname, self.config.gettext_compact)]
for toctree in self.env.tocs[docname].traverse(addnodes.toctree):
for node, msg in extract_messages(toctree):
node.uid = '' # type: ignore # Hack UUID model
catalog.add(msg, node)
for node, msg in extract_messages(doctree):
catalog.add(msg, node)
if 'index' in self.env.config.gettext_additional_targets:
# Extract translatable messages from index entries.
for node, entries in traverse_translatable_index(doctree):
for typ, msg, tid, main, key_ in entries:
for m in split_index_msg(typ, msg):
if typ == 'pair' and m in pairindextypes.values():
# avoid built-in translated message was incorporated
# in 'sphinx.util.nodes.process_index_entry'
continue
catalog.add(m, node)
def write_doc(self, docname, doctree):
# type: (str, nodes.document) -> None
catalog = self.catalogs[find_catalog(docname, self.config.gettext_compact)]
for toctree in self.env.tocs[docname].traverse(addnodes.toctree):
for node, msg in extract_messages(toctree):
node.uid = '' # type: ignore # Hack UUID model
catalog.add(msg, node)
for node, msg in extract_messages(doctree):
catalog.add(msg, node)
if 'index' in self.env.config.gettext_additional_targets:
# Extract translatable messages from index entries.
for node, entries in traverse_translatable_index(doctree):
for typ, msg, tid, main, key_ in entries:
for m in split_index_msg(typ, msg):
if typ == 'pair' and m in pairindextypes.values():
# avoid built-in translated message was incorporated
# in 'sphinx.util.nodes.process_index_entry'
continue
catalog.add(m, node)
def write_doc(self, docname, doctree):
catalog = self.catalogs[find_catalog(docname, self.config.gettext_compact)]
for node, msg in extract_messages(doctree):
catalog.add(msg, node)
if "index" in self.env.config.gettext_additional_targets:
# Extract translatable messages from index entries.
for node, entries in traverse_translatable_index(doctree):
for typ, msg, tid, main in entries:
for m in split_index_msg(typ, msg):
if typ == "pair" and m in pairindextypes.values():
# avoid built-in translated message was incorporated
# in 'sphinx.util.nodes.process_index_entry'
continue
catalog.add(m, node)
def write_doc(self, docname, doctree):
catalog = self.catalogs[find_catalog(docname,
self.config.gettext_compact)]
for node, msg in extract_messages(doctree):
catalog.add(msg, node)
# Extract translatable messages from index entries.
for node, entries in traverse_translatable_index(doctree):
for typ, msg, tid, main in entries:
for m in split_index_msg(typ, msg):
if typ == 'pair' and m in pairindextypes.values():
# avoid built-in translated message was incorporated
# in 'sphinx.util.nodes.process_index_entry'
continue
catalog.add(m, node)
def test_extract_messages():
text = dedent(
"""
.. admonition:: admonition title
admonition body
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.title, 1,
)
text = dedent(
"""
.. figure:: foo.jpg
this is title
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.caption, 1,
)
text = dedent(
"""
.. rubric:: spam
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.rubric, 1,
)
text = dedent(
"""
| spam
| egg
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
text = dedent(
"""
section
=======
+----------------+
| | **Title 1** |
| | Message 1 |
+----------------+
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
text = dedent(
"""
* | **Title 1**
| Message 1
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
def test_extract_messages():
text = dedent(
"""
.. admonition:: admonition title
admonition body
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.title, 1,
)
text = dedent(
"""
.. figure:: foo.jpg
this is title
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.caption, 1,
)
text = dedent(
"""
.. rubric:: spam
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.rubric, 1,
)
text = dedent(
"""
| spam
| egg
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
text = dedent(
"""
section
=======
+----------------+
| | **Title 1** |
| | Message 1 |
+----------------+
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
text = dedent(
"""
* | **Title 1**
| Message 1
"""
)
yield (
assert_node_count,
extract_messages(_get_doctree(text)),
nodes.line, 2,
)
def write_doc(self, docname, doctree):
catalog = self.catalogs[find_catalog(docname,
self.config.gettext_compact)]
for node, msg in extract_messages(doctree):
catalog.add(msg, node)
def apply(self, **kwargs):
# type: (Any) -> None
settings, source = self.document.settings, self.document['source']
msgstr = u''
# XXX check if this is reliable
assert source.startswith(self.env.srcdir)
docname = path.splitext(relative_path(path.join(self.env.srcdir, 'dummy'),
source))[0]
textdomain = find_catalog(docname, self.config.gettext_compact)
# fetch translations
dirs = [path.join(self.env.srcdir, directory)
for directory in self.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, self.config.language, textdomain)
if not has_catalog:
return
# phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
patch = publish_msgstr(self.app, msgstr, source,
node.line, self.config, settings)
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
# Original section name (reference target name) should be kept to refer
# from other nodes which is still not translated or uses explicit target
# name like "`text to display <explicit target name_>`_"..
# So, `old_name` is still exist in `names`.
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * if _id is None:
#
# _id is None means:
#
# 1. _id was not provided yet.
#
# 2. _id was duplicated.
#
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * if _id is provided: bellow process
if _id:
if not explicit:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
#
# Note: msgnode that is a second parameter of the
# `note_implicit_target` is not necessary here because
# section_node has been noted previously on rst parsing by
# `docutils.parsers.rst.states.RSTState.new_subsection()`
# and already has `system_message` if needed.
self.document.note_implicit_target(section_node)
# replace target's refname to new target name
matcher = NodeMatcher(nodes.target, refname=old_name)
for old_target in self.document.traverse(matcher): # type: nodes.target
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
gloss_entries = self.env.temp_data.setdefault('gloss_entries', set())
for _id in node['names']:
if _id in gloss_entries:
gloss_entries.remove(_id)
parts = split_term_classifiers(msgstr)
patch = publish_msgstr(self.app, parts[0], source,
node.line, self.config, settings)
patch = make_glossary_term(self.env, patch, parts[1],
source, node.line, _id)
node['ids'] = patch['ids']
node['names'] = patch['names']
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True # to avoid double translation
# phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False): # to avoid double translation
continue # skip if the node is already translated by phase1
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# update translatable nodes
if isinstance(node, addnodes.translatable):
node.apply_translated_message(msg, msgstr)
continue
# update meta nodes
if isinstance(node, nodes.pending) and is_pending_meta(node):
node.details['nodes'][0]['content'] = msgstr
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
# Structural Subelements phase1
# There is a possibility that only the title node is created.
# see: http://docutils.sourceforge.net/docs/ref/doctree.html#structural-subelements
if isinstance(node, nodes.title):
# This generates: <section ...><title>msgstr</title></section>
msgstr = msgstr + '\n' + '-' * len(msgstr) * 2
patch = publish_msgstr(self.app, msgstr, source,
node.line, self.config, settings)
# Structural Subelements phase2
if isinstance(node, nodes.title):
# get <title> node that placed as a first child
patch = patch.next_node()
# ignore unexpected markups in translation message
unexpected = (
nodes.paragraph, # expected form of translation
nodes.title # generated by above "Subelements phase2"
) # type: Tuple[Type[nodes.Element], ...]
# following types are expected if
# config.gettext_additional_targets is configured
unexpected += LITERAL_TYPE_NODES
unexpected += IMAGE_TYPE_NODES
if not isinstance(patch, unexpected):
continue # skip
# auto-numbered foot note reference should use original 'ids'.
def list_replace_or_append(lst, old, new):
# type: (List[N], N, N) -> None
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
is_autofootnote_ref = NodeMatcher(nodes.footnote_reference, auto=Any)
old_foot_refs = node.traverse(is_autofootnote_ref) # type: List[nodes.footnote_reference] # NOQA
new_foot_refs = patch.traverse(is_autofootnote_ref) # type: List[nodes.footnote_reference] # NOQA
if len(old_foot_refs) != len(new_foot_refs):
old_foot_ref_rawsources = [ref.rawsource for ref in old_foot_refs]
new_foot_ref_rawsources = [ref.rawsource for ref in new_foot_refs]
logger.warning(__('inconsistent footnote references in translated message.' +
' original: {0}, translated: {1}')
.format(old_foot_ref_rawsources, new_foot_ref_rawsources),
location=node)
old_foot_namerefs = {} # type: Dict[str, List[nodes.footnote_reference]]
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for newf in new_foot_refs:
refname = newf.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
continue
oldf = refs.pop(0)
newf['ids'] = oldf['ids']
for id in newf['ids']:
self.document.ids[id] = newf
if newf['auto'] == 1:
# autofootnote_refs
list_replace_or_append(self.document.autofootnote_refs, oldf, newf)
else:
# symbol_footnote_refs
list_replace_or_append(self.document.symbol_footnote_refs, oldf, newf)
if refname:
footnote_refs = self.document.footnote_refs.setdefault(refname, [])
list_replace_or_append(footnote_refs, oldf, newf)
refnames = self.document.refnames.setdefault(refname, [])
list_replace_or_append(refnames, oldf, newf)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
is_refnamed_ref = NodeMatcher(nodes.reference, refname=Any)
old_refs = node.traverse(is_refnamed_ref) # type: List[nodes.reference]
new_refs = patch.traverse(is_refnamed_ref) # type: List[nodes.reference]
if len(old_refs) != len(new_refs):
old_ref_rawsources = [ref.rawsource for ref in old_refs]
new_ref_rawsources = [ref.rawsource for ref in new_refs]
logger.warning(__('inconsistent references in translated message.' +
' original: {0}, translated: {1}')
.format(old_ref_rawsources, new_ref_rawsources),
location=node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for newr in new_refs:
if not self.document.has_name(newr['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
newr['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(newr)
# refnamed footnote should use original 'ids'.
is_refnamed_footnote_ref = NodeMatcher(nodes.footnote_reference, refname=Any)
old_foot_refs = node.traverse(is_refnamed_footnote_ref)
new_foot_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {} # type: Dict[str, List[str]]
if len(old_foot_refs) != len(new_foot_refs):
old_foot_ref_rawsources = [ref.rawsource for ref in old_foot_refs]
new_foot_ref_rawsources = [ref.rawsource for ref in new_foot_refs]
logger.warning(__('inconsistent footnote references in translated message.' +
' original: {0}, translated: {1}')
.format(old_foot_ref_rawsources, new_foot_ref_rawsources),
location=node)
for oldf in old_foot_refs:
refname_ids_map.setdefault(oldf["refname"], []).append(oldf["ids"])
for newf in new_foot_refs:
refname = newf["refname"]
if refname_ids_map.get(refname):
newf["ids"] = refname_ids_map[refname].pop(0)
# citation should use original 'ids'.
is_citation_ref = NodeMatcher(nodes.citation_reference, refname=Any)
old_cite_refs = node.traverse(is_citation_ref) # type: List[nodes.citation_reference] # NOQA
new_cite_refs = patch.traverse(is_citation_ref) # type: List[nodes.citation_reference] # NOQA
refname_ids_map = {}
if len(old_cite_refs) != len(new_cite_refs):
old_cite_ref_rawsources = [ref.rawsource for ref in old_cite_refs]
new_cite_ref_rawsources = [ref.rawsource for ref in new_cite_refs]
logger.warning(__('inconsistent citation references in translated message.' +
' original: {0}, translated: {1}')
.format(old_cite_ref_rawsources, new_cite_ref_rawsources),
location=node)
for oldc in old_cite_refs:
refname_ids_map.setdefault(oldc["refname"], []).append(oldc["ids"])
for newc in new_cite_refs:
refname = newc["refname"]
if refname_ids_map.get(refname):
newc["ids"] = refname_ids_map[refname].pop()
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_xrefs = node.traverse(addnodes.pending_xref)
new_xrefs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_xrefs) != len(new_xrefs):
old_xref_rawsources = [xref.rawsource for xref in old_xrefs]
new_xref_rawsources = [xref.rawsource for xref in new_xrefs]
logger.warning(__('inconsistent term references in translated message.' +
' original: {0}, translated: {1}')
.format(old_xref_rawsources, new_xref_rawsources),
location=node)
def get_ref_key(node):
# type: (addnodes.pending_xref) -> Tuple[str, str, str]
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],)
for old in old_xrefs:
key = get_ref_key(old)
if key:
xref_reftarget_map[key] = old.attributes
for new in new_xrefs:
key = get_ref_key(new)
# Copy attributes to keep original node behavior. Especially
# copying 'reftarget', 'py:module', 'py:class' are needed.
for k, v in xref_reftarget_map.get(key, {}).items():
# Note: This implementation overwrite all attributes.
# if some attributes `k` should not be overwritten,
# you should provide exclude list as:
# `if k not in EXCLUDE_LIST: new[k] = v`
new[k] = v
# update leaves
for child in patch.children:
child.parent = node
node.children = patch.children
# for highlighting that expects .rawsource and .astext() are same.
if isinstance(node, LITERAL_TYPE_NODES):
node.rawsource = node.astext()
if isinstance(node, IMAGE_TYPE_NODES):
node.update_all_atts(patch)
node['translated'] = True # to avoid double translation
if 'index' in self.config.gettext_additional_targets:
# Extract and translate messages for index entries.
for node, entries in traverse_translatable_index(self.document):
new_entries = [] # type: List[Tuple[str, str, str, str, str]]
for type, msg, tid, main, key_ in entries:
msg_parts = split_index_msg(type, msg)
msgstr_parts = []
for part in msg_parts:
msgstr = catalog.gettext(part)
if not msgstr:
msgstr = part
msgstr_parts.append(msgstr)
new_entries.append((type, ';'.join(msgstr_parts), tid, main, None))
node['raw_entries'] = entries
node['entries'] = new_entries
# remove translated attribute that is used for avoiding double translation.
for translated in self.document.traverse(NodeMatcher(translated=Any)): # type: nodes.Element # NOQA
translated.delattr('translated')
def write_doc(self, docname, doctree):
catalog = self.catalogs[docname.split(SEP, 1)[0]]
for node, msg in extract_messages(doctree):
catalog.setdefault(node.uid, msg)
def test_extract_messages(rst, node_cls, count):
msg = extract_messages(_get_doctree(dedent(rst)))
assert_node_count(msg, node_cls, count)
def apply(self):
env = self.document.settings.env
settings, source = self.document.settings, self.document['source']
# XXX check if this is reliable
assert source.startswith(env.srcdir)
docname = path.splitext(relative_path(env.srcdir, source))[0]
textdomain = find_catalog(docname,
self.document.settings.gettext_compact)
# fetch translations
dirs = [path.join(env.srcdir, directory)
for directory in env.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, env.config.language,
textdomain)
if not has_catalog:
return
parser = RSTParser()
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
patch = patch[0]
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
# auto-numbered foot note reference should use original 'ids'.
def is_autonumber_footnote_ref(node):
return isinstance(node, nodes.footnote_reference) and \
node.get('auto') == 1
old_foot_refs = node.traverse(is_autonumber_footnote_ref)
new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
env.warn_node('inconsistent footnote references in '
'translated message', node)
for old, new in zip(old_foot_refs, new_foot_refs):
new['ids'] = old['ids']
for id in new['ids']:
self.document.ids[id] = new
self.document.autofootnote_refs.remove(old)
self.document.note_autofootnote_ref(new)
# reference should use original 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * section refname is not translatable.
# * inline reference "`Python <...>`_" has no 'refname'.
def is_refnamed_ref(node):
return isinstance(node, nodes.reference) and \
'refname' in node
old_refs = node.traverse(is_refnamed_ref)
new_refs = patch.traverse(is_refnamed_ref)
applied_refname_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
for new in new_refs:
if new['refname'] in applied_refname_map:
# 2nd appearance of the reference
new['refname'] = applied_refname_map[new['refname']]
elif old_refs:
# 1st appearance of the reference in old_refs
old = old_refs.pop(0)
refname = old['refname']
new['refname'] = refname
applied_refname_map[new['refname']] = refname
else:
# the reference is not found in old_refs
applied_refname_map[new['refname']] = new['refname']
self.document.note_refname(new)
# refnamed footnote and citation should use original 'ids'.
def is_refnamed_footnote_ref(node):
footnote_ref_classes = (nodes.footnote_reference,
nodes.citation_reference)
return isinstance(node, footnote_ref_classes) and \
'refname' in node
old_refs = node.traverse(is_refnamed_footnote_ref)
new_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
for old in old_refs:
refname_ids_map[old["refname"]] = old["ids"]
for new in new_refs:
refname = new["refname"]
if refname in refname_ids_map:
new["ids"] = refname_ids_map[refname]
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_refs = node.traverse(addnodes.pending_xref)
new_refs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent term references in '
'translated message', node)
for old in old_refs:
key = old["reftype"], old["refdomain"]
xref_reftarget_map[key] = old["reftarget"]
for new in new_refs:
key = new["reftype"], new["refdomain"]
if key in xref_reftarget_map:
new['reftarget'] = xref_reftarget_map[key]
# update leaves
for child in patch.children:
child.parent = node
node.children = patch.children
# Extract and translate messages for index entries.
for node, entries in traverse_translatable_index(self.document):
new_entries = []
for type, msg, tid, main in entries:
msg_parts = split_index_msg(type, msg)
msgstr_parts = []
for part in msg_parts:
msgstr = catalog.gettext(part)
if not msgstr:
msgstr = part
msgstr_parts.append(msgstr)
new_entries.append((type, ';'.join(msgstr_parts), tid, main))
node['raw_entries'] = entries
node['entries'] = new_entries
def apply(self, **kwargs: Any) -> None:
settings, source = self.document.settings, self.document['source']
msgstr = ''
# XXX check if this is reliable
assert source.startswith(self.env.srcdir)
docname = path.splitext(relative_path(path.join(self.env.srcdir, 'dummy'),
source))[0]
textdomain = docname_to_domain(docname, self.config.gettext_compact)
# fetch translations
dirs = [path.join(self.env.srcdir, directory)
for directory in self.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, self.config.language, textdomain)
if not has_catalog:
return
# phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
patch = publish_msgstr(self.app, msgstr, source,
node.line, self.config, settings)
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title) and isinstance(node.parent, nodes.section):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
# Original section name (reference target name) should be kept to refer
# from other nodes which is still not translated or uses explicit target
# name like "`text to display <explicit target name_>`_"..
# So, `old_name` is still exist in `names`.
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * if _id is None:
#
# _id is None means:
#
# 1. _id was not provided yet.
#
# 2. _id was duplicated.
#
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * if _id is provided: bellow process
if _id:
if not explicit:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
#
# Note: msgnode that is a second parameter of the
# `note_implicit_target` is not necessary here because
# section_node has been noted previously on rst parsing by
# `docutils.parsers.rst.states.RSTState.new_subsection()`
# and already has `system_message` if needed.
self.document.note_implicit_target(section_node)
# replace target's refname to new target name
matcher = NodeMatcher(nodes.target, refname=old_name)
for old_target in self.document.traverse(matcher): # type: nodes.target
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
for _id in node['ids']:
parts = split_term_classifiers(msgstr)
patch = publish_msgstr(self.app, parts[0], source,
node.line, self.config, settings)
patch = make_glossary_term(self.env, patch, parts[1],
source, node.line, _id,
self.document)
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True # to avoid double translation
# phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False): # to avoid double translation
continue # skip if the node is already translated by phase1
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# update translatable nodes
if isinstance(node, addnodes.translatable):
node.apply_translated_message(msg, msgstr)
continue
# update meta nodes
if isinstance(node, nodes.pending) and is_pending_meta(node):
node.details['nodes'][0]['content'] = msgstr
continue
if isinstance(node, nodes.image) and node.get('alt') == msg:
node['alt'] = msgstr
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
# Structural Subelements phase1
# There is a possibility that only the title node is created.
# see: https://docutils.sourceforge.io/docs/ref/doctree.html#structural-subelements
if isinstance(node, nodes.title):
# This generates: <section ...><title>msgstr</title></section>
msgstr = msgstr + '\n' + '=' * len(msgstr) * 2
patch = publish_msgstr(self.app, msgstr, source,
node.line, self.config, settings)
# Structural Subelements phase2
if isinstance(node, nodes.title):
# get <title> node that placed as a first child
patch = patch.next_node()
# ignore unexpected markups in translation message
unexpected: Tuple[Type[Element], ...] = (
nodes.paragraph, # expected form of translation
nodes.title # generated by above "Subelements phase2"
)
# following types are expected if
# config.gettext_additional_targets is configured
unexpected += LITERAL_TYPE_NODES
unexpected += IMAGE_TYPE_NODES
if not isinstance(patch, unexpected):
continue # skip
# auto-numbered foot note reference should use original 'ids'.
def list_replace_or_append(lst: List[N], old: N, new: N) -> None:
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
is_autofootnote_ref = NodeMatcher(nodes.footnote_reference, auto=Any)
old_foot_refs: List[nodes.footnote_reference] = node.traverse(is_autofootnote_ref)
new_foot_refs: List[nodes.footnote_reference] = patch.traverse(is_autofootnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
old_foot_ref_rawsources = [ref.rawsource for ref in old_foot_refs]
new_foot_ref_rawsources = [ref.rawsource for ref in new_foot_refs]
logger.warning(__('inconsistent footnote references in translated message.' +
' original: {0}, translated: {1}')
.format(old_foot_ref_rawsources, new_foot_ref_rawsources),
location=node)
old_foot_namerefs: Dict[str, List[nodes.footnote_reference]] = {}
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for newf in new_foot_refs:
refname = newf.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
newf.parent.remove(newf)
continue
oldf = refs.pop(0)
newf['ids'] = oldf['ids']
for id in newf['ids']:
self.document.ids[id] = newf
if newf['auto'] == 1:
# autofootnote_refs
list_replace_or_append(self.document.autofootnote_refs, oldf, newf)
else:
# symbol_footnote_refs
list_replace_or_append(self.document.symbol_footnote_refs, oldf, newf)
if refname:
footnote_refs = self.document.footnote_refs.setdefault(refname, [])
list_replace_or_append(footnote_refs, oldf, newf)
refnames = self.document.refnames.setdefault(refname, [])
list_replace_or_append(refnames, oldf, newf)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
is_refnamed_ref = NodeMatcher(nodes.reference, refname=Any)
old_refs: List[nodes.reference] = node.traverse(is_refnamed_ref)
new_refs: List[nodes.reference] = patch.traverse(is_refnamed_ref)
if len(old_refs) != len(new_refs):
old_ref_rawsources = [ref.rawsource for ref in old_refs]
new_ref_rawsources = [ref.rawsource for ref in new_refs]
logger.warning(__('inconsistent references in translated message.' +
' original: {0}, translated: {1}')
.format(old_ref_rawsources, new_ref_rawsources),
location=node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for newr in new_refs:
if not self.document.has_name(newr['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
newr['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(newr)
# refnamed footnote should use original 'ids'.
is_refnamed_footnote_ref = NodeMatcher(nodes.footnote_reference, refname=Any)
old_foot_refs = node.traverse(is_refnamed_footnote_ref)
new_foot_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map: Dict[str, List[str]] = {}
if len(old_foot_refs) != len(new_foot_refs):
old_foot_ref_rawsources = [ref.rawsource for ref in old_foot_refs]
new_foot_ref_rawsources = [ref.rawsource for ref in new_foot_refs]
logger.warning(__('inconsistent footnote references in translated message.' +
' original: {0}, translated: {1}')
.format(old_foot_ref_rawsources, new_foot_ref_rawsources),
location=node)
for oldf in old_foot_refs:
refname_ids_map.setdefault(oldf["refname"], []).append(oldf["ids"])
for newf in new_foot_refs:
refname = newf["refname"]
if refname_ids_map.get(refname):
newf["ids"] = refname_ids_map[refname].pop(0)
# citation should use original 'ids'.
is_citation_ref = NodeMatcher(nodes.citation_reference, refname=Any)
old_cite_refs: List[nodes.citation_reference] = node.traverse(is_citation_ref)
new_cite_refs: List[nodes.citation_reference] = patch.traverse(is_citation_ref)
refname_ids_map = {}
if len(old_cite_refs) != len(new_cite_refs):
old_cite_ref_rawsources = [ref.rawsource for ref in old_cite_refs]
new_cite_ref_rawsources = [ref.rawsource for ref in new_cite_refs]
logger.warning(__('inconsistent citation references in translated message.' +
' original: {0}, translated: {1}')
.format(old_cite_ref_rawsources, new_cite_ref_rawsources),
location=node)
for oldc in old_cite_refs:
refname_ids_map.setdefault(oldc["refname"], []).append(oldc["ids"])
for newc in new_cite_refs:
refname = newc["refname"]
if refname_ids_map.get(refname):
newc["ids"] = refname_ids_map[refname].pop()
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_xrefs = node.traverse(addnodes.pending_xref)
new_xrefs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_xrefs) != len(new_xrefs):
old_xref_rawsources = [xref.rawsource for xref in old_xrefs]
new_xref_rawsources = [xref.rawsource for xref in new_xrefs]
logger.warning(__('inconsistent term references in translated message.' +
' original: {0}, translated: {1}')
.format(old_xref_rawsources, new_xref_rawsources),
location=node)
def get_ref_key(node: addnodes.pending_xref) -> Optional[Tuple[str, str, str]]:
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],)
def doctree_resolved(app, doctree, docname):
def abbreviating():
remove_items = []
new_items = {}
for k, v in trans_finder.master_dic.items():
ref_list = RefList(msg=v)
new_v = ref_list.quotedToAbbrev(k)
has_new_v = (new_v is not None) and (len(new_v) > 0)
if has_new_v:
new_entry = {k: new_v}
new_items.update(new_entry)
has_remove_items = (len(remove_items) > 0)
if has_remove_items:
for k in remove_items:
dd(f'Delete from dictionary:[{k}]')
del trans_finder.master_dic[k]
is_writing_changes = (len(new_items) > 0)
if is_writing_changes:
trans_finder.master_dic.update(new_items)
dic_file = '/Users/hoangduytran/blender_manual/test_dic.json'
print(
f'Writing changes to: {dic_file}, number of records:{len(new_items)}'
)
trans_finder.writeJSONDic(dict_list=trans_finder.master_dic,
file_name=dic_file)
def checkDictForMultipleMeaningsInTrans():
pattern = re.compile(r'(\w+(/\w+)+)')
k: str = None
v: str = None
for k, v in trans_finder.master_dic.items():
k_found_list = pattern.findall(k)
v_found_list = pattern.findall(v)
is_same = (len(k_found_list) == len(v_found_list))
if is_same:
continue
k_is_single_word = (len(k.split(' ')) == 1)
if k_is_single_word:
continue
PP(k_found_list)
print('-' * 3)
PP(v_found_list)
print('-' * 3)
print(f'{k}\n\n{v}')
print('-' * 40)
def trimmingText(text):
txt_has_trimmable_ending = (cm.TRIMMABLE_ENDING.search(text)
is not None)
txt_has_trimmable_beginning = (cm.TRIMMABLE_BEGINNING.search(text)
is not None)
is_trim = (txt_has_trimmable_ending or txt_has_trimmable_beginning)
if not is_trim:
return text, False
text = cm.TRIMMABLE_BEGINNING.sub('', text)
text = cm.TRIMMABLE_ENDING.sub('', text)
return text, True
def removeDictBeginAndEndingPuncts():
remove_set = {}
add_set = {}
for k, v in trans_finder.master_dic.items():
# is_debug = ('Cut to' in k)
# if is_debug:
# dd('DEBUG')
trimmed_k, is_trimmed_k = trimmingText(k)
trimmed_v, is_trimmed_v = trimmingText(v)
changed = (is_trimmed_k or is_trimmed_v)
if changed:
remove_entry = {k: v}
remove_set.update(remove_entry)
add_entry = {trimmed_k: trimmed_v}
add_set.update(add_entry)
print(f'[{k}]')
print(f'[{trimmed_k}]')
print('-' * 3)
print(f'[{v}]')
print(f'[{trimmed_v}]')
print('-' * 40)
changed = False
for k, v in remove_set.items():
remove_entry = {k: v}
print(f'remove:{remove_entry}')
del trans_finder.master_dic[k]
changed = True
for k, v in add_set.items():
add_entry = {k: v}
trans_finder.master_dic.update(add_entry)
print(f'added: {add_entry}')
changed = True
if changed:
new_dict = cleanupLeadingTrailingPunct(trans_finder.master_dic)
test_to_file = '/Users/hoangduytran/blender_manual/ref_dict_0005.json'
trans_finder.writeJSONDic(dict_list=new_dict,
file_name=test_to_file)
def removeDuplication(txt_with_punct):
# is_debug = (txt_with_punct.endswith('::'))
# if is_debug:
# dd('DEBUG')
cropped_txt, begin_with_punctuations, ending_with_punctuations = cm.beginAndEndPunctuation(
txt_with_punct, is_single=True)
trans = trans_finder.isInList(cropped_txt)
is_repeat = (trans is not None)
if not is_repeat:
cropped_txt, begin_with_punctuations, ending_with_punctuations = cm.beginAndEndPunctuation(
txt_with_punct, is_single=False)
trans = trans_finder.isInList(cropped_txt)
is_repeat = (trans is not None)
return is_repeat
def cleanupLeadingTrailingPunct(d_dict):
return_dict = {}
for k, v in d_dict.items():
trimmed_k = str(k)
trimmed_v = str(v)
found_k = cm.WORD_WITHOUT_QUOTE.search(k)
found_v = cm.WORD_WITHOUT_QUOTE.search(v)
if found_k:
trimmed_k = found_k.group(1)
if found_v:
trimmed_v = found_v.group(1)
entry = {trimmed_k: trimmed_v}
return_dict.update(entry)
return return_dict
def refToDictItems(ref_list):
ref_dict = {}
ref: RefRecord = None
interest_ref = [
RefType.REF,
RefType.DOC,
RefType.GA,
RefType.TERM,
]
for ref in ref_list:
# print(ref)
type = ref.getOrigin().getRefType()
first_ref = ref.getRefItemByIndex(0)
ref_text = first_ref.getText()
is_debug = ('Poor mans steadycam' in ref_text)
if is_debug:
dd('DEBUG')
en_part = None
vn_part = None
d_dict = {}
if type == RefType.MENUSELECTION:
print(f'MENUSELECTION:{type}')
text_list = cm.MENU_TEXT_REVERSE.findall(ref_text)
length = len(text_list)
i_index = 0
for i in range(length):
tran = text_list[i_index]
if i_index + 1 < length:
orig = text_list[i_index + 1]
else:
print('ERROR: Orig is NOT THERE, use original')
orig = ref.getOrigin().getText()
entry = {orig: tran}
print(f'menu:{entry}')
d_dict.update(entry)
i_index += 2
if i_index >= length:
break
elif type == RefType.ABBR:
print(f'ABBR:{type}')
text_list = cm.ABBREV_TEXT_REVERSE.findall(ref_text)
abbr = text_list[0]
defin = text_list[1]
has_further_explanation = (': ' in defin)
if has_further_explanation:
exp_list = defin.split(': ')
orig_part = exp_list[0]
further_exp = exp_list[1]
print(
f'abbr:{abbr}; orig_part:{orig_part}; further_exp:{further_exp}'
)
if abbr.isascii():
entry = {abbr: f'{orig_part}, {further_exp}'}
elif orig_part.isascii():
entry = {orig_part: f'{further_exp}, {abbr}'}
else:
entry = {further_exp: f'{orig_part}, {abbr}'}
d_dict.update(entry)
else:
print(f'abbr:{abbr}; defin:{defin}')
if defin.isascii():
entry = {defin: abbr}
else:
entry = {abbr: defin}
d_dict.update(entry)
elif type in interest_ref:
print(f'GENERIC_REF:{type}')
text_list = cm.REF_TEXT_REVERSE.findall(ref_text)
has_text = (len(text_list) > 0)
if not has_text:
origin_text = ref.getOrigin().getText()
print(f'ERROR: origin_text:{origin_text}')
# print(f'{text_list}, appeared to be empty!!!')
else:
vn_part, en_part = text_list[0]
print(f'en_part:{en_part} vn_part:{vn_part}')
entry = {en_part: vn_part}
d_dict.update(entry)
else:
dd(f'{type} is not the type we are looking for.')
ref_dict.update(d_dict)
return_dict = cleanupLeadingTrailingPunct(d_dict)
return return_dict
def listDictRefsToDict():
interest_ref_list = [
RefType.MENUSELECTION,
RefType.REF,
RefType.DOC,
RefType.GA,
RefType.TERM,
RefType.ABBR,
]
ref_dict = {}
ref_dict_filename = '/Users/hoangduytran/blender_manual/ref_dict_refsonly.json'
for k, v in trans_finder.master_dic.items():
ref_list = RefList(msg=v, keep_orig=False, tf=trans_finder)
ref_list.parseMessage()
inter_ref_list = ref_list.getListOfRefType(interest_ref_list)
has_ref = (len(inter_ref_list) > 0)
if not has_ref:
continue
current_ref_dict = refToDictItems(inter_ref_list)
ref_dict.update(current_ref_dict)
has_dict_content = (len(ref_dict) > 0)
if has_dict_content:
trans_finder.writeJSONDic(dict_list=ref_dict,
file_name=ref_dict_filename)
def tranRef(msg, is_keep_original):
ref_list = RefList(msg=msg,
keep_orig=is_keep_original,
tf=trans_finder)
ref_list.parseMessage()
ref_list.translateRefList()
tran = ref_list.getTranslation()
# trans_finder.addDictEntry((msg, tran))
# print("Got translation from REF_LIST")
return tran
# def fuzzyTextSimilar(txt1 : str, txt2 : str, accept_ratio):
# try:
# similar_ratio = LE.ratio(txt1, txt2)
# is_similar = (similar_ratio >= accept_ratio)
# return is_similar
# except Exception as e:
# print(e)
# return False
def getTimeNow(self):
local_time = timezone('Europe/London')
fmt = '%Y-%m-%d %H:%M%z'
loc_dt = local_time.localize(datetime.datetime.now())
formatted_dt = loc_dt.strftime(fmt)
return formatted_dt
# is_running = runAppOrNot()
# if not is_running:
# return
# correctingDictionary()
# checkDictKeyboard()
# checkDictRef()
# checkNonTranslatedDictWords()
# checkDictForMultipleMeaningsInTrans()
# removeDictBeginAndEndingPuncts()
# listDictRefsToDict()
# trans_finder.saveMasterDict()
# exit(0)
try:
is_debug = ('vr_scene_inspection' in docname)
if is_debug:
dd('DEBUG')
ex_env_key = 'EX_PO_TRANS'
is_ex_env_set = (ex_env_key in os.environ)
if not is_ex_env_set:
return
ex_env_key_value = os.environ[ex_env_key]
is_ex_set_true = (ex_env_key_value.lower() == 'true')
if not is_ex_set_true:
return
debug_file = cm.debug_file
if debug_file:
is_debug_file = (debug_file in docname)
if not is_debug_file:
return
build_dir = "build/rstdoc"
po_vi_dir = "locale/vi/LC_MESSAGES"
po_file_path = "{}.po".format(docname)
local_path = os.path.dirname(os.path.abspath(__file__))
blender_docs_path = cm.BLENDER_DOCS # os.path.dirname(local_path)
locale_vi_path = "locale/vi/LC_MESSAGES"
po_path = os.path.join(blender_docs_path,
os.path.join(locale_vi_path, po_file_path))
if not os.path.isfile(po_path):
msg = f'po_path: {po_path} NOT FOUND!'
print(msg)
raise Exception(msg)
exit(0)
# #loading local po file to get translation if any
po_dic, current_po_cat = trans_finder.loadPOAsDic(po_path)
trans_finder.flatPOFile(po_path)
rst_output_location = os.path.join(blender_docs_path, build_dir)
output_path = os.path.join(rst_output_location, po_file_path)
local_time = timezone(TIME_ZONE)
time_now = local_time.localize(datetime.datetime.now())
local_locale = locale.getlocale()[0]
current_header = current_po_cat._get_header_comment()
new_po_cat = Catalog(locale="vi",
header_comment=current_header,
project=current_po_cat.project,
version=current_po_cat.version,
copyright_holder=YOUR_ID,
creation_date=current_po_cat.creation_date,
revision_date=time_now,
last_translator=YOUR_ID,
language_team=YOUR_TRANSLATION_TEAM)
dd("#" * 80)
dd("filename: {}".format(output_path))
# msgid = "Lines should be less than 120 characters long."
# msgstr = "Số chữ trong các dòng phải ít hơn 120 ký tự de lam gi."
# trans_finder.addDictEntry((msgid, msgstr), False)
# exit(0)
for node, msg in extract_messages(doctree):
msg = msg.strip()
dd("=" * 80)
dd("msgid:[{}]".format(msg))
# clean up po file
is_inline = isinstance(node, nodes.inline)
is_emphasis = isinstance(node, nodes.emphasis)
is_title = isinstance(node, nodes.title)
is_term = isinstance(node, nodes.term)
is_rubric = isinstance(node, nodes.rubric)
is_field_name = isinstance(node, nodes.field_name)
is_reference = isinstance(node, nodes.reference)
is_strong = isinstance(node, nodes.strong)
is_keep_original = (is_inline or is_emphasis or is_title or is_term
or is_rubric or is_field_name or is_reference
or is_strong)
tran = None
# is_debug = ('Get involved in discussions' in msg)
# if is_debug:
# dd('DEBUG')
is_ignore = ig.isIgnored(msg)
if is_ignore:
print(f'IGNORED: {msg}')
continue
# is_added = False
tran, is_ignore = trans_finder.findTranslation(msg)
if is_ignore:
continue
has_translation = (tran is not None)
if not has_translation:
is_debug = ('is based on the OpenXR specification' in msg)
if is_debug:
dd('Debug')
ref_list = RefList(msg=msg,
keep_orig=is_keep_original,
tf=trans_finder)
ref_list.parseMessage()
ref_list.translateRefList()
tran = ref_list.getTranslation()
# tran = tranRef(msg, is_keep_original)
has_translation = (tran is not None)
if not has_translation:
tran = po_dic[msg]
has_translation = (tran is not None)
if has_translation:
has_month = ('Tháng ' in tran)
has_original = (msg.lower() in tran.lower())
has_link = (cm.REF_LINK.search(tran) is not None)
can_ignore = (has_month or has_original or has_link)
is_repeat = is_keep_original and not can_ignore
if is_repeat:
print('Repeating MSG')
tran = cm.matchCase(msg, tran)
tran = f'{tran} -- {msg}'
print(f'Repeating MSG:{tran}')
if tran is not None:
new_po_cat.add(msg, string=tran)
else:
new_po_cat.add(msg, string="")
print(f'msgid \"{msg}\"')
if tran is not None:
print(f'msgstr \"{tran}\"')
else:
print('msgstr \"\"')
print("Output to the path:", new_po_cat, output_path)
c.dump_po(output_path, new_po_cat)
# dd('DEBUG')
except Exception as e:
df.LOG(f'{e}', error=True)
def apply(self):
env = self.document.settings.env
settings, source = self.document.settings, self.document['source']
# XXX check if this is reliable
assert source.startswith(env.srcdir)
docname = path.splitext(relative_path(env.srcdir, source))[0]
textdomain = find_catalog(docname,
self.document.settings.gettext_compact)
# fetch translations
dirs = [
path.join(env.srcdir, directory)
for directory in env.config.locale_dirs
]
catalog, has_catalog = init_locale(dirs, env.config.language,
textdomain)
if not has_catalog:
return
parser = RSTParser()
#phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
patch = patch[0]
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
if old_name in names:
names.remove(old_name)
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * _id is None:
#
# _id is None means _id was duplicated.
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * _id is provided: bellow process
if not explicit and _id:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
self.document.note_implicit_target(section_node,
section_node)
# replace target's refname to new target name
def is_named_target(node):
return isinstance(node, nodes.target) and \
node.get('refname') == old_name
for old_target in self.document.traverse(is_named_target):
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
gloss_entries = env.temp_data.setdefault(
'gloss_entries', set())
ids = []
termnodes = []
for _id in node['names']:
if _id in gloss_entries:
gloss_entries.remove(_id)
_id, _, new_termnodes = \
make_termnodes_from_paragraph_node(env, patch, _id)
ids.append(_id)
termnodes.extend(new_termnodes)
if termnodes and ids:
patch = make_term_from_paragraph_node(termnodes, ids)
node['ids'] = patch['ids']
node['names'] = patch['names']
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True
#phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False):
continue
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
patch = patch[0]
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
# auto-numbered foot note reference should use original 'ids'.
def is_autonumber_footnote_ref(node):
return isinstance(node, nodes.footnote_reference) and \
node.get('auto') == 1
def list_replace_or_append(lst, old, new):
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
old_foot_refs = node.traverse(is_autonumber_footnote_ref)
new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
env.warn_node(
'inconsistent footnote references in '
'translated message', node)
old_foot_namerefs = {}
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for new in new_foot_refs:
refname = new.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
continue
old = refs.pop(0)
new['ids'] = old['ids']
for id in new['ids']:
self.document.ids[id] = new
list_replace_or_append(self.document.autofootnote_refs, old,
new)
if refname:
list_replace_or_append(
self.document.footnote_refs.setdefault(refname, []),
old, new)
list_replace_or_append(
self.document.refnames.setdefault(refname, []), old,
new)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
def is_refnamed_ref(node):
return isinstance(node, nodes.reference) and \
'refname' in node
old_refs = node.traverse(is_refnamed_ref)
new_refs = patch.traverse(is_refnamed_ref)
if len(old_refs) != len(new_refs):
env.warn_node(
'inconsistent references in '
'translated message', node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for new in new_refs:
if not self.document.has_name(new['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
new['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(new)
# refnamed footnote and citation should use original 'ids'.
def is_refnamed_footnote_ref(node):
footnote_ref_classes = (nodes.footnote_reference,
nodes.citation_reference)
return isinstance(node, footnote_ref_classes) and \
'refname' in node
old_refs = node.traverse(is_refnamed_footnote_ref)
new_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {}
if len(old_refs) != len(new_refs):
env.warn_node(
'inconsistent references in '
'translated message', node)
for old in old_refs:
refname_ids_map[old["refname"]] = old["ids"]
for new in new_refs:
refname = new["refname"]
if refname in refname_ids_map:
new["ids"] = refname_ids_map[refname]
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_refs = node.traverse(addnodes.pending_xref)
new_refs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_refs) != len(new_refs):
env.warn_node(
'inconsistent term references in '
'translated message', node)
def get_ref_key(node):
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],
)
def apply(self):
env = self.document.settings.env
settings, source = self.document.settings, self.document['source']
# XXX check if this is reliable
assert source.startswith(env.srcdir)
docname = path.splitext(relative_path(env.srcdir, source))[0]
textdomain = find_catalog(docname,
self.document.settings.gettext_compact)
# fetch translations
dirs = [path.join(env.srcdir, directory)
for directory in env.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, env.config.language,
textdomain)
if not has_catalog:
return
parser = RSTParser()
#phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
patch = patch[0]
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
if old_name in names:
names.remove(old_name)
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * _id is None:
#
# _id is None means _id was duplicated.
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * _id is provided: bellow process
if not explicit and _id:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
self.document.note_implicit_target(
section_node, section_node)
# replace target's refname to new target name
def is_named_target(node):
return isinstance(node, nodes.target) and \
node.get('refname') == old_name
for old_target in self.document.traverse(is_named_target):
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
gloss_entries = env.temp_data.setdefault('gloss_entries', set())
ids = []
termnodes = []
for _id in node['names']:
if _id in gloss_entries:
gloss_entries.remove(_id)
_id, _, new_termnodes = \
make_termnodes_from_paragraph_node(env, patch, _id)
ids.append(_id)
termnodes.extend(new_termnodes)
if termnodes and ids:
patch = make_term_from_paragraph_node(termnodes, ids)
node['ids'] = patch['ids']
node['names'] = patch['names']
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True
#phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False):
continue
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
patch = patch[0]
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
# auto-numbered foot note reference should use original 'ids'.
def is_autonumber_footnote_ref(node):
return isinstance(node, nodes.footnote_reference) and \
node.get('auto') == 1
def list_replace_or_append(lst, old, new):
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
old_foot_refs = node.traverse(is_autonumber_footnote_ref)
new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
env.warn_node('inconsistent footnote references in '
'translated message', node)
old_foot_namerefs = {}
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for new in new_foot_refs:
refname = new.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
continue
old = refs.pop(0)
new['ids'] = old['ids']
for id in new['ids']:
self.document.ids[id] = new
list_replace_or_append(
self.document.autofootnote_refs, old, new)
if refname:
list_replace_or_append(
self.document.footnote_refs.setdefault(refname, []),
old, new)
list_replace_or_append(
self.document.refnames.setdefault(refname, []),
old, new)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
def is_refnamed_ref(node):
return isinstance(node, nodes.reference) and \
'refname' in node
old_refs = node.traverse(is_refnamed_ref)
new_refs = patch.traverse(is_refnamed_ref)
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for new in new_refs:
if not self.document.has_name(new['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
new['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(new)
# refnamed footnote and citation should use original 'ids'.
def is_refnamed_footnote_ref(node):
footnote_ref_classes = (nodes.footnote_reference,
nodes.citation_reference)
return isinstance(node, footnote_ref_classes) and \
'refname' in node
old_refs = node.traverse(is_refnamed_footnote_ref)
new_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
for old in old_refs:
refname_ids_map[old["refname"]] = old["ids"]
for new in new_refs:
refname = new["refname"]
if refname in refname_ids_map:
new["ids"] = refname_ids_map[refname]
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_refs = node.traverse(addnodes.pending_xref)
new_refs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent term references in '
'translated message', node)
def get_ref_key(node):
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],)
for old in old_refs:
key = get_ref_key(old)
if key:
xref_reftarget_map[key] = old["reftarget"]
for new in new_refs:
key = get_ref_key(new)
if key in xref_reftarget_map:
new['reftarget'] = xref_reftarget_map[key]
# update leaves
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True
# Extract and translate messages for index entries.
for node, entries in traverse_translatable_index(self.document):
new_entries = []
for type, msg, tid, main in entries:
msg_parts = split_index_msg(type, msg)
msgstr_parts = []
for part in msg_parts:
msgstr = catalog.gettext(part)
if not msgstr:
msgstr = part
msgstr_parts.append(msgstr)
new_entries.append((type, ';'.join(msgstr_parts), tid, main))
node['raw_entries'] = entries
node['entries'] = new_entries
def test_extract_messages(rst, node_cls, count):
msg = extract_messages(_get_doctree(dedent(rst)))
assert_node_count(msg, node_cls, count)
def apply(self):
env = self.document.settings.env
settings, source = self.document.settings, self.document['source']
# XXX check if this is reliable
assert source.startswith(env.srcdir)
docname = path.splitext(relative_path(path.join(env.srcdir, 'dummy'),
source))[0]
textdomain = find_catalog(docname,
self.document.settings.gettext_compact)
# fetch translations
dirs = [path.join(env.srcdir, directory)
for directory in env.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, env.config.language,
textdomain,
charset=env.config.source_encoding)
if not has_catalog:
return
parser = RSTParser()
# phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' '*3)
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
try:
patch = patch[0]
except IndexError: # empty node
pass
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
# Original section name (reference target name) should be kept to refer
# from other nodes which is still not translated or uses explicit target
# name like "`text to display <explicit target name_>`_"..
# So, `old_name` is still exist in `names`.
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * if _id is None:
#
# _id is None means:
#
# 1. _id was not provided yet.
#
# 2. _id was duplicated.
#
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * if _id is provided: bellow process
if _id:
if not explicit:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
#
# Note: msgnode that is a second parameter of the
# `note_implicit_target` is not necessary here because
# section_node has been noted previously on rst parsing by
# `docutils.parsers.rst.states.RSTState.new_subsection()`
# and already has `system_message` if needed.
self.document.note_implicit_target(section_node)
# replace target's refname to new target name
def is_named_target(node):
return isinstance(node, nodes.target) and \
node.get('refname') == old_name
for old_target in self.document.traverse(is_named_target):
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
gloss_entries = env.temp_data.setdefault('gloss_entries', set())
ids = []
termnodes = []
for _id in node['names']:
if _id in gloss_entries:
gloss_entries.remove(_id)
_id, _, new_termnodes = \
make_termnodes_from_paragraph_node(env, patch, _id)
ids.append(_id)
termnodes.extend(new_termnodes)
if termnodes and ids:
patch = make_term_from_paragraph_node(termnodes, ids)
node['ids'] = patch['ids']
node['names'] = patch['names']
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True
# phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False):
continue
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' '*3)
patch = new_document(source, settings)
CustomLocaleReporter(node.source, node.line).set_reporter(patch)
parser.parse(msgstr, patch)
try:
patch = patch[0]
except IndexError: # empty node
pass
# XXX doctest and other block markup
if not isinstance(
patch,
(nodes.paragraph,) + LITERAL_TYPE_NODES + IMAGE_TYPE_NODES):
continue # skip for now
# auto-numbered foot note reference should use original 'ids'.
def is_autonumber_footnote_ref(node):
return isinstance(node, nodes.footnote_reference) and \
node.get('auto') == 1
def list_replace_or_append(lst, old, new):
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
old_foot_refs = node.traverse(is_autonumber_footnote_ref)
new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
env.warn_node('inconsistent footnote references in '
'translated message', node)
old_foot_namerefs = {}
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for new in new_foot_refs:
refname = new.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
continue
old = refs.pop(0)
new['ids'] = old['ids']
for id in new['ids']:
self.document.ids[id] = new
list_replace_or_append(
self.document.autofootnote_refs, old, new)
if refname:
list_replace_or_append(
self.document.footnote_refs.setdefault(refname, []),
old, new)
list_replace_or_append(
self.document.refnames.setdefault(refname, []),
old, new)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
def is_refnamed_ref(node):
return isinstance(node, nodes.reference) and \
'refname' in node
old_refs = node.traverse(is_refnamed_ref)
new_refs = patch.traverse(is_refnamed_ref)
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for new in new_refs:
if not self.document.has_name(new['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
new['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(new)
# refnamed footnote and citation should use original 'ids'.
def is_refnamed_footnote_ref(node):
footnote_ref_classes = (nodes.footnote_reference,
nodes.citation_reference)
return isinstance(node, footnote_ref_classes) and \
'refname' in node
old_refs = node.traverse(is_refnamed_footnote_ref)
new_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent references in '
'translated message', node)
for old in old_refs:
refname_ids_map[old["refname"]] = old["ids"]
for new in new_refs:
refname = new["refname"]
if refname in refname_ids_map:
new["ids"] = refname_ids_map[refname]
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_refs = node.traverse(addnodes.pending_xref)
new_refs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_refs) != len(new_refs):
env.warn_node('inconsistent term references in '
'translated message', node)
def get_ref_key(node):
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],)
for old in old_refs:
key = get_ref_key(old)
if key:
xref_reftarget_map[key] = old.attributes
for new in new_refs:
key = get_ref_key(new)
# Copy attributes to keep original node behavior. Especially
# copying 'reftarget', 'py:module', 'py:class' are needed.
for k, v in xref_reftarget_map.get(key, {}).items():
# Note: This implementation overwrite all attributes.
# if some attributes `k` should not be overwritten,
# you should provide exclude list as:
# `if k not in EXCLUDE_LIST: new[k] = v`
new[k] = v
# update leaves
for child in patch.children:
child.parent = node
node.children = patch.children
# for highlighting that expects .rawsource and .astext() are same.
if isinstance(node, LITERAL_TYPE_NODES):
node.rawsource = node.astext()
if isinstance(node, IMAGE_TYPE_NODES):
node.update_all_atts(patch)
node['translated'] = True
if 'index' in env.config.gettext_additional_targets:
# Extract and translate messages for index entries.
for node, entries in traverse_translatable_index(self.document):
new_entries = []
for type, msg, tid, main in entries:
msg_parts = split_index_msg(type, msg)
msgstr_parts = []
for part in msg_parts:
msgstr = catalog.gettext(part)
if not msgstr:
msgstr = part
msgstr_parts.append(msgstr)
new_entries.append((type, ';'.join(msgstr_parts), tid, main))
node['raw_entries'] = entries
node['entries'] = new_entries
def apply(self):
# type: () -> None
settings, source = self.document.settings, self.document['source']
# XXX check if this is reliable
assert source.startswith(self.env.srcdir)
docname = path.splitext(
relative_path(path.join(self.env.srcdir, 'dummy'), source))[0]
textdomain = find_catalog(docname,
self.document.settings.gettext_compact)
# fetch translations
dirs = [
path.join(self.env.srcdir, directory)
for directory in self.config.locale_dirs
]
catalog, has_catalog = init_locale(dirs, self.config.language,
textdomain)
if not has_catalog:
return
# phase1: replace reference ids with translated names
for node, msg in extract_messages(self.document):
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg or not msgstr.strip():
# as-of-yet untranslated
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
patch = publish_msgstr(self.app, msgstr, source, node.line,
self.config, settings)
# XXX doctest and other block markup
if not isinstance(patch, nodes.paragraph):
continue # skip for now
processed = False # skip flag
# update title(section) target name-id mapping
if isinstance(node, nodes.title):
section_node = node.parent
new_name = nodes.fully_normalize_name(patch.astext())
old_name = nodes.fully_normalize_name(node.astext())
if old_name != new_name:
# if name would be changed, replace node names and
# document nameids mapping with new name.
names = section_node.setdefault('names', [])
names.append(new_name)
# Original section name (reference target name) should be kept to refer
# from other nodes which is still not translated or uses explicit target
# name like "`text to display <explicit target name_>`_"..
# So, `old_name` is still exist in `names`.
_id = self.document.nameids.get(old_name, None)
explicit = self.document.nametypes.get(old_name, None)
# * if explicit: _id is label. title node need another id.
# * if not explicit:
#
# * if _id is None:
#
# _id is None means:
#
# 1. _id was not provided yet.
#
# 2. _id was duplicated.
#
# old_name entry still exists in nameids and
# nametypes for another duplicated entry.
#
# * if _id is provided: bellow process
if _id:
if not explicit:
# _id was not duplicated.
# remove old_name entry from document ids database
# to reuse original _id.
self.document.nameids.pop(old_name, None)
self.document.nametypes.pop(old_name, None)
self.document.ids.pop(_id, None)
# re-entry with new named section node.
#
# Note: msgnode that is a second parameter of the
# `note_implicit_target` is not necessary here because
# section_node has been noted previously on rst parsing by
# `docutils.parsers.rst.states.RSTState.new_subsection()`
# and already has `system_message` if needed.
self.document.note_implicit_target(section_node)
# replace target's refname to new target name
def is_named_target(node):
# type: (nodes.Node) -> bool
return isinstance(node, nodes.target) and \
node.get('refname') == old_name
for old_target in self.document.traverse(is_named_target):
old_target['refname'] = new_name
processed = True
# glossary terms update refid
if isinstance(node, nodes.term):
gloss_entries = self.env.temp_data.setdefault(
'gloss_entries', set())
for _id in node['names']:
if _id in gloss_entries:
gloss_entries.remove(_id)
parts = split_term_classifiers(msgstr)
patch = publish_msgstr(self.app, parts[0], source,
node.line, self.config, settings)
patch = make_glossary_term(self.env, patch, parts[1],
source, node.line, _id)
node['ids'] = patch['ids']
node['names'] = patch['names']
processed = True
# update leaves with processed nodes
if processed:
for child in patch.children:
child.parent = node
node.children = patch.children
node['translated'] = True
# phase2: translation
for node, msg in extract_messages(self.document):
if node.get('translated', False):
continue
msgstr = catalog.gettext(msg)
# XXX add marker to untranslated parts
if not msgstr or msgstr == msg: # as-of-yet untranslated
continue
# update translatable nodes
if isinstance(node, addnodes.translatable):
node.apply_translated_message(msg, msgstr)
continue
# update meta nodes
if is_pending_meta(node):
node.details['nodes'][0]['content'] = msgstr
continue
# Avoid "Literal block expected; none found." warnings.
# If msgstr ends with '::' then it cause warning message at
# parser.parse() processing.
# literal-block-warning is only appear in avobe case.
if msgstr.strip().endswith('::'):
msgstr += '\n\n dummy literal'
# dummy literal node will discard by 'patch = patch[0]'
# literalblock need literal block notation to avoid it become
# paragraph.
if isinstance(node, LITERAL_TYPE_NODES):
msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
patch = publish_msgstr(self.app, msgstr, source, node.line,
self.config, settings)
# XXX doctest and other block markup
if not isinstance(
patch,
(nodes.paragraph, ) + LITERAL_TYPE_NODES + IMAGE_TYPE_NODES):
continue # skip for now
# auto-numbered foot note reference should use original 'ids'.
def is_autonumber_footnote_ref(node):
# type: (nodes.Node) -> bool
return isinstance(node, nodes.footnote_reference) and \
node.get('auto') == 1
def list_replace_or_append(lst, old, new):
# type: (List, Any, Any) -> None
if old in lst:
lst[lst.index(old)] = new
else:
lst.append(new)
old_foot_refs = node.traverse(is_autonumber_footnote_ref)
new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
if len(old_foot_refs) != len(new_foot_refs):
logger.warning(
'inconsistent footnote references in translated message',
location=node)
old_foot_namerefs = {
} # type: Dict[unicode, List[nodes.footnote_reference]]
for r in old_foot_refs:
old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
for new in new_foot_refs:
refname = new.get('refname')
refs = old_foot_namerefs.get(refname, [])
if not refs:
continue
old = refs.pop(0)
new['ids'] = old['ids']
for id in new['ids']:
self.document.ids[id] = new
list_replace_or_append(self.document.autofootnote_refs, old,
new)
if refname:
list_replace_or_append(
self.document.footnote_refs.setdefault(refname, []),
old, new)
list_replace_or_append(
self.document.refnames.setdefault(refname, []), old,
new)
# reference should use new (translated) 'refname'.
# * reference target ".. _Python: ..." is not translatable.
# * use translated refname for section refname.
# * inline reference "`Python <...>`_" has no 'refname'.
def is_refnamed_ref(node):
# type: (nodes.Node) -> bool
return isinstance(node, nodes.reference) and \
'refname' in node
old_refs = node.traverse(is_refnamed_ref)
new_refs = patch.traverse(is_refnamed_ref)
if len(old_refs) != len(new_refs):
logger.warning('inconsistent references in translated message',
location=node)
old_ref_names = [r['refname'] for r in old_refs]
new_ref_names = [r['refname'] for r in new_refs]
orphans = list(set(old_ref_names) - set(new_ref_names))
for new in new_refs:
if not self.document.has_name(new['refname']):
# Maybe refname is translated but target is not translated.
# Note: multiple translated refnames break link ordering.
if orphans:
new['refname'] = orphans.pop(0)
else:
# orphan refnames is already empty!
# reference number is same in new_refs and old_refs.
pass
self.document.note_refname(new)
# refnamed footnote and citation should use original 'ids'.
def is_refnamed_footnote_ref(node):
# type: (nodes.Node) -> bool
footnote_ref_classes = (nodes.footnote_reference,
nodes.citation_reference)
return isinstance(node, footnote_ref_classes) and \
'refname' in node
old_refs = node.traverse(is_refnamed_footnote_ref)
new_refs = patch.traverse(is_refnamed_footnote_ref)
refname_ids_map = {}
if len(old_refs) != len(new_refs):
logger.warning('inconsistent references in translated message',
location=node)
for old in old_refs:
refname_ids_map[old["refname"]] = old["ids"]
for new in new_refs:
refname = new["refname"]
if refname in refname_ids_map:
new["ids"] = refname_ids_map[refname]
# Original pending_xref['reftarget'] contain not-translated
# target name, new pending_xref must use original one.
# This code restricts to change ref-targets in the translation.
old_refs = node.traverse(addnodes.pending_xref)
new_refs = patch.traverse(addnodes.pending_xref)
xref_reftarget_map = {}
if len(old_refs) != len(new_refs):
logger.warning(
'inconsistent term references in translated message',
location=node)
def get_ref_key(node):
# type: (nodes.Node) -> Tuple[unicode, unicode, unicode]
case = node["refdomain"], node["reftype"]
if case == ('std', 'term'):
return None
else:
return (
node["refdomain"],
node["reftype"],
node['reftarget'],
)
def write_doc(self, docname, doctree):
catalog = self.catalogs[find_catalog(docname,
self.config.gettext_compact)]
for node, msg in extract_messages(doctree):
catalog.add(msg, node)