def apply(self): env = self.document.settings.env settings, source = self.document.settings, self.document['source'] # XXX check if this is reliable assert source.startswith(env.srcdir) docname = path.splitext(relative_path(path.join(env.srcdir, 'dummy'), source))[0] textdomain = find_catalog(docname, self.document.settings.gettext_compact) # fetch translations dirs = [path.join(env.srcdir, directory) for directory in env.config.locale_dirs] catalog, has_catalog = init_locale(dirs, env.config.language, textdomain, charset=env.config.source_encoding) if not has_catalog: return parser = RSTParser() # phase1: replace reference ids with translated names for node, msg in extract_messages(self.document): msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg or not msgstr.strip(): # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' # literalblock need literal block notation to avoid it become # paragraph. if isinstance(node, LITERAL_TYPE_NODES): msgstr = '::\n\n' + indent(msgstr, ' '*3) patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) try: patch = patch[0] except IndexError: # empty node pass # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now processed = False # skip flag # update title(section) target name-id mapping if isinstance(node, nodes.title): section_node = node.parent new_name = nodes.fully_normalize_name(patch.astext()) old_name = nodes.fully_normalize_name(node.astext()) if old_name != new_name: # if name would be changed, replace node names and # document nameids mapping with new name. names = section_node.setdefault('names', []) names.append(new_name) # Original section name (reference target name) should be kept to refer # from other nodes which is still not translated or uses explicit target # name like "`text to display <explicit target name_>`_".. # So, `old_name` is still exist in `names`. _id = self.document.nameids.get(old_name, None) explicit = self.document.nametypes.get(old_name, None) # * if explicit: _id is label. title node need another id. # * if not explicit: # # * if _id is None: # # _id is None means: # # 1. _id was not provided yet. # # 2. _id was duplicated. # # old_name entry still exists in nameids and # nametypes for another duplicated entry. # # * if _id is provided: bellow process if _id: if not explicit: # _id was not duplicated. # remove old_name entry from document ids database # to reuse original _id. self.document.nameids.pop(old_name, None) self.document.nametypes.pop(old_name, None) self.document.ids.pop(_id, None) # re-entry with new named section node. # # Note: msgnode that is a second parameter of the # `note_implicit_target` is not necessary here because # section_node has been noted previously on rst parsing by # `docutils.parsers.rst.states.RSTState.new_subsection()` # and already has `system_message` if needed. self.document.note_implicit_target(section_node) # replace target's refname to new target name def is_named_target(node): return isinstance(node, nodes.target) and \ node.get('refname') == old_name for old_target in self.document.traverse(is_named_target): old_target['refname'] = new_name processed = True # glossary terms update refid if isinstance(node, nodes.term): gloss_entries = env.temp_data.setdefault('gloss_entries', set()) ids = [] termnodes = [] for _id in node['names']: if _id in gloss_entries: gloss_entries.remove(_id) _id, _, new_termnodes = \ make_termnodes_from_paragraph_node(env, patch, _id) ids.append(_id) termnodes.extend(new_termnodes) if termnodes and ids: patch = make_term_from_paragraph_node(termnodes, ids) node['ids'] = patch['ids'] node['names'] = patch['names'] processed = True # update leaves with processed nodes if processed: for child in patch.children: child.parent = node node.children = patch.children node['translated'] = True # phase2: translation for node, msg in extract_messages(self.document): if node.get('translated', False): continue msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' # literalblock need literal block notation to avoid it become # paragraph. if isinstance(node, LITERAL_TYPE_NODES): msgstr = '::\n\n' + indent(msgstr, ' '*3) patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) try: patch = patch[0] except IndexError: # empty node pass # XXX doctest and other block markup if not isinstance( patch, (nodes.paragraph,) + LITERAL_TYPE_NODES + IMAGE_TYPE_NODES): continue # skip for now # auto-numbered foot note reference should use original 'ids'. def is_autonumber_footnote_ref(node): return isinstance(node, nodes.footnote_reference) and \ node.get('auto') == 1 def list_replace_or_append(lst, old, new): if old in lst: lst[lst.index(old)] = new else: lst.append(new) old_foot_refs = node.traverse(is_autonumber_footnote_ref) new_foot_refs = patch.traverse(is_autonumber_footnote_ref) if len(old_foot_refs) != len(new_foot_refs): env.warn_node('inconsistent footnote references in ' 'translated message', node) old_foot_namerefs = {} for r in old_foot_refs: old_foot_namerefs.setdefault(r.get('refname'), []).append(r) for new in new_foot_refs: refname = new.get('refname') refs = old_foot_namerefs.get(refname, []) if not refs: continue old = refs.pop(0) new['ids'] = old['ids'] for id in new['ids']: self.document.ids[id] = new list_replace_or_append( self.document.autofootnote_refs, old, new) if refname: list_replace_or_append( self.document.footnote_refs.setdefault(refname, []), old, new) list_replace_or_append( self.document.refnames.setdefault(refname, []), old, new) # reference should use new (translated) 'refname'. # * reference target ".. _Python: ..." is not translatable. # * use translated refname for section refname. # * inline reference "`Python <...>`_" has no 'refname'. def is_refnamed_ref(node): return isinstance(node, nodes.reference) and \ 'refname' in node old_refs = node.traverse(is_refnamed_ref) new_refs = patch.traverse(is_refnamed_ref) if len(old_refs) != len(new_refs): env.warn_node('inconsistent references in ' 'translated message', node) old_ref_names = [r['refname'] for r in old_refs] new_ref_names = [r['refname'] for r in new_refs] orphans = list(set(old_ref_names) - set(new_ref_names)) for new in new_refs: if not self.document.has_name(new['refname']): # Maybe refname is translated but target is not translated. # Note: multiple translated refnames break link ordering. if orphans: new['refname'] = orphans.pop(0) else: # orphan refnames is already empty! # reference number is same in new_refs and old_refs. pass self.document.note_refname(new) # refnamed footnote and citation should use original 'ids'. def is_refnamed_footnote_ref(node): footnote_ref_classes = (nodes.footnote_reference, nodes.citation_reference) return isinstance(node, footnote_ref_classes) and \ 'refname' in node old_refs = node.traverse(is_refnamed_footnote_ref) new_refs = patch.traverse(is_refnamed_footnote_ref) refname_ids_map = {} if len(old_refs) != len(new_refs): env.warn_node('inconsistent references in ' 'translated message', node) for old in old_refs: refname_ids_map[old["refname"]] = old["ids"] for new in new_refs: refname = new["refname"] if refname in refname_ids_map: new["ids"] = refname_ids_map[refname] # Original pending_xref['reftarget'] contain not-translated # target name, new pending_xref must use original one. # This code restricts to change ref-targets in the translation. old_refs = node.traverse(addnodes.pending_xref) new_refs = patch.traverse(addnodes.pending_xref) xref_reftarget_map = {} if len(old_refs) != len(new_refs): env.warn_node('inconsistent term references in ' 'translated message', node) def get_ref_key(node): case = node["refdomain"], node["reftype"] if case == ('std', 'term'): return None else: return ( node["refdomain"], node["reftype"], node['reftarget'],) for old in old_refs: key = get_ref_key(old) if key: xref_reftarget_map[key] = old.attributes for new in new_refs: key = get_ref_key(new) # Copy attributes to keep original node behavior. Especially # copying 'reftarget', 'py:module', 'py:class' are needed. for k, v in xref_reftarget_map.get(key, {}).items(): # Note: This implementation overwrite all attributes. # if some attributes `k` should not be overwritten, # you should provide exclude list as: # `if k not in EXCLUDE_LIST: new[k] = v` new[k] = v # update leaves for child in patch.children: child.parent = node node.children = patch.children # for highlighting that expects .rawsource and .astext() are same. if isinstance(node, LITERAL_TYPE_NODES): node.rawsource = node.astext() if isinstance(node, IMAGE_TYPE_NODES): node.update_all_atts(patch) node['translated'] = True if 'index' in env.config.gettext_additional_targets: # Extract and translate messages for index entries. for node, entries in traverse_translatable_index(self.document): new_entries = [] for type, msg, tid, main in entries: msg_parts = split_index_msg(type, msg) msgstr_parts = [] for part in msg_parts: msgstr = catalog.gettext(part) if not msgstr: msgstr = part msgstr_parts.append(msgstr) new_entries.append((type, ';'.join(msgstr_parts), tid, main)) node['raw_entries'] = entries node['entries'] = new_entries
def apply(self): env = self.document.settings.env settings, source = self.document.settings, self.document['source'] # XXX check if this is reliable assert source.startswith(env.srcdir) docname = path.splitext(relative_path(env.srcdir, source))[0] textdomain = find_catalog(docname, self.document.settings.gettext_compact) # fetch translations dirs = [ path.join(env.srcdir, directory) for directory in env.config.locale_dirs ] catalog, has_catalog = init_locale(dirs, env.config.language, textdomain) if not has_catalog: return parser = RSTParser() #phase1: replace reference ids with translated names for node, msg in extract_messages(self.document): msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg or not msgstr.strip(): # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now processed = False # skip flag # update title(section) target name-id mapping if isinstance(node, nodes.title): section_node = node.parent new_name = nodes.fully_normalize_name(patch.astext()) old_name = nodes.fully_normalize_name(node.astext()) if old_name != new_name: # if name would be changed, replace node names and # document nameids mapping with new name. names = section_node.setdefault('names', []) names.append(new_name) if old_name in names: names.remove(old_name) _id = self.document.nameids.get(old_name, None) explicit = self.document.nametypes.get(old_name, None) # * if explicit: _id is label. title node need another id. # * if not explicit: # # * _id is None: # # _id is None means _id was duplicated. # old_name entry still exists in nameids and # nametypes for another duplicated entry. # # * _id is provided: bellow process if not explicit and _id: # _id was not duplicated. # remove old_name entry from document ids database # to reuse original _id. self.document.nameids.pop(old_name, None) self.document.nametypes.pop(old_name, None) self.document.ids.pop(_id, None) # re-entry with new named section node. self.document.note_implicit_target(section_node, section_node) # replace target's refname to new target name def is_named_target(node): return isinstance(node, nodes.target) and \ node.get('refname') == old_name for old_target in self.document.traverse(is_named_target): old_target['refname'] = new_name processed = True # glossary terms update refid if isinstance(node, nodes.term): gloss_entries = env.temp_data.setdefault( 'gloss_entries', set()) ids = [] termnodes = [] for _id in node['names']: if _id in gloss_entries: gloss_entries.remove(_id) _id, _, new_termnodes = \ make_termnodes_from_paragraph_node(env, patch, _id) ids.append(_id) termnodes.extend(new_termnodes) if termnodes and ids: patch = make_term_from_paragraph_node(termnodes, ids) node['ids'] = patch['ids'] node['names'] = patch['names'] processed = True # update leaves with processed nodes if processed: for child in patch.children: child.parent = node node.children = patch.children node['translated'] = True #phase2: translation for node, msg in extract_messages(self.document): if node.get('translated', False): continue msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now # auto-numbered foot note reference should use original 'ids'. def is_autonumber_footnote_ref(node): return isinstance(node, nodes.footnote_reference) and \ node.get('auto') == 1 def list_replace_or_append(lst, old, new): if old in lst: lst[lst.index(old)] = new else: lst.append(new) old_foot_refs = node.traverse(is_autonumber_footnote_ref) new_foot_refs = patch.traverse(is_autonumber_footnote_ref) if len(old_foot_refs) != len(new_foot_refs): env.warn_node( 'inconsistent footnote references in ' 'translated message', node) old_foot_namerefs = {} for r in old_foot_refs: old_foot_namerefs.setdefault(r.get('refname'), []).append(r) for new in new_foot_refs: refname = new.get('refname') refs = old_foot_namerefs.get(refname, []) if not refs: continue old = refs.pop(0) new['ids'] = old['ids'] for id in new['ids']: self.document.ids[id] = new list_replace_or_append(self.document.autofootnote_refs, old, new) if refname: list_replace_or_append( self.document.footnote_refs.setdefault(refname, []), old, new) list_replace_or_append( self.document.refnames.setdefault(refname, []), old, new) # reference should use new (translated) 'refname'. # * reference target ".. _Python: ..." is not translatable. # * use translated refname for section refname. # * inline reference "`Python <...>`_" has no 'refname'. def is_refnamed_ref(node): return isinstance(node, nodes.reference) and \ 'refname' in node old_refs = node.traverse(is_refnamed_ref) new_refs = patch.traverse(is_refnamed_ref) if len(old_refs) != len(new_refs): env.warn_node( 'inconsistent references in ' 'translated message', node) old_ref_names = [r['refname'] for r in old_refs] new_ref_names = [r['refname'] for r in new_refs] orphans = list(set(old_ref_names) - set(new_ref_names)) for new in new_refs: if not self.document.has_name(new['refname']): # Maybe refname is translated but target is not translated. # Note: multiple translated refnames break link ordering. if orphans: new['refname'] = orphans.pop(0) else: # orphan refnames is already empty! # reference number is same in new_refs and old_refs. pass self.document.note_refname(new) # refnamed footnote and citation should use original 'ids'. def is_refnamed_footnote_ref(node): footnote_ref_classes = (nodes.footnote_reference, nodes.citation_reference) return isinstance(node, footnote_ref_classes) and \ 'refname' in node old_refs = node.traverse(is_refnamed_footnote_ref) new_refs = patch.traverse(is_refnamed_footnote_ref) refname_ids_map = {} if len(old_refs) != len(new_refs): env.warn_node( 'inconsistent references in ' 'translated message', node) for old in old_refs: refname_ids_map[old["refname"]] = old["ids"] for new in new_refs: refname = new["refname"] if refname in refname_ids_map: new["ids"] = refname_ids_map[refname] # Original pending_xref['reftarget'] contain not-translated # target name, new pending_xref must use original one. # This code restricts to change ref-targets in the translation. old_refs = node.traverse(addnodes.pending_xref) new_refs = patch.traverse(addnodes.pending_xref) xref_reftarget_map = {} if len(old_refs) != len(new_refs): env.warn_node( 'inconsistent term references in ' 'translated message', node) def get_ref_key(node): case = node["refdomain"], node["reftype"] if case == ('std', 'term'): return None else: return ( node["refdomain"], node["reftype"], node['reftarget'], )
def apply(self): env = self.document.settings.env settings, source = self.document.settings, self.document['source'] # XXX check if this is reliable assert source.startswith(env.srcdir) docname = path.splitext(relative_path(env.srcdir, source))[0] textdomain = find_catalog(docname, self.document.settings.gettext_compact) # fetch translations dirs = [path.join(env.srcdir, directory) for directory in env.config.locale_dirs] catalog, has_catalog = init_locale(dirs, env.config.language, textdomain) if not has_catalog: return parser = RSTParser() #phase1: replace reference ids with translated names for node, msg in extract_messages(self.document): msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg or not msgstr.strip(): # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now processed = False # skip flag # update title(section) target name-id mapping if isinstance(node, nodes.title): section_node = node.parent new_name = nodes.fully_normalize_name(patch.astext()) old_name = nodes.fully_normalize_name(node.astext()) if old_name != new_name: # if name would be changed, replace node names and # document nameids mapping with new name. names = section_node.setdefault('names', []) names.append(new_name) if old_name in names: names.remove(old_name) _id = self.document.nameids.get(old_name, None) explicit = self.document.nametypes.get(old_name, None) # * if explicit: _id is label. title node need another id. # * if not explicit: # # * _id is None: # # _id is None means _id was duplicated. # old_name entry still exists in nameids and # nametypes for another duplicated entry. # # * _id is provided: bellow process if not explicit and _id: # _id was not duplicated. # remove old_name entry from document ids database # to reuse original _id. self.document.nameids.pop(old_name, None) self.document.nametypes.pop(old_name, None) self.document.ids.pop(_id, None) # re-entry with new named section node. self.document.note_implicit_target( section_node, section_node) # replace target's refname to new target name def is_named_target(node): return isinstance(node, nodes.target) and \ node.get('refname') == old_name for old_target in self.document.traverse(is_named_target): old_target['refname'] = new_name processed = True # glossary terms update refid if isinstance(node, nodes.term): gloss_entries = env.temp_data.setdefault('gloss_entries', set()) ids = [] termnodes = [] for _id in node['names']: if _id in gloss_entries: gloss_entries.remove(_id) _id, _, new_termnodes = \ make_termnodes_from_paragraph_node(env, patch, _id) ids.append(_id) termnodes.extend(new_termnodes) if termnodes and ids: patch = make_term_from_paragraph_node(termnodes, ids) node['ids'] = patch['ids'] node['names'] = patch['names'] processed = True # update leaves with processed nodes if processed: for child in patch.children: child.parent = node node.children = patch.children node['translated'] = True #phase2: translation for node, msg in extract_messages(self.document): if node.get('translated', False): continue msgstr = catalog.gettext(msg) # XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated continue # Avoid "Literal block expected; none found." warnings. # If msgstr ends with '::' then it cause warning message at # parser.parse() processing. # literal-block-warning is only appear in avobe case. if msgstr.strip().endswith('::'): msgstr += '\n\n dummy literal' # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup if not isinstance(patch, nodes.paragraph): continue # skip for now # auto-numbered foot note reference should use original 'ids'. def is_autonumber_footnote_ref(node): return isinstance(node, nodes.footnote_reference) and \ node.get('auto') == 1 def list_replace_or_append(lst, old, new): if old in lst: lst[lst.index(old)] = new else: lst.append(new) old_foot_refs = node.traverse(is_autonumber_footnote_ref) new_foot_refs = patch.traverse(is_autonumber_footnote_ref) if len(old_foot_refs) != len(new_foot_refs): env.warn_node('inconsistent footnote references in ' 'translated message', node) old_foot_namerefs = {} for r in old_foot_refs: old_foot_namerefs.setdefault(r.get('refname'), []).append(r) for new in new_foot_refs: refname = new.get('refname') refs = old_foot_namerefs.get(refname, []) if not refs: continue old = refs.pop(0) new['ids'] = old['ids'] for id in new['ids']: self.document.ids[id] = new list_replace_or_append( self.document.autofootnote_refs, old, new) if refname: list_replace_or_append( self.document.footnote_refs.setdefault(refname, []), old, new) list_replace_or_append( self.document.refnames.setdefault(refname, []), old, new) # reference should use new (translated) 'refname'. # * reference target ".. _Python: ..." is not translatable. # * use translated refname for section refname. # * inline reference "`Python <...>`_" has no 'refname'. def is_refnamed_ref(node): return isinstance(node, nodes.reference) and \ 'refname' in node old_refs = node.traverse(is_refnamed_ref) new_refs = patch.traverse(is_refnamed_ref) if len(old_refs) != len(new_refs): env.warn_node('inconsistent references in ' 'translated message', node) old_ref_names = [r['refname'] for r in old_refs] new_ref_names = [r['refname'] for r in new_refs] orphans = list(set(old_ref_names) - set(new_ref_names)) for new in new_refs: if not self.document.has_name(new['refname']): # Maybe refname is translated but target is not translated. # Note: multiple translated refnames break link ordering. if orphans: new['refname'] = orphans.pop(0) else: # orphan refnames is already empty! # reference number is same in new_refs and old_refs. pass self.document.note_refname(new) # refnamed footnote and citation should use original 'ids'. def is_refnamed_footnote_ref(node): footnote_ref_classes = (nodes.footnote_reference, nodes.citation_reference) return isinstance(node, footnote_ref_classes) and \ 'refname' in node old_refs = node.traverse(is_refnamed_footnote_ref) new_refs = patch.traverse(is_refnamed_footnote_ref) refname_ids_map = {} if len(old_refs) != len(new_refs): env.warn_node('inconsistent references in ' 'translated message', node) for old in old_refs: refname_ids_map[old["refname"]] = old["ids"] for new in new_refs: refname = new["refname"] if refname in refname_ids_map: new["ids"] = refname_ids_map[refname] # Original pending_xref['reftarget'] contain not-translated # target name, new pending_xref must use original one. # This code restricts to change ref-targets in the translation. old_refs = node.traverse(addnodes.pending_xref) new_refs = patch.traverse(addnodes.pending_xref) xref_reftarget_map = {} if len(old_refs) != len(new_refs): env.warn_node('inconsistent term references in ' 'translated message', node) def get_ref_key(node): case = node["refdomain"], node["reftype"] if case == ('std', 'term'): return None else: return ( node["refdomain"], node["reftype"], node['reftarget'],) for old in old_refs: key = get_ref_key(old) if key: xref_reftarget_map[key] = old["reftarget"] for new in new_refs: key = get_ref_key(new) if key in xref_reftarget_map: new['reftarget'] = xref_reftarget_map[key] # update leaves for child in patch.children: child.parent = node node.children = patch.children node['translated'] = True # Extract and translate messages for index entries. for node, entries in traverse_translatable_index(self.document): new_entries = [] for type, msg, tid, main in entries: msg_parts = split_index_msg(type, msg) msgstr_parts = [] for part in msg_parts: msgstr = catalog.gettext(part) if not msgstr: msgstr = part msgstr_parts.append(msgstr) new_entries.append((type, ';'.join(msgstr_parts), tid, main)) node['raw_entries'] = entries node['entries'] = new_entries