def write_toc(self, node: Node, indentlevel: int = 4) -> List[str]: parts = [] # type: List[str] if isinstance(node, nodes.list_item) and self.isdocnode(node): compact_paragraph = cast(addnodes.compact_paragraph, node[0]) reference = cast(nodes.reference, compact_paragraph[0]) link = reference['refuri'] title = html.escape(reference.astext()).replace('"', '"') item = '<section title="%(title)s" ref="%(ref)s">' % \ {'title': title, 'ref': link} parts.append(' ' * 4 * indentlevel + item) bullet_list = cast(nodes.bullet_list, node[1]) list_items = cast(Iterable[nodes.list_item], bullet_list) for list_item in list_items: parts.extend(self.write_toc(list_item, indentlevel + 1)) parts.append(' ' * 4 * indentlevel + '</section>') elif isinstance(node, nodes.list_item): for subnode in node: parts.extend(self.write_toc(subnode, indentlevel)) elif isinstance(node, nodes.reference): link = node['refuri'] title = html.escape(node.astext()).replace('"', '"') item = section_template % {'title': title, 'ref': link} item = ' ' * 4 * indentlevel + item parts.append(item.encode('ascii', 'xmlcharrefreplace').decode()) elif isinstance(node, nodes.bullet_list): for subnode in node: parts.extend(self.write_toc(subnode, indentlevel)) elif isinstance(node, addnodes.compact_paragraph): for subnode in node: parts.extend(self.write_toc(subnode, indentlevel)) return parts
def make_xrefs(self, rolename: str, domain: str, target: str, innernode: "Type[TextlikeNode]" = nodes.emphasis, contnode: Node = None, env: BuildEnvironment = None) -> List[Node]: delims = r'(\s*[\[\]\(\),](?:\s*or\s)?\s*|\s+or\s+)' delims_re = re.compile(delims) sub_targets = re.split(delims, target) split_contnode = bool(contnode and contnode.astext() == target) results = [] for sub_target in filter(None, sub_targets): if split_contnode: contnode = nodes.Text(sub_target) if delims_re.match(sub_target): results.append(contnode or innernode(sub_target, sub_target)) else: results.append( self.make_xref(rolename, domain, sub_target, innernode, contnode, env)) return results
def select_doc_nodes(self, doctree: nodes.Node) -> List[nodes.Node]: """Select the nodes for which entries in the TOC are desired This is a separate method so that it might be overriden by subclasses wanting to add other types of nodes to the TOC. """ return doctree.traverse(addnodes.desc)
def get_rst_title(rst_doc: Node) -> Optional[str]: """ Given some RST, extract what docutils thinks is the title """ if rst_doc: for title in rst_doc.traverse(nodes.title): return title.astext() return None
def is_translatable(node: Node) -> bool: if isinstance(node, addnodes.translatable): return True # image node marked as translatable or having alt text if isinstance(node, nodes.image) and (node.get('translatable') or node.get('alt')): return True if isinstance(node, nodes.Inline) and 'translatable' not in node: # type: ignore # inline node must not be translated if 'translatable' is not set return False if isinstance(node, nodes.TextElement): if not node.source: logger.debug('[i18n] SKIP %r because no node.source: %s', get_full_module_name(node), repr_domxml(node)) return False # built-in message if isinstance(node, IGNORED_NODES) and 'translatable' not in node: logger.debug( "[i18n] SKIP %r because node is in IGNORED_NODES " "and no node['translatable']: %s", get_full_module_name(node), repr_domxml(node)) return False if not node.get('translatable', True): # not(node['translatable'] == True or node['translatable'] is None) logger.debug("[i18n] SKIP %r because not node['translatable']: %s", get_full_module_name(node), repr_domxml(node)) return False # <field_name>orphan</field_name> # XXX ignore all metadata (== docinfo) if isinstance(node, nodes.field_name) and node.children[0] == 'orphan': logger.debug('[i18n] SKIP %r because orphan node: %s', get_full_module_name(node), repr_domxml(node)) return False return True if is_pending_meta(node) or isinstance(node, addnodes.meta): # docutils-0.17 or older return True elif isinstance(node, addnodes.docutils_meta): # docutils-0.18+ return True return False
def locate_in_toc(self, app: Sphinx, node: nodes.Node) -> Optional[nodes.Node]: toc = app.env.tocs[app.env.docname] ref = self.get_ref(node) for node in toc.traverse(nodes.reference): node_ref = node.get('anchorname') if not node_ref or node_ref[0] != "#": continue if node_ref[1:] == ref: return node.parent.parent
def dispatch_visit(self, node: Node) -> None: if isinstance(node, nodes.comment): raise nodes.SkipNode elif isinstance(node, nodes.raw): if 'html' in node.get('format', '').split(): # Some people might put content in raw HTML that should be searched, # so we just amateurishly strip HTML tags and index the remaining # content nodetext = re.sub(r'(?is)<style.*?</style>', '', node.astext()) nodetext = re.sub(r'(?is)<script.*?</script>', '', nodetext) nodetext = re.sub(r'<[^<]+?>', '', nodetext) self.found_words.extend(self.lang.split(nodetext)) raise nodes.SkipNode elif isinstance(node, nodes.Text): self.found_words.extend(self.lang.split(node.astext())) elif isinstance(node, nodes.title): self.found_title_words.extend(self.lang.split(node.astext())) elif isinstance(node, addnodes.meta) and self.is_meta_keywords(node): keywords = node['content'] keywords = [keyword.strip() for keyword in keywords.split(',')] self.found_words.extend(keywords)
def register_tabs_as_label(app: Sphinx, document: nodes.Node) -> None: docname = app.env.docname labels = app.env.domaindata['std']['labels'] anonlabels = app.env.domaindata['std']['anonlabels'] for node in document.traverse(nxt_tab_head): if node.label_id in labels: logger.warning(__('duplicate label %s, other instance in %s'), node.label_id, app.env.doc2path(labels[node.label_id][0]), location=node) anonlabels[node.label_id] = docname, node.label_id labels[node.label_id] = docname, node.label_id, node.astext()
def post_process_images(self, doctree: Node) -> None: for node in doctree.traverse(RevealjsNode): elm = getattr(node, 'revealit_el', None) if elm: for img in elm.images.values(): if img not in self.env.images: # non-existing URI; let it alone continue self.images[img] = self.env.images[img][1] super().post_process_images(doctree)
def resolve_any_xref( self, env: BuildEnvironment, from_doc_name: str, builder: Builder, target: str, node: nodes.Node, cont_node: nodes.Node) -> List[Tuple[str, nodes.Node]]: modname = node.get('lua:module') class_name = node.get('lua:class') results: List[Tuple[str, nodes.Node]] = [] # always search in "refspecific" mode with the :any: role matches = self.find_obj(env, modname, class_name, target, None, 1) for name, obj in matches: if obj[1] == 'module': results.append( ('lua:mod', self._make_module_refnode(builder, from_doc_name, name, cont_node))) else: results.append(('lua:' + self.role_for_objtype(obj[1]), make_refnode(builder, from_doc_name, obj[0], name, cont_node, name))) return results
def write_toc(node: nodes.Node, parent: etree.Element): if isinstance(node, addnodes.compact_paragraph) or \ isinstance(node, nodes.bullet_list): for subnode in node: write_toc(subnode, parent) elif isinstance(node, nodes.list_item): item = etree.SubElement(parent, 'topic') for subnode in node: write_toc(subnode, item) elif isinstance(node, nodes.reference): parent.attrib['label'] = node.astext() parent.attrib['href'] = base_dir + node['refuri']
def write_doc(self, docname: str, doctree: Node) -> None: # work around multiple string % tuple issues in docutils; # replace tuples in attribute values with lists doctree = doctree.deepcopy() for node in doctree.traverse(nodes.Element): for att, value in node.attributes.items(): if isinstance(value, tuple): node.attributes[att] = list(value) value = node.attributes[att] if isinstance(value, list): for i, val in enumerate(value): if isinstance(val, tuple): value[i] = list(val) destination = StringOutput(encoding='utf-8') self.writer.write(doctree, destination) outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix) ensuredir(path.dirname(outfilename)) try: with open(outfilename, 'w', encoding='utf-8') as f: f.write(self.writer.output) except OSError as err: logger.warning(__("error writing file %s: %s"), outfilename, err)
def process_doc(self, env: BuildEnvironment, docname: str, document: Node) -> None: """Process a document after it is read by the environment.""" entries = self.entries.setdefault(env.docname, []) for node in list(document.findall(addnodes.index)): try: for entry in node['entries']: split_index_msg(entry[0], entry[1]) except ValueError as exc: logger.warning(str(exc), location=node) node.parent.remove(node) else: for entry in node['entries']: entries.append(entry)
def _run_doctests_on_graphtik_document(app: Sphinx, doctree: nodes.Node): """Callback of `doctree-resolved`` event. """ try: docname = app.env.docname from ._graphtikbuilder import get_graphtik_builder if _should_work(app) and any(doctree.traverse(graphtik_node)): log.info(__("Graphtik-ing document %r..."), docname) graphtik_builder = get_graphtik_builder(app) graphtik_builder.test_doc(docname, doctree) except Exception as ex: log.error("General failure of Graphtik-sphinx extension: %s", ex, exc_info=True) raise
def get_refnodes(self, doctree: Node, result: List[Dict[str, Any]]) -> List[Dict[str, Any]]: # NOQA """Collect section titles, their depth in the toc and the refuri.""" # XXX: is there a better way than checking the attribute # toctree-l[1-8] on the parent node? if isinstance(doctree, nodes.reference) and doctree.get('refuri'): refuri = doctree['refuri'] if refuri.startswith('http://') or refuri.startswith('https://') \ or refuri.startswith('irc:') or refuri.startswith('mailto:'): return result classes = doctree.parent.attributes['classes'] for level in range(8, 0, -1): # or range(1, 8)? if (self.toctree_template % level) in classes: result.append({ 'level': level, 'refuri': html.escape(refuri), 'text': ssp(html.escape(doctree.astext())) }) break elif isinstance(doctree, nodes.Element): for elem in doctree: result = self.get_refnodes(elem, result) return result
def add_uids(doctree: Node, condition: Any) -> Iterator[Node]: """Add a unique id to every node in the `doctree` which matches the condition and yield the nodes. :param doctree: A :class:`docutils.nodes.document` instance. :param condition: A callable which returns either ``True`` or ``False`` for a given node. """ for node in doctree.findall(condition): node.uid = uuid4().hex yield node
def _update_ref(self, node: Node, labelid: str) -> None: source_attr = self.env.exercise_list[labelid] source_node = source_attr.get("node", Node) if is_linked_node(source_node): default_title = "Solution to " target_labelid = source_node.get("target_label", "") target_attr = self.env.exercise_list[target_labelid] target_node = target_attr.get("node", Node) if is_enumerable_node( target_node) and node.astext() == default_title: node.replace(node[0], source_node[0][0][0]) return if is_unenumerable_node( target_node) and node.astext() == default_title: if target_attr.get("title"): if self._has_math_child(target_node[0]): title = self._update_title(target_node[0]) title.insert(0, nodes.Text(default_title, default_title)) node.replace(node[0], title) else: text = target_attr.get("title", "") node[0].insert(len(node[0]), nodes.Text(text, text)) else: node[0].insert(len(node[0]), nodes.Text("Exercise", "Exercise")) else: # If no node.astext() simply add "Exercise" if is_enumerable_node(source_node) and not node.astext(): text = nodes.Text("Exercise", "Exercise") node[0].insert(0, text) return if ":math:" in node.astext(): title = self._update_title(source_node[0]) node.replace(node[0], title)
def fix_refuris(self, tree: Node) -> None: # fix refuris with double anchor fname = self.config.master_doc + self.out_suffix for refnode in tree.traverse(nodes.reference): if 'refuri' not in refnode: continue refuri = refnode['refuri'] hashindex = refuri.find('#') if hashindex < 0: continue hashindex = refuri.find('#', hashindex + 1) if hashindex >= 0: refnode['refuri'] = fname + refuri[hashindex:]
def doctree_read(app: Sphinx, doctree: Node) -> None: env = app.builder.env resolve_target = getattr(env.config, 'linkcode_resolve', None) if not callable(env.config.linkcode_resolve): raise LinkcodeError( "Function `linkcode_resolve` is not given in conf.py") domain_keys = { 'py': ['module', 'fullname'], 'c': ['names'], 'cpp': ['names'], 'js': ['object', 'fullname'], } for objnode in list(doctree.traverse(addnodes.desc)): domain = objnode.get('domain') uris: Set[str] = set() for signode in objnode: if not isinstance(signode, addnodes.desc_signature): continue # Convert signode to a specified format info = {} for key in domain_keys.get(domain, []): value = signode.get(key) if not value: value = '' info[key] = value if not info: continue # Call user code to resolve the link uri = resolve_target(domain, info) if not uri: # no source continue if uri in uris or not uri: # only one link per name, please continue uris.add(uri) inline = nodes.inline('', _('[source]'), classes=['viewcode-link']) onlynode = addnodes.only(expr='html') onlynode += nodes.reference('', '', inline, internal=False, refuri=uri) signode += onlynode
def get_enumerable_node_type(self, node: Node) -> str: """Get type of enumerable nodes.""" def has_child(node: Element, cls: "Type") -> bool: return any(isinstance(child, cls) for child in node) if isinstance(node, nodes.section): return 'section' elif isinstance(node, nodes.container): if node.get('literal_block') and has_child(node, nodes.literal_block): return 'code-block' else: return None else: figtype, _ = self.enumerable_nodes.get(node.__class__, (None, None)) return figtype
def doctree_read(app: Sphinx, document: Node) -> None: """ Read the doctree and apply updates to sphinx-exercise nodes """ domain = cast(StandardDomain, app.env.get_domain("std")) # Traverse sphinx-exercise nodes for node in document.traverse(): if is_extension_node(node): name = node.get("names", [])[0] label = document.nameids[name] docname = app.env.docname section_name = node.attributes.get("title") domain.anonlabels[name] = docname, label domain.labels[name] = docname, label, section_name
def _add_pending_source_cross_reference(app: Sphinx, signode: Node, fullname: str) -> None: """ Adds a pending source cross reference to the signature in the doctree, `signode`. The viewcode and linkcode extensions walk the doctree once parsed and then add this node, however since sphinx_c_autodoc already has to add logic, the `module` option to the directives it seems more practical to just create the full pending cross reference here, and then viewcode is an extension which will populate this cross refernce. Args: app (Sphinx): The sphinx app currently doing the processing. signode (Node): The signature node to apply the source code cross reference to. fullname (str): The dotted fullname of the C construct. """ module = signode.get("module") if module is None: return source_page = _get_source_page_name(module) # Using the `viewcode-link` to be consistent with the python versions in # case someone else wants to walk the tree and do other links inline = nodes.inline("", "[source]", classes=["viewcode-link"]) # Limit this cross referencing only to html html_node = addnodes.only(expr="html") html_node += addnodes.pending_xref( "", inline, reftype=f"{C_DOMAIN_LINK_PREFIX}viewcode", refdomain="std", refexplicit=False, reftarget=source_page, refid=f"{C_DOMAIN_LINK_PREFIX}{fullname}", refdoc=app.builder.env.docname, module=module, fullname=fullname, ) signode += html_node
def repr_domxml(node: Node, length: int = 80) -> str: """ return DOM XML representation of the specified node like: '<paragraph translatable="False"><inline classes="versionmodified">New in version...' :param nodes.Node node: target node :param int length: length of return value to be striped. if false-value is specified, repr_domxml returns full of DOM XML representation. :return: DOM XML representation """ try: text = node.asdom().toxml() except Exception: text = str(node) if length and len(text) > length: text = text[:length] + '...' return text
def line_of_end(node: nodes.Node) -> Optional[int]: next_node = node.next_node(descend=False, siblings=True, ascend=True) while next_node: if next_node.line: return line_of_start(next_node) next_node = next_node.next_node( # Some nodes' line attr is always None, but their children has # valid line attr descend=True, # If node and its children have not valid line attr, try use line # of next node ascend=True, siblings=True) # No line found, return the max line of source file if node.source: with open(node.source) as f: return sum(1 for line in f) raise AttributeError('None source attr of node %s' % node)
def process_only_nodes(document: Node, tags: "Tags") -> None: """Filter ``only`` nodes which do not match *tags*.""" for node in document.findall(addnodes.only): try: ret = tags.eval_condition(node['expr']) except Exception as err: logger.warning(__('exception while evaluating only directive expression: %s'), err, location=node) node.replace_self(node.children or nodes.comment()) else: if ret: node.replace_self(node.children or nodes.comment()) else: # A comment on the comment() nodes being inserted: replacing by [] would # result in a "Losing ids" exception if there is a target node before # the only node, so we make sure docutils can transfer the id to # something, even if it's just a comment and will lose the id anyway... node.replace_self(nodes.comment())
def test_doc(self, docname: str, doctree: Node) -> None: # Get all applicable nodes doc_nodes = list(doctree.traverse(self.condition)) if not doc_nodes: return print() print(f"{self.name} testing: {docname}") print() self.total_tries += 1 try: with tempfile.TemporaryDirectory( ) as tempdir, contextlib.ExitStack() as stack: ctx = {"cwd": tempdir} for node in doc_nodes: # type: Element filename = self.get_filename_for_node(node, docname) line_number = self.get_line_number(node) if "consoletest-literalinclude" in node: print() print("Copying", node["source"], node["filepath"]) print() shutil.copyfile( node["source"], os.path.join(ctx["cwd"], *node["filepath"]), ) elif "consoletest_commands" in node: for command in node["consoletest_commands"]: print() print("Running", command) print() stack.enter_context(command) command.run(ctx) print() print("No more tempdir") print() except: self.total_failures += 1
def handle_missing_xref(app: Sphinx, env, node: nodes.Node, contnode: nodes.Node) -> None: # Ignore missing reference warnings for the wheel_filename module if node.get("reftarget", '').startswith("docutils."): raise NoUri if node.get("reftarget", '').startswith("sphinx.ext.autodoc."): raise NoUri if node.get("reftarget", '').startswith("sphinx.ext.autosummary."): raise NoUri if node.get("reftarget", '').startswith("sphinx_toolbox._data_documenter."): # TODO: redirect raise NoUri if node.get("reftarget", '') in { "spam", "lobster", "foo", "typing_extensions", "bs4.BeautifulSoup", "pytest_regressions.file_regression.FileRegressionFixture", "sphinx_toolbox.patched_autosummary", "sphinx_toolbox.autodoc_augment_defaults", "sphinx_toolbox.autodoc_typehints", "sphinx_toolbox.autotypeddict", "sphinx_toolbox.autoprotocol", "sphinx_toolbox.utils._T", "sphinx_toolbox.testing.EventManager", # TODO "sphinx.registry.SphinxComponentRegistry", "sphinx.config.Config", "sphinx.config.Config.latex_elements", "sphinx.util.docfields.TypedField", "sphinx.writers.html.HTMLTranslator", "sphinx.writers.latex.LaTeXTranslator", "sphinx.domains.python.PyXRefRole", "sphinx.domains.std.GenericObject", "sphinx.domains.changeset.VersionChange", "sphinx.directives.code.CodeBlock", "sphinx.roles.Abbreviation", "autodoc.Documenter", # TODO: why not sphinx.ext.autodoc.Documenter? }: raise NoUri if node.get("reftarget", '').startswith("consolekit.terminal_colours.Fore."): raise NoUri
def match(self, node: Node) -> bool: try: if self.classes and not isinstance(node, self.classes): return False if self.attrs: if not isinstance(node, nodes.Element): return False for key, value in self.attrs.items(): if key not in node: return False elif value is Any: continue elif node.get(key) != value: return False return True except Exception: # for non-Element nodes return False
def is_gettext_additional_targets(node: Node) -> bool: # This func judges if passed `node` is one of the following, which can be # tranlated by `gettext_addtional_targets` setting and is not supported by # this sphinx extension: # - toctree (with hidden attribute as True) # - literal-block # - raw # - image # hidden toc element if isinstance(node, toctree) and node.get("hidden", None) is True: return True # literal_block / raw / image nodes are not supported since they are not # supported due to implementation issue, e.g. compatibility with code-block # highlight HTML writing elif (isinstance(node, literal_block) or isinstance(node, raw) or isinstance(node, image)): return True else: return False
def visit_title(self, node: nodes.Node) -> None: symbol = self.current_symbol has_parent = True if symbol is None: has_parent = False symbol = self.push_symbol() name = node.astext() line = (node.line or 1) - 1 symbol.name = name symbol.range.start.line = line symbol.range.end.line = line symbol.range.end.character = len(name) - 1 symbol.selection_range.start.line = line symbol.selection_range.end.line = line symbol.selection_range.end.character = len(name) - 1 if not has_parent: self.pop_symbol()
def add_node(self, env: BuildEnvironment, node: Node, targetnode: Node, lineno: int): """ Add a node. :param env: The Sphinx build environment. :param node: :param targetnode: :param lineno: """ if not hasattr(env, self.attr_name): setattr(env, self.attr_name, []) all_nodes = getattr(env, self.attr_name) all_nodes.append({ "docname": env.docname, "lineno": lineno, "installation_node": node.deepcopy(), "target": targetnode, })