def write_toc(self, node: Node, indentlevel: int = 4) -> List[str]:
parts = [] # type: List[str]
if isinstance(node, nodes.list_item) and self.isdocnode(node):
compact_paragraph = cast(addnodes.compact_paragraph, node[0])
reference = cast(nodes.reference, compact_paragraph[0])
link = reference['refuri']
title = html.escape(reference.astext()).replace('"', '"')
item = '<section title="%(title)s" ref="%(ref)s">' % \
{'title': title, 'ref': link}
parts.append(' ' * 4 * indentlevel + item)
bullet_list = cast(nodes.bullet_list, node[1])
list_items = cast(Iterable[nodes.list_item], bullet_list)
for list_item in list_items:
parts.extend(self.write_toc(list_item, indentlevel + 1))
parts.append(' ' * 4 * indentlevel + '</section>')
elif isinstance(node, nodes.list_item):
for subnode in node:
parts.extend(self.write_toc(subnode, indentlevel))
elif isinstance(node, nodes.reference):
link = node['refuri']
title = html.escape(node.astext()).replace('"', '"')
item = section_template % {'title': title, 'ref': link}
item = ' ' * 4 * indentlevel + item
parts.append(item.encode('ascii', 'xmlcharrefreplace').decode())
elif isinstance(node, nodes.bullet_list):
for subnode in node:
parts.extend(self.write_toc(subnode, indentlevel))
elif isinstance(node, addnodes.compact_paragraph):
for subnode in node:
parts.extend(self.write_toc(subnode, indentlevel))
return parts
def make_xrefs(self,
rolename: str,
domain: str,
target: str,
innernode: "Type[TextlikeNode]" = nodes.emphasis,
contnode: Node = None,
env: BuildEnvironment = None) -> List[Node]:
delims = r'(\s*[\[\]\(\),](?:\s*or\s)?\s*|\s+or\s+)'
delims_re = re.compile(delims)
sub_targets = re.split(delims, target)
split_contnode = bool(contnode and contnode.astext() == target)
results = []
for sub_target in filter(None, sub_targets):
if split_contnode:
contnode = nodes.Text(sub_target)
if delims_re.match(sub_target):
results.append(contnode or innernode(sub_target, sub_target))
else:
results.append(
self.make_xref(rolename, domain, sub_target, innernode,
contnode, env))
return results
def select_doc_nodes(self, doctree: nodes.Node) -> List[nodes.Node]:
"""Select the nodes for which entries in the TOC are desired
This is a separate method so that it might be overriden by
subclasses wanting to add other types of nodes to the TOC.
"""
return doctree.traverse(addnodes.desc)
def get_rst_title(rst_doc: Node) -> Optional[str]:
""" Given some RST, extract what docutils thinks is the title """
if rst_doc:
for title in rst_doc.traverse(nodes.title):
return title.astext()
return None
def is_translatable(node: Node) -> bool:
if isinstance(node, addnodes.translatable):
return True
# image node marked as translatable or having alt text
if isinstance(node, nodes.image) and (node.get('translatable')
or node.get('alt')):
return True
if isinstance(node,
nodes.Inline) and 'translatable' not in node: # type: ignore
# inline node must not be translated if 'translatable' is not set
return False
if isinstance(node, nodes.TextElement):
if not node.source:
logger.debug('[i18n] SKIP %r because no node.source: %s',
get_full_module_name(node), repr_domxml(node))
return False # built-in message
if isinstance(node, IGNORED_NODES) and 'translatable' not in node:
logger.debug(
"[i18n] SKIP %r because node is in IGNORED_NODES "
"and no node['translatable']: %s", get_full_module_name(node),
repr_domxml(node))
return False
if not node.get('translatable', True):
# not(node['translatable'] == True or node['translatable'] is None)
logger.debug("[i18n] SKIP %r because not node['translatable']: %s",
get_full_module_name(node), repr_domxml(node))
return False
# <field_name>orphan</field_name>
# XXX ignore all metadata (== docinfo)
if isinstance(node, nodes.field_name) and node.children[0] == 'orphan':
logger.debug('[i18n] SKIP %r because orphan node: %s',
get_full_module_name(node), repr_domxml(node))
return False
return True
if is_pending_meta(node) or isinstance(node, addnodes.meta):
# docutils-0.17 or older
return True
elif isinstance(node, addnodes.docutils_meta):
# docutils-0.18+
return True
return False
def locate_in_toc(self, app: Sphinx,
node: nodes.Node) -> Optional[nodes.Node]:
toc = app.env.tocs[app.env.docname]
ref = self.get_ref(node)
for node in toc.traverse(nodes.reference):
node_ref = node.get('anchorname')
if not node_ref or node_ref[0] != "#":
continue
if node_ref[1:] == ref:
return node.parent.parent
def dispatch_visit(self, node: Node) -> None:
if isinstance(node, nodes.comment):
raise nodes.SkipNode
elif isinstance(node, nodes.raw):
if 'html' in node.get('format', '').split():
# Some people might put content in raw HTML that should be searched,
# so we just amateurishly strip HTML tags and index the remaining
# content
nodetext = re.sub(r'(?is)<style.*?</style>', '', node.astext())
nodetext = re.sub(r'(?is)<script.*?</script>', '', nodetext)
nodetext = re.sub(r'<[^<]+?>', '', nodetext)
self.found_words.extend(self.lang.split(nodetext))
raise nodes.SkipNode
elif isinstance(node, nodes.Text):
self.found_words.extend(self.lang.split(node.astext()))
elif isinstance(node, nodes.title):
self.found_title_words.extend(self.lang.split(node.astext()))
elif isinstance(node, addnodes.meta) and self.is_meta_keywords(node):
keywords = node['content']
keywords = [keyword.strip() for keyword in keywords.split(',')]
self.found_words.extend(keywords)
def register_tabs_as_label(app: Sphinx, document: nodes.Node) -> None:
docname = app.env.docname
labels = app.env.domaindata['std']['labels']
anonlabels = app.env.domaindata['std']['anonlabels']
for node in document.traverse(nxt_tab_head):
if node.label_id in labels:
logger.warning(__('duplicate label %s, other instance in %s'),
node.label_id,
app.env.doc2path(labels[node.label_id][0]),
location=node)
anonlabels[node.label_id] = docname, node.label_id
labels[node.label_id] = docname, node.label_id, node.astext()
def post_process_images(self, doctree: Node) -> None:
for node in doctree.traverse(RevealjsNode):
elm = getattr(node, 'revealit_el', None)
if elm:
for img in elm.images.values():
if img not in self.env.images:
# non-existing URI; let it alone
continue
self.images[img] = self.env.images[img][1]
super().post_process_images(doctree)
def resolve_any_xref(
self, env: BuildEnvironment, from_doc_name: str, builder: Builder,
target: str, node: nodes.Node,
cont_node: nodes.Node) -> List[Tuple[str, nodes.Node]]:
modname = node.get('lua:module')
class_name = node.get('lua:class')
results: List[Tuple[str, nodes.Node]] = []
# always search in "refspecific" mode with the :any: role
matches = self.find_obj(env, modname, class_name, target, None, 1)
for name, obj in matches:
if obj[1] == 'module':
results.append(
('lua:mod',
self._make_module_refnode(builder, from_doc_name, name,
cont_node)))
else:
results.append(('lua:' + self.role_for_objtype(obj[1]),
make_refnode(builder, from_doc_name, obj[0],
name, cont_node, name)))
return results
def write_toc(node: nodes.Node, parent: etree.Element):
if isinstance(node, addnodes.compact_paragraph) or \
isinstance(node, nodes.bullet_list):
for subnode in node:
write_toc(subnode, parent)
elif isinstance(node, nodes.list_item):
item = etree.SubElement(parent, 'topic')
for subnode in node:
write_toc(subnode, item)
elif isinstance(node, nodes.reference):
parent.attrib['label'] = node.astext()
parent.attrib['href'] = base_dir + node['refuri']
def write_doc(self, docname: str, doctree: Node) -> None:
# work around multiple string % tuple issues in docutils;
# replace tuples in attribute values with lists
doctree = doctree.deepcopy()
for node in doctree.traverse(nodes.Element):
for att, value in node.attributes.items():
if isinstance(value, tuple):
node.attributes[att] = list(value)
value = node.attributes[att]
if isinstance(value, list):
for i, val in enumerate(value):
if isinstance(val, tuple):
value[i] = list(val)
destination = StringOutput(encoding='utf-8')
self.writer.write(doctree, destination)
outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix)
ensuredir(path.dirname(outfilename))
try:
with open(outfilename, 'w', encoding='utf-8') as f:
f.write(self.writer.output)
except OSError as err:
logger.warning(__("error writing file %s: %s"), outfilename, err)
def process_doc(self, env: BuildEnvironment, docname: str, document: Node) -> None:
"""Process a document after it is read by the environment."""
entries = self.entries.setdefault(env.docname, [])
for node in list(document.findall(addnodes.index)):
try:
for entry in node['entries']:
split_index_msg(entry[0], entry[1])
except ValueError as exc:
logger.warning(str(exc), location=node)
node.parent.remove(node)
else:
for entry in node['entries']:
entries.append(entry)
def _run_doctests_on_graphtik_document(app: Sphinx, doctree: nodes.Node):
"""Callback of `doctree-resolved`` event. """
try:
docname = app.env.docname
from ._graphtikbuilder import get_graphtik_builder
if _should_work(app) and any(doctree.traverse(graphtik_node)):
log.info(__("Graphtik-ing document %r..."), docname)
graphtik_builder = get_graphtik_builder(app)
graphtik_builder.test_doc(docname, doctree)
except Exception as ex:
log.error("General failure of Graphtik-sphinx extension: %s", ex, exc_info=True)
raise
def get_refnodes(self, doctree: Node, result: List[Dict[str, Any]]) -> List[Dict[str, Any]]: # NOQA
"""Collect section titles, their depth in the toc and the refuri."""
# XXX: is there a better way than checking the attribute
# toctree-l[1-8] on the parent node?
if isinstance(doctree, nodes.reference) and doctree.get('refuri'):
refuri = doctree['refuri']
if refuri.startswith('http://') or refuri.startswith('https://') \
or refuri.startswith('irc:') or refuri.startswith('mailto:'):
return result
classes = doctree.parent.attributes['classes']
for level in range(8, 0, -1): # or range(1, 8)?
if (self.toctree_template % level) in classes:
result.append({
'level': level,
'refuri': html.escape(refuri),
'text': ssp(html.escape(doctree.astext()))
})
break
elif isinstance(doctree, nodes.Element):
for elem in doctree:
result = self.get_refnodes(elem, result)
return result
def add_uids(doctree: Node, condition: Any) -> Iterator[Node]:
"""Add a unique id to every node in the `doctree` which matches the
condition and yield the nodes.
:param doctree:
A :class:`docutils.nodes.document` instance.
:param condition:
A callable which returns either ``True`` or ``False`` for a given node.
"""
for node in doctree.findall(condition):
node.uid = uuid4().hex
yield node
def _update_ref(self, node: Node, labelid: str) -> None:
source_attr = self.env.exercise_list[labelid]
source_node = source_attr.get("node", Node)
if is_linked_node(source_node):
default_title = "Solution to "
target_labelid = source_node.get("target_label", "")
target_attr = self.env.exercise_list[target_labelid]
target_node = target_attr.get("node", Node)
if is_enumerable_node(
target_node) and node.astext() == default_title:
node.replace(node[0], source_node[0][0][0])
return
if is_unenumerable_node(
target_node) and node.astext() == default_title:
if target_attr.get("title"):
if self._has_math_child(target_node[0]):
title = self._update_title(target_node[0])
title.insert(0, nodes.Text(default_title,
default_title))
node.replace(node[0], title)
else:
text = target_attr.get("title", "")
node[0].insert(len(node[0]), nodes.Text(text, text))
else:
node[0].insert(len(node[0]),
nodes.Text("Exercise", "Exercise"))
else:
# If no node.astext() simply add "Exercise"
if is_enumerable_node(source_node) and not node.astext():
text = nodes.Text("Exercise", "Exercise")
node[0].insert(0, text)
return
if ":math:" in node.astext():
title = self._update_title(source_node[0])
node.replace(node[0], title)
def fix_refuris(self, tree: Node) -> None:
# fix refuris with double anchor
fname = self.config.master_doc + self.out_suffix
for refnode in tree.traverse(nodes.reference):
if 'refuri' not in refnode:
continue
refuri = refnode['refuri']
hashindex = refuri.find('#')
if hashindex < 0:
continue
hashindex = refuri.find('#', hashindex + 1)
if hashindex >= 0:
refnode['refuri'] = fname + refuri[hashindex:]
def doctree_read(app: Sphinx, doctree: Node) -> None:
env = app.builder.env
resolve_target = getattr(env.config, 'linkcode_resolve', None)
if not callable(env.config.linkcode_resolve):
raise LinkcodeError(
"Function `linkcode_resolve` is not given in conf.py")
domain_keys = {
'py': ['module', 'fullname'],
'c': ['names'],
'cpp': ['names'],
'js': ['object', 'fullname'],
}
for objnode in list(doctree.traverse(addnodes.desc)):
domain = objnode.get('domain')
uris: Set[str] = set()
for signode in objnode:
if not isinstance(signode, addnodes.desc_signature):
continue
# Convert signode to a specified format
info = {}
for key in domain_keys.get(domain, []):
value = signode.get(key)
if not value:
value = ''
info[key] = value
if not info:
continue
# Call user code to resolve the link
uri = resolve_target(domain, info)
if not uri:
# no source
continue
if uri in uris or not uri:
# only one link per name, please
continue
uris.add(uri)
inline = nodes.inline('', _('[source]'), classes=['viewcode-link'])
onlynode = addnodes.only(expr='html')
onlynode += nodes.reference('',
'',
inline,
internal=False,
refuri=uri)
signode += onlynode
def get_enumerable_node_type(self, node: Node) -> str:
"""Get type of enumerable nodes."""
def has_child(node: Element, cls: "Type") -> bool:
return any(isinstance(child, cls) for child in node)
if isinstance(node, nodes.section):
return 'section'
elif isinstance(node, nodes.container):
if node.get('literal_block') and has_child(node, nodes.literal_block):
return 'code-block'
else:
return None
else:
figtype, _ = self.enumerable_nodes.get(node.__class__, (None, None))
return figtype
def doctree_read(app: Sphinx, document: Node) -> None:
"""
Read the doctree and apply updates to sphinx-exercise nodes
"""
domain = cast(StandardDomain, app.env.get_domain("std"))
# Traverse sphinx-exercise nodes
for node in document.traverse():
if is_extension_node(node):
name = node.get("names", [])[0]
label = document.nameids[name]
docname = app.env.docname
section_name = node.attributes.get("title")
domain.anonlabels[name] = docname, label
domain.labels[name] = docname, label, section_name
def _add_pending_source_cross_reference(app: Sphinx, signode: Node,
fullname: str) -> None:
"""
Adds a pending source cross reference to the signature in the doctree, `signode`.
The viewcode and linkcode extensions walk the doctree once parsed and
then add this node, however since sphinx_c_autodoc already has to add
logic, the `module` option to the directives it seems more practical to
just create the full pending cross reference here, and then viewcode is
an extension which will populate this cross refernce.
Args:
app (Sphinx):
The sphinx app currently doing the processing.
signode (Node):
The signature node to apply the source code cross reference to.
fullname (str):
The dotted fullname of the C construct.
"""
module = signode.get("module")
if module is None:
return
source_page = _get_source_page_name(module)
# Using the `viewcode-link` to be consistent with the python versions in
# case someone else wants to walk the tree and do other links
inline = nodes.inline("", "[source]", classes=["viewcode-link"])
# Limit this cross referencing only to html
html_node = addnodes.only(expr="html")
html_node += addnodes.pending_xref(
"",
inline,
reftype=f"{C_DOMAIN_LINK_PREFIX}viewcode",
refdomain="std",
refexplicit=False,
reftarget=source_page,
refid=f"{C_DOMAIN_LINK_PREFIX}{fullname}",
refdoc=app.builder.env.docname,
module=module,
fullname=fullname,
)
signode += html_node
def repr_domxml(node: Node, length: int = 80) -> str:
"""
return DOM XML representation of the specified node like:
'<paragraph translatable="False"><inline classes="versionmodified">New in version...'
:param nodes.Node node: target node
:param int length:
length of return value to be striped. if false-value is specified, repr_domxml
returns full of DOM XML representation.
:return: DOM XML representation
"""
try:
text = node.asdom().toxml()
except Exception:
text = str(node)
if length and len(text) > length:
text = text[:length] + '...'
return text
def line_of_end(node: nodes.Node) -> Optional[int]:
next_node = node.next_node(descend=False, siblings=True, ascend=True)
while next_node:
if next_node.line:
return line_of_start(next_node)
next_node = next_node.next_node(
# Some nodes' line attr is always None, but their children has
# valid line attr
descend=True,
# If node and its children have not valid line attr, try use line
# of next node
ascend=True,
siblings=True)
# No line found, return the max line of source file
if node.source:
with open(node.source) as f:
return sum(1 for line in f)
raise AttributeError('None source attr of node %s' % node)
def process_only_nodes(document: Node, tags: "Tags") -> None:
"""Filter ``only`` nodes which do not match *tags*."""
for node in document.findall(addnodes.only):
try:
ret = tags.eval_condition(node['expr'])
except Exception as err:
logger.warning(__('exception while evaluating only directive expression: %s'), err,
location=node)
node.replace_self(node.children or nodes.comment())
else:
if ret:
node.replace_self(node.children or nodes.comment())
else:
# A comment on the comment() nodes being inserted: replacing by [] would
# result in a "Losing ids" exception if there is a target node before
# the only node, so we make sure docutils can transfer the id to
# something, even if it's just a comment and will lose the id anyway...
node.replace_self(nodes.comment())
def test_doc(self, docname: str, doctree: Node) -> None:
# Get all applicable nodes
doc_nodes = list(doctree.traverse(self.condition))
if not doc_nodes:
return
print()
print(f"{self.name} testing: {docname}")
print()
self.total_tries += 1
try:
with tempfile.TemporaryDirectory(
) as tempdir, contextlib.ExitStack() as stack:
ctx = {"cwd": tempdir}
for node in doc_nodes: # type: Element
filename = self.get_filename_for_node(node, docname)
line_number = self.get_line_number(node)
if "consoletest-literalinclude" in node:
print()
print("Copying", node["source"], node["filepath"])
print()
shutil.copyfile(
node["source"],
os.path.join(ctx["cwd"], *node["filepath"]),
)
elif "consoletest_commands" in node:
for command in node["consoletest_commands"]:
print()
print("Running", command)
print()
stack.enter_context(command)
command.run(ctx)
print()
print("No more tempdir")
print()
except:
self.total_failures += 1
def handle_missing_xref(app: Sphinx, env, node: nodes.Node,
contnode: nodes.Node) -> None:
# Ignore missing reference warnings for the wheel_filename module
if node.get("reftarget", '').startswith("docutils."):
raise NoUri
if node.get("reftarget", '').startswith("sphinx.ext.autodoc."):
raise NoUri
if node.get("reftarget", '').startswith("sphinx.ext.autosummary."):
raise NoUri
if node.get("reftarget",
'').startswith("sphinx_toolbox._data_documenter."):
# TODO: redirect
raise NoUri
if node.get("reftarget", '') in {
"spam",
"lobster",
"foo",
"typing_extensions",
"bs4.BeautifulSoup",
"pytest_regressions.file_regression.FileRegressionFixture",
"sphinx_toolbox.patched_autosummary",
"sphinx_toolbox.autodoc_augment_defaults",
"sphinx_toolbox.autodoc_typehints",
"sphinx_toolbox.autotypeddict",
"sphinx_toolbox.autoprotocol",
"sphinx_toolbox.utils._T",
"sphinx_toolbox.testing.EventManager", # TODO
"sphinx.registry.SphinxComponentRegistry",
"sphinx.config.Config",
"sphinx.config.Config.latex_elements",
"sphinx.util.docfields.TypedField",
"sphinx.writers.html.HTMLTranslator",
"sphinx.writers.latex.LaTeXTranslator",
"sphinx.domains.python.PyXRefRole",
"sphinx.domains.std.GenericObject",
"sphinx.domains.changeset.VersionChange",
"sphinx.directives.code.CodeBlock",
"sphinx.roles.Abbreviation",
"autodoc.Documenter", # TODO: why not sphinx.ext.autodoc.Documenter?
}:
raise NoUri
if node.get("reftarget",
'').startswith("consolekit.terminal_colours.Fore."):
raise NoUri
def match(self, node: Node) -> bool:
try:
if self.classes and not isinstance(node, self.classes):
return False
if self.attrs:
if not isinstance(node, nodes.Element):
return False
for key, value in self.attrs.items():
if key not in node:
return False
elif value is Any:
continue
elif node.get(key) != value:
return False
return True
except Exception:
# for non-Element nodes
return False
def is_gettext_additional_targets(node: Node) -> bool:
# This func judges if passed `node` is one of the following, which can be
# tranlated by `gettext_addtional_targets` setting and is not supported by
# this sphinx extension:
# - toctree (with hidden attribute as True)
# - literal-block
# - raw
# - image
# hidden toc element
if isinstance(node, toctree) and node.get("hidden", None) is True:
return True
# literal_block / raw / image nodes are not supported since they are not
# supported due to implementation issue, e.g. compatibility with code-block
# highlight HTML writing
elif (isinstance(node, literal_block) or isinstance(node, raw)
or isinstance(node, image)):
return True
else:
return False
def visit_title(self, node: nodes.Node) -> None:
symbol = self.current_symbol
has_parent = True
if symbol is None:
has_parent = False
symbol = self.push_symbol()
name = node.astext()
line = (node.line or 1) - 1
symbol.name = name
symbol.range.start.line = line
symbol.range.end.line = line
symbol.range.end.character = len(name) - 1
symbol.selection_range.start.line = line
symbol.selection_range.end.line = line
symbol.selection_range.end.character = len(name) - 1
if not has_parent:
self.pop_symbol()
def add_node(self, env: BuildEnvironment, node: Node, targetnode: Node,
lineno: int):
"""
Add a node.
:param env: The Sphinx build environment.
:param node:
:param targetnode:
:param lineno:
"""
if not hasattr(env, self.attr_name):
setattr(env, self.attr_name, [])
all_nodes = getattr(env, self.attr_name)
all_nodes.append({
"docname": env.docname,
"lineno": lineno,
"installation_node": node.deepcopy(),
"target": targetnode,
})
