def apply(self):
glue_domain = NbGlueDomain.from_env(self.app.env) # type: NbGlueDomain
for paste_node in self.document.traverse(PasteNode):
if paste_node.key not in glue_domain:
SPHINX_LOGGER.warning(
(f"Couldn't find key `{paste_node.key}` "
"in keys defined across all pages."),
location=(paste_node.source, paste_node.line),
)
continue
# Grab the output for this key
output = glue_domain.get(paste_node.key)
out_node = paste_node.create_node(output=output,
document=self.document,
env=self.app.env)
if out_node is None:
SPHINX_LOGGER.warning(
("Couldn't find compatible output format for key "
f"`{paste_node.key}`"),
location=(paste_node.source, paste_node.line),
)
else:
paste_node.replace_self(out_node)
def parse(self, inputstring: str, document: nodes.document):
self.reporter = document.reporter
self.env = document.settings.env # type: BuildEnvironment
converter = get_nb_converter(
self.env.doc2path(self.env.docname, False),
self.env,
inputstring.splitlines(keepends=True),
)
if converter is None:
# Read the notebook as a text-document
super().parse(inputstring, document=document)
return
try:
ntbk = converter.func(inputstring)
except Exception as error:
SPHINX_LOGGER.error(
"MyST-NB: Conversion to notebook failed: %s",
error,
# exc_info=True,
location=(self.env.docname, 1),
)
return
# add outputs to notebook from the cache
if self.env.config["jupyter_execute_notebooks"] != "off":
ntbk = generate_notebook_outputs(
self.env,
ntbk,
show_traceback=self.env.config["execution_show_tb"])
# Parse the notebook content to a list of syntax tokens and an env
# containing global data like reference definitions
md_parser, env, tokens = nb_to_tokens(
ntbk,
self.env.myst_config if converter is None else converter.config,
self.env.config["nb_render_plugin"],
)
# Write the notebook's output to disk
path_doc = nb_output_to_disc(ntbk, document)
# Update our glue key list with new ones defined in this page
glue_domain = NbGlueDomain.from_env(self.env)
glue_domain.add_notebook(ntbk, path_doc)
# Render the Markdown tokens to docutils AST.
tokens_to_docutils(md_parser, env, tokens, document)
def test_parser(sphinx_run, clean_doctree, file_regression):
sphinx_run.build()
# print(sphinx_run.status())
assert sphinx_run.warnings() == ""
doctree = clean_doctree(sphinx_run.get_resolved_doctree("with_glue"))
file_regression.check(doctree.pformat(), extension=".xml", encoding="utf8")
glue_domain = NbGlueDomain.from_env(sphinx_run.app.env)
assert set(glue_domain.cache) == {
"key_text1",
"key_float",
"key_undisplayed",
"key_df",
"key_plt",
"sym_eq",
}
glue_domain.clear_doc("with_glue")
assert glue_domain.cache == {}
assert glue_domain.docmap == {}
def parse(self, inputstring: str, document: nodes.document):
self.reporter = document.reporter
self.env = document.settings.env
self.config = self.default_config.copy()
try:
new_cfg = document.settings.env.config.myst_config
self.config.update(new_cfg)
except AttributeError:
pass
try:
ntbk = string_to_notebook(inputstring, self.env)
except Exception as err:
SPHINX_LOGGER.error("Notebook load failed for %s: %s",
self.env.docname, err)
return
if not ntbk:
# Read the notebook as a text-document
to_docutils(inputstring, options=self.config, document=document)
return
# add outputs to notebook from the cache
if self.env.config["jupyter_execute_notebooks"] != "off":
ntbk = add_notebook_outputs(
self.env,
ntbk,
show_traceback=self.env.config["execution_show_tb"])
# Parse the notebook content to a list of syntax tokens and an env
# containing global data like reference definitions
md_parser, env, tokens = nb_to_tokens(ntbk)
# Write the notebook's output to disk
path_doc = nb_output_to_disc(ntbk, document)
# Update our glue key list with new ones defined in this page
glue_domain = NbGlueDomain.from_env(self.env)
glue_domain.add_notebook(ntbk, path_doc)
# Render the Markdown tokens to docutils AST.
tokens_to_docutils(md_parser, env, tokens, document)
def test_parser(sphinx_run, file_regression):
sphinx_run.build()
# print(sphinx_run.status())
assert sphinx_run.warnings() == ""
document = sphinx_run.get_doctree()
transformer = Transformer(document)
transformer.add_transforms([CellOutputsToNodes, transform.PasteNodesToDocutils])
transformer.apply_transforms()
file_regression.check(document.pformat(), extension=".xml")
glue_domain = NbGlueDomain.from_env(sphinx_run.app.env)
assert set(glue_domain.cache) == {
"key_text1",
"key_float",
"key_undisplayed",
"key_df",
"key_plt",
"sym_eq",
}
glue_domain.clear_doc("with_glue")
assert glue_domain.cache == {}
assert glue_domain.docmap == {}
def test_parser(mock_document, get_notebook, file_regression):
parser = NotebookParser()
parser.parse(get_notebook("with_glue.ipynb").read_text(), mock_document)
transformer = Transformer(mock_document)
transformer.add_transforms(
[CellOutputsToNodes, transform.PasteNodesToDocutils])
transformer.apply_transforms()
file_regression.check(mock_document.pformat(), extension=".xml")
glue_domain = NbGlueDomain.from_env(mock_document.document.settings.env)
assert set(glue_domain.cache) == {
"key_text1",
"key_float",
"key_undisplayed",
"key_df",
"key_plt",
"sym_eq",
}
glue_domain.clear_doc(mock_document.settings.env.docname)
assert glue_domain.cache == {}
assert glue_domain.docmap == {}
def __init__(self, tmp_path):
self.docname = "source/nb"
self.dependencies = defaultdict(set)
self.domaindata = {}
self.domains = {
NbGlueDomain.name: NbGlueDomain(self),
MathDomain.name: MathDomain(self),
}
self._tmp_path = tmp_path
class app:
class builder:
name = "html"
class config:
language = None
env = self
srcdir = tmp_path / "source"
outdir = tmp_path / "build" / "outdir"
self.app = app
def parse(self, inputstring, document):
# de-serialize the notebook
ntbk = nbf.reads(inputstring, nbf.NO_CONVERT)
# This is a contaner for top level markdown tokens
# which we will add to as we walk the document
mkdown_tokens = [] # type: list[BlockToken]
# First we ensure that we are using a 'clean' global context
# for parsing, which is setup with the MyST parsing tokens
# the logger will report on duplicate link/footnote definitions, etc
parse_context = ParseContext(
find_blocks=SphinxNBRenderer.default_block_tokens,
find_spans=SphinxNBRenderer.default_span_tokens,
logger=SPHINX_LOGGER,
)
set_parse_context(parse_context)
for cell_index, nb_cell in enumerate(ntbk.cells):
# Skip empty cells
if len(nb_cell["source"].strip()) == 0:
continue
# skip cells tagged for removal
tags = nb_cell.metadata.get("tags", [])
if "remove_cell" in tags:
continue
if nb_cell["cell_type"] == "markdown":
# we add the document path and cell index
# to the source lines, so they can be included in the error logging
# NOTE: currently the logic to report metadata is not written
# into SphinxRenderer, but this will be introduced in a later update
lines = SourceLines(
nb_cell["source"],
uri=document["source"],
metadata={"cell_index": cell_index},
standardize_ends=True,
)
# parse the source markdown text;
# at this point span/inline level tokens are not yet processed, but
# link/footnote definitions are collected/stored in the global context
mkdown_tokens.extend(tokenize_block(lines))
# TODO for md cells, think of a way to implement the previous
# `if "hide_input" in tags:` logic
elif nb_cell["cell_type"] == "code":
# here we do nothing but store the cell as a custom token
mkdown_tokens.append(
NbCodeCell(
cell=nb_cell,
position=Position(
line_start=0,
uri=document["source"],
data={"cell_index": cell_index},
),
))
# Now all definitions have been gathered, we walk the tokens and
# process any inline text
for token in mkdown_tokens + list(
get_parse_context().foot_definitions.values()):
token.expand_spans()
# If there are widgets, this will embed the state of all widgets in a script
if contains_widgets(ntbk):
mkdown_tokens.insert(0,
JupyterWidgetState(state=get_widgets(ntbk)))
# create the front matter token
front_matter = FrontMatter(content=ntbk.metadata, position=None)
# Finally, we create the top-level markdown document
markdown_doc = Document(
children=mkdown_tokens,
front_matter=front_matter,
link_definitions=parse_context.link_definitions,
footnotes=parse_context.foot_definitions,
footref_order=parse_context.foot_references,
)
self.reporter = document.reporter
self.config = self.default_config.copy()
try:
new_cfg = document.settings.env.config.myst_config
self.config.update(new_cfg)
except AttributeError:
pass
# Remove all the mime prefixes from "glue" step.
# This way, writing properly captures the glued images
replace_mime = []
for cell in ntbk.cells:
if hasattr(cell, "outputs"):
for out in cell.outputs:
if "data" in out:
# Only do the mimebundle replacing for the scrapbook outputs
mime_prefix = (out.get("metadata",
{}).get("scrapbook",
{}).get("mime_prefix"))
if mime_prefix:
out["data"] = {
key.replace(mime_prefix, ""): val
for key, val in out["data"].items()
}
replace_mime.append(out)
# Write the notebook's output to disk. This changes metadata in notebook cells
path_doc = Path(document.settings.env.docname)
doc_relpath = path_doc.parent
doc_filename = path_doc.name
build_dir = Path(document.settings.env.app.outdir).parent
output_dir = build_dir.joinpath("jupyter_execute", doc_relpath)
write_notebook_output(ntbk, str(output_dir), doc_filename)
# Now add back the mime prefixes to the right outputs so they aren't rendered
# until called from the role/directive
for out in replace_mime:
out["data"] = {
f"{GLUE_PREFIX}{key}": val
for key, val in out["data"].items()
}
# Update our glue key list with new ones defined in this page
glue_domain = NbGlueDomain.from_env(document.settings.env)
glue_domain.add_notebook(ntbk, path_doc)
# render the Markdown AST to docutils AST
renderer = SphinxNBRenderer(parse_context=parse_context,
document=document,
current_node=None)
renderer.render(markdown_doc)
