def create_myst_config(app):
from sphinx.util import logging
from sphinx.util.console import bold
from myst_parser.main import MdParserConfig
logger = logging.getLogger(__name__)
values = {
name: app.config[f"myst_{name}"]
for name in MdParserConfig().as_dict().keys() if name != "renderer"
}
try:
app.env.myst_config = MdParserConfig(**values)
logger.info(
bold("myst v%s:") + " %s", __version__, app.env.myst_config)
except (TypeError, ValueError) as error:
logger.error("myst configuration invalid: %s", error.args[0])
app.env.myst_config = MdParserConfig()
# https://docs.mathjax.org/en/v2.7-latest/options/preprocessors/tex2jax.html#configure-tex2jax
if app.env.myst_config.override_mathjax:
app.config.mathjax_config = {
"tex2jax": {
"inlineMath": [["\\(", "\\)"]],
"displayMath": [["\\[", "\\]"]],
"processRefs": False,
"processEnvironments": False,
}
}
def parse(
self,
inputstring: str,
document: nodes.document,
):
"""
Parse source text.
Args:
inputstring: The source string to parse
document: The root docutils node to add AST elements to
"""
try:
config = document.settings.env.myst_config
except Exception:
config = MdParserConfig(renderer="docutils")
parser = default_parser(config)
parser.options["document"] = document
env = AttrDict()
tokens = parser.parse(inputstring, env)
if not tokens or tokens[0].type != "front_matter":
# we always add front matter, so that we can merge it with global keys,
# specified in the sphinx configuration
tokens = [
Token(
type="front_matter",
tag="",
nesting=0,
content="{}", # noqa: P103
map=[0, 0],
),
] + tokens
parser.renderer.render(tokens, parser.options, env)
def print_anchors(args=None):
""" """
parser = argparse.ArgumentParser()
parser.add_argument(
"input",
nargs="?",
type=argparse.FileType("r"),
default=sys.stdin,
help="Input file (default stdin)",
)
parser.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
default=sys.stdout,
help="Output file (default stdout)",
)
parser.add_argument("-l",
"--level",
type=int,
default=2,
help="Maximum heading level.")
args = parser.parse_args(args)
parser = default_parser(
MdParserConfig(renderer="html", heading_anchors=args.level))
def _filter_plugin(state):
state.tokens = [
t for t in state.tokens
if t.type.startswith("heading_") and int(t.tag[1]) <= args.level
]
parser.use(lambda p: p.core.ruler.push("filter", _filter_plugin))
text = parser.render(args.input.read())
args.output.write(text)
def parse_markdown(
self, text: str, parent: Optional[nodes.Node] = None
) -> List[nodes.Node]:
"""Parse text as CommonMark, in a new document."""
parser = default_parser(MdParserConfig(commonmark_only=True))
# setup parent node
if parent is None:
parent = nodes.container()
self.add_source_and_line(parent)
parser.options["current_node"] = parent
# setup containing document
new_doc = make_document(self.node.source)
new_doc.settings = self.document.settings
new_doc.reporter = self.document.reporter
parser.options["document"] = new_doc
# use the node docname, where possible, to deal with single document builds
with mock.patch.dict(
self.env.temp_data, {"docname": self.env.path2doc(self.node.source)}
):
parser.render(text)
# TODO is there any transforms we should retroactively carry out?
return parent.children
def test_definition_lists(line, title, input, expected):
document = to_docutils(
input, MdParserConfig(enable_extensions=["deflist"]), in_sphinx_env=True
)
print(document.pformat())
assert "\n".join(
[ll.rstrip() for ll in document.pformat().splitlines()]
) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
def parse_markdown(
self, text: str, parent: Optional[nodes.Node] = None
) -> List[nodes.Node]:
"""Parse text as CommonMark, in a new document."""
parser = default_parser(MdParserConfig(commonmark_only=True))
parent = parent or nodes.container()
parser.options["current_node"] = parent
parser.render(text)
# TODO is there any transforms we should retroactively carry out?
return parent.children
def create_myst_config(app):
from sphinx.util import logging
from sphinx.util.console import bold
from myst_parser.main import MdParserConfig
logger = logging.getLogger(__name__)
values = {
name: app.config[f"myst_{name}"]
for name in MdParserConfig().as_dict().keys() if name != "renderer"
}
try:
app.env.myst_config = MdParserConfig(**values)
logger.info(
bold("myst v%s:") + " %s", __version__, app.env.myst_config)
except (TypeError, ValueError) as error:
logger.error("myst configuration invalid: %s", error.args[0])
app.env.myst_config = MdParserConfig()
def test_basic(line, title, input, expected):
document = make_document("source/path")
messages = []
def observer(msg_node):
if msg_node["level"] > 1:
messages.append(msg_node.astext())
document.reporter.attach_observer(observer)
to_docutils(input, MdParserConfig(renderer="docutils"), document=document)
assert "\n".join(messages).rstrip() == expected.rstrip()
def test_containers(line, title, input, expected, monkeypatch):
monkeypatch.setattr(SphinxRenderer, "_random_label", lambda self: "mock-uuid")
document = to_docutils(
input, MdParserConfig(enable_extensions=["colon_fence"]), in_sphinx_env=True
)
print(document.pformat())
_actual, _expected = [
"\n".join([ll.rstrip() for ll in text.splitlines()])
for text in (document.pformat(), expected)
]
assert _actual == _expected
def test_render(line, title, input, expected):
dct = yaml.safe_load(input)
dct.setdefault("metadata", {})
ntbk = nbformat.from_dict(dct)
md, env, tokens = nb_to_tokens(ntbk, MdParserConfig(), "default")
document = make_document()
with mock_sphinx_env(document=document):
tokens_to_docutils(md, env, tokens, document)
output = document.pformat().rstrip()
if output != expected.rstrip():
print(output)
assert output == expected.rstrip()
def create_myst_config(app):
from sphinx.util import logging
# Ignore type checkers because the attribute is dynamically assigned
from sphinx.util.console import bold # type: ignore[attr-defined]
from myst_parser.main import MdParserConfig
logger = logging.getLogger(__name__)
values = {
name: app.config[f"myst_{name}"]
for name in MdParserConfig().as_dict().keys() if name != "renderer"
}
try:
app.env.myst_config = MdParserConfig(**values)
logger.info(
bold("myst v%s:") + " %s", __version__, app.env.myst_config)
except (TypeError, ValueError) as error:
logger.error("myst configuration invalid: %s", error.args[0])
app.env.myst_config = MdParserConfig()
def create_myst_config(app):
from sphinx.util import logging
# Ignore type checkers because the attribute is dynamically assigned
from sphinx.util.console import bold # type: ignore[attr-defined]
from myst_parser.main import MdParserConfig
logger = logging.getLogger(__name__)
# TODO remove deprecations after v0.13.0
deprecations = {
"myst_admonition_enable": "colon_fence",
"myst_figure_enable": "colon_fence",
"myst_dmath_enable": "dollarmath",
"myst_amsmath_enable": "amsmath",
"myst_deflist_enable": "deflist",
"myst_html_img_enable": "html_image",
}
for old, new in deprecations.items():
if app.config[old]:
logger.warning(
f'config `{old}` is deprecated, please add "{new}" to '
"`myst_enable_extensions = []`")
app.config["myst_enable_extensions"].append(new)
values = {
name: app.config[f"myst_{name}"]
for name in MdParserConfig().as_dict().keys() if name != "renderer"
}
try:
app.env.myst_config = MdParserConfig(**values)
logger.info(
bold("myst v%s:") + " %s", __version__, app.env.myst_config)
except (TypeError, ValueError) as error:
logger.error("myst configuration invalid: %s", error.args[0])
app.env.myst_config = MdParserConfig()
def parse(
self, inputstring: str, document: nodes.document, renderer: str = "sphinx"
):
"""Parse source text.
:param inputstring: The source string to parse
:param document: The root docutils node to add AST elements to
"""
if renderer == "sphinx":
config = document.settings.env.myst_config
else:
config = MdParserConfig()
parser = default_parser(config)
parser.options["document"] = document
parser.render(inputstring)
def setup_sphinx(app):
"""Initialize all settings and transforms in Sphinx."""
# we do this separately to setup,
# so that it can be called by external packages like myst_nb
from myst_parser.myst_refs import MystReferenceResolver
from myst_parser.mathjax import override_mathjax
from myst_parser.main import MdParserConfig
app.add_post_transform(MystReferenceResolver)
for name, default in MdParserConfig().as_dict().items():
if not name == "renderer":
app.add_config_value(f"myst_{name}", default, "env")
app.connect("builder-inited", create_myst_config)
app.connect("builder-inited", override_mathjax)
def parse(self, inputstring: str, document: nodes.document) -> None:
"""Parse source text.
:param inputstring: The source string to parse
:param document: The root docutils node to add AST elements to
"""
config = MdParserConfig(renderer="docutils", enable_extensions=['linkify'])
parser = default_parser(config)
parser.options["document"] = document
env = AttrDict()
tokens = parser.parse(inputstring, env)
if not tokens or tokens[0].type != "front_matter":
# we always add front matter, so that we can merge it with global keys,
# specified in the sphinx configuration
tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens
parser.renderer.render(tokens, parser.options, env)
def test_reporting(line, title, input, expected):
dct = yaml.safe_load(input)
dct.setdefault("metadata", {})
ntbk = nbformat.from_dict(dct)
md, env, tokens = nb_to_tokens(ntbk, MdParserConfig(), "default")
document = make_document("source/path")
messages = []
def observer(msg_node):
if msg_node["level"] > 1:
messages.append(msg_node.astext())
document.reporter.attach_observer(observer)
with mock_sphinx_env(document=document):
tokens_to_docutils(md, env, tokens, document)
assert "\n".join(messages).rstrip() == expected.rstrip()
def to_sphinx(
filename: Iterable[str],
parser_config: Optional[MdParserConfig] = None,
options=None,
env=None,
document=None,
conf=None,
srcdir=None,
with_builder="singlehtml",
):
"""Render text to the docutils AST (before transforms)
:param text: the text to render
:param options: options to update the parser with
:param env: The sandbox environment for the parse
(will contain e.g. reference definitions)
:param document: the docutils root node to use (otherwise a new one will be created)
:param in_sphinx_env: initialise a minimal sphinx environment (useful for testing)
:param conf: the sphinx conf.py as a dictionary
:param srcdir: to parse to the mock sphinx env
:returns: docutils document
"""
from myst_parser.docutils_renderer import make_document
md = default_parser(parser_config or MdParserConfig())
if options:
md.options.update(options)
md.options["document"] = document or make_document()
force_all = False
with mock_sphinx_env_compat(
conf=conf,
srcdir=srcdir,
document=md.options["document"],
with_builder=with_builder,
) as app:
app.build(force_all, (filename, ))
filehtml = Path(filename).with_suffix(".html").name
output = (Path(app.outdir) / filehtml).read_text()
return get_div_body(output)
def setup_sphinx(app: "Sphinx"):
"""Initialize all settings and transforms in Sphinx."""
# we do this separately to setup,
# so that it can be called by external packages like myst_nb
from myst_parser.directives import FigureMarkdown, SubstitutionReferenceRole
from myst_parser.main import MdParserConfig
from myst_parser.mathjax import override_mathjax
from myst_parser.myst_refs import MystReferenceResolver
app.add_role("sub-ref", SubstitutionReferenceRole())
app.add_directive("figure-md", FigureMarkdown)
app.add_post_transform(MystReferenceResolver)
for name, default in MdParserConfig().as_dict().items():
if not name == "renderer":
app.add_config_value(f"myst_{name}", default, "env")
app.connect("builder-inited", create_myst_config)
app.connect("builder-inited", override_mathjax)
def test_parse(test_name, text, should_warn, file_regression):
with mock_sphinx_env(
conf={"extensions": ["myst_parser"]},
srcdir="root",
with_builder=True,
raise_on_warning=True,
) as app: # type: Sphinx
app.env.myst_config = MdParserConfig()
document = parse(app, text, docname="index")
if should_warn:
with pytest.raises(SphinxWarning):
app.env.apply_post_transforms(document, "index")
else:
app.env.apply_post_transforms(document, "index")
content = document.pformat()
# windows fix
content = content.replace("root" + os.sep + "index.md", "root/index.md")
file_regression.check(content, basename=test_name, extension=".xml")
def replace_admonition_in_cell_source(cell_str):
"""Returns cell source with admonition replaced by its generated HTML.
"""
config = MdParserConfig(renderer="docutils")
parser = default_parser(config)
tokens = parser.parse(cell_str)
admonition_tokens = [
t for t in tokens if t.type == "fence" and t.info in all_directive_names
]
cell_lines = cell_str.splitlines()
new_cell_str = cell_str
for t in admonition_tokens:
adm_begin, adm_end = t.map
adm_src = "\n".join(cell_lines[adm_begin:adm_end])
adm_doc = parser.render(adm_src)
adm_html = admonition_html(adm_doc)
new_cell_str = new_cell_str.replace(adm_src, adm_html)
return new_cell_str