def parse(self, inputstring: str, document: nodes.document):
"""Parse source text.
:param inputstring: The source string to parse
:param document: The root docutils node to add AST elements to
"""
self.config = self.default_config.copy()
try:
new_cfg = document.settings.env.config.myst_config
self.config.update(new_cfg)
except AttributeError:
pass
# TODO raise errors or log error with sphinx?
try:
for s in self.config["disable_syntax"]:
assert isinstance(s, str)
except (AssertionError, TypeError):
raise TypeError("disable_syntax not of type List[str]")
allowed_delimiters = ["brackets", "kramdown", "dollars", "julia"]
if not self.config["math_delimiters"] in allowed_delimiters:
raise ValueError(
f"math_delimiters config not an allowed name: {allowed_delimiters}"
)
to_docutils(
inputstring,
options=self.config,
document=document,
disable_syntax=self.config["disable_syntax"] or [],
math_delimiters=self.config["math_delimiters"],
)
def test_render(line, title, input, expected, tmp_path):
tmp_path.joinpath("other.md").write_text("a\nb\nc")
tmp_path.joinpath("fmatter.md").write_text("---\na: 1\n---\nb")
document = make_document(str(tmp_path / "test.md"))
to_docutils(input, document=document, in_sphinx_env=True, srcdir=str(tmp_path))
output = document.pformat().replace(str(tmp_path) + os.sep, "tmpdir" + "/").rstrip()
print(output)
assert output == expected.rstrip()
def test_basic(line, title, input, expected):
document = make_document("source/path")
messages = []
def observer(msg_node):
if msg_node["level"] > 1:
messages.append(msg_node.astext())
document.reporter.attach_observer(observer)
to_docutils(input, MdParserConfig(renderer="docutils"), document=document)
assert "\n".join(messages).rstrip() == expected.rstrip()
def test_errors(line, title, input, expected, tmp_path):
tmp_path.joinpath("bad.md").write_text("{a}`b`")
document = make_document(str(tmp_path / "test.md"))
messages = []
def observer(msg_node):
if msg_node["level"] > 1:
messages.append(msg_node.astext().replace(str(tmp_path), "tmpdir"))
document.reporter.attach_observer(observer)
document.reporter.halt_level = 6
to_docutils(input,
document=document,
in_sphinx_env=True,
srcdir=str(tmp_path))
assert "\n".join(messages).rstrip() == expected.rstrip()
def _run_myst_to_html(self, content, source_path=None, bib_files=None):
"""Execute the MyST parser and return output."""
if source_path and (
self.force_sphinx
or bib_files
or any(
ext in ("dollarmath", "amsmath")
for ext in self.parser_config.enable_extensions
)
):
sphinx_conf = dict(
extensions=[
"myst_parser",
"sphinx.ext.autosectionlabel",
"sphinx.ext.mathjax",
"sphinxcontrib.bibtex",
],
bibtex_bibfiles=bib_files,
master_doc=os.path.basename(source_path).split(".")[0],
myst_enable_extensions=self.parser_config.enable_extensions,
)
# FIXME: See https://github.com/executablebooks/MyST-Parser/issues/327
# return main.to_docutils(
# in_sphinx_env=True,
return to_sphinx(
source_path,
parser_config=self.parser_config,
conf=sphinx_conf,
srcdir=os.path.dirname(source_path),
)
else:
# return main.to_html(content, config=self.parser_config)
return main.to_docutils(content, parser_config=self.parser_config)
def run(self):
filename = self.arguments[0]
pointer = self.arguments[1]
env = self.state.document.settings.env
path = os.path.join(os.path.dirname(env.doc2path(env.docname)),
filename)
env.note_dependency(path)
try:
with open(path, encoding='utf-8') as f:
schema = json.load(f)
description = jsonpointer.resolve_pointer(
schema, f'{pointer}/description')
except FileNotFoundError:
raise self.error(f'JSON Schema file not found: {path}')
except PermissionError:
raise self.error(f'JSON Schema file not readable: {path}')
except json.decoder.JSONDecodeError:
raise self.error(f'JSON Schema file not valid: {path}')
except jsonpointer.JsonPointerException:
raise self.error(
f"Pointer '{pointer}/description' not found: {path}")
block_quote = nodes.block_quote(
'',
*to_docutils(description).children,
classes=['directive--field-description'])
return [block_quote]
def test_definition_lists(line, title, input, expected):
document = to_docutils(
input, MdParserConfig(enable_extensions=["deflist"]), in_sphinx_env=True
)
print(document.pformat())
assert "\n".join(
[ll.rstrip() for ll in document.pformat().splitlines()]
) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
def test_sphinx_roles(line, title, input, expected):
if title.startswith("SKIP"):
pytest.skip(title)
document = to_docutils(input, in_sphinx_env=True)
print(document.pformat())
assert "\n".join([l.rstrip()
for l in document.pformat().splitlines()]) == "\n".join(
[l.rstrip() for l in expected.splitlines()])
def test_errors(line, title, input, expected, tmp_path):
if title.startswith("Non-existent path") and os.name == "nt":
pytest.skip("tmp_path not converted correctly on Windows")
tmp_path.joinpath("bad.md").write_text("{a}`b`")
document = make_document(str(tmp_path / "test.md"))
messages = []
def observer(msg_node):
if msg_node["level"] > 1:
messages.append(
msg_node.astext().replace(str(tmp_path) + os.sep, "tmpdir" + "/")
)
document.reporter.attach_observer(observer)
document.reporter.halt_level = 6
to_docutils(input, document=document, in_sphinx_env=True, srcdir=str(tmp_path))
assert "\n".join(messages).rstrip() == expected.rstrip()
def cell(self, text, morecols=0, source=None):
entry = nodes.entry(morecols=morecols)
if not isinstance(text, str):
text = str(text)
for child in to_docutils(text).children[:]:
child.source = source
entry += child
return entry
def test_docutils_directives(line, title, input, expected):
# TODO fix skipped directives
# TODO test domain directives
if title.startswith("SKIP"):
pytest.skip(title)
document = to_docutils(input)
print(document.pformat())
assert "\n".join(
[ll.rstrip() for ll in document.pformat().splitlines()]
) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
def parse(self, inputstring: str, document: nodes.document):
self.reporter = document.reporter
self.env = document.settings.env
self.config = self.default_config.copy()
try:
new_cfg = document.settings.env.config.myst_config
self.config.update(new_cfg)
except AttributeError:
pass
try:
ntbk = string_to_notebook(inputstring, self.env)
except Exception as err:
SPHINX_LOGGER.error("Notebook load failed for %s: %s",
self.env.docname, err)
return
if not ntbk:
# Read the notebook as a text-document
to_docutils(inputstring, options=self.config, document=document)
return
# add outputs to notebook from the cache
if self.env.config["jupyter_execute_notebooks"] != "off":
ntbk = add_notebook_outputs(
self.env,
ntbk,
show_traceback=self.env.config["execution_show_tb"])
# Parse the notebook content to a list of syntax tokens and an env
# containing global data like reference definitions
md_parser, env, tokens = nb_to_tokens(ntbk)
# Write the notebook's output to disk
path_doc = nb_output_to_disc(ntbk, document)
# Update our glue key list with new ones defined in this page
glue_domain = NbGlueDomain.from_env(self.env)
glue_domain.add_notebook(ntbk, path_doc)
# Render the Markdown tokens to docutils AST.
tokens_to_docutils(md_parser, env, tokens, document)
def test_containers(line, title, input, expected, monkeypatch):
monkeypatch.setattr(SphinxRenderer, "_random_label", lambda self: "mock-uuid")
document = to_docutils(
input, MdParserConfig(enable_extensions=["colon_fence"]), in_sphinx_env=True
)
print(document.pformat())
_actual, _expected = [
"\n".join([ll.rstrip() for ll in text.splitlines()])
for text in (document.pformat(), expected)
]
assert _actual == _expected
def test_sphinx_roles(line, title, input, expected):
if title.startswith("SKIP"):
pytest.skip(title)
document = to_docutils(input, in_sphinx_env=True)
print(document.pformat())
_actual, _expected = [
"\n".join([ll.rstrip() for ll in text.splitlines()])
for text in (document.pformat(), expected)
]
# sphinx 3 adds a parent key
_actual = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', _actual)
assert _actual == _expected
def test_sphinx_directives(line, title, input, expected):
# TODO fix skipped directives
# TODO test domain directives
if title.startswith("SKIP"):
pytest.skip(title)
if title.startswith("SPHINX3") and sphinx.version_info[0] < 3:
pytest.skip(title)
document = to_docutils(input, in_sphinx_env=True)
print(document.pformat())
_actual, _expected = [
"\n".join([ll.rstrip() for ll in text.splitlines()])
for text in (document.pformat(), expected)
]
assert _actual == _expected
def transform(self):
"""Re-parse field body as myst"""
if not self._body.children:
return
# parse body as myst
doc = to_docutils(self.body)
if not doc.children:
return
# replace old body with new
self._body.children.clear()
for node in doc.children:
# breakpoint()
self._body.append(node)
def run(self):
config = self.state.document.settings.env.config
language = config.overrides.get('language', 'en')
try:
headers = config.codelist_headers[language]
except KeyError:
raise self.error(
f"codelist_headers in conf.py is missing a '{language}' key")
filename = self.arguments[0]
code = self.arguments[1]
env = self.state.document.settings.env
path = os.path.join(os.path.dirname(env.doc2path(env.docname)),
filename)
env.note_dependency(path)
try:
with open(path, encoding='utf-8') as f:
reader = csv.DictReader(f)
description = next(row[headers['description']]
for row in reader
if row[headers['code']] == code)
except FileNotFoundError:
raise self.error(f'CSV codelist file not found: {path}')
except PermissionError:
raise self.error(f'CSV codelist file not readable: {path}')
except KeyError as e:
raise self.error(
f"Column {e} not found ({', '.join(reader.fieldnames)}): {path}"
)
except StopIteration:
raise self.error(
f"Value '{code}' not found in column '{headers['code']}': {path}"
)
block_quote = nodes.block_quote(
'',
*to_docutils(description).children,
classes=['directive--code-description'])
return [block_quote]
def test_docutils_roles(line, title, input, expected):
document = to_docutils(input)
print(document.pformat())
assert "\n".join(
[ll.rstrip() for ll in document.pformat().splitlines()]
) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
def test_tables(line, title, input, expected):
document = to_docutils(input, in_sphinx_env=True)
print(document.pformat())
assert "\n".join(
[ll.rstrip() for ll in document.pformat().splitlines()]
) == "\n".join([ll.rstrip() for ll in expected.splitlines()])
