def read_fenced_cell(token, cell_index, cell_type):
"""Return cell options and body"""
try:
_, options, body_lines = parse_directive_text(
directive_class=MockDirective,
argument_str="",
content=token.content,
validate_options=False,
)
except DirectiveParsingError as err:
raise MystMetadataParsingError(
"{0} cell {1} at line {2} could not be read: {3}".format(
cell_type, cell_index, token.map[0] + 1, err))
return options, body_lines
def read_fenced_cell(token, cell_index, cell_type):
from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text
try:
_, options, body_lines = parse_directive_text(
directive_class=MockDirective,
first_line="",
content=token.content,
validate_options=False,
)
except DirectiveParsingError as err:
raise MystMetadataParsingError(
"{0} cell {1} at line {2} could not be read: {3}".format(
cell_type, cell_index, token.map[0] + 1, err))
return options, body_lines
def test_parsing(klass, arguments, content, data_regression):
arguments, options, body_lines = parse_directive_text(klass, arguments, content)
data_regression.check(
{"arguments": arguments, "options": options, "body": body_lines}
)
def test_parsing_errors(descript, klass, arguments, content):
with pytest.raises(DirectiveParsingError):
parse_directive_text(klass, arguments, content)
def myst_to_notebook(
text,
code_directive=CODE_DIRECTIVE,
raw_directive=RAW_DIRECTIVE,
ignore_bad_meta=False,
store_line_numbers=False,
):
"""Convert text written in the myst format to a notebook.
:param text: the file text
:param code_directive: the name of the directive to search for containing code cells
:param raw_directive: the name of the directive to search for containing raw cells
:param ignore_bad_meta: ignore metadata that cannot be parsed as JSON/YAML
:param store_line_numbers: add a `_source_lines` key to cell metadata,
mapping to the source text.
NOTE: we assume here that all of these directives are at the top-level,
i.e. not nested in other directives.
"""
from mistletoe.base_elements import SourceLines
from mistletoe.parse_context import (
ParseContext,
get_parse_context,
set_parse_context,
)
from mistletoe.block_tokens import Document, CodeFence
from myst_parser.block_tokens import BlockBreak
from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text
from myst_parser.docutils_renderer import DocutilsRenderer
code_directive = "{{{0}}}".format(code_directive)
raw_directive = "{{{0}}}".format(raw_directive)
original_context = get_parse_context()
parse_context = ParseContext(
find_blocks=DocutilsRenderer.default_block_tokens,
find_spans=DocutilsRenderer.default_span_tokens,
)
if isinstance(text, SourceLines):
lines = text
else:
lines = SourceLines(text, standardize_ends=True)
try:
set_parse_context(parse_context)
doc = Document.read(lines, front_matter=True)
metadata_nb = {}
try:
metadata_nb = doc.front_matter.get_data() if doc.front_matter else {}
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
if not ignore_bad_meta:
raise MystMetadataParsingError("Notebook metadata: {}".format(error))
nbf_version = nbf.v4
kwargs = {"metadata": nbf.from_dict(metadata_nb)}
notebook = nbf_version.new_notebook(**kwargs)
current_line = 0 if not doc.front_matter else doc.front_matter.position.line_end
md_metadata = {}
for item in doc.walk(["CodeFence", "BlockBreak"]):
if isinstance(item.node, BlockBreak):
token = item.node # type: BlockBreak
source = _fmt_md(
"".join(lines.lines[current_line:token.position.line_start - 1])
)
if source:
md_metadata = nbf.from_dict(md_metadata)
if store_line_numbers:
md_metadata["_source_lines"] = [
current_line,
token.position.line_start - 1,
]
notebook.cells.append(
nbf_version.new_markdown_cell(
source=source, metadata=md_metadata,
)
)
if token.content:
md_metadata = {}
try:
md_metadata = json.loads(token.content.strip())
except Exception as err:
if not ignore_bad_meta:
raise MystMetadataParsingError(
"markdown cell {0} at {1} could not be read: {2}".format(
len(notebook.cells) + 1, token.position, err
)
)
if not isinstance(md_metadata, dict):
if not ignore_bad_meta:
raise MystMetadataParsingError(
"markdown cell {0} at {1} is not a dict".format(
len(notebook.cells) + 1, token.position
)
)
else:
md_metadata = {}
current_line = token.position.line_start
if isinstance(item.node, CodeFence) and item.node.language in [
code_directive,
raw_directive,
]:
token = item.node # type: CodeFence
# Note: we ignore anything after the directive on the first line
# this is reserved for the optional lexer name
# TODO: could log warning about if token.arguments != lexer name
options, body_lines = {}, []
try:
_, options, body_lines = parse_directive_text(
directive_class=MockDirective,
argument_str="",
content=token.children[0].content,
validate_options=False,
)
except DirectiveParsingError as err:
if not ignore_bad_meta:
raise MystMetadataParsingError(
"Code cell {0} at {1} could not be read: {2}".format(
len(notebook.cells) + 1, token.position, err
)
)
md_source = _fmt_md(
"".join(lines.lines[current_line:token.position.line_start - 1])
)
if md_source:
md_metadata = nbf.from_dict(md_metadata)
if store_line_numbers:
md_metadata["_source_lines"] = [
current_line,
token.position.line_start - 1,
]
notebook.cells.append(
nbf_version.new_markdown_cell(
source=md_source, metadata=md_metadata,
)
)
current_line = token.position.line_end
md_metadata = {}
cell_metadata = nbf.from_dict(options)
if store_line_numbers:
cell_metadata["_source_lines"] = [
token.position.line_start,
token.position.line_end,
]
if item.node.language == code_directive:
notebook.cells.append(
nbf_version.new_code_cell(
source="\n".join(body_lines), metadata=cell_metadata,
)
)
if item.node.language == raw_directive:
notebook.cells.append(
nbf_version.new_raw_cell(
source="\n".join(body_lines), metadata=cell_metadata,
)
)
# add the final markdown cell (if present)
if lines.lines[current_line:]:
md_metadata = nbf.from_dict(md_metadata)
if store_line_numbers:
md_metadata["_source_lines"] = [current_line, len(lines.lines)]
notebook.cells.append(
nbf_version.new_markdown_cell(
source=_fmt_md("".join(lines.lines[current_line:])),
metadata=md_metadata,
)
)
finally:
set_parse_context(original_context)
return notebook
def render_directive(self, token):
"""Render special fenced code blocks as directives."""
name = token.language[1:-1]
# TODO directive name white/black lists
content = token.children[0].content
self.document.current_line = token.position.line_start
# get directive class
directive_class, messages = directives.directive(
name, self.language_module, self.document
) # type: (Directive, list)
if not directive_class:
error = self.reporter.error(
"Unknown directive type '{}'\n".format(name),
# nodes.literal_block(content, content),
line=token.position.line_start,
)
self.current_node += [error] + messages
return
try:
arguments, options, body_lines = parse_directive_text(
directive_class, token.arguments, content
)
except DirectiveParsingError as error:
error = self.reporter.error(
"Directive '{}':\n{}".format(name, error),
nodes.literal_block(content, content),
line=token.position.line_start,
)
self.current_node += [error]
return
# initialise directive
if issubclass(directive_class, Include):
directive_instance = MockIncludeDirective(
self,
name=name,
klass=directive_class,
arguments=arguments,
options=options,
body=body_lines,
token=token,
)
else:
state_machine = MockStateMachine(self, token.position.line_start)
state = MockState(
self, state_machine, token.position.line_start, token=token
)
directive_instance = directive_class(
name=name,
# the list of positional arguments
arguments=arguments,
# a dictionary mapping option names to values
options=options,
# the directive content line by line
content=StringList(body_lines, self.document["source"]),
# the absolute line number of the first line of the directive
lineno=token.position.line_start,
# the line offset of the first line of the content
content_offset=0, # TODO get content offset from `parse_directive_text`
# a string containing the entire directive
block_text="\n".join(body_lines),
state=state,
state_machine=state_machine,
)
# run directive
try:
result = directive_instance.run()
except DirectiveError as error:
msg_node = self.reporter.system_message(
error.level, error.msg, line=token.position.line_start
)
msg_node += nodes.literal_block(content, content)
result = [msg_node]
except MockingError as exc:
error = self.reporter.error(
"Directive '{}' cannot be mocked:\n{}: {}".format(
name, exc.__class__.__name__, exc
),
nodes.literal_block(content, content),
line=token.position.line_start,
)
self.current_node += [error]
return
assert isinstance(
result, list
), 'Directive "{}" must return a list of nodes.'.format(name)
for i in range(len(result)):
assert isinstance(
result[i], nodes.Node
), 'Directive "{}" returned non-Node object (index {}): {}'.format(
name, i, result[i]
)
self.current_node += result