def read_fenced_cell(token, cell_index, cell_type): """Return cell options and body""" try: _, options, body_lines = parse_directive_text( directive_class=MockDirective, argument_str="", content=token.content, validate_options=False, ) except DirectiveParsingError as err: raise MystMetadataParsingError( "{0} cell {1} at line {2} could not be read: {3}".format( cell_type, cell_index, token.map[0] + 1, err)) return options, body_lines
def read_fenced_cell(token, cell_index, cell_type): from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text try: _, options, body_lines = parse_directive_text( directive_class=MockDirective, first_line="", content=token.content, validate_options=False, ) except DirectiveParsingError as err: raise MystMetadataParsingError( "{0} cell {1} at line {2} could not be read: {3}".format( cell_type, cell_index, token.map[0] + 1, err)) return options, body_lines
def test_parsing(klass, arguments, content, data_regression): arguments, options, body_lines = parse_directive_text(klass, arguments, content) data_regression.check( {"arguments": arguments, "options": options, "body": body_lines} )
def test_parsing_errors(descript, klass, arguments, content): with pytest.raises(DirectiveParsingError): parse_directive_text(klass, arguments, content)
def myst_to_notebook( text, code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, ignore_bad_meta=False, store_line_numbers=False, ): """Convert text written in the myst format to a notebook. :param text: the file text :param code_directive: the name of the directive to search for containing code cells :param raw_directive: the name of the directive to search for containing raw cells :param ignore_bad_meta: ignore metadata that cannot be parsed as JSON/YAML :param store_line_numbers: add a `_source_lines` key to cell metadata, mapping to the source text. NOTE: we assume here that all of these directives are at the top-level, i.e. not nested in other directives. """ from mistletoe.base_elements import SourceLines from mistletoe.parse_context import ( ParseContext, get_parse_context, set_parse_context, ) from mistletoe.block_tokens import Document, CodeFence from myst_parser.block_tokens import BlockBreak from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text from myst_parser.docutils_renderer import DocutilsRenderer code_directive = "{{{0}}}".format(code_directive) raw_directive = "{{{0}}}".format(raw_directive) original_context = get_parse_context() parse_context = ParseContext( find_blocks=DocutilsRenderer.default_block_tokens, find_spans=DocutilsRenderer.default_span_tokens, ) if isinstance(text, SourceLines): lines = text else: lines = SourceLines(text, standardize_ends=True) try: set_parse_context(parse_context) doc = Document.read(lines, front_matter=True) metadata_nb = {} try: metadata_nb = doc.front_matter.get_data() if doc.front_matter else {} except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error: if not ignore_bad_meta: raise MystMetadataParsingError("Notebook metadata: {}".format(error)) nbf_version = nbf.v4 kwargs = {"metadata": nbf.from_dict(metadata_nb)} notebook = nbf_version.new_notebook(**kwargs) current_line = 0 if not doc.front_matter else doc.front_matter.position.line_end md_metadata = {} for item in doc.walk(["CodeFence", "BlockBreak"]): if isinstance(item.node, BlockBreak): token = item.node # type: BlockBreak source = _fmt_md( "".join(lines.lines[current_line:token.position.line_start - 1]) ) if source: md_metadata = nbf.from_dict(md_metadata) if store_line_numbers: md_metadata["_source_lines"] = [ current_line, token.position.line_start - 1, ] notebook.cells.append( nbf_version.new_markdown_cell( source=source, metadata=md_metadata, ) ) if token.content: md_metadata = {} try: md_metadata = json.loads(token.content.strip()) except Exception as err: if not ignore_bad_meta: raise MystMetadataParsingError( "markdown cell {0} at {1} could not be read: {2}".format( len(notebook.cells) + 1, token.position, err ) ) if not isinstance(md_metadata, dict): if not ignore_bad_meta: raise MystMetadataParsingError( "markdown cell {0} at {1} is not a dict".format( len(notebook.cells) + 1, token.position ) ) else: md_metadata = {} current_line = token.position.line_start if isinstance(item.node, CodeFence) and item.node.language in [ code_directive, raw_directive, ]: token = item.node # type: CodeFence # Note: we ignore anything after the directive on the first line # this is reserved for the optional lexer name # TODO: could log warning about if token.arguments != lexer name options, body_lines = {}, [] try: _, options, body_lines = parse_directive_text( directive_class=MockDirective, argument_str="", content=token.children[0].content, validate_options=False, ) except DirectiveParsingError as err: if not ignore_bad_meta: raise MystMetadataParsingError( "Code cell {0} at {1} could not be read: {2}".format( len(notebook.cells) + 1, token.position, err ) ) md_source = _fmt_md( "".join(lines.lines[current_line:token.position.line_start - 1]) ) if md_source: md_metadata = nbf.from_dict(md_metadata) if store_line_numbers: md_metadata["_source_lines"] = [ current_line, token.position.line_start - 1, ] notebook.cells.append( nbf_version.new_markdown_cell( source=md_source, metadata=md_metadata, ) ) current_line = token.position.line_end md_metadata = {} cell_metadata = nbf.from_dict(options) if store_line_numbers: cell_metadata["_source_lines"] = [ token.position.line_start, token.position.line_end, ] if item.node.language == code_directive: notebook.cells.append( nbf_version.new_code_cell( source="\n".join(body_lines), metadata=cell_metadata, ) ) if item.node.language == raw_directive: notebook.cells.append( nbf_version.new_raw_cell( source="\n".join(body_lines), metadata=cell_metadata, ) ) # add the final markdown cell (if present) if lines.lines[current_line:]: md_metadata = nbf.from_dict(md_metadata) if store_line_numbers: md_metadata["_source_lines"] = [current_line, len(lines.lines)] notebook.cells.append( nbf_version.new_markdown_cell( source=_fmt_md("".join(lines.lines[current_line:])), metadata=md_metadata, ) ) finally: set_parse_context(original_context) return notebook
def render_directive(self, token): """Render special fenced code blocks as directives.""" name = token.language[1:-1] # TODO directive name white/black lists content = token.children[0].content self.document.current_line = token.position.line_start # get directive class directive_class, messages = directives.directive( name, self.language_module, self.document ) # type: (Directive, list) if not directive_class: error = self.reporter.error( "Unknown directive type '{}'\n".format(name), # nodes.literal_block(content, content), line=token.position.line_start, ) self.current_node += [error] + messages return try: arguments, options, body_lines = parse_directive_text( directive_class, token.arguments, content ) except DirectiveParsingError as error: error = self.reporter.error( "Directive '{}':\n{}".format(name, error), nodes.literal_block(content, content), line=token.position.line_start, ) self.current_node += [error] return # initialise directive if issubclass(directive_class, Include): directive_instance = MockIncludeDirective( self, name=name, klass=directive_class, arguments=arguments, options=options, body=body_lines, token=token, ) else: state_machine = MockStateMachine(self, token.position.line_start) state = MockState( self, state_machine, token.position.line_start, token=token ) directive_instance = directive_class( name=name, # the list of positional arguments arguments=arguments, # a dictionary mapping option names to values options=options, # the directive content line by line content=StringList(body_lines, self.document["source"]), # the absolute line number of the first line of the directive lineno=token.position.line_start, # the line offset of the first line of the content content_offset=0, # TODO get content offset from `parse_directive_text` # a string containing the entire directive block_text="\n".join(body_lines), state=state, state_machine=state_machine, ) # run directive try: result = directive_instance.run() except DirectiveError as error: msg_node = self.reporter.system_message( error.level, error.msg, line=token.position.line_start ) msg_node += nodes.literal_block(content, content) result = [msg_node] except MockingError as exc: error = self.reporter.error( "Directive '{}' cannot be mocked:\n{}: {}".format( name, exc.__class__.__name__, exc ), nodes.literal_block(content, content), line=token.position.line_start, ) self.current_node += [error] return assert isinstance( result, list ), 'Directive "{}" must return a list of nodes.'.format(name) for i in range(len(result)): assert isinstance( result[i], nodes.Node ), 'Directive "{}" returned non-Node object (index {}): {}'.format( name, i, result[i] ) self.current_node += result