def repl(match_obj): match = match_obj.group(1) if not match: # @@ return match_obj.group(0) if match.isspace(): # @ @ return match return pf.convert_text( pf.Plain(*pf.convert_text(match)[0].content), input_format='panflute', output_format=doc.format)
def highlighting(output_format): return pf.convert_text( '`_`{.cpp}', output_format=output_format, extra_args=[ '--highlight-style', f.name, '--template', os.path.join(datadir, 'template', 'highlighting') ])
def run_test(name, action): input_fn = os.path.join('tests', name + '.md') # Read markdown, convert to JSON and then to elements with open(input_fn, encoding='utf-8') as f: md = f.read() print('~' * 80) print(' ' * 30, 'INPUT') print('~' * 80) print(md) print('~' * 80, '\n') print('... Parsing markdown') doc = pf.convert_text(md, output_format='doc') doc.format = 'markdown' assert type(doc) == pf.Doc print(' Done.') # Walk through AST sys.path.append('filters') print('... Importing module') mod = importlib.import_module(name) print(' Done.') f_action = mod.__dict__[action] print('... Applying filters') altered = doc.walk(f_action, doc) print(' Done.') # Convert AST into JSON print('... Converting document into JSON') with io.StringIO() as f: pf.dump(altered, f) contents = f.getvalue() print(' Done.') # Convert JSON into markdown print('... Converting JSON into markdown') md = pf.convert_text(contents, input_format='json', output_format='markdown') print(' Done.') print('~' * 80) print(' ' * 30, 'OUTPUT') print('~' * 80) print(md) print('~' * 80, '\n')
def assert3(*extra_args, stdin): """ filters=None, search_dirs=None, data_dir=True, sys_path=True, panfl_=False """ sys.argv[1:] = [] sys.argv.append('markdown') _stdout = io.StringIO() pf.stdio(*extra_args, input_stream=io.StringIO(stdin), output_stream=_stdout) _stdout = pf.convert_text(_stdout.getvalue(), 'json', 'markdown') assert _stdout == out1
def parse_table_list(markdown, table_list,doc): """ read table in list and return panflute table format """ # make functions local to_table_row = pf.TableRow if markdown: to_table_cell = lambda x: pf.TableCell(*pf.convert_text(x)) else: to_table_cell = lambda x: pf.TableCell( pf.Plain(pf.Str(x))) return [to_table_row(*[to_table_cell(x) for x in row]) for row in table_list]
def action(elem, doc): if isinstance(elem, pf.Para) and is_include_line(elem): fn = get_filename(elem) if not os.path.isfile(fn): return with open(fn) as f: raw = f.read() new_elems = pf.convert_text(raw) div = pf.Div(*new_elems, attributes={'source': fn}) return div
def action(elem, doc): if isinstance(elem, pf.Para) and is_include_line(elem): fn = get_filename(elem) if not os.path.isfile(fn): return with open(fn) as f: raw = f.read() new_elems = pf.convert_text(raw) # Alternative A: return new_elems
def codeblock(elem, doc): if not isinstance(elem, pf.CodeBlock): return None is_raw = not elem.classes if is_raw: elem.classes.append('default') elem.attributes['style'] = 'color: inherit' if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']): return None if escape_char not in elem.text: return None datadir = doc.get_metadata('datadir') syntaxdir = os.path.join(datadir, 'syntax') text = pf.convert_text( elem, input_format='panflute', output_format=doc.format, extra_args=[ '--syntax-definition', os.path.join(syntaxdir, 'isocpp.xml') ]) def repl(match_obj): match = match_obj.group(1) if not match: # @@ return match_obj.group(0) if match.isspace(): # @ @ return match return pf.convert_text( pf.Plain(*pf.convert_text(match)[0].content), input_format='panflute', output_format=doc.format) result = pf.RawBlock(escape_span.sub(repl, text), doc.format) return pf.Div( pf.RawBlock('{\\renewcommand{\\NormalTok}[1]{#1}', 'latex'), result, pf.RawBlock('}', 'latex')) if is_raw else result
def _create_images(doc, icons, size): # Generate the LaTeX image code images = [] for icon in icons: # Get the apps dirs from pkg_resources import get_distribution import appdirs folder = appdirs.AppDirs( "pandoc_latex_tip", version=get_distribution("pandoc_latex_tip").version ).user_cache_dir # Get the image from the App cache folder image_dir = os.path.join( folder, icon["collection"], icon["version"], icon["variant"], icon["color"] ) image = os.path.join(image_dir, icon["extended-name"] + ".png") # Create the image if not existing in the cache try: if not os.path.isfile(image): # Create the image in the cache category = _category( icon["collection"], icon["version"], icon["variant"] ) doc.get_icon_font[category]["font"].export_icon( icon["extended-name"], 512, color=icon["color"], export_dir=image_dir, ) # Add the LaTeX image image = Image( url=image, attributes={"width": size + "pt", "height": size + "pt"} ) if icon["link"] == "": elem = image else: elem = Link(image, url=icon["link"]) images.append( convert_text( Plain(elem), input_format="panflute", output_format="latex" ) ) except TypeError: debug( "[WARNING] pandoc-latex-tip: icon name " + icon["name"] + " does not exist in variant " + icon["variant"] + " for collection " + icon["collection"] + "-" + icon["version"] ) except FileNotFoundError: debug("[WARNING] pandoc-latex-tip: error in generating image") return images
def action(elem, doc): global entryEnter global options if isinstance(elem, pf.Para): includeType = is_include_line(elem) if includeType == 0: return # Try to read inherited options from temp file if options is None: try: with open(temp_filename, 'r') as f: options = json.load(f) except: options = {} pass # pandoc options pandoc_options = doc.get_metadata('pandoc-options') if not pandoc_options: if 'pandoc-options' in options: pandoc_options = options['pandoc-options'] else: # default options pandoc_options = ['--filter=pandoc-include'] else: # Replace em-dash to double dashes in smart typography for i in range(len(pandoc_options)): pandoc_options[i] = pandoc_options[i].replace('\u2013', '--') options['pandoc-options'] = pandoc_options # The entry file's directory entry = doc.get_metadata('include-entry') if not entryEnter and entry: os.chdir(entry) entryEnter = True fn = get_filename(elem, includeType) if not os.path.isfile(fn): raise ValueError('Included file not found: ' + fn + ' ' + entry + ' ' + os.getcwd()) with open(fn, encoding="utf-8") as f: raw = f.read() # Save current path cur_path = os.getcwd() # Change to included file's path so that sub-include's path is correct target = os.path.dirname(fn) # Empty means relative to current dir if not target: target = '.' os.chdir(target) # save options with open(temp_filename, 'w+') as f: json.dump(options, f) # Add recursive include support new_elems = None new_metadata = None if includeType == 1: new_elems = pf.convert_text(raw, extra_args=pandoc_options) # Get metadata (Recursive header include) new_metadata = pf.convert_text( raw, standalone=True, extra_args=pandoc_options).get_metadata() else: # Read header from yaml new_metadata = yaml.load(raw) new_metadata = OrderedDict(new_metadata) # Merge metadata for key in new_metadata: if not key in doc.get_metadata(): doc.metadata[key] = new_metadata[key] # delete temp file os.remove(temp_filename) # Restore to current path os.chdir(cur_path) # Alternative A: return new_elems
def finalize(doc): doc.content = pf.convert_text('\n'.join(b.text for b in doc.code_blocks))
def get_caption(options): '''parsed as markdown into panflute AST if non-empty.''' return panflute.convert_text(str( options['caption']))[0].content if 'caption' in options else None
def finalize(doc): def init_code_elems(elem, doc): if isinstance(elem, pf.Header) and doc.format == 'latex': elem.walk(lambda elem, doc: elem.classes.append('raw') if any( isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]) else None) # Mark code elements within colored divspan as default. if any(isinstance(elem, cls) for cls in [pf.Div, pf.Span]) and \ any(cls in elem.classes for cls in ['add', 'rm', 'ednote']): elem.walk(lambda elem, doc: elem.classes.insert(0, 'default') if any( isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]) else None) if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]): return None # As `walk` performs post-order traversal, this is # guaranteed to run before the 'raw' code path. if not elem.classes: if isinstance(elem, pf.Code): cls = doc.get_metadata('highlighting.inline-code', 'default') elif isinstance(elem, pf.CodeBlock): cls = doc.get_metadata('highlighting.code-block', 'default') elem.classes.append(cls) doc.walk(init_code_elems) def collect_code_elems(elem, doc): if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]): return None if 'raw' in elem.classes: return None if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']): return None code_elems.append(elem) code_elems = [] doc.walk(collect_code_elems) if not code_elems: return def intersperse(lst, item): result = [item] * (len(lst) * 2 - 1) result[0::2] = lst return result datadir = doc.get_metadata('datadir') text = pf.convert_text(intersperse([ pf.Plain(elem) if isinstance(elem, pf.Code) else elem for elem in code_elems ], pf.Plain(pf.RawInline('---', doc.format))), input_format='panflute', output_format=doc.format, extra_args=[ '--syntax-definition', os.path.join(datadir, 'syntax', 'isocpp.xml') ]) # Workaround for https://github.com/jgm/skylighting/issues/91. if doc.format == 'latex': text = text.replace('<', '\\textless{}') \ .replace('>', '\\textgreater{}') if doc.format == 'latex': texts = text.split('\n\n---\n\n') elif doc.format == 'html': texts = text.split('\n---\n') assert (len(code_elems) == len(texts)) def convert(elem, text): def repl2(match): if match.isspace(): # @ @ return match result = convert.cache.get(match) if result is not None: return result if doc.format == 'latex': # Undo `escapeLaTeX` from https://github.com/jgm/skylighting match = match.replace('\\textbackslash{}', '\\') \ .replace('\\{', '{') \ .replace('\\}', '}') \ .replace('\\VerbBar{}', '|') \ .replace('\\_', '_') \ .replace('\\&', '&') \ .replace('\\%', '%') \ .replace('\\#', '#') \ .replace('\\textasciigrave{}', '`') \ .replace('\\textquotesingle{}', '\'') \ .replace('{-}', '-') \ .replace('\\textasciitilde{}', '~') \ .replace('\\^{}', '^') # Undo the workaround escaping. match = match.replace('\\textless{}', '<') \ .replace('\\textgreater{}', '>') elif doc.format == 'html': match = html.unescape(match) result = pf.convert_text( pf.Plain(*pf.convert_text(match)[0].content).walk( divspan, doc).walk(init_code_elems, doc), input_format='panflute', output_format=doc.format, extra_args=[ '--syntax-definition', os.path.join(datadir, 'syntax', 'isocpp.xml') ]) convert.cache[match] = result return result def repl(match_obj): groups = match_obj.groups() if not any(groups): return match_obj.group() group = groups[0] if group is not None: return embedded_md.sub(repl, repl2(group)) group = groups[1] if group is not None: return repl2(group) if isinstance(elem, pf.Code): result = pf.RawInline(embedded_md.sub(repl, text), doc.format) elif isinstance(elem, pf.CodeBlock): result = pf.RawBlock(embedded_md.sub(repl, text), doc.format) if 'diff' not in elem.classes: return result # For HTML, this is handled via CSS in `data/templates/wg21.html`. command = '\\renewcommand{{\\{}}}[1]{{\\textcolor[HTML]{{{}}}{{#1}}}}' uc = command.format('NormalTok', doc.get_metadata('uccolor')) add = command.format('VariableTok', doc.get_metadata('addcolor')) rm = command.format('StringTok', doc.get_metadata('rmcolor')) if isinstance(elem, pf.Code): return pf.Span(pf.RawInline(uc, 'latex'), pf.RawInline(add, 'latex'), pf.RawInline(rm, 'latex'), result) elif isinstance(elem, pf.CodeBlock): return pf.Div(pf.RawBlock('{', 'latex'), pf.RawBlock(uc, 'latex'), pf.RawBlock(add, 'latex'), pf.RawBlock(rm, 'latex'), result, pf.RawBlock('}', 'latex')) convert.cache = {} def code_elem(elem, doc): if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]): return None if 'raw' in elem.classes: return None if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']): return None return convert(*next(converted)) converted = zip(code_elems, texts) doc.walk(code_elem)
def test_sharp_sharp(self): definition = r"Example ##" doc = Doc(*convert_text(definition)) pandoc_numbering.main(doc) self.assertEqual(doc.content[0].content[-1].text, "#")
def format_table(table, doc): # type: (Table, Doc) -> Element """ originally adapted from: `pandoc-tablenos <https://github.com/tomduck/pandoc-tablenos>`_ """ if not isinstance(table, pf.Table): return None div = None # type: pf.Div if (isinstance(table.parent, pf.Div) and LABELLED_TABLE_CLASS in table.parent.classes): div = table.parent if div is None: return None attributes = convert_attributes(div.attributes) if "align" in div.attributes: align_text = attributes["align"] align = [{ "l": "AlignLeft", "r": "AlignRight", "c": "AlignCenter" }.get(a, None) for a in align_text] if None in align: raise ValueError("table '{0}' alignment must contain only l,r,c:" " {1}".format(div.identifier, align_text)) table.alignment = align attributes["align"] = align if "widths" in div.attributes: widths = attributes["widths"] try: widths = [float(w) for w in widths] except Exception: raise ValueError("table '{0}' widths must be a list of numbers:" " {1}".format(div.identifier, widths)) table.width = widths attributes["widths"] = widths if doc.format in ("tex", "latex"): # TODO placement table.caption.append( pf.RawInline("\\label{{{0}}}".format(div.identifier), format="tex")) return table if doc.format in ("rst", ): # pandoc 2.6 doesn't output table options if attributes: tbl_doc = pf.Doc(table) tbl_doc.api_version = doc.api_version tbl_str = pf.convert_text(tbl_doc, input_format="panflute", output_format="rst") tbl_lines = tbl_str.splitlines() if tbl_lines[1].strip() == "": tbl_lines.insert(1, " :align: center") if "widths" in attributes: # in rst widths must be integers widths = " ".join([str(int(w * 10)) for w in table.width]) tbl_lines.insert(1, " :widths: {}".format(widths)) # TODO rst column alignment, see # https://cloud-sptheme.readthedocs.io/en/latest/lib/cloud_sptheme.ext.table_styling.html return [ pf.Para( pf.RawInline(".. _`{0}`:".format(div.identifier), format="rst")), pf.RawBlock("\n".join(tbl_lines) + "\n\n", format=doc.format), ] return [ pf.Para( pf.RawInline(".. _`{0}`:".format(div.identifier), format="rst")), table, ] if doc.format in ("html", "html5"): return _wrap_in_anchor(table, div.identifier, inline=False)
def test_doctest_no_kernel_configured(tmp_path): res = Path.resolve(Path(__file__)).parent doc = convert_text(Path(res / "no_kernel_configured.md").read_text(), standalone=True) with pytest.raises(RuntimeError): run_doctest(doc)
def test_doctest_missing_ref(tmp_path): res = Path.resolve(Path(__file__)).parent doc = convert_text(Path(res / "missing_ref.md").read_text(), standalone=True) with pytest.raises(ValueError): run_doctest(doc)
def run(self): url = self.renderer(self.input, self.cmd_renderer) pfcaption = (pf.convert_text(self.caption))[0].content return pf.Para( pf.Image(*pfcaption, url=url, title=u'fig:' + self.caption))
def ast_to_markdown(ast): """convert panflute AST to Markdown""" return panflute.convert_text(ast, input_format='panflute', output_format='markdown')
def action(elem, doc): global entryEnter global options # Try to read inherited options from temp file if options is None: options = parseOptions(doc) # The entry file's directory entry = doc.get_metadata('include-entry') if not entryEnter and entry: os.chdir(entry) entryEnter = True if isinstance(elem, pf.Para): includeType, name, config = is_include_line(elem) if includeType == 0: return # Enable shell-style wildcards files = glob.glob(name) if len(files) == 0: eprint('[Warn] included file not found: ' + name) # order include_order = options['include-order'] if include_order == 'natural': files = natsorted(files) elif include_order == 'alphabetical': files = sorted(files) elif include_order == 'default': pass else: raise ValueError('Invalid file order: ' + include_order) elements = [] for fn in files: if not os.path.isfile(fn): continue raw = read_file(fn, config) # Save current path cur_path = os.getcwd() # Change to included file's path so that sub-include's path is correct target = os.path.dirname(fn) # Empty means relative to current dir if not target: target = '.' currentPath = options["current-path"] options["current-path"] = os.path.normpath( os.path.join(currentPath, target)) os.chdir(target) # pass options by temp files with open(TEMP_FILE, 'w+') as f: json.dump(options, f) # Add recursive include support new_elems = None new_metadata = None if includeType == 1: # Set file format if "format" in config: fmt = config["format"] else: fmt = formatFromPath(fn) # default use markdown if fmt is None: fmt = "markdown" # copy since pf will modify this argument pandoc_options = list(options["pandoc-options"]) if "raw" in config: rawFmt = config.get("raw") # raw block new_elems = [pf.RawBlock(raw, format=rawFmt)] else: new_elems = pf.convert_text(raw, input_format=fmt, extra_args=pandoc_options) # Get metadata (Recursive header include) new_metadata = pf.convert_text( raw, input_format=fmt, standalone=True, extra_args=pandoc_options).get_metadata(builtin=False) else: # Read header from yaml # Use pf to preserve all info new_metadata = pf.convert_text( f"---\n{raw}\n---", standalone=True).get_metadata(builtin=False) # Merge metadata if new_metadata is not None: for key in new_metadata.content: if not key in doc.metadata.content: doc.metadata[key] = new_metadata[key] # delete temp file (the file might have been deleted in subsequent executions) if os.path.exists(TEMP_FILE): os.remove(TEMP_FILE) # Restore to current path os.chdir(cur_path) options["current-path"] = currentPath # incremement headings increment = config.get('incrementSection', 0) if increment: for elem in new_elems: if isinstance(elem, pf.Header): elem.level += increment if new_elems != None: elements += new_elems return elements elif isinstance(elem, pf.CodeBlock): includeType, name, config = is_code_include(elem) if includeType == 0: return # Enable shell-style wildcards files = glob.glob(name) if len(files) == 0: eprint('[Warn] included file not found: ' + name) codes = [] for fn in files: codes.append(read_file(fn, config)) elem.text = "\n".join(codes) elif isinstance(elem, pf.Image): rewritePath = options.get("rewrite-path", True) if not rewritePath: return url = elem.url # try to parse the url first result = urlparse(url) # url if result.scheme != "": return # absolute path if os.path.isabs(url): return # rewrite relative path elem.url = os.path.join(options["current-path"], url)
def apply_filter(in_object, filter_func=None, out_format="panflute", in_format="markdown", strip_meta=False, strip_blank_lines=False, replace_api_version=True, dry_run=False, **kwargs): # type: (list[str], FunctionType) -> str """convenience function to apply a panflute filter(s) to a string, list of string lines, pandoc AST or panflute.Doc Parameters ---------- in_object: str or list[str] or dict can also be panflute.Doc filter_func: the filter function or a list of filter functions out_format: str for use by pandoc or, if 'panflute', return the panflute.Doc in_format="markdown": str strip_meta=False: bool strip the document metadata before final conversion strip_blank_lines: bool strip_ends: bool strip any blank lines or space from the start and end replace_api_version: bool for dict input only, if True, find the api_version of the available pandoc and reformat the json as appropriate dry_run: bool If True, return the Doc object, before applying the filter kwargs: to parse to filter func Returns ------- str """ if isinstance(in_object, pf.Doc): pass elif isinstance(in_object, dict): if not in_format == "json": raise AssertionError("the in_format for a dict should be json, " "not {}".format(in_format)) if "meta" not in in_object: raise ValueError( "the in_object does contain a 'meta' key") if "blocks" not in in_object: raise ValueError( "the in_object does contain a 'blocks' key") if "pandoc-api-version" not in in_object: raise ValueError( "the in_object does contain a 'pandoc-api-version' key") if replace_api_version: # run pandoc on a null object, to get the correct api version null_raw = pf.run_pandoc("", args=["-t", "json"]) null_stream = io.StringIO(null_raw) api_version = pf.load(null_stream).api_version # see panflute.load, w.r.t to legacy version if api_version is None: in_object = [{'unMeta': in_object["meta"]}, in_object["blocks"]] else: ans = OrderedDict() ans['pandoc-api-version'] = api_version ans['meta'] = in_object["meta"] ans['blocks'] = in_object["blocks"] in_object = ans in_str = json.dumps(in_object) elif isinstance(in_object, (list, tuple)): in_str = "\n".join(in_object) elif isinstance(in_object, string_types): in_str = in_object else: raise TypeError("object not accepted: {}".format(in_object)) if not isinstance(in_object, pf.Doc): doc = pf.convert_text( in_str, input_format=in_format, standalone=True) # f = io.StringIO(in_json) # doc = pf.load(f) else: doc = in_object doc.format = out_format if dry_run: return doc if not isinstance(filter_func, (list, tuple, set)): filter_func = [filter_func] out_doc = doc for func in filter_func: out_doc = func(out_doc, **kwargs) # type: Doc # post-process Doc if strip_meta: out_doc.metadata = {} if out_format == "panflute": return out_doc # create out str # with io.StringIO() as f: # pf.dump(doc, f) # jsonstr = f.getvalue() # jsonstr = json.dumps(out_doc.to_json() out_str = pf.convert_text(out_doc, input_format="panflute", output_format=out_format) # post-process final str if strip_blank_lines: out_str = out_str.replace("\n\n", "\n") return out_str
def raw(fmt, text, element_type=RawBlock): '''Return a Raw pandoc element in the given format.''' if fmt not in ['tex', 'latex', 'html', 'context']: return convert_text(text) return element_type(text, fmt)
def pypandoc_filter(body, input_format, output_format, extra_args, outputfile): from pypandoc import convert_text return convert_text(body, output_format, input_format, extra_args=extra_args, outputfile=outputfile)
def pre_stitch_ast(source: str) -> dict: return json.loads( pf.convert_text(knitty_preprosess(source), input_format='markdown', output_format='json'))
def finalize(doc): """ Finalize document. Arguments --------- doc: pandoc document """ # Loop on all listings definition if doc.format in {"tex", "latex"}: # Add header-includes if necessary if "header-includes" not in doc.metadata: doc.metadata["header-includes"] = MetaList() # Convert header-includes to MetaList if necessary elif not isinstance(doc.metadata["header-includes"], MetaList): doc.metadata["header-includes"] = MetaList( doc.metadata["header-includes"]) doc.metadata["header-includes"].append( MetaInlines(RawInline(r"\usepackage{tocloft}", "tex"))) doc.metadata["header-includes"].append( MetaInlines(RawInline(r"\usepackage{etoolbox}", "tex"))) i = 0 listof = [] for category, definition in doc.defined.items(): if definition["listing-title"] is not None: if doc.format in {"tex", "latex"}: latex_category = re.sub("[^a-z]+", "", category) latex = (r"\newlistof{%s}{%s}{%s}" r"\renewcommand{\cft%stitlefont}{\cfttoctitlefont}" r"\setlength{\cft%snumwidth}{\cftfignumwidth}" r"\setlength{\cft%sindent}{\cftfigindent}" % ( latex_category, latex_category, convert_text( Plain(*definition["listing-title"]), input_format="panflute", output_format="latex", ), latex_category, latex_category, latex_category, )) doc.metadata["header-includes"].append( MetaInlines(RawInline(latex, "tex"))) listof.append(r"\listof%s" % latex_category) else: classes = ["pandoc-numbering-listing"] + definition["classes"] if definition["listing-unnumbered"]: classes.append("unnumbered") if definition["listing-unlisted"]: classes.append("unlisted") if definition["listing-identifier"] is False: header = Header(*definition["listing-title"], level=1, classes=classes) elif definition["listing-identifier"] is True: header = Header(*definition["listing-title"], level=1, classes=classes) header = convert_text( convert_text(header, input_format="panflute", output_format="markdown"), output_format="panflute", )[0] else: header = Header( *definition["listing-title"], level=1, classes=classes, identifier=definition["listing-identifier"]) doc.content.insert(i, header) i = i + 1 table = table_other(doc, category, definition) if table: doc.content.insert(i, table) i = i + 1 if doc.format in {"tex", "latex"}: header = ( r"\ifdef{\mainmatter}" r"{\let\oldmainmatter\mainmatter\renewcommand{\mainmatter}[0]{%s\oldmainmatter}}" r"{}") doc.metadata["header-includes"].append( MetaInlines(RawInline(header % "\n".join(listof), "tex"))) latex = r"\ifdef{\mainmatter}{}{%s}" doc.content.insert(0, RawBlock(latex % "\n".join(listof), "tex"))
def convert_text(self, text=None, input_fmt='markdown', extra_args=None): '''Converts text in input_fmt to self.fmt''' if text is None: text = self.text return convert_text(text, input_fmt, self.fmt, False, extra_args)
def convert2table(options, data, doc,element): """ provided to pf.yaml_filter to parse its content as pandoc table. """ logstring("convert2table: received " +str(options) + ":" + str(data), doc) # prepare table in list from data/include raw_table_list = read_data(options.get('include', None), data,doc) # delete element if table is empty (by returning []) # element unchanged if include is invalid (by returning None) try: assert raw_table_list and raw_table_list is not None except AssertionError: logstring("pantable: table is empty or include is invalid", doc) # [] means delete the current element; None means kept as is return raw_table_list # regularize table: all rows should have same length table_list, number_of_columns = regularize_table_list(raw_table_list,doc) # Initialize the `options` output from `pf.yaml_filter` # parse width width = get_width(options, number_of_columns,doc) # auto-width when width is not specified if width is None: width = auto_width(get_table_width( options,doc), number_of_columns, table_list,doc) # delete element if table is empty (by returning []) # width remains None only when table is empty try: assert width is not None except AssertionError: logstring("pantable: table is empty",doc) return [] # parse alignment alignment = parse_alignment(options.get( 'alignment', None), number_of_columns,doc) header = options.get('header', True) markdown = options.get('markdown', True) # Boaz: change default to True # get caption: parsed as markdown into panflute AST if non-empty. caption_ = options.get('caption','') caption = pf.convert_text(caption_)[ 0].content if caption_ else None # parse list to panflute table table_body = parse_table_list(markdown, table_list,doc) # extract header row header_row = table_body.pop(0) if ( len(table_body) > 1 and header ) else None T = pf.Table( *table_body, caption=caption, alignment=alignment, width=width, header=header_row, ) id = options.get("identifier",options.get("id","")) if not id: return T if doc.format=="latex": return [T,pf.Para(pf.RawInline(fr"\label{{{id}}}",format="latex"))] if doc.format=="html": return [pf.Para(pf.RawInline(fr'<a name="{id}"></a>',format="html")),T] return T
def prepare(doc): date = doc.get_metadata('date') if date == 'today': doc.metadata['date'] = datetime.date.today().isoformat() datadir = doc.get_metadata('datadir') def highlighting(output_format): return pf.convert_text( '`_`{.default}', output_format=output_format, extra_args=[ '--highlight-style', os.path.join(datadir, 'syntax', 'wg21.theme'), '--template', os.path.join(datadir, 'template', 'highlighting') ]) doc.metadata['highlighting-macros'] = pf.MetaBlocks( pf.RawBlock(highlighting('latex'), 'latex')) doc.metadata['highlighting-css'] = pf.MetaBlocks( pf.RawBlock(highlighting('html'), 'html')) def intersperse(lst, item): result = [item] * (len(lst) * 2 - 1) result[0::2] = lst return result def codeblock(elem, doc): if not isinstance(elem, pf.CodeBlock): return None if not elem.classes: elem.classes.append('default') codeblocks.append(elem) codeblocks = [] doc.walk(codeblock) texts = pf.convert_text( intersperse(codeblocks, pf.Plain(pf.RawInline('---', doc.format))), input_format='panflute', output_format=doc.format, extra_args=[ '--syntax-definition', os.path.join(datadir, 'syntax', 'isocpp.xml') ]).split('\n---\n') assert(len(codeblocks) == len(texts)) def convert(elem, text): if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']): return elem def repl(match_obj): match = match_obj.group(1) if not match: # @@ return match_obj.group(0) if match.isspace(): # @ @ return match result = convert.cache.get(match) if result is not None: return result if doc.format == 'latex': # Undo `escapeLaTeX` from https://github.com/jgm/skylighting match = match.replace('\\textbackslash{}', '\\') \ .replace('\\{', '{') \ .replace('\\}', '}') result = pf.convert_text( pf.Plain(*pf.convert_text(match)[0].content).walk(divspan, doc), input_format='panflute', output_format=doc.format) convert.cache[match] = result return result result = pf.RawBlock(embedded.sub(repl, text), doc.format) if 'diff' not in elem.classes: return result # For HTML, this is handled via CSS in `data/template/wg21.html`. command = '\\renewcommand{{\\{}}}[1]{{\\textcolor[HTML]{{{}}}{{#1}}}}' return pf.Div( pf.RawBlock('{', 'latex'), pf.RawBlock(command.format('NormalTok', doc.get_metadata('uccolor')), 'latex'), pf.RawBlock(command.format('VariableTok', doc.get_metadata('addcolor')), 'latex'), pf.RawBlock(command.format('StringTok', doc.get_metadata('rmcolor')), 'latex'), result, pf.RawBlock('}', 'latex')) convert.cache = {} prepare.converted = [convert(elem, text) for elem, text in zip(codeblocks, texts)]
def finalize(doc): text = '\n\n'.join(doc.code_blocks) doc.content = pf.convert_text(text)
def markdown_to_table_cell(string): return panflute.TableCell(*panflute.convert_text(string))
def conversion(cls, markdown, fmt="markdown"): doc = convert_text(markdown, standalone=True) doc.format = fmt pandoc_latex_admonition.main(doc) return doc
def conversion(markdown, format="markdown"): doc = convert_text(markdown, standalone=True) doc.format = format pandoc_numbering.main(doc) return doc
def to_json(text): return pf.convert_text(text, 'markdown', 'json')
def tikz(elem, doc): """ Add admonition to elem Arguments --------- elem: The current element doc: The pandoc document Returns ------- The modified element """ # Is it in the right format and is it Div or a CodeBlock? if doc.format in ["beamer"] and elem.tag in ["Span"]: # Is there a latex-admonition-color attribute? if "beamer-arrow-node" in elem.classes: text = convert_text(Plain(*elem.content), input_format="panflute", output_format="latex") options = ["anchor=base"] color = get_color(elem, doc) if color: options.append(f"fill={color}") (from_value, to_value) = get_range(elem, doc) if from_value or to_value: display = f"\\only<{from_value}-{to_value}>" else: display = "" return RawInline( f"\\tikz[baseline]{{" f"{display}{{\\node[{','.join(options)}] " f"({elem.identifier}) " f"{{{text}}}" f";}}}}", format="tex", ) if "beamer-arrow-edge" in elem.classes: angles = [] if "angle_src" in elem.attributes: try: angle = elem.attributes["angle_src"] angle = int(angle) angles.append(f"out={angle}") except ValueError: debug( f"pandoc-beamer-arrow: angle_src '{angle}' is not correct" ) if "angle_dest" in elem.attributes: try: angle = elem.attributes["angle_dest"] angle = int(angle) angles.append(f"in={angle}") except ValueError: debug( f"pandoc-beamer-arrow: angle_dest '{angle}' is not correct" ) options = ["->"] color = get_color(elem, doc) if color: options.append(color) if "linewidth" in elem.attributes: try: linewidth = elem.attributes["linewidth"] linewidth = int(linewidth) options.append(f"line width={linewidth}pt") except ValueError: debug( f"pandoc-beamer-arrow: linewidth '{linewidth}' is not correct" ) (from_value, to_value) = get_range(elem, doc) if from_value or to_value: display = f"<{from_value}-{to_value}>" else: display = "" return RawInline( f"\\begin{{tikzpicture}}[overlay]" f"\\path[{','.join(options)}]{display} " f"({elem.attributes['src']}) " f"edge " f"[{','.join(angles)}] " f"({elem.attributes['dest']});" f"\\end{{tikzpicture}}", format="tex", )
thumb_url = None if config.has_option('wordpress','thumbnail'): thumbnail = config.get('wordpress', 'thumbnail') here = path.dirname( postfile ) thumb_url = path.join( here, thumbnail ) # Make path absolute # Wordpress related, create the post WP = Client( url, username, password ) post = WordPressPost() # Take markdown, convert to HTML and put it as post content # Makes intermediate convertion to Panflute AST to apply the filters. postdocument = pf.convert_text(postcontent, input_format='markdown', output_format='panflute', standalone=True) pf.run_filters( [ imageURLs, codeBlocks ], doc = postdocument ) content = pf.convert_text(postdocument, input_format='panflute', output_format='html') # Set post metadata post.title = title post.content = content post.post_status = post_status post.terms_names = terms_names if not thumb_url == None: thumb_mime = checkImage(thumb_url)
def test_all(): md = 'Some *markdown* **text** ~xyz~' c_md = pf.convert_text(md) b_md = [ pf.Para(pf.Str("Some"), pf.Space, pf.Emph(pf.Str("markdown")), pf.Space, pf.Strong(pf.Str("text")), pf.Space, pf.Subscript(pf.Str("xyz"))) ] print("Benchmark MD:") print(b_md) print("Converted MD:") print(c_md) assert repr(c_md) == repr(b_md) with io.StringIO() as f: doc = pf.Doc(*c_md) pf.dump(doc, f) c_md_dump = f.getvalue() with io.StringIO() as f: doc = pf.Doc(*b_md) pf.dump(doc, f) b_md_dump = f.getvalue() assert c_md_dump == b_md_dump # ---------------------- print() tex = r'Some $x^y$ or $x_n = \sqrt{a + b}$ \textit{a}' c_tex = pf.convert_text(tex) b_tex = [ pf.Para(pf.Str("Some"), pf.Space, pf.Math("x^y", format='InlineMath'), pf.Space, pf.Str("or"), pf.Space, pf.Math(r"x_n = \sqrt{a + b}", format='InlineMath'), pf.Space, pf.RawInline(r"\textit{a}", format='tex')) ] print("Benchmark TEX:") print(b_tex) print("Converted TEX:") print(c_tex) assert repr(c_tex) == repr(b_tex) with io.StringIO() as f: doc = pf.Doc(*c_tex) pf.dump(doc, f) c_tex_dump = f.getvalue() with io.StringIO() as f: doc = pf.Doc(*b_tex) pf.dump(doc, f) b_tex_dump = f.getvalue() assert c_tex_dump == b_tex_dump print("\nBack and forth conversions... md->json->md") md = 'Some *markdown* **text** ~xyz~' print("[MD]", md) md2json = pf.convert_text(md, input_format='markdown', output_format='json') print("[JSON]", md2json) md2json2md = pf.convert_text(md2json, input_format='json', output_format='markdown') print("[MD]", md2json2md) assert md == md2json2md print("\nBack and forth conversions... md->panflute->md") md = 'Some *markdown* **text** ~xyz~' print("[MD]", md) md2panflute = pf.convert_text(md, input_format='markdown', output_format='panflute') print("[PANFLUTE]", md2panflute) md2panflute2md = pf.convert_text(md2panflute, input_format='panflute', output_format='markdown') print("[MD]", md2panflute2md) assert md == md2panflute2md print("\nBack and forth conversions... md->panflute(standalone)->md") md = 'Some *markdown* **text** ~xyz~' print("[MD]", md) md2panflute = pf.convert_text(md, input_format='markdown', output_format='panflute', standalone=True) print("[PANFLUTE]", md2panflute) md2panflute2md = pf.convert_text(md2panflute, input_format='panflute', output_format='markdown') print("[MD]", md2panflute2md) assert md == md2panflute2md print( "\nBack and forth conversions... md table -> json(standalone) -> md table" ) md = """ --- --- x y --- --- """ print("[MD]", repr(md)) md2json = pf.convert_text(md, input_format='markdown', output_format='json', standalone=True) print("[json]", md2json) md2json2md = pf.convert_text(md2json, input_format='json', output_format='markdown') print("[MD]", repr(md2json2md)) assert md == md2json2md print( "\nBack and forth conversions... md table -> panflute(standalone) -> md table" ) print("[MD]", repr(md)) md2panflute = pf.convert_text(md, input_format='markdown', output_format='panflute', standalone=True) print("[PANFLUTE]", md2panflute) md2panflute2md = pf.convert_text(md2panflute, input_format='panflute', output_format='markdown') print("[MD]", repr(md2panflute2md)) assert md == md2panflute2md
def panflute_filter(body, input_format, output_format, extra_args, outputfile): from panflute import convert_text return convert_text(body, input_format=input_format, output_format=output_format, extra_args=extra_args)
def conversion(markdown, fmt="markdown"): doc = convert_text(markdown, standalone=True) doc.format = fmt pandoc_beamer_arrow.main(doc) return doc
def plain(elem: Union[panflute.Div, panflute.Header], doc: panflute.Doc) -> Optional[panflute.Div]: """Assemble the plain text version of the acronym list The base plain text output is a bulleted list of acronyms in the following format:: - {short}: {long} in a new :class:`panflute.Div` with the identifier “acronym-list”. If the given element is a :class:`panflute.Div`, the list is placed under a level 1 header with the text “Acronyms” unless the ``name`` or ``level`` attributes are set in which case, the request is honored. The list is sorted by the short version of the acronyms by default unless the ``sort`` attribute is set to “false” (case insensitive) in which case the order is unspecified. If an attribute cannot be interpreted, it is omitted and a warning is logged. Parameters ---------- elem: :class:`panflute.Div` or :class:`panflute.Header` The element to replace doc: :class:`panflute.Doc` The document under consideration. Returns ------- :class:`panflute.Div`, optional: The replacement for the block. """ logger = logging.getLogger(__name__ + ".plain_text") if "acronyms" not in doc.metadata: return None if isinstance(elem, panflute.Header): header = elem elif isinstance(elem, panflute.Div): header = panflute.Header(panflute.Str( elem.attributes.get("name", "Acronyms")), level=elem.attributes.get("level", 1)) else: cls = type(elem) logger.warning(f"Unknown element type {cls}") return None if "sort" in elem.attributes: sort = elem.attributes["sort"].lower() if sort not in ("true", "false"): sort = "true" logger.warning(f"Unknown 'sort' option '{sort}'") else: sort = "true" if sort == "true": acronyms = sorted(doc.acronyms.values(), key=lambda x: x["short"]) else: acronyms = doc.acronyms.values() acrolist = [ panflute.ListItem( panflute.Plain(panflute.Strong(panflute.Str(acro["short"])), panflute.Str(":"), panflute.Space, *panflute.convert_text(acro["long"])[0].content)) for acro in acronyms if acro["list"] ] return panflute.Div(header, panflute.BulletList(*acrolist), identifier="acronym-list")
def process_raw_spans(container, doc): # type: (Span, Doc) -> Element if not isinstance(container, (pf.Span, pf.Div)): return None hide_raw = doc.get_metadata(IPUB_META_ROUTE + ".hide_raw", False) if CONVERTED_OTHER_CLASS in container.classes and isinstance( container, pf.Span): if doc.format == "rst" and container.attributes["format"] == "latex": if container.attributes["tag"] in ["todo"]: return pf.Str("\n\n.. {}:: {}\n\n".format( container.attributes["tag"], container.attributes["content"])) if container.attributes["tag"] == "ensuremath": return pf.RawInline(":math:`{}`".format( container.attributes["content"]), format="rst") return pf.RawInline(container.attributes.get("original"), format=container.attributes["format"]) if CONVERTED_DIRECTIVE_CLASS in container.classes and isinstance( container, pf.Div): # convert the directive head, which will be e.g. # Para(Str(..) Space Str(toctree::) SoftBreak Str(:maxdepth:) Space Str(2) SoftBreak Str(:numbered:)) # noqa # we need to spilt on the soft breaks, # place them on a new line and re-indent them if doc.format in ("rst"): # split into lines by soft breaks header_lines = [ list(y) for x, y in itertools.groupby( container.content[0].content, lambda z: isinstance(z, pf.SoftBreak)) if not x ] # wrap each line in a Para and convert block with pandoc head_doc = pf.Doc(*[pf.Para(*l) for l in header_lines]) head_doc.api_version = doc.api_version head_str = pf.convert_text(head_doc, input_format="panflute", output_format=doc.format) # remove blank lines and indent head_str = head_str.replace("\n\n", "\n ") + "\n\n" head_block = pf.RawBlock(head_str, format=doc.format) if len(container.content) == 1: return head_block # split into lines by soft breaks, we use indicators to tell # us where to indent in the converted text body_blocks = [] for block in container.content[1:]: new_elements = [pf.RawInline("%^*", format=doc.format)] for el in block.content: if isinstance(el, pf.SoftBreak): new_elements.append( pf.RawInline("?&@", format=doc.format)) else: new_elements.append(el) block.content = new_elements body_blocks.append(block) # convert body content with pandoc body_doc = pf.Doc(*body_blocks) body_doc.api_version = doc.api_version body_str = pf.convert_text(body_doc, input_format="panflute", output_format=doc.format) # raise ValueError(body_blocks) body_str = body_str.replace("%^*", " ").replace("?&@", "\n ") # ensure all lines are indented correctly # (doesn't occur by default?) body_str = ("\n".join([ " " + l.lstrip() if l.strip() else l for l in body_str.splitlines() ]) + "\n\n") body_block = pf.RawBlock(body_str, format=doc.format) return [head_block, body_block] elif (doc.format in ("html", "html5") and container.attributes["format"] == "rst"): if hide_raw: return [] head_para = pf.Para(*[ pf.RawInline("<br>" + " " * 4) if isinstance(c, pf.SoftBreak) else c for c in container.content[0].content ]) head_str = pf.convert_text(head_para, input_format="panflute", output_format=doc.format) if len(container.content) > 1: body_doc = pf.Doc(*container.content[1:]) body_doc.api_version = doc.api_version body_str = pf.convert_text(body_doc, input_format="panflute", output_format=doc.format) body_str = ('<p></p><div style="margin-left: 20px">' "{0}</div>").format(body_str) else: body_str = "" return pf.RawBlock( '<div {0} style="background-color:rgba(10, 225, 10, .2)">' "{1}{2}" "</div>".format(container.attributes.get("directive", ""), head_str, body_str), format="html", ) elif doc.format in ( "tex", "latex") and container.attributes["format"] == "rst": if hide_raw: return [] directive = container.attributes.get("directive", "") inline = container.attributes.get("inline", "") # TODO handle directive with options and/or inline body # e.g. .. figure:: path/to/figure # :centre: box_open = ( "\\begin{{mdframed}}" "[frametitle={{{0}}},frametitlerule=true]".format(directive)) if inline: box_open += "\n\\mdfsubtitle{{{0}}}".format(inline) box_close = "\\end{mdframed}" if len(container.content) == 1: return pf.RawBlock(box_open + box_close, format="tex") else: return ([pf.RawBlock(box_open, format="tex")] + list(container.content[1:]) + [pf.RawBlock(box_close, format="tex")]) return pf.RawBlock( pf.stringify(pf.Doc(*container.content)), format=container.attributes["format"], ) if CONVERTED_OTHER_CLASS in container.classes and isinstance( container, pf.Div): return pf.RawBlock( pf.stringify(pf.Doc(*container.content)), format=container.attributes["format"], )