def repl(match_obj):
match = match_obj.group(1)
if not match: # @@
return match_obj.group(0)
if match.isspace(): # @ @
return match
return pf.convert_text(
pf.Plain(*pf.convert_text(match)[0].content),
input_format='panflute',
output_format=doc.format)
def highlighting(output_format):
return pf.convert_text(
'`_`{.cpp}',
output_format=output_format,
extra_args=[
'--highlight-style', f.name,
'--template', os.path.join(datadir, 'template', 'highlighting')
])
def run_test(name, action):
input_fn = os.path.join('tests', name + '.md')
# Read markdown, convert to JSON and then to elements
with open(input_fn, encoding='utf-8') as f:
md = f.read()
print('~' * 80)
print(' ' * 30, 'INPUT')
print('~' * 80)
print(md)
print('~' * 80, '\n')
print('... Parsing markdown')
doc = pf.convert_text(md, output_format='doc')
doc.format = 'markdown'
assert type(doc) == pf.Doc
print(' Done.')
# Walk through AST
sys.path.append('filters')
print('... Importing module')
mod = importlib.import_module(name)
print(' Done.')
f_action = mod.__dict__[action]
print('... Applying filters')
altered = doc.walk(f_action, doc)
print(' Done.')
# Convert AST into JSON
print('... Converting document into JSON')
with io.StringIO() as f:
pf.dump(altered, f)
contents = f.getvalue()
print(' Done.')
# Convert JSON into markdown
print('... Converting JSON into markdown')
md = pf.convert_text(contents, input_format='json', output_format='markdown')
print(' Done.')
print('~' * 80)
print(' ' * 30, 'OUTPUT')
print('~' * 80)
print(md)
print('~' * 80, '\n')
def assert3(*extra_args, stdin):
"""
filters=None, search_dirs=None, data_dir=True, sys_path=True, panfl_=False
"""
sys.argv[1:] = []
sys.argv.append('markdown')
_stdout = io.StringIO()
pf.stdio(*extra_args, input_stream=io.StringIO(stdin), output_stream=_stdout)
_stdout = pf.convert_text(_stdout.getvalue(), 'json', 'markdown')
assert _stdout == out1
def parse_table_list(markdown, table_list,doc):
"""
read table in list and return panflute table format
"""
# make functions local
to_table_row = pf.TableRow
if markdown:
to_table_cell = lambda x: pf.TableCell(*pf.convert_text(x))
else:
to_table_cell = lambda x: pf.TableCell(
pf.Plain(pf.Str(x)))
return [to_table_row(*[to_table_cell(x) for x in row]) for row in table_list]
def action(elem, doc):
if isinstance(elem, pf.Para) and is_include_line(elem):
fn = get_filename(elem)
if not os.path.isfile(fn):
return
with open(fn) as f:
raw = f.read()
new_elems = pf.convert_text(raw)
div = pf.Div(*new_elems, attributes={'source': fn})
return div
def action(elem, doc):
if isinstance(elem, pf.Para) and is_include_line(elem):
fn = get_filename(elem)
if not os.path.isfile(fn):
return
with open(fn) as f:
raw = f.read()
new_elems = pf.convert_text(raw)
# Alternative A:
return new_elems
def codeblock(elem, doc):
if not isinstance(elem, pf.CodeBlock):
return None
is_raw = not elem.classes
if is_raw:
elem.classes.append('default')
elem.attributes['style'] = 'color: inherit'
if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']):
return None
if escape_char not in elem.text:
return None
datadir = doc.get_metadata('datadir')
syntaxdir = os.path.join(datadir, 'syntax')
text = pf.convert_text(
elem,
input_format='panflute',
output_format=doc.format,
extra_args=[
'--syntax-definition', os.path.join(syntaxdir, 'isocpp.xml')
])
def repl(match_obj):
match = match_obj.group(1)
if not match: # @@
return match_obj.group(0)
if match.isspace(): # @ @
return match
return pf.convert_text(
pf.Plain(*pf.convert_text(match)[0].content),
input_format='panflute',
output_format=doc.format)
result = pf.RawBlock(escape_span.sub(repl, text), doc.format)
return pf.Div(
pf.RawBlock('{\\renewcommand{\\NormalTok}[1]{#1}', 'latex'),
result,
pf.RawBlock('}', 'latex')) if is_raw else result
def _create_images(doc, icons, size):
# Generate the LaTeX image code
images = []
for icon in icons:
# Get the apps dirs
from pkg_resources import get_distribution
import appdirs
folder = appdirs.AppDirs(
"pandoc_latex_tip", version=get_distribution("pandoc_latex_tip").version
).user_cache_dir
# Get the image from the App cache folder
image_dir = os.path.join(
folder, icon["collection"], icon["version"], icon["variant"], icon["color"]
)
image = os.path.join(image_dir, icon["extended-name"] + ".png")
# Create the image if not existing in the cache
try:
if not os.path.isfile(image):
# Create the image in the cache
category = _category(
icon["collection"], icon["version"], icon["variant"]
)
doc.get_icon_font[category]["font"].export_icon(
icon["extended-name"],
512,
color=icon["color"],
export_dir=image_dir,
)
# Add the LaTeX image
image = Image(
url=image, attributes={"width": size + "pt", "height": size + "pt"}
)
if icon["link"] == "":
elem = image
else:
elem = Link(image, url=icon["link"])
images.append(
convert_text(
Plain(elem), input_format="panflute", output_format="latex"
)
)
except TypeError:
debug(
"[WARNING] pandoc-latex-tip: icon name "
+ icon["name"]
+ " does not exist in variant "
+ icon["variant"]
+ " for collection "
+ icon["collection"]
+ "-"
+ icon["version"]
)
except FileNotFoundError:
debug("[WARNING] pandoc-latex-tip: error in generating image")
return images
def action(elem, doc):
global entryEnter
global options
if isinstance(elem, pf.Para):
includeType = is_include_line(elem)
if includeType == 0:
return
# Try to read inherited options from temp file
if options is None:
try:
with open(temp_filename, 'r') as f:
options = json.load(f)
except:
options = {}
pass
# pandoc options
pandoc_options = doc.get_metadata('pandoc-options')
if not pandoc_options:
if 'pandoc-options' in options:
pandoc_options = options['pandoc-options']
else:
# default options
pandoc_options = ['--filter=pandoc-include']
else:
# Replace em-dash to double dashes in smart typography
for i in range(len(pandoc_options)):
pandoc_options[i] = pandoc_options[i].replace('\u2013', '--')
options['pandoc-options'] = pandoc_options
# The entry file's directory
entry = doc.get_metadata('include-entry')
if not entryEnter and entry:
os.chdir(entry)
entryEnter = True
fn = get_filename(elem, includeType)
if not os.path.isfile(fn):
raise ValueError('Included file not found: ' + fn + ' ' + entry +
' ' + os.getcwd())
with open(fn, encoding="utf-8") as f:
raw = f.read()
# Save current path
cur_path = os.getcwd()
# Change to included file's path so that sub-include's path is correct
target = os.path.dirname(fn)
# Empty means relative to current dir
if not target:
target = '.'
os.chdir(target)
# save options
with open(temp_filename, 'w+') as f:
json.dump(options, f)
# Add recursive include support
new_elems = None
new_metadata = None
if includeType == 1:
new_elems = pf.convert_text(raw, extra_args=pandoc_options)
# Get metadata (Recursive header include)
new_metadata = pf.convert_text(
raw, standalone=True,
extra_args=pandoc_options).get_metadata()
else:
# Read header from yaml
new_metadata = yaml.load(raw)
new_metadata = OrderedDict(new_metadata)
# Merge metadata
for key in new_metadata:
if not key in doc.get_metadata():
doc.metadata[key] = new_metadata[key]
# delete temp file
os.remove(temp_filename)
# Restore to current path
os.chdir(cur_path)
# Alternative A:
return new_elems
def finalize(doc):
doc.content = pf.convert_text('\n'.join(b.text for b in doc.code_blocks))
def get_caption(options):
'''parsed as markdown into panflute AST if non-empty.'''
return panflute.convert_text(str(
options['caption']))[0].content if 'caption' in options else None
def finalize(doc):
def init_code_elems(elem, doc):
if isinstance(elem, pf.Header) and doc.format == 'latex':
elem.walk(lambda elem, doc: elem.classes.append('raw') if any(
isinstance(elem, cls)
for cls in [pf.Code, pf.CodeBlock]) else None)
# Mark code elements within colored divspan as default.
if any(isinstance(elem, cls) for cls in [pf.Div, pf.Span]) and \
any(cls in elem.classes for cls in ['add', 'rm', 'ednote']):
elem.walk(lambda elem, doc: elem.classes.insert(0, 'default')
if any(
isinstance(elem, cls)
for cls in [pf.Code, pf.CodeBlock]) else None)
if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]):
return None
# As `walk` performs post-order traversal, this is
# guaranteed to run before the 'raw' code path.
if not elem.classes:
if isinstance(elem, pf.Code):
cls = doc.get_metadata('highlighting.inline-code', 'default')
elif isinstance(elem, pf.CodeBlock):
cls = doc.get_metadata('highlighting.code-block', 'default')
elem.classes.append(cls)
doc.walk(init_code_elems)
def collect_code_elems(elem, doc):
if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]):
return None
if 'raw' in elem.classes:
return None
if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']):
return None
code_elems.append(elem)
code_elems = []
doc.walk(collect_code_elems)
if not code_elems:
return
def intersperse(lst, item):
result = [item] * (len(lst) * 2 - 1)
result[0::2] = lst
return result
datadir = doc.get_metadata('datadir')
text = pf.convert_text(intersperse([
pf.Plain(elem) if isinstance(elem, pf.Code) else elem
for elem in code_elems
], pf.Plain(pf.RawInline('---', doc.format))),
input_format='panflute',
output_format=doc.format,
extra_args=[
'--syntax-definition',
os.path.join(datadir, 'syntax', 'isocpp.xml')
])
# Workaround for https://github.com/jgm/skylighting/issues/91.
if doc.format == 'latex':
text = text.replace('<', '\\textless{}') \
.replace('>', '\\textgreater{}')
if doc.format == 'latex':
texts = text.split('\n\n---\n\n')
elif doc.format == 'html':
texts = text.split('\n---\n')
assert (len(code_elems) == len(texts))
def convert(elem, text):
def repl2(match):
if match.isspace(): # @ @
return match
result = convert.cache.get(match)
if result is not None:
return result
if doc.format == 'latex':
# Undo `escapeLaTeX` from https://github.com/jgm/skylighting
match = match.replace('\\textbackslash{}', '\\') \
.replace('\\{', '{') \
.replace('\\}', '}') \
.replace('\\VerbBar{}', '|') \
.replace('\\_', '_') \
.replace('\\&', '&') \
.replace('\\%', '%') \
.replace('\\#', '#') \
.replace('\\textasciigrave{}', '`') \
.replace('\\textquotesingle{}', '\'') \
.replace('{-}', '-') \
.replace('\\textasciitilde{}', '~') \
.replace('\\^{}', '^')
# Undo the workaround escaping.
match = match.replace('\\textless{}', '<') \
.replace('\\textgreater{}', '>')
elif doc.format == 'html':
match = html.unescape(match)
result = pf.convert_text(
pf.Plain(*pf.convert_text(match)[0].content).walk(
divspan, doc).walk(init_code_elems, doc),
input_format='panflute',
output_format=doc.format,
extra_args=[
'--syntax-definition',
os.path.join(datadir, 'syntax', 'isocpp.xml')
])
convert.cache[match] = result
return result
def repl(match_obj):
groups = match_obj.groups()
if not any(groups):
return match_obj.group()
group = groups[0]
if group is not None:
return embedded_md.sub(repl, repl2(group))
group = groups[1]
if group is not None:
return repl2(group)
if isinstance(elem, pf.Code):
result = pf.RawInline(embedded_md.sub(repl, text), doc.format)
elif isinstance(elem, pf.CodeBlock):
result = pf.RawBlock(embedded_md.sub(repl, text), doc.format)
if 'diff' not in elem.classes:
return result
# For HTML, this is handled via CSS in `data/templates/wg21.html`.
command = '\\renewcommand{{\\{}}}[1]{{\\textcolor[HTML]{{{}}}{{#1}}}}'
uc = command.format('NormalTok', doc.get_metadata('uccolor'))
add = command.format('VariableTok', doc.get_metadata('addcolor'))
rm = command.format('StringTok', doc.get_metadata('rmcolor'))
if isinstance(elem, pf.Code):
return pf.Span(pf.RawInline(uc,
'latex'), pf.RawInline(add, 'latex'),
pf.RawInline(rm, 'latex'), result)
elif isinstance(elem, pf.CodeBlock):
return pf.Div(pf.RawBlock('{', 'latex'), pf.RawBlock(uc, 'latex'),
pf.RawBlock(add, 'latex'), pf.RawBlock(rm, 'latex'),
result, pf.RawBlock('}', 'latex'))
convert.cache = {}
def code_elem(elem, doc):
if not any(isinstance(elem, cls) for cls in [pf.Code, pf.CodeBlock]):
return None
if 'raw' in elem.classes:
return None
if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']):
return None
return convert(*next(converted))
converted = zip(code_elems, texts)
doc.walk(code_elem)
def test_sharp_sharp(self):
definition = r"Example ##"
doc = Doc(*convert_text(definition))
pandoc_numbering.main(doc)
self.assertEqual(doc.content[0].content[-1].text, "#")
def format_table(table, doc):
# type: (Table, Doc) -> Element
"""
originally adapted from:
`pandoc-tablenos <https://github.com/tomduck/pandoc-tablenos>`_
"""
if not isinstance(table, pf.Table):
return None
div = None # type: pf.Div
if (isinstance(table.parent, pf.Div)
and LABELLED_TABLE_CLASS in table.parent.classes):
div = table.parent
if div is None:
return None
attributes = convert_attributes(div.attributes)
if "align" in div.attributes:
align_text = attributes["align"]
align = [{
"l": "AlignLeft",
"r": "AlignRight",
"c": "AlignCenter"
}.get(a, None) for a in align_text]
if None in align:
raise ValueError("table '{0}' alignment must contain only l,r,c:"
" {1}".format(div.identifier, align_text))
table.alignment = align
attributes["align"] = align
if "widths" in div.attributes:
widths = attributes["widths"]
try:
widths = [float(w) for w in widths]
except Exception:
raise ValueError("table '{0}' widths must be a list of numbers:"
" {1}".format(div.identifier, widths))
table.width = widths
attributes["widths"] = widths
if doc.format in ("tex", "latex"):
# TODO placement
table.caption.append(
pf.RawInline("\\label{{{0}}}".format(div.identifier),
format="tex"))
return table
if doc.format in ("rst", ):
# pandoc 2.6 doesn't output table options
if attributes:
tbl_doc = pf.Doc(table)
tbl_doc.api_version = doc.api_version
tbl_str = pf.convert_text(tbl_doc,
input_format="panflute",
output_format="rst")
tbl_lines = tbl_str.splitlines()
if tbl_lines[1].strip() == "":
tbl_lines.insert(1, " :align: center")
if "widths" in attributes:
# in rst widths must be integers
widths = " ".join([str(int(w * 10)) for w in table.width])
tbl_lines.insert(1, " :widths: {}".format(widths))
# TODO rst column alignment, see
# https://cloud-sptheme.readthedocs.io/en/latest/lib/cloud_sptheme.ext.table_styling.html
return [
pf.Para(
pf.RawInline(".. _`{0}`:".format(div.identifier),
format="rst")),
pf.RawBlock("\n".join(tbl_lines) + "\n\n", format=doc.format),
]
return [
pf.Para(
pf.RawInline(".. _`{0}`:".format(div.identifier),
format="rst")),
table,
]
if doc.format in ("html", "html5"):
return _wrap_in_anchor(table, div.identifier, inline=False)
def test_doctest_no_kernel_configured(tmp_path):
res = Path.resolve(Path(__file__)).parent
doc = convert_text(Path(res / "no_kernel_configured.md").read_text(),
standalone=True)
with pytest.raises(RuntimeError):
run_doctest(doc)
def test_doctest_missing_ref(tmp_path):
res = Path.resolve(Path(__file__)).parent
doc = convert_text(Path(res / "missing_ref.md").read_text(),
standalone=True)
with pytest.raises(ValueError):
run_doctest(doc)
def run(self):
url = self.renderer(self.input, self.cmd_renderer)
pfcaption = (pf.convert_text(self.caption))[0].content
return pf.Para(
pf.Image(*pfcaption, url=url, title=u'fig:' + self.caption))
def ast_to_markdown(ast):
"""convert panflute AST to Markdown"""
return panflute.convert_text(ast,
input_format='panflute',
output_format='markdown')
def action(elem, doc):
global entryEnter
global options
# Try to read inherited options from temp file
if options is None:
options = parseOptions(doc)
# The entry file's directory
entry = doc.get_metadata('include-entry')
if not entryEnter and entry:
os.chdir(entry)
entryEnter = True
if isinstance(elem, pf.Para):
includeType, name, config = is_include_line(elem)
if includeType == 0:
return
# Enable shell-style wildcards
files = glob.glob(name)
if len(files) == 0:
eprint('[Warn] included file not found: ' + name)
# order
include_order = options['include-order']
if include_order == 'natural':
files = natsorted(files)
elif include_order == 'alphabetical':
files = sorted(files)
elif include_order == 'default':
pass
else:
raise ValueError('Invalid file order: ' + include_order)
elements = []
for fn in files:
if not os.path.isfile(fn):
continue
raw = read_file(fn, config)
# Save current path
cur_path = os.getcwd()
# Change to included file's path so that sub-include's path is correct
target = os.path.dirname(fn)
# Empty means relative to current dir
if not target:
target = '.'
currentPath = options["current-path"]
options["current-path"] = os.path.normpath(
os.path.join(currentPath, target))
os.chdir(target)
# pass options by temp files
with open(TEMP_FILE, 'w+') as f:
json.dump(options, f)
# Add recursive include support
new_elems = None
new_metadata = None
if includeType == 1:
# Set file format
if "format" in config:
fmt = config["format"]
else:
fmt = formatFromPath(fn)
# default use markdown
if fmt is None:
fmt = "markdown"
# copy since pf will modify this argument
pandoc_options = list(options["pandoc-options"])
if "raw" in config:
rawFmt = config.get("raw")
# raw block
new_elems = [pf.RawBlock(raw, format=rawFmt)]
else:
new_elems = pf.convert_text(raw,
input_format=fmt,
extra_args=pandoc_options)
# Get metadata (Recursive header include)
new_metadata = pf.convert_text(
raw,
input_format=fmt,
standalone=True,
extra_args=pandoc_options).get_metadata(builtin=False)
else:
# Read header from yaml
# Use pf to preserve all info
new_metadata = pf.convert_text(
f"---\n{raw}\n---",
standalone=True).get_metadata(builtin=False)
# Merge metadata
if new_metadata is not None:
for key in new_metadata.content:
if not key in doc.metadata.content:
doc.metadata[key] = new_metadata[key]
# delete temp file (the file might have been deleted in subsequent executions)
if os.path.exists(TEMP_FILE):
os.remove(TEMP_FILE)
# Restore to current path
os.chdir(cur_path)
options["current-path"] = currentPath
# incremement headings
increment = config.get('incrementSection', 0)
if increment:
for elem in new_elems:
if isinstance(elem, pf.Header):
elem.level += increment
if new_elems != None:
elements += new_elems
return elements
elif isinstance(elem, pf.CodeBlock):
includeType, name, config = is_code_include(elem)
if includeType == 0:
return
# Enable shell-style wildcards
files = glob.glob(name)
if len(files) == 0:
eprint('[Warn] included file not found: ' + name)
codes = []
for fn in files:
codes.append(read_file(fn, config))
elem.text = "\n".join(codes)
elif isinstance(elem, pf.Image):
rewritePath = options.get("rewrite-path", True)
if not rewritePath:
return
url = elem.url
# try to parse the url first
result = urlparse(url)
# url
if result.scheme != "":
return
# absolute path
if os.path.isabs(url):
return
# rewrite relative path
elem.url = os.path.join(options["current-path"], url)
def apply_filter(in_object, filter_func=None,
out_format="panflute", in_format="markdown",
strip_meta=False, strip_blank_lines=False,
replace_api_version=True, dry_run=False,
**kwargs):
# type: (list[str], FunctionType) -> str
"""convenience function to apply a panflute filter(s)
to a string, list of string lines, pandoc AST or panflute.Doc
Parameters
----------
in_object: str or list[str] or dict
can also be panflute.Doc
filter_func:
the filter function or a list of filter functions
out_format: str
for use by pandoc or, if 'panflute', return the panflute.Doc
in_format="markdown": str
strip_meta=False: bool
strip the document metadata before final conversion
strip_blank_lines: bool
strip_ends: bool
strip any blank lines or space from the start and end
replace_api_version: bool
for dict input only, if True,
find the api_version of the available pandoc and
reformat the json as appropriate
dry_run: bool
If True, return the Doc object, before applying the filter
kwargs:
to parse to filter func
Returns
-------
str
"""
if isinstance(in_object, pf.Doc):
pass
elif isinstance(in_object, dict):
if not in_format == "json":
raise AssertionError("the in_format for a dict should be json, "
"not {}".format(in_format))
if "meta" not in in_object:
raise ValueError(
"the in_object does contain a 'meta' key")
if "blocks" not in in_object:
raise ValueError(
"the in_object does contain a 'blocks' key")
if "pandoc-api-version" not in in_object:
raise ValueError(
"the in_object does contain a 'pandoc-api-version' key")
if replace_api_version:
# run pandoc on a null object, to get the correct api version
null_raw = pf.run_pandoc("", args=["-t", "json"])
null_stream = io.StringIO(null_raw)
api_version = pf.load(null_stream).api_version
# see panflute.load, w.r.t to legacy version
if api_version is None:
in_object = [{'unMeta': in_object["meta"]},
in_object["blocks"]]
else:
ans = OrderedDict()
ans['pandoc-api-version'] = api_version
ans['meta'] = in_object["meta"]
ans['blocks'] = in_object["blocks"]
in_object = ans
in_str = json.dumps(in_object)
elif isinstance(in_object, (list, tuple)):
in_str = "\n".join(in_object)
elif isinstance(in_object, string_types):
in_str = in_object
else:
raise TypeError("object not accepted: {}".format(in_object))
if not isinstance(in_object, pf.Doc):
doc = pf.convert_text(
in_str, input_format=in_format, standalone=True)
# f = io.StringIO(in_json)
# doc = pf.load(f)
else:
doc = in_object
doc.format = out_format
if dry_run:
return doc
if not isinstance(filter_func, (list, tuple, set)):
filter_func = [filter_func]
out_doc = doc
for func in filter_func:
out_doc = func(out_doc, **kwargs) # type: Doc
# post-process Doc
if strip_meta:
out_doc.metadata = {}
if out_format == "panflute":
return out_doc
# create out str
# with io.StringIO() as f:
# pf.dump(doc, f)
# jsonstr = f.getvalue()
# jsonstr = json.dumps(out_doc.to_json()
out_str = pf.convert_text(out_doc,
input_format="panflute",
output_format=out_format)
# post-process final str
if strip_blank_lines:
out_str = out_str.replace("\n\n", "\n")
return out_str
def raw(fmt, text, element_type=RawBlock):
'''Return a Raw pandoc element in the given format.'''
if fmt not in ['tex', 'latex', 'html', 'context']:
return convert_text(text)
return element_type(text, fmt)
def pypandoc_filter(body, input_format, output_format, extra_args, outputfile):
from pypandoc import convert_text
return convert_text(body, output_format, input_format, extra_args=extra_args, outputfile=outputfile)
def pre_stitch_ast(source: str) -> dict:
return json.loads(
pf.convert_text(knitty_preprosess(source),
input_format='markdown',
output_format='json'))
def finalize(doc):
"""
Finalize document.
Arguments
---------
doc: pandoc document
"""
# Loop on all listings definition
if doc.format in {"tex", "latex"}:
# Add header-includes if necessary
if "header-includes" not in doc.metadata:
doc.metadata["header-includes"] = MetaList()
# Convert header-includes to MetaList if necessary
elif not isinstance(doc.metadata["header-includes"], MetaList):
doc.metadata["header-includes"] = MetaList(
doc.metadata["header-includes"])
doc.metadata["header-includes"].append(
MetaInlines(RawInline(r"\usepackage{tocloft}", "tex")))
doc.metadata["header-includes"].append(
MetaInlines(RawInline(r"\usepackage{etoolbox}", "tex")))
i = 0
listof = []
for category, definition in doc.defined.items():
if definition["listing-title"] is not None:
if doc.format in {"tex", "latex"}:
latex_category = re.sub("[^a-z]+", "", category)
latex = (r"\newlistof{%s}{%s}{%s}"
r"\renewcommand{\cft%stitlefont}{\cfttoctitlefont}"
r"\setlength{\cft%snumwidth}{\cftfignumwidth}"
r"\setlength{\cft%sindent}{\cftfigindent}" % (
latex_category,
latex_category,
convert_text(
Plain(*definition["listing-title"]),
input_format="panflute",
output_format="latex",
),
latex_category,
latex_category,
latex_category,
))
doc.metadata["header-includes"].append(
MetaInlines(RawInline(latex, "tex")))
listof.append(r"\listof%s" % latex_category)
else:
classes = ["pandoc-numbering-listing"] + definition["classes"]
if definition["listing-unnumbered"]:
classes.append("unnumbered")
if definition["listing-unlisted"]:
classes.append("unlisted")
if definition["listing-identifier"] is False:
header = Header(*definition["listing-title"],
level=1,
classes=classes)
elif definition["listing-identifier"] is True:
header = Header(*definition["listing-title"],
level=1,
classes=classes)
header = convert_text(
convert_text(header,
input_format="panflute",
output_format="markdown"),
output_format="panflute",
)[0]
else:
header = Header(
*definition["listing-title"],
level=1,
classes=classes,
identifier=definition["listing-identifier"])
doc.content.insert(i, header)
i = i + 1
table = table_other(doc, category, definition)
if table:
doc.content.insert(i, table)
i = i + 1
if doc.format in {"tex", "latex"}:
header = (
r"\ifdef{\mainmatter}"
r"{\let\oldmainmatter\mainmatter\renewcommand{\mainmatter}[0]{%s\oldmainmatter}}"
r"{}")
doc.metadata["header-includes"].append(
MetaInlines(RawInline(header % "\n".join(listof), "tex")))
latex = r"\ifdef{\mainmatter}{}{%s}"
doc.content.insert(0, RawBlock(latex % "\n".join(listof), "tex"))
def convert_text(self, text=None, input_fmt='markdown', extra_args=None):
'''Converts text in input_fmt to self.fmt'''
if text is None:
text = self.text
return convert_text(text, input_fmt, self.fmt, False, extra_args)
def convert2table(options, data, doc,element):
"""
provided to pf.yaml_filter to parse its content as pandoc table.
"""
logstring("convert2table: received " +str(options) + ":" + str(data), doc)
# prepare table in list from data/include
raw_table_list = read_data(options.get('include', None), data,doc)
# delete element if table is empty (by returning [])
# element unchanged if include is invalid (by returning None)
try:
assert raw_table_list and raw_table_list is not None
except AssertionError:
logstring("pantable: table is empty or include is invalid", doc)
# [] means delete the current element; None means kept as is
return raw_table_list
# regularize table: all rows should have same length
table_list, number_of_columns = regularize_table_list(raw_table_list,doc)
# Initialize the `options` output from `pf.yaml_filter`
# parse width
width = get_width(options, number_of_columns,doc)
# auto-width when width is not specified
if width is None:
width = auto_width(get_table_width(
options,doc), number_of_columns, table_list,doc)
# delete element if table is empty (by returning [])
# width remains None only when table is empty
try:
assert width is not None
except AssertionError:
logstring("pantable: table is empty",doc)
return []
# parse alignment
alignment = parse_alignment(options.get(
'alignment', None), number_of_columns,doc)
header = options.get('header', True)
markdown = options.get('markdown', True) # Boaz: change default to True
# get caption: parsed as markdown into panflute AST if non-empty.
caption_ = options.get('caption','')
caption = pf.convert_text(caption_)[
0].content if caption_ else None
# parse list to panflute table
table_body = parse_table_list(markdown, table_list,doc)
# extract header row
header_row = table_body.pop(0) if (
len(table_body) > 1 and header
) else None
T = pf.Table(
*table_body,
caption=caption,
alignment=alignment,
width=width,
header=header_row,
)
id = options.get("identifier",options.get("id",""))
if not id: return T
if doc.format=="latex":
return [T,pf.Para(pf.RawInline(fr"\label{{{id}}}",format="latex"))]
if doc.format=="html":
return [pf.Para(pf.RawInline(fr'<a name="{id}"></a>',format="html")),T]
return T
def prepare(doc):
date = doc.get_metadata('date')
if date == 'today':
doc.metadata['date'] = datetime.date.today().isoformat()
datadir = doc.get_metadata('datadir')
def highlighting(output_format):
return pf.convert_text(
'`_`{.default}',
output_format=output_format,
extra_args=[
'--highlight-style', os.path.join(datadir, 'syntax', 'wg21.theme'),
'--template', os.path.join(datadir, 'template', 'highlighting')
])
doc.metadata['highlighting-macros'] = pf.MetaBlocks(
pf.RawBlock(highlighting('latex'), 'latex'))
doc.metadata['highlighting-css'] = pf.MetaBlocks(
pf.RawBlock(highlighting('html'), 'html'))
def intersperse(lst, item):
result = [item] * (len(lst) * 2 - 1)
result[0::2] = lst
return result
def codeblock(elem, doc):
if not isinstance(elem, pf.CodeBlock):
return None
if not elem.classes:
elem.classes.append('default')
codeblocks.append(elem)
codeblocks = []
doc.walk(codeblock)
texts = pf.convert_text(
intersperse(codeblocks, pf.Plain(pf.RawInline('---', doc.format))),
input_format='panflute',
output_format=doc.format,
extra_args=[
'--syntax-definition', os.path.join(datadir, 'syntax', 'isocpp.xml')
]).split('\n---\n')
assert(len(codeblocks) == len(texts))
def convert(elem, text):
if not any(cls in elem.classes for cls in ['cpp', 'default', 'diff']):
return elem
def repl(match_obj):
match = match_obj.group(1)
if not match: # @@
return match_obj.group(0)
if match.isspace(): # @ @
return match
result = convert.cache.get(match)
if result is not None:
return result
if doc.format == 'latex':
# Undo `escapeLaTeX` from https://github.com/jgm/skylighting
match = match.replace('\\textbackslash{}', '\\') \
.replace('\\{', '{') \
.replace('\\}', '}')
result = pf.convert_text(
pf.Plain(*pf.convert_text(match)[0].content).walk(divspan, doc),
input_format='panflute',
output_format=doc.format)
convert.cache[match] = result
return result
result = pf.RawBlock(embedded.sub(repl, text), doc.format)
if 'diff' not in elem.classes:
return result
# For HTML, this is handled via CSS in `data/template/wg21.html`.
command = '\\renewcommand{{\\{}}}[1]{{\\textcolor[HTML]{{{}}}{{#1}}}}'
return pf.Div(
pf.RawBlock('{', 'latex'),
pf.RawBlock(command.format('NormalTok', doc.get_metadata('uccolor')), 'latex'),
pf.RawBlock(command.format('VariableTok', doc.get_metadata('addcolor')), 'latex'),
pf.RawBlock(command.format('StringTok', doc.get_metadata('rmcolor')), 'latex'),
result,
pf.RawBlock('}', 'latex'))
convert.cache = {}
prepare.converted = [convert(elem, text) for elem, text in zip(codeblocks, texts)]
def finalize(doc):
text = '\n\n'.join(doc.code_blocks)
doc.content = pf.convert_text(text)
def markdown_to_table_cell(string):
return panflute.TableCell(*panflute.convert_text(string))
def conversion(cls, markdown, fmt="markdown"):
doc = convert_text(markdown, standalone=True)
doc.format = fmt
pandoc_latex_admonition.main(doc)
return doc
def conversion(markdown, format="markdown"):
doc = convert_text(markdown, standalone=True)
doc.format = format
pandoc_numbering.main(doc)
return doc
def to_json(text):
return pf.convert_text(text, 'markdown', 'json')
def to_json(text):
return pf.convert_text(text, 'markdown', 'json')
def tikz(elem, doc):
"""
Add admonition to elem
Arguments
---------
elem:
The current element
doc:
The pandoc document
Returns
-------
The modified element
"""
# Is it in the right format and is it Div or a CodeBlock?
if doc.format in ["beamer"] and elem.tag in ["Span"]:
# Is there a latex-admonition-color attribute?
if "beamer-arrow-node" in elem.classes:
text = convert_text(Plain(*elem.content),
input_format="panflute",
output_format="latex")
options = ["anchor=base"]
color = get_color(elem, doc)
if color:
options.append(f"fill={color}")
(from_value, to_value) = get_range(elem, doc)
if from_value or to_value:
display = f"\\only<{from_value}-{to_value}>"
else:
display = ""
return RawInline(
f"\\tikz[baseline]{{"
f"{display}{{\\node[{','.join(options)}] "
f"({elem.identifier}) "
f"{{{text}}}"
f";}}}}",
format="tex",
)
if "beamer-arrow-edge" in elem.classes:
angles = []
if "angle_src" in elem.attributes:
try:
angle = elem.attributes["angle_src"]
angle = int(angle)
angles.append(f"out={angle}")
except ValueError:
debug(
f"pandoc-beamer-arrow: angle_src '{angle}' is not correct"
)
if "angle_dest" in elem.attributes:
try:
angle = elem.attributes["angle_dest"]
angle = int(angle)
angles.append(f"in={angle}")
except ValueError:
debug(
f"pandoc-beamer-arrow: angle_dest '{angle}' is not correct"
)
options = ["->"]
color = get_color(elem, doc)
if color:
options.append(color)
if "linewidth" in elem.attributes:
try:
linewidth = elem.attributes["linewidth"]
linewidth = int(linewidth)
options.append(f"line width={linewidth}pt")
except ValueError:
debug(
f"pandoc-beamer-arrow: linewidth '{linewidth}' is not correct"
)
(from_value, to_value) = get_range(elem, doc)
if from_value or to_value:
display = f"<{from_value}-{to_value}>"
else:
display = ""
return RawInline(
f"\\begin{{tikzpicture}}[overlay]"
f"\\path[{','.join(options)}]{display} "
f"({elem.attributes['src']}) "
f"edge "
f"[{','.join(angles)}] "
f"({elem.attributes['dest']});"
f"\\end{{tikzpicture}}",
format="tex",
)
thumb_url = None
if config.has_option('wordpress','thumbnail'):
thumbnail = config.get('wordpress', 'thumbnail')
here = path.dirname( postfile )
thumb_url = path.join( here, thumbnail ) # Make path absolute
# Wordpress related, create the post
WP = Client( url, username, password )
post = WordPressPost()
# Take markdown, convert to HTML and put it as post content
# Makes intermediate convertion to Panflute AST to apply the filters.
postdocument = pf.convert_text(postcontent, input_format='markdown',
output_format='panflute',
standalone=True)
pf.run_filters( [ imageURLs, codeBlocks ], doc = postdocument )
content = pf.convert_text(postdocument, input_format='panflute',
output_format='html')
# Set post metadata
post.title = title
post.content = content
post.post_status = post_status
post.terms_names = terms_names
if not thumb_url == None:
thumb_mime = checkImage(thumb_url)
def test_all():
md = 'Some *markdown* **text** ~xyz~'
c_md = pf.convert_text(md)
b_md = [
pf.Para(pf.Str("Some"), pf.Space, pf.Emph(pf.Str("markdown")),
pf.Space, pf.Strong(pf.Str("text")), pf.Space,
pf.Subscript(pf.Str("xyz")))
]
print("Benchmark MD:")
print(b_md)
print("Converted MD:")
print(c_md)
assert repr(c_md) == repr(b_md)
with io.StringIO() as f:
doc = pf.Doc(*c_md)
pf.dump(doc, f)
c_md_dump = f.getvalue()
with io.StringIO() as f:
doc = pf.Doc(*b_md)
pf.dump(doc, f)
b_md_dump = f.getvalue()
assert c_md_dump == b_md_dump
# ----------------------
print()
tex = r'Some $x^y$ or $x_n = \sqrt{a + b}$ \textit{a}'
c_tex = pf.convert_text(tex)
b_tex = [
pf.Para(pf.Str("Some"), pf.Space, pf.Math("x^y", format='InlineMath'),
pf.Space, pf.Str("or"), pf.Space,
pf.Math(r"x_n = \sqrt{a + b}", format='InlineMath'), pf.Space,
pf.RawInline(r"\textit{a}", format='tex'))
]
print("Benchmark TEX:")
print(b_tex)
print("Converted TEX:")
print(c_tex)
assert repr(c_tex) == repr(b_tex)
with io.StringIO() as f:
doc = pf.Doc(*c_tex)
pf.dump(doc, f)
c_tex_dump = f.getvalue()
with io.StringIO() as f:
doc = pf.Doc(*b_tex)
pf.dump(doc, f)
b_tex_dump = f.getvalue()
assert c_tex_dump == b_tex_dump
print("\nBack and forth conversions... md->json->md")
md = 'Some *markdown* **text** ~xyz~'
print("[MD]", md)
md2json = pf.convert_text(md,
input_format='markdown',
output_format='json')
print("[JSON]", md2json)
md2json2md = pf.convert_text(md2json,
input_format='json',
output_format='markdown')
print("[MD]", md2json2md)
assert md == md2json2md
print("\nBack and forth conversions... md->panflute->md")
md = 'Some *markdown* **text** ~xyz~'
print("[MD]", md)
md2panflute = pf.convert_text(md,
input_format='markdown',
output_format='panflute')
print("[PANFLUTE]", md2panflute)
md2panflute2md = pf.convert_text(md2panflute,
input_format='panflute',
output_format='markdown')
print("[MD]", md2panflute2md)
assert md == md2panflute2md
print("\nBack and forth conversions... md->panflute(standalone)->md")
md = 'Some *markdown* **text** ~xyz~'
print("[MD]", md)
md2panflute = pf.convert_text(md,
input_format='markdown',
output_format='panflute',
standalone=True)
print("[PANFLUTE]", md2panflute)
md2panflute2md = pf.convert_text(md2panflute,
input_format='panflute',
output_format='markdown')
print("[MD]", md2panflute2md)
assert md == md2panflute2md
print(
"\nBack and forth conversions... md table -> json(standalone) -> md table"
)
md = """ --- ---
x y
--- ---
"""
print("[MD]", repr(md))
md2json = pf.convert_text(md,
input_format='markdown',
output_format='json',
standalone=True)
print("[json]", md2json)
md2json2md = pf.convert_text(md2json,
input_format='json',
output_format='markdown')
print("[MD]", repr(md2json2md))
assert md == md2json2md
print(
"\nBack and forth conversions... md table -> panflute(standalone) -> md table"
)
print("[MD]", repr(md))
md2panflute = pf.convert_text(md,
input_format='markdown',
output_format='panflute',
standalone=True)
print("[PANFLUTE]", md2panflute)
md2panflute2md = pf.convert_text(md2panflute,
input_format='panflute',
output_format='markdown')
print("[MD]", repr(md2panflute2md))
assert md == md2panflute2md
def panflute_filter(body, input_format, output_format, extra_args, outputfile):
from panflute import convert_text
return convert_text(body, input_format=input_format, output_format=output_format, extra_args=extra_args)
def conversion(markdown, fmt="markdown"):
doc = convert_text(markdown, standalone=True)
doc.format = fmt
pandoc_beamer_arrow.main(doc)
return doc
def plain(elem: Union[panflute.Div, panflute.Header],
doc: panflute.Doc) -> Optional[panflute.Div]:
"""Assemble the plain text version of the acronym list
The base plain text output is a bulleted list of acronyms in the
following format::
- {short}: {long}
in a new :class:`panflute.Div` with the identifier “acronym-list”.
If the given element is a :class:`panflute.Div`, the list is placed
under a level 1 header with the text “Acronyms” unless the ``name``
or ``level`` attributes are set in which case, the request is
honored. The list is sorted by the short version of the acronyms by
default unless the ``sort`` attribute is set to “false” (case
insensitive) in which case the order is unspecified. If an
attribute cannot be interpreted, it is omitted and a warning is
logged.
Parameters
----------
elem: :class:`panflute.Div` or :class:`panflute.Header`
The element to replace
doc: :class:`panflute.Doc`
The document under consideration.
Returns
-------
:class:`panflute.Div`, optional:
The replacement for the block.
"""
logger = logging.getLogger(__name__ + ".plain_text")
if "acronyms" not in doc.metadata:
return None
if isinstance(elem, panflute.Header):
header = elem
elif isinstance(elem, panflute.Div):
header = panflute.Header(panflute.Str(
elem.attributes.get("name", "Acronyms")),
level=elem.attributes.get("level", 1))
else:
cls = type(elem)
logger.warning(f"Unknown element type {cls}")
return None
if "sort" in elem.attributes:
sort = elem.attributes["sort"].lower()
if sort not in ("true", "false"):
sort = "true"
logger.warning(f"Unknown 'sort' option '{sort}'")
else:
sort = "true"
if sort == "true":
acronyms = sorted(doc.acronyms.values(), key=lambda x: x["short"])
else:
acronyms = doc.acronyms.values()
acrolist = [
panflute.ListItem(
panflute.Plain(panflute.Strong(panflute.Str(acro["short"])),
panflute.Str(":"), panflute.Space,
*panflute.convert_text(acro["long"])[0].content))
for acro in acronyms if acro["list"]
]
return panflute.Div(header,
panflute.BulletList(*acrolist),
identifier="acronym-list")
def process_raw_spans(container, doc):
# type: (Span, Doc) -> Element
if not isinstance(container, (pf.Span, pf.Div)):
return None
hide_raw = doc.get_metadata(IPUB_META_ROUTE + ".hide_raw", False)
if CONVERTED_OTHER_CLASS in container.classes and isinstance(
container, pf.Span):
if doc.format == "rst" and container.attributes["format"] == "latex":
if container.attributes["tag"] in ["todo"]:
return pf.Str("\n\n.. {}:: {}\n\n".format(
container.attributes["tag"],
container.attributes["content"]))
if container.attributes["tag"] == "ensuremath":
return pf.RawInline(":math:`{}`".format(
container.attributes["content"]),
format="rst")
return pf.RawInline(container.attributes.get("original"),
format=container.attributes["format"])
if CONVERTED_DIRECTIVE_CLASS in container.classes and isinstance(
container, pf.Div):
# convert the directive head, which will be e.g.
# Para(Str(..) Space Str(toctree::) SoftBreak Str(:maxdepth:) Space Str(2) SoftBreak Str(:numbered:)) # noqa
# we need to spilt on the soft breaks,
# place them on a new line and re-indent them
if doc.format in ("rst"):
# split into lines by soft breaks
header_lines = [
list(y) for x, y in itertools.groupby(
container.content[0].content,
lambda z: isinstance(z, pf.SoftBreak)) if not x
]
# wrap each line in a Para and convert block with pandoc
head_doc = pf.Doc(*[pf.Para(*l) for l in header_lines])
head_doc.api_version = doc.api_version
head_str = pf.convert_text(head_doc,
input_format="panflute",
output_format=doc.format)
# remove blank lines and indent
head_str = head_str.replace("\n\n", "\n ") + "\n\n"
head_block = pf.RawBlock(head_str, format=doc.format)
if len(container.content) == 1:
return head_block
# split into lines by soft breaks, we use indicators to tell
# us where to indent in the converted text
body_blocks = []
for block in container.content[1:]:
new_elements = [pf.RawInline("%^*", format=doc.format)]
for el in block.content:
if isinstance(el, pf.SoftBreak):
new_elements.append(
pf.RawInline("?&@", format=doc.format))
else:
new_elements.append(el)
block.content = new_elements
body_blocks.append(block)
# convert body content with pandoc
body_doc = pf.Doc(*body_blocks)
body_doc.api_version = doc.api_version
body_str = pf.convert_text(body_doc,
input_format="panflute",
output_format=doc.format)
# raise ValueError(body_blocks)
body_str = body_str.replace("%^*", " ").replace("?&@", "\n ")
# ensure all lines are indented correctly
# (doesn't occur by default?)
body_str = ("\n".join([
" " + l.lstrip() if l.strip() else l
for l in body_str.splitlines()
]) + "\n\n")
body_block = pf.RawBlock(body_str, format=doc.format)
return [head_block, body_block]
elif (doc.format in ("html", "html5")
and container.attributes["format"] == "rst"):
if hide_raw:
return []
head_para = pf.Para(*[
pf.RawInline("<br>" +
" " * 4) if isinstance(c, pf.SoftBreak) else c
for c in container.content[0].content
])
head_str = pf.convert_text(head_para,
input_format="panflute",
output_format=doc.format)
if len(container.content) > 1:
body_doc = pf.Doc(*container.content[1:])
body_doc.api_version = doc.api_version
body_str = pf.convert_text(body_doc,
input_format="panflute",
output_format=doc.format)
body_str = ('<p></p><div style="margin-left: 20px">'
"{0}</div>").format(body_str)
else:
body_str = ""
return pf.RawBlock(
'<div {0} style="background-color:rgba(10, 225, 10, .2)">'
"{1}{2}"
"</div>".format(container.attributes.get("directive", ""),
head_str, body_str),
format="html",
)
elif doc.format in (
"tex", "latex") and container.attributes["format"] == "rst":
if hide_raw:
return []
directive = container.attributes.get("directive", "")
inline = container.attributes.get("inline", "")
# TODO handle directive with options and/or inline body
# e.g. .. figure:: path/to/figure
# :centre:
box_open = (
"\\begin{{mdframed}}"
"[frametitle={{{0}}},frametitlerule=true]".format(directive))
if inline:
box_open += "\n\\mdfsubtitle{{{0}}}".format(inline)
box_close = "\\end{mdframed}"
if len(container.content) == 1:
return pf.RawBlock(box_open + box_close, format="tex")
else:
return ([pf.RawBlock(box_open, format="tex")] +
list(container.content[1:]) +
[pf.RawBlock(box_close, format="tex")])
return pf.RawBlock(
pf.stringify(pf.Doc(*container.content)),
format=container.attributes["format"],
)
if CONVERTED_OTHER_CLASS in container.classes and isinstance(
container, pf.Div):
return pf.RawBlock(
pf.stringify(pf.Doc(*container.content)),
format=container.attributes["format"],
)
