def process_with_common_mark(raw_text):
"""Exec given text, add own processing based on commonmark's ast."""
parser = cm.Parser()
ast = parser.parse(raw_text)
ast = inject_subsup_tags(ast)
return cm.HtmlRenderer().render(ast)
def docstring(app, what, name, obj, options, lines):
md = '\n'.join(lines)
ast = commonmark.Parser().parse(md)
rst = commonmark.ReStructuredTextRenderer().render(ast)
lines.clear()
for line in rst.splitlines():
lines.append(line)
def convert_md_to_python(info):
"Converts markdown data to Python object"
parser = commonmark.Parser()
parse_data = parser.parse(info)
json_format = commonmark.dumpJSON(parse_data)
py_object = json.loads(json_format)
return py_object
def docstring(app, what, name, obj, options, lines):
"""Converts doc-strings from (Commonmark) Markdown to reStructuredText."""
md = '\n'.join(lines)
ast = commonmark.Parser().parse(md)
rst = commonmark.ReStructuredTextRenderer().render(ast)
lines.clear()
lines += rst.splitlines()
def render(self):
with open(self.path, "rt") as f:
content = f.read()
parser = commonmark.Parser()
renderer = HTMLRenderer()
html = renderer.render(parser.parse(content))
return f'<div class="content">{html}</div>'
def docstring(app, what, name, obj, options, lines):
text = []
in_arg = False
arg_txt = ''
arg_name = None
for l in lines:
if arg_name:
if len(l) and l[0]==' ':
# Continues argument description
arg_txt += ' ' + l.strip()
continue
text.append('* `' + arg_name + '`: ' + arg_txt)
arg_name = None
if is_line.match(l):
continue
arg_name = var_name.match(l)
if arg_name:
arg_txt = arg_name.group('desc')
arg_name = arg_name.group('name')
print((arg_txt, arg_name))
else:
text.append(l)
if arg_name:
text.append('* `' + arg_name + '`: ' + arg_txt)
md = '\n'.join(text)
ast = commonmark.Parser().parse(md)
rst = commonmark.ReStructuredTextRenderer().render(ast)
lines.clear()
for line in rst.splitlines():
line = re.sub(r'\$(?P<eqn>[^$]*)\$', r':math:`\1`', line)
print(line)
lines.append(line)
def render_text(text, autocompletes=None, comment=None, unwrap_p=False):
# Render comment text into HTML.
import re
# Put @-mentions in bold.
if autocompletes:
text, _ = match_autocompletes(text, autocompletes,
lambda text: "**" + text + "**")
# Rewrite attachment:### URLs.
if comment is not None:
def get_attachment_url(attachment_id):
try:
return Attachment.objects.get(
id=attachment_id.group(1)).get_absolute_url()
except:
return "about:blank"
text = re.sub("(?<=\()attachment:(\d+)(?=\))", get_attachment_url,
text)
# Render to HTML as if CommonMark.
import commonmark
parsed = commonmark.Parser().parse(text)
text = commonmark.HtmlRenderer({"safe": True}).render(parsed)
if unwrap_p:
# If it's a single paragraph, unwrap it.
text = re.sub(r"^<p>(.*)</p>$", r"\1", text)
return text
def parse(self, src):
self.src = src
self.recipe = Recipe(title=None)
parser = commonmark.Parser()
ast = parser.parse(src)
self.current = ast.first_child
self._parse_title()
self._parse_description()
self._parse_tags_and_yields()
if self.current is not None and self.current.t == 'thematic_break':
self._next_node()
else:
# TODO this divider is required, but we might just continue anyways?
raise RuntimeError(
f"Invalid, expected divider before ingredient list, got {self.current and self.current.t} instead"
)
self._parse_ingredients()
if self.current is not None:
if self.current.t == 'thematic_break':
self._next_node()
else:
# TODO this divider is required, but we might just continue anyways?
raise RuntimeError(
f"Invalid, expected divider before ingredient list, got {self.current and self.current.t} instead"
)
self.recipe.instructions = self._get_source_until()
return self.recipe
def markdown_to_notion(markdown):
if not isinstance(markdown, str):
markdown = str(markdown)
# commonmark doesn't support strikethrough, so we need to handle it ourselves
while markdown.count("~~") >= 2:
markdown = markdown.replace("~~", "<s>", 1)
markdown = markdown.replace("~~", "</s>", 1)
parser = commonmark.Parser()
ast = prepare(parser.parse(markdown))
format = set()
notion = []
for section in ast:
_, ended_format = _extract_text_and_format_from_ast(section)
if ended_format and ended_format in format:
format.remove(ended_format)
if section["type"] == "paragraph":
notion.append(["\n\n"])
for item in section.get("children", []):
literal, new_format = _extract_text_and_format_from_ast(item)
if new_format:
format.add(new_format)
if item["type"] == "html_inline" and literal == "</s>":
format.remove(("s",))
literal = ""
if item["type"] == "softbreak":
literal = "\n"
if literal:
notion.append([literal, [list(f) for f in sorted(format)]] if format else [literal])
# in the ast format, code blocks are meant to be immediately self-closing
if ("c",) in format:
format.remove(("c",))
# remove any trailing newlines from automatic closing paragraph markers
if notion:
notion[-1][0] = notion[-1][0].rstrip("\n")
# consolidate any adjacent text blocks with identical styles
consolidated = []
for item in notion:
if consolidated and _get_format(consolidated[-1], as_set=True) == _get_format(item, as_set=True):
consolidated[-1][0] += item[0]
elif item[0]:
consolidated.append(item)
return consolidated
def _render(self, text):
'''Render CommonMark with ettings taken in account'''
parser = commonmark.Parser()
ast = parser.parse(text)
renderer = HtmlRenderer(self)
html = renderer.render(ast)
return html
def convert_content_to_html(md_content):
"It renders content of the article and converts into html format"
renderer = commonmark.HtmlRenderer()
parser = commonmark.Parser()
syntax_tree = parser.parse(md_content)
html = renderer.render(syntax_tree)
return html
def get_changes_for_version(wanted_version: version.Version) -> str:
"""Get the changelogs for the given version.
If an RC then will only get the changelog for that RC version, otherwise if
its a full release will get the changelog for the release and all its RCs.
"""
with open("CHANGES.md") as f:
changes = f.read()
# First we parse the changelog so that we can split it into sections based
# on the release headings.
ast = commonmark.Parser().parse(changes)
@attr.s(auto_attribs=True)
class VersionSection:
title: str
# These are 0-based.
start_line: int
end_line: Optional[int] = None # Is none if its the last entry
headings: List[VersionSection] = []
for node, _ in ast.walker():
# We look for all text nodes that are in a level 1 heading.
if node.t != "text":
continue
if node.parent.t != "heading" or node.parent.level != 1:
continue
# If we have a previous heading then we update its `end_line`.
if headings:
headings[-1].end_line = node.parent.sourcepos[0][0] - 1
headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1))
changes_by_line = changes.split("\n")
version_changelog = [] # The lines we want to include in the changelog
# Go through each section and find any that match the requested version.
regex = re.compile(r"^Synapse v?(\S+)")
for section in headings:
groups = regex.match(section.title)
if not groups:
continue
heading_version = version.parse(groups.group(1))
heading_base_version = version.parse(heading_version.base_version)
# Check if heading version matches the requested version, or if its an
# RC of the requested version.
if wanted_version not in (heading_version, heading_base_version):
continue
version_changelog.extend(changes_by_line[section.start_line : section.end_line])
return "\n".join(version_changelog)
def markdown_filter(data):
if not data:
return ''
parser = commonmark.Parser()
renderer = CoprHtmlRenderer()
return Markup(renderer.render(parser.parse(data)))
def doc_strings(app, what, name, obj, options, lines):
"""Converts doc-strings from Markdown to reStructuredText."""
tweaked = syntax_tweaks(lines)
markdown = '\n'.join(tweaked)
parsed = commonmark.Parser().parse(markdown)
restruct = commonmark.ReStructuredTextRenderer().render(parsed)
lines.clear()
lines += restruct.splitlines()
def commonmark(value):
parser = CommonMark.Parser()
ast = parser.parse(value)
renderer = CommonMark.HtmlRenderer()
html = renderer.render(ast)
return html
def render_page(self, page):
page_s = self.retrieve_page(page)
if page_s:
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
ast = parser.parse(page_s)
html = renderer.render(ast)
return html
else:
return "<h1><i class='fa fa-warning error'></i> Page Not Found</h1><p>Whoops, the page " + page + " could not be found or is empty."
def count_specified_tag(contents, tag):
"""Count the specified markdown tag in the string contents."""
ast = commonmark.Parser().parse(contents)
tag_count = 0
# iteratively check all of the nodes in the AST of the markdown file
for subnode, enter in ast.walker():
# check to see if the current subnode is an open node of the specified tag
if subnode.t == tag and enter:
tag_count += 1
return tag_count
def __init__(self, item):
super().__init__()
self.item = item
self.ast = commonmark.Parser().parse(self.item.fspath.open().read())
self.stack = [(
0,
self.item,
self.item.name,
)]
self.collected = []
def _markdown(text: str) -> str:
"""Convert a Markdown snippet to Org-mode."""
# Bazel (including Stardoc) interprets all files as Latin-1,
# https://docs.bazel.build/versions/4.1.0/build-ref.html#BUILD_files.
# However, our files all use UTF-8, leading to double encoding. Reverse
# that effect here.
text = text.strip().encode('latin-1').decode('utf-8')
document = commonmark.Parser().parse(text)
text = _OrgRenderer().render(document)
return text + '\n'
def count_specified_tag(contents, tag):
"""Counts the specified markdown tag in the string contents"""
ast = commonmark.Parser().parse(contents)
tag_count = 0
for subnode, enter in ast.walker():
# Check to see if the current subnode is an open node of the specified tag
if subnode.t == tag and enter:
tag_count += 1
return tag_count
def __init__(self, parser=None, style_name=None):
if parser is None:
parser = commonmark.Parser()
if style_name is None:
style_name = 'native'
self.parser = parser
self.style_name = style_name
self.list_level = -1
self.counters = {}
self.footnotes = []
def test_smart_dashes(self):
md = 'a - b -- c --- d ---- e ----- f'
EM = '\u2014'
EN = '\u2013'
expected_html = ('<p>' + 'a - ' + 'b ' + EN + ' ' + 'c ' + EM + ' ' +
'd ' + EN + EN + ' ' + 'e ' + EM + EN + ' ' +
'f</p>\n')
parser = commonmark.Parser(options=dict(smart=True))
ast = parser.parse(md)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
self.assertEqual(html, expected_html)
def commonmark_safe(text):
ast = commonmark.Parser().parse(text)
walker = ast.walker()
# Remove images
for node, entering in walker:
if node.t == 'image':
node.unlink()
html = commonmark.HtmlRenderer({'safe': True}).render(ast)
return mark_safe(html)
def markdown(value, arg=''):
try:
import commonmark
except ImportError:
logging.warning("Markdown package not installed.")
return force_text(value)
else:
parser = commonmark.Parser()
ast = parser.parse(value)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
return mark_safe(html)
def replace_images(msg, addr):
parser = commonmark.Parser()
ast = parser.parse(msg["data"]["raw"])
ripper = html_image_ripper()
ripper.feed(msg["data"]["cooked"])
for cur, entering in ast.walker():
if cur.t == "image" and entering:
cur.t = "link"
dest = ripper.images.pop(0)
if dest.startswith("/"):
dest = addr + dest
cur.destination = dest
renderer = commonmark_extensions.plaintext.CommonMarkToCommonMarkRenderer()
return renderer.render(ast)
def run_renderer(renderer, ext, wrap=lambda x: x):
r = template
# fix the {% extends "..." %} file extension.
r = re.sub(r"^(\s*\{%\s*extends\s+[\"'][^\"']*)([\"']\s*%\})",
lambda m: m.group(1) + "." + ext + m.group(2), r)
# Run CommonMark on each block separately.
r = re.sub(
r"(\{%\s*block [^%]+\s*%\})\s*([\s\S]*?)\s*(\{%\s*endblock\s*%\})",
lambda m: m.group(1) + wrap(
renderer.render(commonmark.Parser().parse(m.group(2)))
) + m.group(3), r)
return r
def parseHeaders(text: str): parser = commonmark.Parser() parsed = parser.parse(text) def isSuitableCodeBlock(n): return n.t == "code_block" and n.info == "yaml" res = [b.literal for b in crawl(parsed, isSuitableCodeBlock)] if len(res) > 2: res1 = res[:2] elif len(res) < 1: return None, None elif len(res) == 1: return res[0], None return res
def docstring(
app,
what,
name,
obj,
options,
lines,
): # pylint: disable=unused-argument, too-many-arguments
"""Helper function to render docstring using markdown"""
md = "\n".join(lines)
ast = commonmark.Parser().parse(md)
rst = commonmark.ReStructuredTextRenderer().render(ast)
lines.clear()
for line in rst.splitlines():
lines.append(line)
def process_markdown(text):
parser = commonmark.Parser()
ast = parser.parse(text)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
parts = []
for line in html.splitlines():
if line.startswith("<h1>"):
title = line[4:-5]
parts.append([title])
elif line.startswith("<h2>"):
parts.append([])
else:
parts[-1].append(line)
return parts
def render(self, context):
md = self.variable.resolve(context)
if not context.autoescape:
# Auto-escaping is off, so we're in the text portion
# of a notification email. Return the raw markdown.
return md
else:
# Auto-escaping is on, so we're in the HTML portion
# of a notification email. Rather than returning the
# raw Markdown, which will look funny because e.g.
# line breaks will be ignored when it is placed within
# HTML, render the Markdown to HTML. Turn on safe mode
# since the content can't be trusted.
import commonmark
return commonmark.HtmlRenderer({ "safe": True })\
.render(commonmark.Parser().parse(md))
