def render_text(text, autocompletes=None, comment=None, unwrap_p=False):
# Render comment text into HTML.
import re
# Put @-mentions in bold.
if autocompletes:
text, _ = match_autocompletes(text, autocompletes,
lambda text: "**" + text + "**")
# Rewrite attachment:### URLs.
if comment is not None:
def get_attachment_url(attachment_id):
try:
return Attachment.objects.get(
id=attachment_id.group(1)).get_absolute_url()
except:
return "about:blank"
text = re.sub("(?<=\()attachment:(\d+)(?=\))", get_attachment_url,
text)
# Render to HTML as if CommonMark.
import commonmark
parsed = commonmark.Parser().parse(text)
text = commonmark.HtmlRenderer({"safe": True}).render(parsed)
if unwrap_p:
# If it's a single paragraph, unwrap it.
text = re.sub(r"^<p>(.*)</p>$", r"\1", text)
return text
def process_with_common_mark(raw_text):
"""Exec given text, add own processing based on commonmark's ast."""
parser = cm.Parser()
ast = parser.parse(raw_text)
ast = inject_subsup_tags(ast)
return cm.HtmlRenderer().render(ast)
def convert_content_to_html(md_content):
"It renders content of the article and converts into html format"
renderer = commonmark.HtmlRenderer()
parser = commonmark.Parser()
syntax_tree = parser.parse(md_content)
html = renderer.render(syntax_tree)
return html
def commonmark(value):
parser = CommonMark.Parser()
ast = parser.parse(value)
renderer = CommonMark.HtmlRenderer()
html = renderer.render(ast)
return html
def render_page(self, page):
page_s = self.retrieve_page(page)
if page_s:
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
ast = parser.parse(page_s)
html = renderer.render(ast)
return html
else:
return "<h1><i class='fa fa-warning error'></i> Page Not Found</h1><p>Whoops, the page " + page + " could not be found or is empty."
def commonmark_safe(text):
ast = commonmark.Parser().parse(text)
walker = ast.walker()
# Remove images
for node, entering in walker:
if node.t == 'image':
node.unlink()
html = commonmark.HtmlRenderer({'safe': True}).render(ast)
return mark_safe(html)
def test_smart_dashes(self):
md = 'a - b -- c --- d ---- e ----- f'
EM = '\u2014'
EN = '\u2013'
expected_html = ('<p>' + 'a - ' + 'b ' + EN + ' ' + 'c ' + EM + ' ' +
'd ' + EN + EN + ' ' + 'e ' + EM + EN + ' ' +
'f</p>\n')
parser = commonmark.Parser(options=dict(smart=True))
ast = parser.parse(md)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
self.assertEqual(html, expected_html)
def markdown(value, arg=''):
try:
import commonmark
except ImportError:
logging.warning("Markdown package not installed.")
return force_text(value)
else:
parser = commonmark.Parser()
ast = parser.parse(value)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
return mark_safe(html)
def process_markdown(text):
parser = commonmark.Parser()
ast = parser.parse(text)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
parts = []
for line in html.splitlines():
if line.startswith("<h1>"):
title = line[4:-5]
parts.append([title])
elif line.startswith("<h2>"):
parts.append([])
else:
parts[-1].append(line)
return parts
def render(self, context):
md = self.variable.resolve(context)
if not context.autoescape:
# Auto-escaping is off, so we're in the text portion
# of a notification email. Return the raw markdown.
return md
else:
# Auto-escaping is on, so we're in the HTML portion
# of a notification email. Rather than returning the
# raw Markdown, which will look funny because e.g.
# line breaks will be ignored when it is placed within
# HTML, render the Markdown to HTML. Turn on safe mode
# since the content can't be trusted.
import commonmark
return commonmark.HtmlRenderer({ "safe": True })\
.render(commonmark.Parser().parse(md))
def markdown(value):
# Renders the string using CommonMark in safe mode, which blocks
# raw HTML in the input and also some links using a blacklist,
# plus a second pass filtering using a whitelist for allowed
# tags and URL schemes.
import commonmark
ast = commonmark.Parser().parse(value)
html = commonmark.HtmlRenderer({'safe': True}).render(ast)
import html5lib, urllib.parse
def filter_url(url):
try:
urlp = urllib.parse.urlparse(url)
except Exception as e:
# invalid URL
return None
if urlp.scheme not in ("http", "https"):
return None
return url
valid_tags = set(
'strong em a code p h1 h2 h3 h4 h5 h6 pre br hr img ul ol li span blockquote'
.split())
valid_tags = set('{http://www.w3.org/1999/xhtml}' + tag
for tag in valid_tags)
dom = html5lib.HTMLParser().parseFragment(html)
for node in dom.iter():
if node.tag not in valid_tags and node.tag != 'DOCUMENT_FRAGMENT':
node.tag = '{http://www.w3.org/1999/xhtml}span'
for name, val in list(node.attrib.items()):
if name.lower() in ("href", "src"):
val = filter_url(val)
if val is None:
node.attrib.pop(name)
else:
node.set(name, val)
else:
# No other attributes are permitted.
node.attrib.pop(name)
html = html5lib.serialize(dom,
quote_attr_values="always",
omit_optional_tags=False,
alphabetical_attributes=True)
return safestring.mark_safe(html)
def parse_and_render_recipe(input, js_path_prefix="", css_path_prefix=""):
"""
Parse a recipe from markdown
Args:
input: str, The recipe as markdown
"""
parser = commonmark.Parser()
ast = parser.parse(input)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
soup = BeautifulSoup(html, 'html.parser')
# print(soup.prettify())
steps = soup.find("h2", text="Zubereitung")
if (nextPart := steps.findNextSibling("h2")) is not None:
steps_upto = soup.index(nextPart)
def a_article(a_id):
article = Article.query.get(int(a_id))
if not article:
flask.abort(404)
parser = commonmark.Parser()
ast = parser.parse(article.text)
renderer = commonmark.HtmlRenderer()
html = renderer.render(ast)
article_dict = {
"title": article.title,
"time": str(article.time),
"text": html,
"author": article.user.name
}
return render_template("article.html",
id=a_id,
tag_dict=get_tag_dict(),
article_dict=article_dict)
def render_markdown(filename):
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
with open(filename) as f:
doc = f.read()
ast = parser.parse(doc)
doc = renderer.render(ast)
# add internal links
find_section = lambda x: re.search(
r"<h([2-6])>(?:<.+>)?(.+?)(?:<.+>)?<\/h\1>", x)
sub = lambda x: x.lower().replace(" ", "-").translate(
{ord(c): ""
for c in "{}/"})
section = find_section(doc)
while section:
start, end = section.span()
gr = section.groups()
sec = '<h{0} id="{1}">{2}</h{0}>'.format(gr[0], sub(gr[1]), gr[1])
doc = doc[:start] + sec + doc[end:]
section = find_section(doc)
return doc
def main():
parser = argparse.ArgumentParser(
description="Process Markdown according to "
"the CommonMark specification.")
if sys.version_info < (3, 0):
reload(sys) # noqa
sys.setdefaultencoding('utf-8')
parser.add_argument('infile',
nargs="?",
type=argparse.FileType('r'),
default=sys.stdin,
help="Input Markdown file to parse, defaults to STDIN")
parser.add_argument('-o',
nargs="?",
type=argparse.FileType('w'),
default=sys.stdout,
help="Output HTML/JSON file, defaults to STDOUT")
parser.add_argument('-a', action="store_true", help="Print formatted AST")
parser.add_argument('-aj', action="store_true", help="Output JSON AST")
args = parser.parse_args()
parser = commonmark.Parser()
f = args.infile
o = args.o
lines = []
for line in f:
lines.append(line)
data = "".join(lines)
ast = parser.parse(data)
if not args.a and not args.aj:
renderer = commonmark.HtmlRenderer()
o.write(renderer.render(ast))
exit()
if args.a:
# print ast
commonmark.dumpAST(ast)
exit()
# o.write(ast.to_JSON())
o.write(commonmark.dumpJSON(ast))
exit()
def __init__(self):
self.path = os.path.join(settings.BASE_DIR, "NEWS.md")
self.parser = commonmark.Parser()
self.renderer = commonmark.HtmlRenderer()
self.cookie_name = "news_current"
def render_body(self):
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
ast = parser.parse(self.body)
return renderer.render(ast)
def html_for_entry(entry):
ast = commonmark.Parser().parse(entry)
rewrite_ast(ast)
renderer = commonmark.HtmlRenderer()
return ''.join(['<article>\n', renderer.render(ast), '</article>\n'])
class HtmlEscapingRenderer(commonmark.HtmlRenderer):
def __init__(self, allow_html: bool = False):
super().__init__()
self.allow_html = allow_html
def lit(self, s):
if self.allow_html:
return super().lit(s)
return super().lit(s.replace("<", "<").replace(">", ">"))
def image(self, node, entering):
prev = self.allow_html
self.allow_html = True
super().image(node, entering)
self.allow_html = prev
md_parser = commonmark.Parser()
yes_html_renderer = commonmark.HtmlRenderer()
no_html_renderer = HtmlEscapingRenderer()
def render(message: str, allow_html: bool = False) -> str:
parsed = md_parser.parse(message)
if allow_html:
return yes_html_renderer.render(parsed)
else:
return no_html_renderer.render(parsed)
def render_from_markdown(template, template_context):
# Render the Markdown first. Markdown has different text escaping rules
# (backslash-escaping of certain symbols only), and we can't add that
# logic to Django's template auto-escaping. So we render the Markdown
# first, which gives HTML. That HTML can be treated as a regular Django
# template (with regular HTML autoescaping).
#
# (If we did it in the other order, we'd have to disable Django's
# HTML autoescaping and then have some other method to prevent the
# use of variables in the template from generating Markdown tags.)
#
# Do this within each {% block %}...{% endblock %} tag, since we
# don't want to create HTML <p>s around content that doesn't occur
# within a block. Assumes there are no nested blocks.
#
# We turn off CommonMark's safe mode, however, since we trust the
# template. (Safe mode prohibits HTML inlines and also prevents some
# unsafe URLs, but that's up to the caller.)
# CommonMark replaces non-URL-safe characters in link URLs with
# their %-escaped code. Monkey-patch the CommonMark library to
# not do that for { and } so that template variables within links
# remain a template variable and don't turn into %7B%7Bvarname%7D%7D.
# Do this prior to parsing.
from commonmark import common, inlines
def fixed_normalize_uri(uri):
return common.normalize_uri(uri).replace("%7B",
"{").replace("%7D", "}")
inlines.normalize_uri = fixed_normalize_uri
# Build the HTML and text templates.
def run_renderer(renderer, ext, wrap=lambda x: x):
r = template
# fix the {% extends "..." %} file extension.
r = re.sub(r"^(\s*\{%\s*extends\s+[\"'][^\"']*)([\"']\s*%\})",
lambda m: m.group(1) + "." + ext + m.group(2), r)
# Run CommonMark on each block separately.
r = re.sub(
r"(\{%\s*block [^%]+\s*%\})\s*([\s\S]*?)\s*(\{%\s*endblock\s*%\})",
lambda m: m.group(1) + wrap(
renderer.render(commonmark.Parser().parse(m.group(2)))
) + m.group(3), r)
return r
# Render to HTML, put the extends tag back with an .html extension.
html_body = run_renderer(commonmark.HtmlRenderer({"safe": False}), 'html')
# For the text portion, we'll render using a special renderer, and we'll
# wrap each block in the Django template directive to turn off auto-escaping.
text_body = run_renderer(
commonmark_extensions.plaintext.PlainTextRenderer(),
'txt',
wrap=lambda block: "{% autoescape off %}" + block +
"{% endautoescape %}")
# Now render as Django templates.
html_body = Template(html_body).render(Context(template_context)).strip()
text_body = Template(text_body).render(Context(template_context)).strip()
return text_body, html_body
def parse_commonmark(value):
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
ast = parser.parse(value)
return renderer.render(ast)
def init_app(app):
parser = commonmark.Parser # Not an instance because not thread-safe(?)
renderer = commonmark.HtmlRenderer({'softbreak': '<br/>'})
app.extensions['markdown'] = UDataMarkdown(app, parser, renderer)
app.add_template_filter(mdstrip)
def __init__(self, *args, **kwargs):
super(CompileCommonMark, self).__init__(*args, **kwargs)
if commonmark is not None:
self.parser = commonmark.Parser()
self.renderer = commonmark.HtmlRenderer()
import contextlib
import datetime
import time
import pickle
import functools
import os
import re
import string
import random
from ruamel import yaml
from website import querylog
import commonmark
commonmark_parser = commonmark.Parser()
commonmark_renderer = commonmark.HtmlRenderer()
from bs4 import BeautifulSoup
from flask_helpers import render_template
from flask import g
IS_WINDOWS = os.name == 'nt'
class Timer:
"""A quick and dirty timer."""
def __init__(self, name):
self.name = name
def __enter__(self):
self.start = time.time()
def __exit__(self, type, value, tb):
delta = time.time() - self.start
print(f'{self.name}: {delta}s')
def markdown(input):
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
ast = parser.parse(input)
return renderer.render(ast)
def markdown_filter(data):
parser = commonmark.Parser()
renderer = commonmark.HtmlRenderer()
return renderer.render(parser.parse(data))
import commonmark as wrapped_lib
parser = wrapped_lib.Parser()
renderer = wrapped_lib.HtmlRenderer(options={"safe": True})
def convert_commonmark(markdown: str) -> str:
return renderer.render(parser.parse(markdown))
