def readme():
with open(resource_path('.') + '/README.md',
'r') as fp, open(resource_path('.') + '/help.md', 'r') as hp:
d = {
'readme': commonmark.commonmark(fp.read()),
'help': commonmark.commonmark(hp.read())
}
return d
def test_regex_vulnerability_link_label(self):
i = 200
while i <= 2000:
s = commonmark.commonmark('[' + ('\\' * i) + '\n')
self.assertEqual(s, '<p>' + '[' + ('\\' * (i // 2)) + '</p>\n',
'[\\\\... %d deep' % (i, ))
i *= 10
def test_regex_vulnerability_link_destination(self):
i = 200
while i <= 2000:
s = commonmark.commonmark(('[](' * i) + '\n')
self.assertEqual(s, '<p>' + ('[](' * i) + '</p>\n',
'[]( %d deep' % (i, ))
i *= 10
def image(self):
soup = bs(commonmark.commonmark(self.long_desc))
img = soup.find('img', src=True, onload=False, onerror=False)
if img:
return img['src']
else:
return 'none'
async def wit_parser_function(self, apsdroid, config, message):
c = message.connector
text = ""
notfound = ""
for i in regex.finditer(r'#(?P<wit>\d+)', message.text, regex.MULTILINE):
ids = i.group(0)[1:]
try:
value = self.wit.get_work_item(id=ids, project=self.projectid)
except Exception as e:
notfound += f"[{ids}](http:// '{e}'), "
continue
text += f"* [link]({value._links.additional_properties['html']['href']}) - {ids} - {value.fields['System.Title']}\n"
if len(text) > 0:
text = f"@{message.user}: I have found follwing WITs:\n{text}"
notfound = notfound[:-2]
if len(notfound) > 0:
text += f"\n"
text += f"Following WITs not found: {notfound}"
text += f"\n"
text = commonmark.commonmark(text)
await message.respond(text)
def transform_content(self,content,fromType,targetType,output):
if fromType==Article.TEXT_MARKDOWN and targetType==Article.TEXT_HTML:
output.write(commonmark.commonmark(content))
elif fromType==Article.TEXT_HTML and targetType==Article.TEXT_HTML:
output.write(content)
else:
raise ValueError('Unable to convert {fromType} to {targetType}'.format(fromType=fromType,targetType=targetType))
def tree_render(node: model.ContentNode, site: model.Site):
if isinstance(node, model.Page):
if node.dir:
node.destination_path.parent.mkdir()
for child in node.children:
tree_render(child, site)
else:
node.destination_path.parent.mkdir()
with open(node.destination_path, "w") as file:
file_loader = FileSystemLoader(site.templates_path)
env = Environment(loader=file_loader)
env.filters["visible"] = helpers.visible
try:
template = env.get_template(node.template)
except AttributeError:
template = env.get_template("default.html")
try:
node.content = commonmark.commonmark(node.content)
except AttributeError:
pass
output = template.render(page=node, site=site)
file.writelines(output)
elif isinstance(node, model.Folder):
node.destination_path.mkdir()
for child in node.children:
tree_render(child, site)
for item in node.ressources:
copyfile(item.source_path, item.destination_path)
def read_content(filename: Path) -> Dict[str, str]:
"""Read content and metadata from file into a dictionary."""
text = fread(filename)
content = {"slug": filename.stem}
# Read headers from top of file, then separate out the rest of the text
end = 0
for key, val, end in read_headers(text):
content[key] = val
text = text[end:]
# Convert Markdown content to HTML: get the title from the original text, then parse the
# Markdown as HTML, then replace any links to other Markdown files with ones to the generated
# HTML files (using regex that matches <a href="filename.md">, where filename is anything
# without whitespace or a colon). The links should all be relative to the directory where the
# current file is, allowing "..." to go up a level.
if filename.suffix == ".md":
if "title" not in content: # only overwrite if title not already found
content["title"] = get_title(text)
text = commonmark.commonmark(text)
text = re.sub(
r'<a href="([^:\s]+).md">',
lambda match: f'<a href="{match.group(1)}.html">',
text,
)
content.update({"content": text})
return content
def markdown(md, autoescape=True):
if autoescape:
esc = conditional_escape
else:
esc = lambda x: x
html = commonmark.commonmark(esc(md))
return mark_safe(html)
def test_regex_vulnerability_link_label(self):
i = 200
while i <= 2000:
s = commonmark.commonmark('[' + ('\\' * i) + '\n')
self.assertEqual(s, '<p>' + '[' + ('\\' * (i // 2)) + '</p>\n',
'[\\\\... %d deep' % (i,))
i *= 10
def test_regex_vulnerability_link_destination(self):
i = 200
while i <= 2000:
s = commonmark.commonmark(('[](' * i) + '\n')
self.assertEqual(s, '<p>' + ('[](' * i) + '</p>\n',
'[]( %d deep' % (i,))
i *= 10
def mail_template(template_name_or_list, **kwargs):
multi_part_content = flask.render_template(template_name_or_list, **kwargs)
parts = multi_part_content.split('---multi-part:xFE+Ab7j+w,mdIL%---')
subject, markdown = map(lambda p: p.strip(), parts)
return MailParts(normalizews(subject), markdown,
commonmark.commonmark(markdown))
def readme(path):
readme = _safe_read(Path(path) / 'README.md')
try:
readme = readme[readme.find('## Getting Started'):]
except IndexError:
pass
return commonmark.commonmark(readme)
def to_html(self) -> str:
try:
with open("../page/" + self.location, "r") as markdown:
return material_card(commonmark(markdown.read()))
except FileNotFoundError:
return material_card(
description(f"File {self.location} not found", color=red))
def rendered_content(self) -> str:
"""Returns the rendered version of raw_content, or just raw_content."""
if self._rendered_content:
return self._rendered_content
elif self._media_type == "text/markdown" and self.raw_content:
return commonmark(self.raw_content).strip()
return self.raw_content
def cleanApiExplorerRecord(item):
try:
item['_id'] = ObjectId(item.pop('id'))
if ('description' in item and item['description'] == 'unknown'):
item['description'] = None
if ('description' in item and item['description'] != None
and item['description'] != ""):
item['descriptionHtml'] = commonmark.commonmark(
item['description'])
item['descriptionText'] = lxml.html.fromstring(
item['descriptionHtml']).text_content()
else:
item['descriptionHtml'] = None
item['descriptionText'] = None
if ('termsOfService' in item and item['termsOfService'] == 'unknown'):
item['termsOfService'] = None
if ('certified' in item and item['certified']):
item['certified']['date'] = convertDate(item['certified']['date'])
if ('swaggerUrl' in item and item['swaggerUrl'] != None
and item['swaggerUrl'].startswith(
"https://github.optum.com/API-Certification")):
item['swaggerUrl'] = item['swaggerUrl'].replace(
"raw/",
"").replace("https://github.optum.com/API-Certification",
"https://github.optum.com/raw/API-Certification")
if ('apiOwner' in item and item['apiOwner'] != None
and item['apiOwner']['email'] == None):
item['apiOwner']['email'] = ""
return item
except Exception as e:
log.info("Error for the below item:")
log.info("item:" + str(item))
log.info(e)
return None
def format(self, markdown):
"""
Format markdown into HTML, including custom image tags
"""
tags = []
# Find all instance of the dynamic image markdown
for tag in re.finditer(r'\[I:([\w-]+)\]', markdown):
tag_slug = tag.group(1)
try:
image = Image.objects.get(slug=tag_slug)
tag_dict = dict()
tag_dict['start'] = tag.start()
tag_dict['end'] = tag.end()
tag_dict['image'] = image
tags.append(tag_dict)
except ObjectDoesNotExist:
pass
# Replace all of the tags with actual markdown image tags, backwards, to
# prevent changing string positions and messing up substitution
for tag_dict in reversed(tags):
markdown = markdown[:tag_dict['start']] + \
''.format(tag_dict['image'].desc,
tag_dict['image'].get_absolute_url()) + \
markdown[tag_dict['end']:]
return mark_safe(commonmark(markdown))
def markdown_2_html(filename, text):
"""Convert markdown text string to an HTML text string."""
try:
import commonmark
text = commonmark.commonmark(text)
except ImportError as e:
log('WARNING: Cannot render Markdown in {}: {}', filename, str(e))
return text
def to_html(txt, txt_format): #{{{
if txt_format == None:
txt_format = 'html'
if txt_format.lower() in ['html']:
return txt
if txt_format.lower() in ['md', 'markdown']:
if enabled['commonmark']:
return commonmark.commonmark(txt)
def test_unicode(self):
s = commonmark.commonmark('<div>\u2020</div>\n')
self.assertEqual(s, '<div>\u2020</div>\n',
'Unicode works in an HTML block.')
commonmark.commonmark('* unicode: \u2020')
commonmark.commonmark('# unicode: \u2020')
commonmark.commonmark('```\n# unicode: \u2020\n```')
def test_unicode(self):
s = commonmark.commonmark('<div>\u2020</div>\n')
self.assertEqual(s, '<div>\u2020</div>\n',
'Unicode works in an HTML block.')
commonmark.commonmark('* unicode: \u2020')
commonmark.commonmark('# unicode: \u2020')
commonmark.commonmark('```\n# unicode: \u2020\n```')
def respond(self, content: Union[str, MessageEventContent],
event_type: EventType = EventType.ROOM_MESSAGE,
markdown: bool = True) -> Awaitable[EventID]:
if isinstance(content, str):
content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=content)
if markdown:
content.format = Format.HTML
content.formatted_body = commonmark.commonmark(content.body)
return self._client.send_message_event(self.room_id, event_type, content)
def markdown_to_cleaned_html(markdown):
unsafe_html_str = commonmark.commonmark(markdown)
# remove wrapping div
# https://stackoverflow.com/questions/21420922/how-to-use-cleaner-lxml-html-without-returning-div-tag
unsafe_doc = document_fromstring(unsafe_html_str)
clean_doc = _cleaner.clean_html(unsafe_doc)
clean_html_str = "\n".join(
tostring(ch, encoding="unicode") for ch in clean_doc)
return Markup(clean_html_str)
async def essay(request: Request, response: CustomResponse, parameter: str) -> None:
template_name = "essay.html"
template = language_aware_template(template_name, request["language"])
try:
md_content = get_md_content_from_disk(parameter)
except FileNotFoundError:
raise HttpError(HTTPStatus.NOT_FOUND, "Essay not found.")
content = commonmark(md_content)
content = escape_content(content, allowed_tags=ALLOWED_TAGS)
response.html(template, content=content)
def load_md(filename):
try:
file = open(filename + '.md', mode='r', encoding='utf_8')
except OSError:
abort(404)
else:
text = file.read()
content = Markup(commonmark.commonmark(text))
return content
def generatePages():
all_pages = glob.glob("pages/*.md")
for p in all_pages:
page = frontmatter.load(p)
filepath, filename = os.path.split(p)
page.__setattr__("content", commonmark.commonmark(page.content))
createFile("_site/", filename.replace(".md", ".html"),
render_markdown_and_yml("templates/" + page["layout"] + ".html", page))
print("Generating ",filename.replace(".md", ".html"))
pass
def __init__(self, path: Path):
with path.open(mode='r') as read_file:
raw = read_file.read()
self.path = path
self.raw: str = raw
self.topics = res['topics'](raw)
self.data = json.loads(commonmark(self.raw, format='json'))
breakpoint()
def compileFile(inputFilename, outputFilename, up):
with open(inputFilename, "r") as myfile:
contentLines = myfile.readlines()
content = "".join(contentLines)
#content = railroadPattern.sub(diagramProcessor, content)
content = processRailroadPatterns(content)
title = contentLines[0][1:].strip()
html = commonmark.commonmark(content)
result = template.replace("{content}", html).replace("{up}", up).replace("{title}", title)
with open(outputFilename, "w") as myfile:
myfile.write(result)
def markdown(s: str) -> str:
commented_shortcodes = shortcodes.comment_shortcodes(s)
tainted_html = commonmark.commonmark(commented_shortcodes)
# Create a Cleaner that supports parsing of bare links (see filters).
cleaner = bleach.Cleaner(tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
styles=ALLOWED_STYLES,
strip_comments=False,
filters=[bleach.linkifier.LinkifyFilter])
safe_html = cleaner.clean(tainted_html)
return safe_html
def main(changes: str, source_version_control: str) -> None:
changes_html = commonmark.commonmark(changes)
changes_soup = BeautifulSoup(changes_html, "html.parser")
headers = changes_soup.find_all("h2")
latest_tag, *_ = [
header.string for header in headers if header.string != "Unreleased"
]
source_version_control_html = commonmark.commonmark(source_version_control)
source_version_control_soup = BeautifulSoup(source_version_control_html,
"html.parser")
pre_commit_repos = yaml.safe_load(
source_version_control_soup.find(
class_="language-yaml").string)["repos"]
for repo in pre_commit_repos:
pre_commit_rev = repo["rev"]
if not pre_commit_rev == latest_tag:
print(
"Please set the rev in ``source_version_control.md`` to be the latest "
f"one.\nExpected {latest_tag}, got {pre_commit_rev}.\n")
sys.exit(1)
def documentation(page_name: str):
"""Support for documentation pages provided in Markdown are rendered safely according to Commonmark specification."""
if not path.isfile(f"pages/{page_name}.md"):
abort(404)
with open(f"pages/{page_name}.md", 'r') as pagefile:
content = Markup(commonmark(pagefile.read()))
docs = _retrieve_docs_list()
title = page_name.replace("_", " ").title()
return render_template("markdown.html",
content=content,
title=title,
docs=docs), 200
def main(changes: str, the_basics: str) -> None:
changes_html = commonmark.commonmark(changes)
changes_soup = BeautifulSoup(changes_html, "html.parser")
headers = changes_soup.find_all("h2")
tags = [
header.string for header in headers if header.string != "Unreleased"
]
latest_tag = tags[0]
the_basics_html = commonmark.commonmark(the_basics)
the_basics_soup = BeautifulSoup(the_basics_html, "html.parser")
(version_example, ) = [
code_block.string
for code_block in the_basics_soup.find_all(class_="language-console")
if "$ black --version" in code_block.string
]
for tag in tags:
if tag in version_example and tag != latest_tag:
print("Please set the version in the ``black --version`` "
"example from ``the_basics.md`` to be the latest one.\n"
f"Expected {latest_tag}, got {tag}.\n")
sys.exit(1)
def render(self):
"""Pre-render text to Content.rendered."""
text = self.get_and_linkify_tags()
rendered = commonmark(text).strip()
rendered = process_text_links(rendered)
if self.is_nsfw:
rendered = make_nsfw_safe(rendered)
if self.show_preview:
if self.oembed:
rendered = "%s<br>%s" % (rendered, self.oembed.oembed)
if self.opengraph:
rendered = "%s%s" % (rendered,
render_to_string(
"content/_og_preview.html",
{"opengraph": self.opengraph}))
self.rendered = rendered
Content.objects.filter(id=self.id).update(rendered=rendered)
def report(env, node_name, report_id):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
:param env: Search for reports in this environment
:type env: :obj:`string`
:param node_name: Find the reports whose certname match this value
:type node_name: :obj:`string`
:param report_id: The hash or the configuration_version of the desired
report
:type report_id: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
reports = puppetdb.reports(query=query)
try:
report = next(reports)
except StopIteration:
abort(404)
report.version = commonmark.commonmark(report.version)
return render_template('report.html',
report=report,
events=yield_or_stop(report.events()),
logs=report.logs,
metrics=report.metrics,
envs=envs,
current_env=env)
def render(self):
"""Pre-render text to Content.rendered."""
text = self.get_and_linkify_tags()
rendered = commonmark(text).strip()
rendered = process_text_links(rendered)
if self.is_nsfw:
rendered = make_nsfw_safe(rendered)
if self.show_preview:
if self.oembed:
rendered = "%s<br>%s" % (
rendered, self.oembed.oembed
)
if self.opengraph:
rendered = "%s%s" % (
rendered,
render_to_string("content/_og_preview.html", {"opengraph": self.opengraph})
)
self.rendered = rendered
Content.objects.filter(id=self.id).update(rendered=rendered)
def render_markdown(content):
"""
Return a html fragment from markdown text content
Parameters
----------
content : str
A markdown formatted text
Returns
-------
html : str
"""
# commonmark >= 0.8.1; but only optionally. Many other packages may pin it
# to <0.8 due to breaking changes.
try:
import commonmark
except ImportError:
return '<div style="color: red;">' + render_plain(content) + "</div>"
else:
return commonmark.commonmark(content)
def test_output(self):
s = commonmark.commonmark('*hello!*')
self.assertEqual(s, '<p><em>hello!</em></p>\n')
def commonmark(s):
return cm.commonmark(s).replace('<a href="http','<a target="_blank" href="http')
def test_null_string_bug(self):
s = commonmark.commonmark('> sometext\n>\n\n')
self.assertEqual(
s,
'<blockquote>\n<pre><code>sometext\n</code></pre>'
'\n</blockquote>\n')
def test_random_text(self, s):
commonmark.commonmark(s)
def format(self, markdown):
"""
Format markdown into HTML
"""
return mark_safe(commonmark(markdown))
