def markdown(md, autoescape=True):
if autoescape:
esc = conditional_escape
else:
esc = lambda x: x
html = CommonMark.commonmark(esc(md))
return mark_safe(html)
def get(self):
posts = glob("posts/*.md")
output = '<html><head><link rel="stylesheet" href="tufte.css"/></head><body>'
archive_content = "# Archive\n\nHere's an archive of all of my published posts:\n\n"
metadata = {}
ordered_posts = {}
if len(posts) > 0:
for post in posts:
metadata[post] = extract_metadata(post)
for post in posts:
print(metadata[post])
if metadata[post] is not None:
if 'post_number' in metadata[post].keys():
ordered_posts[metadata[post]['post_number']] = post[6:-3]
for i in range(1,len(ordered_posts.keys())+1):
post = ordered_posts[i]
archive_content += "1. [" + post + "](" + post + ")\n"
archive_content += "\n\n"
print(ordered_posts)
output += cm.commonmark(archive_content)
output += '</body></html>'
self.write(output)
def toHTML(self,md,metadata,html,generateTitle=False):
uri = self.weburi + metadata['published'][0]
print('<article xmlns="http://www.w3.org/1999/xhtml" vocab="{}" typeof="{}" resource="{}">'.format(self.vocab,self.typeof,uri),file=html)
print('<script type="application/json+ld">',file=html)
print('{\n"@context" : "http://schema.org/",',file=html)
print('"@id" : "{}",'.format(uri),file=html)
print('"headline" : "{}",'.format(metadata['title'][0]),file=html)
print('"datePublished" : "{}",'.format(metadata['published'][0]),file=html)
print('"dateModified" : "{}",'.format(metadata['updated'][0]),file=html)
if ("keywords" in metadata):
print('"keywords" : [{}],'.format(','.join(['"' + m + '"' for m in metadata['keywords']])),file=html)
html.write('"author" : [ ')
for index,author in enumerate(metadata['author']):
if (index>0):
html.write(', ')
html.write('{{ "@type" : "Person", "name" : "{}" }}'.format(author))
html.write(']\n')
print('}',file=html)
print('</script>',file=html)
if generateTitle:
print("<h1>{}</h1>".format(escape(metadata['title'][0],quote=False)),file=html)
print(CommonMark.commonmark(md),file=html)
print('</article>',file=html)
def read_wiki_page(path):
# Turn the URL-style path into the canonical filesystem-aware path
path = path.replace('/', os.path.sep).lower()
full_path = os.path.join(wiki_bp.root_path, wiki_bp.static_folder, path)
reg_path = _get_md(full_path)
dir_path = _get_md(os.path.join(full_path, '_index'))
if os.path.isfile(reg_path):
full_path = reg_path
elif os.path.isfile(dir_path):
full_path = dir_path
elif os.path.isdir(full_path):
files = os.listdir(full_path)
lines = ['<h1>{}</h1>'.format(fn2title(path) or 'Home')]
for f in files:
if f.startswith('.'):
continue
f = os.path.splitext(f)[0]
wiki_path = '/'.join([path, f]) if path else f
if os.path.isdir(os.path.join(full_path, f)):
wiki_path += '/'
lines.append('<a href="{}">{}</a>'.format(url_for('.wiki', wiki_path=wiki_path), fn2title(f)))
return '\n'.join(lines)
else:
abort(404)
with open(full_path) as fl:
return commonmark.commonmark(fl.read().decode('utf-8'))
def markdown(s):
tainted_html = CommonMark.commonmark(s)
safe_html = bleach.clean(tainted_html,
tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
styles=ALLOWED_STYLES)
return safe_html
def generate_email(self, data, tagline):
email = u"""
Hi {name},
{tagline}
## Outgoing Calls
{outgoing}
## Incoming Calls
{incoming}
Thanks,
The Call Stats Robot
""".format(
name=data['name'],
outgoing=self.stats_markdown(data['outgoing']),
incoming=self.stats_markdown(data['incoming']),
tagline=tagline)
html_email = CommonMark.commonmark(email)
# Send HTML email
html_part = MIMEText(html_email, 'html', 'utf-8')
msg = MIMEMultipart('alternative')
msg['Subject'] = 'Your Call Stats'
msg['From'] = local_settings.SMTP_FROM
msg['To'] = data['email']
msg.attach(html_part)
self.smtp.sendmail(
local_settings.SMTP_FROM,
data['email'], msg.as_string())
def test_unicode(self):
s = CommonMark.commonmark('<div>\u2020</div>\n')
self.assertEqual(s, '<div>\u2020</div>\n',
'Unicode works in an HTML block.')
CommonMark.commonmark('* unicode: \u2020')
CommonMark.commonmark('# unicode: \u2020')
CommonMark.commonmark('```\n# unicode: \u2020\n```')
def test_unicode(self):
s = CommonMark.commonmark('<div>\u2020</div>\n')
self.assertEqual(s, '<div>\u2020</div>\n',
'Unicode works in an HTML block.')
CommonMark.commonmark('* unicode: \u2020')
CommonMark.commonmark('# unicode: \u2020')
CommonMark.commonmark('```\n# unicode: \u2020\n```')
def create_post(request):
print request.FILES.get('file')
print type(request.FILES.get('file'))
image = request.FILES.get('file')
content = request.POST.get('post-input')
published = datetime.now()
is_markdown = json.loads(request.POST.get('is-markdown-post'))
if is_markdown:
contentType = "text/x-markdown"
content = CommonMark.commonmark(content)
else:
contentType = "text/plain"
visibility = request.POST.get('visibility')
c_username = request.user.username
user_object = User.objects.get(username=c_username)
author_object = Author.objects.get(email=user_object)
author_name = author_object.displayName
post_id = uuid.uuid4()
DITTO_HOST = 'http://' + request.get_host() + '/api/posts/' + str(post_id)
title = request.POST.get('title')
description = request.POST.get('description')
categories = request.POST.get('categories')
c = categories.split(' ')
categories_json = json.dumps(c)
new_post = Post(published=published, author=author_object, content=content, contentType=contentType,
visibility=visibility, source=DITTO_HOST, origin=DITTO_HOST, categories=categories, title=title,
description=description, id = post_id)
new_post.save()
if image:
print image.content_type
#image.name = str(uuid.uuid4())
print image.name
print "before creating"
new_image = Img(actual_image = image)
new_image.parent_post = new_post
print "before saving"
new_image.save()
print "after saving"
new_post.content = new_post.content + ' <br> <img src="http://ditto-test.herokuapp.com/ditto/media/images/'+image.name+'" >'
#new_post.content = new_post.content + ' <br> <img src="http://localhost:8000/ditto/media/images/'+image.name+'" >'
new_post.save()
return HttpResponse(request.POST.get('post_body'))
def render(self, context):
text = self.nodelist.render(context)
if self.title:
text = "### %s\n\n%s" % (self.title, text)
#text = markdown(text)
text = CommonMark.commonmark(text)
link_anchor = ANCHOR_PATTERN % self.anchor
text = bleach.linkify(text, callbacks=self.CALLBACKS, skip_pre=True)
text = link_anchor + text + TOP_LINK
return text
def parse_markdown(text):
if not text: return None
text = reDisplayMath.sub(lambda m: f'<math src={quoteattr(m.group(1))} />',
text)
text = reInlineMath.sub(lambda m: f'<math src={quoteattr(m.group(1))} />',
text)
html = CommonMark.commonmark(text)
ast = ElementTree.fromstring(f"<turingarena>{html}</turingarena>")
return list(xml_children(ast))
def markdown(text):
"""Filter for turning text into markdown.
:param text: String to be transformed.
:return: Transformed html string.
Usage:
{{ 'Some *markdown*'|markdown }}
"Some <strong>markdown</strong>"
"""
return Markup(CommonMark.commonmark(text))
def markdown(s: str) -> str:
commented_shortcodes = shortcodes.comment_shortcodes(s)
tainted_html = CommonMark.commonmark(commented_shortcodes)
# Create a Cleaner that supports parsing of bare links (see filters).
cleaner = bleach.Cleaner(tags=ALLOWED_TAGS,
attributes=ALLOWED_ATTRIBUTES,
styles=ALLOWED_STYLES,
strip_comments=False,
filters=[bleach.linkifier.LinkifyFilter])
safe_html = cleaner.clean(tainted_html)
return safe_html
def format_summary(text):
# Some summaries have double-newlines that are probably paragraph breaks.
# Others have newlines at the ends of ~60-column lines that we don't care about.
# Finally, some summaries have single linebreaks that seem to represent paragraphs.
# Which are we dealing with?
def avg(items): return sum(items)/len(items)
avg_line_length = avg([len(line) for line in text.split("\n")+[""]])
if avg_line_length > 100:
# Seems like newlines probably indicate paragraphs. Double them up so that we
# can pass the rest through a renderer.
text = text.replace("\n", "\n\n")
# Turn the text into HTML. This is a fast way to do it that might work nicely.
import CommonMark
return CommonMark.commonmark(text)
def render_markdown(content):
"""
Return a html fragment from markdown text content
Parameters
----------
content : str
A markdown formatted text
Returns
-------
html : str
"""
return CommonMark.commonmark(content)
def render_markdown(content):
"""
Return a html fragment from markdown text content
Parameters
----------
content : str
A markdown formatted text
Returns
-------
html : str
"""
return CommonMark.commonmark(content)
def _convert_to_html(self, template_name: str = '') -> str:
""" Convert from raw markdown text to html.
:return: html formatted text
"""
html = ""
if template_name == "":
html = CommonMark.commonmark(self._raw_text)
else:
template = self._get_template(template_name)
html_parts = self._get_html_parts()
html = template.render(html_parts=html_parts)
return html
def _get_html_parts(self):
# initialize html parts dictionary
ret = {'title': '', 'body': ''}
for l in self._raw_text.split('\n'):
# First H1 size text set to page title
if re.match(r'# .+$', l):
ret['title'] = l.replace('#', '').strip()
break
# self html text set to body html
ret['body'] = CommonMark.commonmark(self._raw_text)
return ret
def markdown_stuff(contentType, markdown):
#checks for block content then escapes content to another
#function before applying and returning markdowned content.
#If not just replace the new line comment with a linebreak
#to make it markdown
clean = html.conditional_escape(contentType)
if markdown:
#markdown block quotes
new_markdown = clean.replace('>', '>')
new_markdown = CommonMark.commonmark(new_markdown)
markdown_text = markdown_unescape(new_markdown)
return markdown_text.replace('\n', '<br/>')
return clean.replace('\n', '<br/>')
def get_thumbnail_url(doc, pagenumber, small):
# Returns a URL to a thumbnail image for a particular page of the document.
# 'small' is a boolean.
# If the document is on DocumentCloud, get the URL to DocumentCloud's thumbnail image.
documentcloud_id = get_documentcloud_document_id(doc)
if documentcloud_id:
# We can use the DocumentCloud API to get the URL to a thumbnail, but in the
# interests of speed, construct the URL ourselves.
#return query_documentcloud_api(documentcloud_id)["document"]["resources"]["page"]["image"].format(
# page=pagenumber,
# size="small" if small else "normal",
#)
return "https://assets.documentcloud.org/documents/%s/pages/%s-p%d-%s.gif" % (
documentcloud_id[0], documentcloud_id[1], pagenumber, "small" if small else "normal")
# If it's a Markdown document, download it, convert it to HTML, then render it to
# a PDF, and then to an image, and return that image as a data: URL.
elif doc.get("format") == "markdown" and os.path.exists("/usr/bin/htmldoc") and os.path.exists("/usr/bin/pdftoppm"):
# Download the Markdown file.
md = get_document_text(doc, pagenumber)
# If we got it...
if md:
import subprocess, base64
# Render the Markdown as HTML.
html = CommonMark.commonmark(md)
# Render the HTML as a PDF.
# TODO: Possible security issue if the Markdown source can generate HTML that
# causes htmldoc to perform network requests or possibly unsafe operations.
pdf = subprocess.check_output(["/usr/bin/htmldoc", "--quiet", "--continuous",
"--size", "4.5x5.8in", # smaller page magnifies the text
"--top", "0", "--right", "1cm", "--bottom", "1cm", "--left", "1cm", # margins
"-t", "pdf14", "-"],
input=html.encode("utf8"))
# Render the PDF and a PNG.
png = subprocess.check_output(["/usr/bin/pdftoppm", "-singlefile", "-r", "60", "-png"],
input=pdf)
# Return a data: URL so we don't have to store/host the image anywhere,
# but we can display it directly.
return "data:image/png;base64," + base64.b64encode(png).decode("ascii")
# No thumbnail image is available for this resource.
return None
def md_escaped(value, inline=True):
if not value:
return ""
formatted = value.strip()
if inline:
formatted = formatted.replace('\n', '<br>')
formatted = CommonMark.commonmark(formatted).strip()
# Remove wrapping <p> tags.
if inline and formatted.startswith('<p>') and formatted.endswith('</p>'):
formatted = formatted[3:-4]
return formatted
def report(env, node_name, report_id):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
:param env: Search for reports in this environment
:type env: :obj:`string`
:param node_name: Find the reports whose certname match this value
:type node_name: :obj:`string`
:param report_id: The hash or the configuration_version of the desired
report
:type report_id: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
reports = puppetdb.reports(query=query)
try:
report = next(reports)
except StopIteration:
abort(404)
report.version = CommonMark.commonmark(report.version)
return render_template(
'report.html',
report=report,
events=yield_or_stop(report.events()),
logs=report.logs,
metrics=report.metrics,
envs=envs,
current_env=env)
def generate_html(self, mdfile=None):
if mdfile is None:
mdf = self.mdfile
else:
mdf = mdfile
filepath = "posts/" + mdf
if mdf[-3:] != ".md":
filepath += ".md"
with open(filepath) as open_md:
content = open_md.readlines()
header = []
markdown = self.__parse_yaml_frontmatter(content)
markdown = ''.join(markdown)
return self.preamble + cm.commonmark(markdown) + self.closing
def post_new(request):
if request.method == "POST":
form = PostForm(data=request.POST)
#print("REQUEST.POST:%s"%request.POST)
imageForm = ImageForm(request.POST, request.FILES)
#print("FILES: %s"%request.FILES['imageFile'])
#print(form)
#print(form.errors)
if form.is_valid():
post = form.save(commit=False)
post.author = Author.objects.get(user=request.user.id)
post.published = timezone.now()
if post.contentType == 'text/x-markdown':
post.content = CommonMark.commonmark(post.content)
post.set_source()
post.set_origin()
post.save()
if request.POST['image_url']:
post.content = post.content + "<br><img src="+"'"+request.POST['image_url']+"'"+"/>"
post.save()
#return redirect('show_posts')
#return render(request, 'authors/index.html', {'form':form})
if imageForm.is_valid():
image = Image(imageFile=request.FILES['imageFile'])
image.post = post
image.save()
content = post.content
print("image url: %s"%image.imageFile.url)
imgUrl = "https://mighty-cliffs-82717.herokuapp.com/media/" + image.imageFile.url
#this will work for mighty cliffs but not for local testing
#imgUrl = post.source + "media/"+image.imageFile.url
print("imgUrl: %s"%imgUrl)
post.content = content + "<br><img src="+"'"+imgUrl+"'"+"/>"
post.save()
print("post.content: %s"%post.content)
return HttpResponseRedirect('/')
else:
return HttpResponseRedirect('/')
else:
form = PostForm()
return render(request, 'authors/index.html', {'form':form})
def compile(content_path, compile_path, stylesheet, html_template):
(mds, other) = sep_filetype(content_path, ".md")
# Prepare dir structure in compile_path
if os.path.isdir(compile_path):
shutil.move(compile_path, compile_path + ".old")
os.mkdir(compile_path)
new_dirs = [os.path.dirname(path.replace(content_path,
compile_path))
for path in (mds + other)]
for d in new_dirs:
os.makedirs(d, exist_ok=True)
# Copy all non md files
for f in other:
new_path = f.replace(content_path, compile_path)
# Coply file to new path, but don't follow symlinks!
shutil.copy2(f, new_path, follow_symlinks=False)
for f in mds:
with open(f, "r") as md_f:
md = md_f.read()
# Going to use simple str.format to build html to avoid deps with
# templating languages
with open(html_template, 'r') as template_f:
template = template_f.read()
compiled_html = template.format(body=CommonMark.commonmark(md),
stylesheet=stylesheet)
new_path = f.replace(content_path, compile_path)
new_path = os.path.splitext(new_path)[0] + ".html"
with open(new_path, "w") as html_f:
html_f.write(compiled_html)
if os.path.isdir(compile_path + ".old"):
shutil.rmtree(compile_path + ".old")
def read_content(filename):
"""Read content and metadata from file into a dictionary."""
# Read file content.
text = fread(filename)
# Read metadata and save it in a dictionary.
date_slug = os.path.basename(filename).split('.')[0]
match = re.search('^(?:(\d\d\d\d-\d\d-\d\d)-)?(.+)$', date_slug)
content = {
'date': match.group(1) or '1970-01-01',
'slug': match.group(2),
'src': filename, # Only for logging purpose
}
# Read headers.
end = 0
for key, val, end in read_headers(text):
content[key] = val
# Separate content from headers.
text = text[end:]
# Convert Markdown content to HTML.
if filename.endswith(('.md', '.mkd', '.mkdn', '.mdown', '.markdown')):
try:
if _test == 'ImportError':
raise ImportError('Error forced by test')
import CommonMark
text = CommonMark.commonmark(text)
except ImportError as e:
log('WARNING: Cannot render Markdown in {}: {}', filename, str(e))
# Update the dictionary with content, summary, and RFC 2822 date.
content.update({
'content': text,
'summary': truncate(text),
'rfc_2822_date': rfc_2822_format(content['date'])
})
return content
def translate(markdown_filepath):
markdown_file = open(markdown_filepath, 'r')
output = ''
block_type = ''
block_content = ''
in_block = False # We're not currently in a block of code
for line in markdown_file.read().split('\n'):
if line[:3] == "```" and not in_block: # Three backticks starts block
in_block = True # We found something to evaluate!
block_type = line[3:]
block_content = ''
# TODO: Finish this
elif line[:3] == "```" and in_block:
in_block = False
output += evaluator.evaluate(block_type, block_content) + '\n'
elif in_block:
block_content += '\n' + line
else:
output += line + '\n'
# TODO: Now, translate the markdown in `output`!
print(cm.commonmark(output))
def generate_email(self, data, tagline):
email = u"""
Hi {name},
{tagline}
## Outgoing Calls
{outgoing}
## Incoming Calls
{incoming}
Thanks,
The Call Stats Robot
""".format(
name=data['name'],
outgoing=self.stats_markdown(data['outgoing']),
incoming=self.stats_markdown(data['incoming']),
tagline=tagline)
html_email = CommonMark.commonmark(email)
# Send HTML email
html_part = MIMEText(html_email, 'html', 'utf-8')
msg = MIMEMultipart('alternative')
msg['Subject'] = 'Your Call Stats'
msg['From'] = local_settings.SMTP_FROM
if isinstance(data['email'], basestring):
data['email'] = [data['email']]
for email in data['email']:
msg['To'] = email
msg.attach(html_part)
self.smtp.sendmail(
local_settings.SMTP_FROM,
email, msg.as_string())
def markdown(content):
"""
A Django template filter to render the given content as Markdown.
"""
return CommonMark.commonmark(content)
def create_comment(request):
# text/x-markdown
# text/plain
comment = request.POST.get('comment-input')
parent_id = request.POST.get('comment-parent-id')
is_markdown = request.POST.get('comment-is-markdown',default='off')
origin = request.POST.get('comment-parent-origin')
if is_markdown == 'on':
comment = CommonMark.commonmark(comment)
is_markdown = True
else:
is_markdown = False
#post_object = Post.objects.get(id=parent_id)
c_username = request.user.username
user_object = User.objects.get(username=c_username)
author_object = Author.objects.get(email=user_object)
#author_name = author_object.displayName
# new_comment = Comment(author_id = author_object, id = post_object, comment = comment, is_markdown = is_markdown, published=published)
# print("comment made")
packet = {}
packet['comment'] = comment
if is_markdown:
packet['contentType'] = 'text/x-markdown'
else:
packet['contentType'] = 'text/plain'
packet['published'] = str(datetime.now())
packet['author'] = {}
packet['author']['id'] = str(author_object.id)
packet['author']['host'] = request.get_host()
packet['author']['displayName'] = author_object.displayName
packet['author']['url'] = author_object.url
packet['author']['github'] = "http://github.com/" + author_object.github
json_packet = json.dumps(packet)
# WE COULD USE THIS IF THEY GAVE US ORIGIN INSTEAD OF AN EMPTY STRING
# if "origin":"http://whereitcamefrom.com/api/posts/zzzzz",
# url1 = origin + "/comments/"
# janky stuff, we should just fix our api for the url
if 'ditto-test' in origin:
url1 = origin + "/comments/"
elif 'mighty' not in origin:
url1 = "http://project-c404.rhcloud.com/api/posts/" + parent_id + "/comments/"
else:
url1 = origin + "/comments/" # + "/api/posts/" + parent_id + "/comments"
print url1
print json_packet
# this works for posting a comment to ourselves
#url1 = "http://" + request.get_host() + "/api/posts/" + parent_id + "/comments/"
req = urllib2.Request(url1)
req.add_header('Content-Type', 'application/json')
foreign_hosts = ForeignHost.objects.filter()
if 'ditto-test' in origin:
base64string = base64.encodestring('%s:%s' % ("admin", "pass")).replace('\n', '')
req.add_header("Authorization", "Basic %s" % base64string)
else:
for host in foreign_hosts:
if host.url in origin:
base64string = base64.encodestring('%s:%s' % (host.username, host.password)).replace('\n', '')
req.add_header("Authorization", "Basic %s" % base64string)
urllib2.urlopen(req, json_packet)
# new_comment.save()
return redirect('/feed')
def fetchData(db, uri, cachetime, tz):
ptr = Query()
table = db.table('events')
settings = db.table('settings')
cal = Calendar.from_ical(requests.get(uri).text)
lastUpdated = settings.search(ptr.name == 'fetchTime')
if len(lastUpdated) == 0:
# We haven't fetched data at all!
log.info("Data fetching has not been completed yet. Fetching...")
settings.insert({
'name':
'fetchTime',
'value':
calendar.timegm(datetime.datetime.utcnow().timetuple())
})
pass
else:
past = datetime.datetime.utcnow() - datetime.timedelta(
seconds=cachetime)
if lastUpdated[0]['value'] < calendar.timegm(past.timetuple()):
# The data is older than an hour, so we refresh it
log.info('Data is considered stale. Fetching...')
table.purge()
pass
else:
# The data is less than an hour old, so we consider it current
log.info('Returning cached data...')
return table.all()
for event in cal.walk('vevent'):
uid = hashlib.sha224(event.decoded('uid')).hexdigest()[:15]
date = event.decoded('dtstart').strftime("%d/%m/%Y")
dateEnd = event.decoded('dtend').strftime("%d/%m/%Y")
if type(event.get('dtstart').dt) == datetime.date:
time = "00:00"
timeEnd = "00:00"
else:
time = utc_to_local(event.get('dtstart').dt, tz).strftime("%H:%M")
timeEnd = utc_to_local(event.get('dtend').dt, tz).strftime("%H:%M")
if len(table.search(ptr.id == uid)) == 0:
log.info("Inserting new event: {}".format(uid))
table.insert({
'id':
uid,
'date':
date,
'dateEnd':
dateEnd,
'time':
time,
'timeEnd':
timeEnd,
'title':
event.decoded('summary'),
'location':
event.decoded('location', 'Online'),
'desc':
CommonMark.commonmark(
event.decoded(
'description',
'This event doesn\'t have a description').decode(
'utf-8')),
'url':
event.decoded('url', ''),
'updated':
'{}'.format(event.decoded('last-modified'))
})
else:
# Event already exists, probably, so we'll check last updated time
lookup = table.search(ptr.id == uid)
if lookup[0]['updated'] != '{}'.format(
event.decoded('last-modified')):
log.info('Found an updated event: {}'.format(uid))
table.update(
{
'date':
date,
'dateEnd':
dateEnd,
'time':
time,
'timeEnd':
timeEnd,
'title':
event.decoded('summary'),
'location':
event.decoded('location', 'Online'),
'desc':
CommonMark.commonmark(
event.decoded(
'description',
'This event doesn\'t have a description')),
'url':
event.decoded('url', ''),
'updated':
'{}'.format(event.decoded('last-modified'))
}, ptr.id == uid)
else:
log.info(
'Event {} has not been updated since insertion'.format(
uid))
settings.update(
{'value': calendar.timegm(datetime.datetime.utcnow().timetuple())},
ptr.name == 'fetchTime')
return table.all()
def test_null_string_bug(self):
s = CommonMark.commonmark('> sometext\n>\n\n')
self.assertEqual(
s, '<blockquote>\n<pre><code>sometext\n</code></pre>'
'\n</blockquote>\n')
def __init__(self, body, encoding='UTF-8'):
self.plain_version = body
import CommonMark
html = CommonMark.commonmark(body)
html = self._template.format(markup=html)
super().__init__(html, encoding)
def comment_new(request):
if request.method == "POST":
try:
form = CommentForm(data=request.POST)
if form.is_valid():
comment = form.save(commit=False)
#comment.post=post
comment.author = Author.objects.get(user=request.user.id)
postid = request.POST.get("post_id", "")
#print("post_id: %s"%postid)
post = Post.objects.get(post_id=postid)
#print("post: %s"%post)
comment.post = post
comment.pub_date = timezone.now()
if comment.contentType == 'text/x-markdown':
comment.comment_text = CommonMark.commonmark(comment.comment_text)
comment.save()
return redirect('../../')
except:
form = CommentForm(data=request.POST)
if form.is_valid():
# TODO: FIX THIS, CHECK FOR OR CREATE GLOBAL AUTHOR
# MUST ALSO PASS IN THE HOST SO WE CAN RETRIEVE THE CORRECT NODE
# Get the author of the comment
#try:
author = Author.objects.get(user=request.user.id)
#except:
# author, status = GlobalAuthor.objects.get_or_create(user=request.user.id)
# Create the author dictionary object
author_dict = {
"id": author.author_id,
"host": author.host,
"displayName": author.user.username, # todo: get the real user name
"url": author.url,
"github": author.github
}
# Create the comment dictionary object
if request.POST.get("contentType") == 'text/x-markdown':
comment = CommonMark.commonmark(request.POST.get("comment_text"))
else:
comment = request.POST.get("comment_text")
comment = {
"author": author_dict,
"comment": comment,
"contentType": request.POST.get("contentType"),
"id": uuid.uuid4().hex,
"published": str(datetime.now())
}
# Get the node url
node_name = request.POST.get("node_name")
try:
node = Node.objects.get(node_name = node_name)
except:
node = Node.objects.get(node_name = "Team 6")
node_url = node.node_url
# Get the node's authentication token
auth_token = "Basic " + str(node.basic_auth_token)
# Get the post id
post_id = request.POST.get("post_id")
# Build the URL to post to
url = node_url + "posts/" + post_id + "/comments/"
#if node_name == 'Team 6':
# url += '/'
print 'URL: '
print url
# Create the request object
req = urllib2.Request(url)
req.add_header('Content-Type', 'application/json')
req.add_header('Authorization', auth_token)
# Jsonify the request
json_request = json.dumps(comment)
#print json_request
# Post to the node
urllib2.urlopen(req, json_request)
return redirect('../../')
else:
form = CommentForm()
return render(request, 'authors/index.html', {'form': form})
def test_output(self):
s = CommonMark.commonmark('*hello!*')
self.assertEqual(s, '<p><em>hello!</em></p>\n')
def test_null_string_bug(self):
s = CommonMark.commonmark('> sometext\n>\n\n')
self.assertEqual(
s,
'<blockquote>\n<pre><code>sometext\n</code></pre>'
'\n</blockquote>\n')
import CommonMark
parser = CommonMark.Parser()
renderer = CommonMark.HtmlRenderer()
ast = parser.parse("Hello *World*")
html = renderer.render(ast)
# json = CommonMark.ASTtoJSON(ast)
# CommonMark.dumpAST(ast)
print(html)
# with syntactic sugar
myhtml = CommonMark.commonmark("Hello *goodbye* I see *I try*!")
print(myhtml)
def markdownyfy(text):
return mark_safe(CommonMark.commonmark(text))
def generate_page(sourcefile, resultfile, genref):
# Generate html DOM from markdown.
markdown = open(sourcefile).read()
htmldoc = CommonMark.commonmark(markdown)
soup = BeautifulSoup(htmldoc, 'html.parser')
# Remove comments.
comments = soup.find_all(string=lambda text: isinstance(text, Comment))
for comment in comments:
comment.extract()
# All h4 sections are actually asides.
admonitions = soup.findAll("h4")
for admonition in admonitions:
p = admonition.find_next_sibling("p")
p['class'] = 'aside'
admonition.extract()
# Colorize the code blocks.
formatter = HtmlFormatter(style='tango')
snippets = soup.findAll("code", {"class": "language-python"})
for snippet in snippets:
code = snippet.contents[0]
highlighted = highlight(code, PythonLexer(), formatter)
newcode = BeautifulSoup(highlighted, 'html.parser')
snippet.parent.replace_with(newcode)
# Generate the HTML in its initial form, including <style>.
htmlfile = open(resultfile, 'w')
htmlfile.write(header)
htmlfile.write('<style>')
htmlfile.write(formatter.get_style_defs('.highlight'))
htmlfile.write('''
.highlight .mb, .highlight .mf, .highlight .mh, .highlight .mi,
.highlight .mo { color: #0063cf; }
''')
htmlfile.write('</style>')
htmlfile.write('<main>\n')
htmlfile.write(forkme)
htmlfile.write(str(soup))
# Generate quickref.
quickref = ''
if genref:
for member in inspect.getmembers(snowy):
name, value = member
if name.startswith('__'):
continue
if not inspect.isfunction(value):
continue
doc = inspect.getdoc(value)
src = inspect.getsource(value)
dsbegin = src.find(r'"""')
dsend = src.rfind(r'"""') + 4
dsbegin = src[:dsbegin].rfind('\n') + 1
src = src[:dsbegin] + src[dsend:]
nlines = len(src.split('\n'))
highlighted_src = highlight(src, PythonLexer(), formatter)
if doc:
doclines = doc.split('\n')
quickref += '<tr>\n'
quickref += f'<td><a href="#{name}">{name}</a></td>\n'
quickref += f'<td>{doclines[0]}</td>\n'
quickref += '<tr>\n'
htmlfile.write(f'<h3>{name}</h3>\n<p>\n')
htmlfile.write(' '.join(doclines))
htmlfile.write('\n</p>\n')
htmlfile.write(highlighted_src)
htmlfile.write('</main>\n')
htmlfile.close()
# Post process HTML by adding anchors, etc.
htmldoc = open(resultfile).read()
htmldoc = htmldoc.replace('$quickref$', quickref)
htmldoc = htmldoc.replace('<h1>', version + '\n<h1>')
soup = BeautifulSoup(htmldoc, 'html.parser')
for tag in 'h2 h3 h4'.split():
headings = soup.find_all(tag)
for heading in headings:
content = heading.contents[0].strip()
id = content.replace(' ', '_').lower()
heading["id"] = id
anchor = soup.new_tag('a', href='#' + id)
anchor.string = content
heading.contents[0].replace_with(anchor)
open(resultfile, 'w').write(str(soup))
def test_random_text(self, s):
CommonMark.commonmark(s)
def deploy_changelog(ctx, source="../CHANGELOG.md"):
content = CommonMark.commonmark(Path(source).read_text())
with template_local_file("index.html.template", "index.html",
{"content": content}):
sudo_put(ctx, "index.html", "/srv/zam/index.html", chown="zam")
def comments(request, uuid):
if request.method == 'GET':
# Does the post actually exist?
try:
p = Post.objects.get(post_id=uuid)
except:
return Response(status=status.HTTP_404_NOT_FOUND)
# Default if not given
page_num = request.GET.get('page', DEFAULT_PAGE_NUM)
page_size = request.GET.get('size', DEFAULT_PAGE_SIZE)
# Get all local and global comments, combine and paginate results
local_comments = Comment.objects.filter(post=p)
global_comments = GlobalComment.objects.filter(post=p)
all_comments = sorted(
chain(local_comments, global_comments),
key=attrgetter('pub_date'))
# Need this here or else the pagination bitches about it being unicode
page_num = int(page_num)
# Get the right page
pages = Paginator(all_comments, page_size)
page = pages.page(page_num+1)
data = page.object_list
response_obj = {}
response_obj['query'] = 'comments'
response_obj['count'] = len(all_comments)
response_obj['size'] = page_size
if page.has_next():
response_obj['next'] = settings.LOCAL_HOST + 'api/posts/' + uuid + '/comments?page=' + str(page_num + 1) + '&size=' + str(page_size)
if page.has_previous():
response_obj['previous'] = settings.LOCAL_HOST + 'api/posts/' + uuid + '/comments?page=' + str(page_num - 1) + '&size=' + str(page_size)
serializer = CommentSerializer(data, many=True)
response_obj['comments'] = serializer.data
return Response(response_obj, status=status.HTTP_200_OK)
elif request.method == 'POST':
try:
try:
# Get an existing global author
author = GlobalAuthor.objects.get(global_author_id=request.data['author']['id'])
print 'Found existing global author...'
except:
# Create a new global author
global_author_name = request.data['author']['displayName']
print 'Creating new global author'
id = request.data['author']['id']
url = request.data['author']['url']
host = request.data['author']['host']
author = GlobalAuthor(global_author_name = global_author_name, url = url, host = host)
author.save();
print 'Successfully created author'
post = Post.objects.get(post_id=uuid)
print 'Found a post...'
comment = GlobalComment(author=author, post=post)
print 'Initialized a comment..'
if request.data['contentType'] == 'text/x-markdown':
markedOne= CommonMark.commonmark(request.data['comment'])
comment.comment_text = markedOne
else:
comment.comment_text = request.data['comment']
print 'added text..'
comment.contentType = request.data['contentType']
print 'Added comment type'
comment.save()
print 'Successfully created comment'
return Response("Comment Successfully Added.", status=status.HTTP_201_CREATED)
except:
return Response("Comment not added, bad JSON format.", status=status.HTTP_400_BAD_REQUEST)
def test_output(self):
s = CommonMark.commonmark('*hello!*')
self.assertEqual(s, '<p><em>hello!</em></p>\n')
def get(self):
parser = PageCreator('index.md')
with open('index.md') as open_md:
self.write(parser.preamble + cm.commonmark(open_md.read()) + parser.closing)
# script to transform markdown into html
import CommonMark
# grab markdown~
md_file = open('md/hallo.md', 'r')
md_string = md_file.read()
md_file.close()
# grab header and footer~
header = open('templates/header.html', 'r')
footer = open('templates/footer.html', 'r')
# convert markdown body~
body = CommonMark.commonmark(md_string)
# stick it all together in a new file~
ht_file = open('ht/hallo.html', 'w')
ht_file.write(header.read() + body + footer.read())
ht_file.close()
<a href="index.html">Home</a>
<a href="entry_index.html">Index</a>
</div>
</html>"""
def ConvertTitleToFilename(title):
return title.lower().replace(" ", "_")
json_data = open("./data/part0_introduction.json").read()
index = {}
entries = json.loads(json_data)
for entry in entries:
title = entry["title"]
body = CommonMark.commonmark(entry["body"])
filename = "{0}.html".format(ConvertTitleToFilename(title))
filepath = "./docs/{0}".format(filename)
index[title] = filename
print("writing {0} to {1}...".format(title, filepath), end="")
with open(filepath, "w") as entry_page:
entry_page.write(ENTRY_PAGE_TEMPLATE.format(title=title, body=body))
print("DONE")
print("writing entry index...", end="")
with open("./docs/entry_index.html", "w") as entry_index_page:
items = ""
ordered_index = OrderedDict(
sorted(index.items(), key=lambda t: t[0].lower()))
for title, filename in ordered_index.items():
def markdown(src):
return mark_safe(CommonMark.commonmark(force_text(src)))
def common_checks_ocds(context, upload_dir, json_data, schema_obj, api=False, cache=True):
schema_name = schema_obj.release_pkg_schema_name
if 'records' in json_data:
schema_name = schema_obj.record_pkg_schema_name
common_checks = common_checks_context(upload_dir, json_data, schema_obj, schema_name, context,
fields_regex=True, api=api, cache=cache)
validation_errors = common_checks['context']['validation_errors']
new_validation_errors = []
for (json_key, values) in validation_errors:
error = json.loads(json_key)
new_message = validation_error_lookup.get(error['message_type'])
if new_message:
error['message_safe'] = conditional_escape(new_message)
else:
if 'message_safe' in error:
error['message_safe'] = mark_safe(error['message_safe'])
else:
error['message_safe'] = conditional_escape(error['message'])
schema_block, ref_info = lookup_schema(schema_obj.get_release_pkg_schema_obj(deref=True), error['path_no_number'])
if schema_block and error['message_type'] != 'required':
if 'description' in schema_block:
error['schema_title'] = escape(schema_block.get('title', ''))
error['schema_description_safe'] = mark_safe(bleach.clean(
CommonMark.commonmark(schema_block['description']),
tags=bleach.sanitizer.ALLOWED_TAGS + ['p']
))
if ref_info:
ref = ref_info['reference']['$ref']
if ref.endswith('release-schema.json'):
ref = ''
else:
ref = ref.strip('#')
ref_path = '/'.join(ref_info['path'])
schema = 'release-schema.json'
else:
ref = ''
ref_path = error['path_no_number']
schema = 'release-package-schema.json'
error['docs_ref'] = format_html('{},{},{}', schema, ref, ref_path)
new_validation_errors.append([json.dumps(error, sort_keys=True), values])
common_checks['context']['validation_errors'] = new_validation_errors
context.update(common_checks['context'])
if schema_name == 'record-package-schema.json':
context['records_aggregates'] = get_records_aggregates(json_data, ignore_errors=bool(validation_errors))
context['schema_url'] = schema_obj.record_pkg_schema_url
else:
additional_codelist_values = get_additional_codelist_values(schema_obj, json_data)
closed_codelist_values = {key: value for key, value in additional_codelist_values.items() if not value['isopen']}
open_codelist_values = {key: value for key, value in additional_codelist_values.items() if value['isopen']}
context.update({
'releases_aggregates': get_releases_aggregates(json_data, ignore_errors=bool(validation_errors)),
'additional_closed_codelist_values': closed_codelist_values,
'additional_open_codelist_values': open_codelist_values
})
context = add_conformance_rule_errors(context, json_data, schema_obj)
return context
def _on_notify_text(self, *args):
html_text = CommonMark.commonmark(self.get_property('text'))
self.web_view.load_html_string(html_text, '')
urlId, cname, format(cmd['inputs']),
' -> %s' % format(cmd['outputs'], output=True)
if len(cmd['outputs']) != 0 else '')
print >> nfp, '<li class="command" value="%i" id="%s">' % (
cmd['cmdId'], urlId)
print >> nfp, (
' <input type="checkbox" class="showDocs" id="showDocs(%s)"></input>'
% (urlId)) if cmd['doc'] != "" else ""
print >> nfp, (
' <label for="showDocs(%s)" class="showDocsLabel"></label>' %
(urlId)) if cmd['doc'] != "" else ""
print >> nfp, ' <code class="signature">[%i] %s</code>' % (
cmd['cmdId'], cdef)
print >> nfp, (
' <div class="docs">%s</div>' %
CommonMark.commonmark(cmd['doc'])) if cmd['doc'] != "" else ""
print >> nfp, '</li>'
nfp -= 2
print >> nfp, '\t</ol>'
print >> nfp, '</li>'
nfp -= 2
print >> nfp, '\t</ul>'
print >> nfp, '</div>'
print >> nfp, '<br />'
print >> tfp, '<h1 class="display-3">SwIPC Types</h1>'
print >> tfp, '<br />'
print >> tfp, '<ul class="list-group">'
tfp += 1
for name, type in sorted(types.items(), key=lambda x: x[0]):
def test_random_text(self, s):
CommonMark.commonmark(s)
import CommonMark
parser = CommonMark.Parser()
renderer = CommonMark.HtmlRenderer()
ast = parser.parse("Hello *World*")
html = renderer.render(ast)
# json = CommonMark.ASTtoJSON(ast)
# CommonMark.dumpAST(ast)
print(html)
# with syntactic sugar
myhtml = CommonMark.commonmark("Hello *goodbye* I see *I try*!")
print(myhtml)
