def create_issue(self, data, **kwargs):
"""
Creates the issue on the remote service and returns an issue ID.
"""
project_id = data.get('project')
if project_id is None:
raise ValueError('Azure DevOps expects project')
client = self.get_client()
title = data['title']
description = data['description']
try:
created_item = client.create_work_item(
instance=self.instance,
project=project_id,
title=title,
# Decriptions cannot easily be seen. So, a comment will be added as well.
description=markdown(description),
comment=markdown(description)
)
except Exception as e:
self.raise_error(e)
project_name = created_item['fields']['System.AreaPath']
return {
'key': six.text_type(created_item['id']),
'title': title,
'description': description,
'metadata': {
'display_name': '%s#%s' % (project_name, created_item['id']),
}
}
def import_file(cls,path=None):
'''
导入博客文件，自动跳过已导入的文件。
'''
path=abspath(path or '~/OneDrive/notes')
note_files=[]
for root,paths,files in os.walk(path):
note_files.extend([join(root,fi) for fi in files
if fi.endswith('.md')])
for fl in cls.find({'filename':{"$in":note_files}},
{'mtime':1,'filename':1}):
if fl['mtime']>=dt_tz(os.path.getmtime(fl['filename'])):
print('文件：%s 已导入，跳过'%(fl['filename']))
note_files.remove(fl['filename'])
renderer=ConvertRenderer()
for filename in note_files:
body='\n'.join(read_file(filename))
renderer.clear()
mistune.markdown(body,renderer=renderer)
cls.update_one({'filename':filename},
{'$set':
{'title':renderer.title,
'abbr':renderer.abbr,
'meta':renderer.meta,
'mtime':dt_tz(os.path.getmtime(filename)),
'ctime':dt_tz(os.path.getctime(filename)),
'body':body}},
True)
def _pre_put_hook(self):
if not self.original_name:
self.original_name = self.name
self.name = clean_name(self.name)
self.category = strip_string(self.category)
self.badge = strip_string(self.badge)
self.barcode = strip_string(self.barcode)
self.brand = strip_string(self.brand)
self.country = strip_string(self.country)
self.material = strip_string(self.material)
self.description_md = self.description_md.strip()
if self.description_md:
self.description_html = mistune.markdown(self.description_md)
self.description = re.sub(
r'(<!--.*?-->|<[^>]*>)', '', self.description_html)
else:
self.description = self.description.strip()
self.description_md = self.description
self.description_html = mistune.markdown(self.description)
if self.description:
self.short_description = u' '.join(self.description.split()[:5])
else:
self.short_description = ''
self.meta_keywords = self.clear_name.replace(' ', ', ')
def test_parse_block_html():
ret = mistune.markdown(
'<div>**foo**</div>', parse_block_html=True, escape=False
)
assert '<div><strong>' in ret
ret = mistune.markdown(
'<span>**foo**</span>', parse_block_html=True, escape=False
)
assert '<strong>' not in ret
def test_use_xhtml():
ret = mistune.markdown('foo\n\n----\n\nbar')
assert '<hr>' in ret
ret = mistune.markdown('foo\n\n----\n\nbar', use_xhtml=True)
assert '<hr />' in ret
ret = mistune.markdown('foo \nbar', use_xhtml=True)
assert '<br />' in ret
ret = mistune.markdown('', use_xhtml=True)
assert '<img src="bar" alt="foo" title="title" />' in ret
def add_post_post():
post = Word(
title = request.form["title"],
content = mistune.markdown(request.form["content"]),
category = mistune.markdown(request.form["category"]),
level = mistune.markdown(request.form["level"]),
author = current_user
)
session.add(post)
session.commit()
return redirect(url_for("words"))
def __init__(self, proto):
self.__proto = proto
self.name = proto.name
self.documentation = mistune.markdown(proto.documentation)
self.example_documentation = mistune.markdown(proto.example_documentation)
self.signature = self._get_signature(proto)
self.attributes = []
for attribute in proto.attribute:
self.attributes.append(Attribute(attribute))
self.outputs = []
for output in proto.output:
self.outputs.append(Output(output))
def test_parse_nested_html():
ret = mistune.markdown(
'<div><a href="http://example.org">**foo**</a></div>',
parse_block_html=True, escape=False
)
assert '<div><a href="http://example.org">' in ret
assert '<strong>' not in ret
ret = mistune.markdown(
'<div><a href="http://example.org">**foo**</a></div>',
parse_block_html=True, parse_inline_html=True, escape=False
)
assert '<div><a href="http://example.org"><strong>' in ret
def _grab_blocks(self):
"""
Parse `self.source` and return a dict mapping block names to their
raw content.
"""
r = BlockRecordRenderer()
mistune.markdown(self.source, renderer=r)
blocks = {}
for block in r.blocks:
name, content = self._parse_raw_block(block)
blocks[name] = content
self.blocks = blocks
def index():
dirs = _get_dirs()
main_index = dict(standard="1377", issue="1990", org="BS", title="BS1377:1990", parts=[])
print dirs
for d in dirs:
files = _get_md_files(d)
print "d=", d
#print "==========", ROOT_PATH + d + "/README.md"
txt = _read_file(ROOT_PATH + d + "/README.md")
html = mistune.markdown(txt)
soup = BeautifulSoup(html)
dic = {}
dic['title'] = str(soup.find("h1").text).strip()
dic['part'] = dic['title'][4:].strip().split(" ")[0].strip()
#dic['sections'] = []
idx = []
for f in files:
#print f
txt = _read_file(ROOT_PATH + f)
html = mistune.markdown(txt)
#print html
soup = BeautifulSoup(html)
print "------------------------------"
title = str(soup.find("h1").text).strip()
toc = []
for h2 in soup.findAll("h2"):
toc.append( dict(title=str(h2.text).strip()) )
#print title, toc
#'toc.append( dict(file=f, title=title, toc=toc) )
#fff = f
#print f, fff
num = f.split("/")[-1][:-3]
print "NO = ", num, type(num)
data = dict(title=title, clauses=toc, file=f, part= str(d)[1:], org="BS", section=num)
idx.append( data )
dic['sections'] = idx
_write_yaml("%s/%s/index.yaml" % (ROOT_PATH, d), dic )
main_index['parts'].append(dic)
#print idx
#break
_write_yaml("%s/index.yaml" % (ROOT_PATH), main_index )
def test_parse_inline_html():
ret = mistune.markdown(
'<div>**foo**</div>', parse_inline_html=True, escape=False
)
assert '<strong>' not in ret
ret = mistune.markdown(
'<span>**foo**</span>', parse_inline_html=True, escape=False
)
assert '<span><strong>' in ret
ret = mistune.markdown(
'<a>http://lepture.com</a>', parse_inline_html=True, escape=False
)
assert 'href' not in ret
def get_html(self):
if self.text_type == 'rst':
from docutils.core import publish_parts
return publish_parts(self.text, writer_name='html')['html_body']
else:
import mistune
return mistune.markdown(self.text)
def edit_post_post(post_id):
post=session.query(Post).get(post_id)
post.title = request.form["title"]
post.content = mistune.markdown(request.form["content"])
session.commit()
return redirect(url_for("posts"))
def create(cls, author, raw_comment, parent):
"""
Create a new comment instance. If the parent is submisison
update comment_count field and save it.
If parent is comment post it as child comment
:param author: RedditUser instance
:type author: RedditUser
:param raw_comment: Raw comment text
:type raw_comment: str
:param parent: Comment or Submission that this comment is child of
:type parent: Comment | Submission
:return: New Comment instance
:rtype: Comment
"""
html_comment = mistune.markdown(raw_comment)
# todo: any exceptions possible?
comment = cls(author=author,
author_name=author.user.username,
raw_comment=raw_comment,
html_comment=html_comment)
if isinstance(parent, Submission):
submission = parent
comment.submission = submission
elif isinstance(parent, Comment):
submission = parent.submission
comment.submission = submission
comment.parent = parent
else:
return
submission.comment_count += 1
submission.save()
return comment
def add_post_post():
members = session.query(User).all()
post = Post(
title=request.form["title"],
content=mistune.markdown(request.form["content"]),
description = request.form["description"],
author=current_user
)
session.add(post)
session.commit()
all_members = []
for member in members:
all_members.append(member.email)
all_members = [str(i) for i in all_members]
describe = request.form["description"] +".... View full message on TeamForum!"
mail=Mail(app)
with mail.connect() as conn:
for email in all_members:
message = Message(subject="A new forum has been created by "+ current_user.name,
body= describe,
sender=("TeamForum", "*****@*****.**"),
recipients=[email]
)
conn.send(message)
flash("You have created a new forum. Team members have been notified.", "info")
return redirect(url_for("posts"))
def main():
md = urllib2.urlopen('https://raw.githubusercontent.com/syhw/wer_are_we/master/README.md').read()
bs = BeautifulSoup(mistune.markdown(md))
wer_data_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/wer.py"))
file_output = "# The file was autogenerated by ../scrapers/wer.py\n\nfrom datetime import date\n\nfrom data.acoustics import speech_recognition, swb_hub_500\nfrom scales import *\n\n"
wer_metrics = []
for table, header in zip(bs.findAll('table'), bs.findAll('h3')):
header = header.getText()
rows = table.findAll('tr')
metric_data = get_metrics(header, rows[0].findAll('th')[:-3], file_output)
metric_names = metric_data[0]
wer_metrics += metric_names
table_data = []
for row in rows:
if row.findAll('td') == []:
continue
measure_data, targets, target_source = add_measures(metric_names, row)
if not targets:
table_data += measure_data
elif not measure_data:
metric_data = get_metrics(header, rows[0].findAll('th')[:-3], file_output, targets = targets, target_source = target_source)
file_output = metric_data[1]
file_output += "".join(sorted(table_data))
file_output = file_output + "\n\nwer_metrics=[" + ", ".join(wer_metrics) + "]"
with open(wer_data_file, 'wb') as f:
f.write(file_output)
def description(self):
"""Extract description from a release."""
if self.release.get('body'):
return markdown(self.release['body'])
elif self.repository.get('description'):
return self.repository['description']
return 'No description provided.'
def edit_post_post(id=1):
post = session.query(Post)
post = post.filter(Post.id == id).first()
post.title=request.form["title"]
post.content=mistune.markdown(request.form["content"])
session.commit()
return redirect(url_for("posts"))
def preview_page():
title = request.form.get('page_title', '')
content_markdown = request.form.get('new_page_content', '')
content_html = mistune.markdown(content_markdown)
preview = {'title': title, 'content': content_html}
return jsonify(preview)
def markdownDeck(deck):
fl = open("decords/{}".format(deck), "r")
xml = mistune.markdown(fl.read())
fl.close()
del (fl)
xml = etree.fromstring(xml)
pass
def generate_post(self, text, out_file_name):
html = mistune.markdown(text, escape=False)
result = self.blog_template.format(
content=html, blog_name=self.blog_name)
with open(out_file_name, "w+") as f:
f.write(result)
def job(job_id):
'''
View a single job (any status) with AJAX callback to update elements, e.g.
- STDOUT/STDERR
- File outputs (figures, etc.), intelligent render handling
- Download link for all files
:param job_id:
:return:
'''
# Get the job object from the database
job = Job.query.get(job_id)
script = job.script
display = {}
cwd = os.path.join(job.path, 'output') # Excution path of the job
if os.path.isdir(cwd): # Execution has begun/finished
files = job.get_output_files()
display = build_display_objects(files)
documentation = script.load_docs()
if documentation:
documentation = mistune.markdown(documentation)
return render_template("public/job.html", script=script, job=job, metadata=script.load_config(), display=display,
documentation=documentation)
def edit_post_post(id):
post = session.query(Post).get(id)
post.title=request.form["title"]
post.content=mistune.markdown(request.form["content"])
session.add(post)
session.commit()
return redirect(url_for("view_post", id=post.id))
def render(self, context):
value = self.nodelist.render(context)
try:
return mark_safe(mistune.markdown(value))
except ImportError:
raise template.TemplateSyntaxError("Error in `markdown` tag: "
"The mistune library isn't installed.")
def render(file):
"""Render HTML from Markdown file content."""
fp = file.open()
content = fp.read()
result = mistune.markdown(content.decode('utf-8'))
fp.close()
return result
def stack_announcement(announcement_name):
ANNMT = u'ANNOUNCEMENT_' + announcement_name
hide = 'true' # This is javascript :-D
message = getattr(config, ANNMT, u'').strip()
prefix = getattr(config, ANNMT + u'_PREFIX', u'')
if message:
# NOTE: This is a non-breakable space, else it gets lost.
message = mistune.markdown(prefix + (u' ' if prefix else u'')
+ message).strip()
priority = getattr(config, ANNMT + u'_PRIORITY', u'').lower()
if priority not in (u'', u'info', u'success', u'error'):
priority = u'info'
if priority in (u'info', u'error'):
hide = 'false'
start_date = getattr(config, ANNMT + u'_START', u'')
end_date = getattr(config, ANNMT + u'_END', u'')
if (not start_date or start_date <= dttoday) and (
not end_date or end_date >= dttoday):
announcements.append((message, priority, hide))
def markdownify():
md = request.form.get("markdown")
if md:
html = mistune.markdown(md)
return html
else:
return "no markdown supplied"
def _renderHtmlWithMistune(text):
options= {
"escape": True,
"use_xhtml": True
}
renderer= _MermaidRenderer(**options)
return mistune.markdown(text, renderer=renderer, **options), renderer.hasDiagram
def read_repo(self, git=None, ls=None):
clone_path = path.join(self.dir_base, str(uuid.uuid4()) if git is not None else 'upstream')
print('github url', git if git is not None else self.upstream)
git_url = urlparse(git if git is not None else self.upstream)
clone_url = None
splitted_path = git_url.path.strip('/').split('/')
if len(splitted_path) > 2:
clone_url = git_url.scheme + '://' + git_url.netloc + '/' + splitted_path[0] + '/' + splitted_path[1] + '.git'
else:
clone_url = git if git is not None else self.upstream
print('clone url', clone_url)
# cloning the repository
Gittle.clone(clone_url, clone_path)
if len(splitted_path) > 2:
readme_file = self.find_readme(clone_path, '/'.join(str(x) for x in splitted_path[2:]))
else:
readme_file = self.find_readme(clone_path)
print(clone_url, readme_file)
try:
with open(path.join(clone_path, readme_file)) as f:
soup = BeautifulSoup(mistune.markdown(f.read()), 'html.parser')
self.save_md(soup, True if git is None else False, ls)
self.stdout.write(self.style.SUCCESS('Successfully read the upstream'))
except Exception as exp:
print('An error happened while reading the repo.', exp)
def edit_entry_post(id):
entry = session.query(Entry)
entry = entry.filter(Entry.id == id).first()
entry.title=request.form["title"]
entry.content= mistune.markdown(request.form["content"])
session.commit()
return redirect(url_for("entries"))
def save(self, *args, **kwargs):
new_hash = xxhash.xxh64(self.content_raw).hexdigest()
if new_hash != self.raw_content_hash or (not self.pk):
self.content_rendered = mistune.markdown(self.content_raw)
super(Appendix, self).save(*args, **kwargs)
self.raw_content_hash = new_hash
def get_html_description(self, obj):
# Parse markdown
return mistune.markdown(obj.description)
def get_file(post_filename):
blob = Blob(post_filename, get_storage())
blog_as_text = blob.download_as_string().decode()
return mistune.markdown(blog_as_text, escape=True, hard_wrap=True)
def convert_to_confluence(markdown, metadata={}):
renderer = ConfluenceRenderer()
content_html = mistune.markdown(markdown, renderer=renderer)
page_html = renderer.layout(content_html)
return page_html, renderer.attachments
def test_linebreak():
ret = mistune.markdown('this **foo** \nis me')
assert '<br>' not in ret
ret = mistune.markdown('this **foo** \nis me', hard_wrap=True)
assert '<br>' in ret
def save(self, *args, **kwargs):
self.content_html = mistune.markdown(self.content)
super().save(*args, **kwargs)
def clean_content(self):
content = self.cleaned_data.get('content')
if len(content) < 10:
raise forms.ValidationError('内容长度太短')
content = mistune.markdown(content)
return content
from flask.json import JSONEncoder
from mistune import markdown
from .constants import OPTIONAL_FIELDS
from .utils import extract_definitions
from .utils import get_specs
from .utils import get_schema_specs
from .utils import parse_definition_docstring
from .utils import get_vendor_extension_fields
from .utils import validate
from .utils import LazyString
from . import __version__
NO_SANITIZER = lambda text: text # noqa
BR_SANITIZER = lambda text: text.replace('\n', '<br/>'
) if text else text # noqa
MK_SANITIZER = lambda text: Markup(markdown(text)) if text else text # noqa
class APIDocsView(MethodView):
"""
The /apidocs
"""
def __init__(self, *args, **kwargs):
view_args = kwargs.pop('view_args', {})
self.config = view_args.get('config')
super(APIDocsView, self).__init__(*args, **kwargs)
def get(self):
"""
The data under /apidocs
json or Swagger UI
from pathlib import Path
from shutil import copyfile
import os
import mistune
scriptDir = os.path.dirname(os.path.abspath(__file__))
DEST_DIR = "{}/../build/doc/".format(scriptDir)
template = open("{}/template.html".format(scriptDir), "r").read()
files = ["{}/../README.md".format(scriptDir), "{}/../virtualKeyNames.md".format(scriptDir), "{}/../CHANGELOG.md".format(scriptDir)]
for mdPath in files:
mdFilename = Path(mdPath).name
file = open(mdPath, "r")
md = file.read()
html = mistune.markdown(md)
html = template.replace("${body}", html).replace("${title}", mdFilename.replace(".md", ""))
# also replace local markdown links with local html links
html = html.replace(".md", ".html")
htmlFilename = DEST_DIR + mdFilename.replace(".md", ".html")
parentDir = Path(htmlFilename).parent
parentDir.mkdir(parents=True, exist_ok=True)
with open(htmlFilename, "w") as htmlFile:
htmlFile.write(html)
# don't forget to copy the css file
copyfile("{}/html.css".format(scriptDir), DEST_DIR + "/html.css")
def html(self):
return markdown(self.text)
def render_type(self, f, depth):
if f["name"] in self.rendered or f["name"] in self.redirects:
return
self.rendered.add(f["name"])
if "doc" not in f:
f["doc"] = ""
f["type"] = copy.deepcopy(f)
f["doc"] = ""
f = f["type"]
if "doc" not in f:
f["doc"] = ""
def extendsfrom(item, ex):
if "extends" in item:
for e in aslist(item["extends"]):
ex.insert(0, self.typemap[e])
extendsfrom(self.typemap[e], ex)
ex = [f]
extendsfrom(f, ex)
enumDesc = {}
if f["type"] == "enum" and isinstance(f["doc"], list):
for e in ex:
for i in e["doc"]:
idx = i.find(":")
if idx > -1:
enumDesc[i[:idx]] = i[idx + 1:]
e["doc"] = [
i for i in e["doc"]
if i.find(":") == -1 or i.find(" ") < i.find(":")
]
f["doc"] = fix_doc(f["doc"])
if f["type"] == "record":
for field in f.get("fields", []):
if "doc" not in field:
field["doc"] = ""
if f["type"] != "documentation":
lines = []
for l in f["doc"].splitlines():
if len(l) > 0 and l[0] == "#":
l = ("#" * depth) + l
lines.append(l)
f["doc"] = "\n".join(lines)
_, frg = urlparse.urldefrag(f["name"])
num = self.toc.add_entry(depth, frg)
doc = "## %s %s\n" % (num, frg)
else:
doc = ""
if self.title is None and f["doc"]:
self.title = f["doc"][0:f["doc"].index("\n")]
if self.title.startswith('# '):
self.title = self.title[2:]
if f["type"] == "documentation":
f["doc"] = number_headings(self.toc, f["doc"])
#if "extends" in f:
# doc += "\n\nExtends "
# doc += ", ".join([" %s" % linkto(ex) for ex in aslist(f["extends"])])
#if f["name"] in self.subs:
# doc += "\n\nExtended by"
# doc += ", ".join([" %s" % linkto(s) for s in self.subs[f["name"]]])
#if f["name"] in self.uses:
# doc += "\n\nReferenced by"
# doc += ", ".join([" [%s.%s](#%s)" % (s[0], s[1], to_id(s[0])) for s in self.uses[f["name"]]])
doc = doc + "\n\n" + f["doc"]
doc = mistune.markdown(doc, renderer=MyRenderer())
if f["type"] == "record":
doc += "<h3>Fields</h3>"
doc += """<table class="table table-striped">"""
doc += "<tr><th>field</th><th>type</th><th>required</th><th>description</th></tr>"
required = []
optional = []
for i in f.get("fields", []):
tp = i["type"]
if isinstance(
tp,
list) and tp[0] == "https://w3id.org/cwl/salad#null":
opt = False
tp = tp[1:]
else:
opt = True
desc = i["doc"]
#if "inherited_from" in i:
# desc = "%s _Inherited from %s_" % (desc, linkto(i["inherited_from"]))
frg = schema.avro_name(i["name"])
tr = "<td><code>%s</code></td><td>%s</td><td>%s</td><td>%s</td>" % (
frg, self.typefmt(
tp, self.redirects), opt, mistune.markdown(desc))
if opt:
required.append(tr)
else:
optional.append(tr)
for i in required + optional:
doc += "<tr>" + i + "</tr>"
doc += """</table>"""
elif f["type"] == "enum":
doc += "<h3>Symbols</h3>"
doc += """<table class="table table-striped">"""
doc += "<tr><th>symbol</th><th>description</th></tr>"
for e in ex:
for i in e.get("symbols", []):
doc += "<tr>"
frg = schema.avro_name(i)
doc += "<td><code>%s</code></td><td>%s</td>" % (
frg, enumDesc.get(frg, ""))
doc += "</tr>"
doc += """</table>"""
f["doc"] = doc
self.typedoc.write(f["doc"])
subs = self.docParent.get(f["name"], []) + self.record_refs.get(
f["name"], [])
if len(subs) == 1:
self.render_type(self.typemap[subs[0]], depth)
else:
for s in subs:
self.render_type(self.typemap[s], depth + 1)
for s in self.docAfter.get(f["name"], []):
self.render_type(self.typemap[s], depth)
def html_body(self):
"""Format and return the markdown for the puzzle body."""
return mistune.markdown(self.get_body(), escape=False)
def view(map_identifier, topic_identifier):
topic_store = get_topic_store()
collaboration_mode = None
if current_user.is_authenticated: # User is logged in
is_map_owner = topic_store.is_topic_map_owner(map_identifier,
current_user.id)
if is_map_owner:
topic_map = topic_store.get_topic_map(map_identifier,
current_user.id)
else:
topic_map = topic_store.get_topic_map(map_identifier)
if topic_map is None:
current_app.logger.warning(
f"Topic map not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}]"
)
abort(404)
collaboration_mode = topic_store.get_collaboration_mode(
map_identifier, current_user.id)
if topic_map.published:
if not is_map_owner and topic_identifier == "home":
flash(
"You are accessing a published topic map of another user.",
"primary")
else:
if not is_map_owner: # The map is private and doesn't belong to the user who is trying to access it
if not collaboration_mode: # The user is not collaborating on the map
abort(403)
else: # User is not logged in
topic_map = topic_store.get_topic_map(map_identifier)
if topic_map is None:
current_app.logger.warning(
f"Topic map not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}]"
)
abort(404)
if not topic_map.published: # User is not logged in and the map is not published
abort(403)
# Determine if (active) scope filtering has been specified in the URL
scope_filtered = request.args.get("filter", type=int)
if scope_filtered is not None:
session["scope_filter"] = scope_filtered
if "scope_filter" in session:
scope_filtered = session["scope_filter"]
else:
session["breadcrumbs"] = []
session["current_scope"] = UNIVERSAL_SCOPE
session["scope_filter"] = 1
# If a context has been specified in the URL, then use that to set the context
scope_identifier = request.args.get("context", type=str)
if scope_identifier and topic_store.topic_exists(map_identifier,
scope_identifier):
session["current_scope"] = scope_identifier
# Get topic
if scope_filtered:
topic = topic_store.get_topic(
map_identifier,
topic_identifier,
scope=session["current_scope"],
resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES,
)
else:
topic = topic_store.get_topic(
map_identifier,
topic_identifier,
scope=session["current_scope"],
resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES,
)
if topic is None:
current_app.logger.warning(
f"Topic not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}], topic identifier: [{topic_identifier}]"
)
session["inexistent_topic_identifier"] = topic_identifier
abort(404)
else:
session.pop("inexistent_topic_identifier", None)
if scope_filtered:
topic_occurrences = topic_store.get_topic_occurrences(
map_identifier,
topic_identifier,
scope=session["current_scope"],
inline_resource_data=RetrievalMode.INLINE_RESOURCE_DATA,
resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES,
)
else:
topic_occurrences = topic_store.get_topic_occurrences(
map_identifier,
topic_identifier,
inline_resource_data=RetrievalMode.INLINE_RESOURCE_DATA,
resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES,
)
occurrences = {
"text": None,
"images": [],
"3d-scenes": [],
"files": [],
"links": [],
"videos": [],
"notes": [],
}
for occurrence in topic_occurrences:
if occurrence.instance_of == "text" and occurrence.scope == session[
"current_scope"]:
if occurrence.resource_data:
occurrences["text"] = mistune.markdown(
occurrence.resource_data.decode())
elif occurrence.instance_of == "image":
occurrences["images"].append({
"title":
occurrence.get_attribute_by_name("title").value,
"url":
occurrence.resource_ref,
})
elif occurrence.instance_of == "3d-scene":
occurrences["3d-scenes"].append({
"title":
occurrence.get_attribute_by_name("title").value,
"url":
occurrence.resource_ref,
})
elif occurrence.instance_of == "file":
occurrences["files"].append({
"title":
occurrence.get_attribute_by_name("title").value,
"url":
occurrence.resource_ref,
})
elif occurrence.instance_of == "url":
occurrences["links"].append({
"title":
occurrence.get_attribute_by_name("title").value,
"url":
occurrence.resource_ref,
})
elif occurrence.instance_of == "video":
occurrences["videos"].append({
"title":
occurrence.get_attribute_by_name("title").value,
"url":
occurrence.resource_ref,
})
elif occurrence.instance_of == "note":
occurrences["notes"].append({
"identifier":
occurrence.identifier,
"title":
occurrence.get_attribute_by_name("title").value,
"timestamp":
maya.parse(
occurrence.get_attribute_by_name(
"modification-timestamp").value),
"text":
mistune.markdown(occurrence.resource_data.decode()),
})
if scope_filtered:
associations = topic_store.get_association_groups(
map_identifier, topic_identifier, scope=session["current_scope"])
else:
associations = topic_store.get_association_groups(
map_identifier, topic_identifier)
creation_date = maya.parse(
topic.get_attribute_by_name("creation-timestamp").value)
modification_date_attribute = topic.get_attribute_by_name(
"modification-timestamp")
modification_date = maya.parse(
modification_date_attribute.value
) if modification_date_attribute else "Undefined"
# Breadcrumbs
if "breadcrumbs" not in session:
session["breadcrumbs"] = []
breadcrumbs = deque(session["breadcrumbs"], BREADCRUMBS_COUNT)
if topic_identifier in breadcrumbs:
breadcrumbs.remove(topic_identifier)
breadcrumbs.append(topic_identifier)
session["breadcrumbs"] = list(breadcrumbs)
return render_template(
"topic/view.html",
topic_map=topic_map,
topic=topic,
occurrences=occurrences,
associations=associations,
creation_date=creation_date,
modification_date=modification_date,
breadcrumbs=breadcrumbs,
collaboration_mode=collaboration_mode,
)
def test_ExpectationSuitePageRenderer_render_expectation_suite_notes():
result = ExpectationSuitePageRenderer._render_expectation_suite_notes(
ExpectationSuite(expectation_suite_name="test", meta={"notes":
"*hi*"}))
# print(RenderedContent.rendered_content_list_to_json(result.text))
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
"*hi*"
]
result = ExpectationSuitePageRenderer._render_expectation_suite_notes(
ExpectationSuite(expectation_suite_name="test",
meta={"notes": ["*alpha*", "_bravo_", "charlie"]}))
# print(RenderedContent.rendered_content_list_to_json(result.text))
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
"*alpha*", "_bravo_", "charlie"
]
result = ExpectationSuitePageRenderer._render_expectation_suite_notes(
ExpectationSuite(expectation_suite_name="test",
meta={
"notes": {
"format": "string",
"content": ["*alpha*", "_bravo_", "charlie"]
}
}))
# print(RenderedContent.rendered_content_list_to_json(result.text))
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
"*alpha*", "_bravo_", "charlie"
]
result = ExpectationSuitePageRenderer._render_expectation_suite_notes(
ExpectationSuite(
expectation_suite_name="test",
meta={"notes": {
"format": "markdown",
"content": "*alpha*"
}}))
# print(RenderedContent.rendered_content_list_to_json(result.text))
try:
mistune.markdown("*test*")
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
{
'content_block_type': 'markdown',
'styling': {
'parent': {}
},
'markdown': '*alpha*'
}
]
except OSError:
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
"*alpha*"
]
result = ExpectationSuitePageRenderer._render_expectation_suite_notes(
ExpectationSuite(expectation_suite_name="test",
meta={
"notes": {
"format": "markdown",
"content": ["*alpha*", "_bravo_", "charlie"]
}
}))
# print(RenderedContent.rendered_content_list_to_json(result.text))
try:
mistune.markdown("*test*")
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
{
'content_block_type': 'markdown',
'styling': {
'parent': {}
},
'markdown': '*alpha*'
}, {
'content_block_type': 'markdown',
'styling': {
'parent': {}
},
'markdown': '_bravo_'
}, {
'content_block_type': 'markdown',
'styling': {
'parent': {}
},
'markdown': 'charlie'
}
]
except OSError:
assert RenderedContent.rendered_content_list_to_json(result.text) == [
'This Expectation suite currently contains 0 total Expectations across 0 columns.',
"*alpha*", "_bravo_", "charlie"
]
def save(self, *args, **kwargs):
if self.is_md:
self.content_html = mistune.markdown(self.content)
else:
self.content_html = self.content
super().save(*args, **kwargs)
def markdown(text):
return Markup(md.markdown(text, escape=True))
#!/usr/bin/python
import subprocess
import os
import mistune
for root, dirs, files in os.walk("/home/karl/jgit/data"):
for file in files:
if file.lower().endswith(".md"):
full_path = os.path.join(root, file)
text = open(full_path).read()
html = mistune.markdown(text, escape=False)
renamed_path = full_path.replace("/", "@")
with open("/tmp/mdtest/" + renamed_path + ".mdorig", "w") as f:
f.write(text)
with open("/tmp/mdtest/" + renamed_path + ".mistune", "w") as f:
f.write(html)
for root, dirs, files in os.walk("/tmp/mdtest"):
for file in files:
if file.lower().endswith(".mdorig"):
full_path = os.path.join(root, file)
old_path = full_path.replace(".mdorig", ".mistune")
new_path = full_path.replace(".mdorig", ".net")
p = subprocess.Popen(["diff", old_path, new_path],
stdout=subprocess.PIPE)
out, err = p.communicate()
rc = p.wait()
print "%s: rc=%d" % (full_path, rc)
if rc != 0:
print out
def generate_html():
for att in CONFIG:
atts[att] = CONFIG[att]
# Generate the HTML for posts and prepare variables for get_pages
for x in get_posts():
atts['date'] = str(datetime.datetime.today().date())
atts['post_title'] = "Preview"
with open(os.path.join(POST_DIR, x)) as f:
if os.path.isfile(
os.path.join(BASE_DIR, POST_DIR,
x.split('.')[0] + ".json")):
with open(
os.path.join(BASE_DIR, POST_DIR,
x.split('.')[0] + '.json')) as s:
j = json.load(s)
t = j['layout']
if t == 'default':
template = POST_TEMPLATE
atts['post_title'] = j['title']
atts['date'] = j['date']
else:
template = Template(t.read())
atts['post_title'] = j['title']
atts['date'] = j['date']
else:
template = POST_TEMPLATE
atts['content'] = mistune.markdown(f.read())
new_filename = atts['date'] + '_' + os.path.splitext(x)[0] + '.html'
open(os.path.join(OUTPUT_DIR, BLOG_DIR, new_filename),
'w').write(template.render(atts))
atts['blog'].append({
'title':
atts['date'] + ' / ' + atts['post_title'],
'link':
'blog/' + atts['date'] + '_' +
atts['post_title'].replace(' ', '-') + '.html'
})
atts['blog'] = sorted(atts['blog'], key=byTitle_key, reverse=True)
# Generate the HTML for pages
for x in get_pages():
with open(os.path.join(PAGE_DIR, x)) as f:
if os.path.isfile(
os.path.join(BASE_DIR, PAGE_DIR,
x.split('.')[0] + ".json")):
with (open(
os.path.join(BASE_DIR, PAGE_DIR,
x.split('.')[0] + '.json'))) as s:
j = json.load(s)
t = j['layout']
if t == 'default':
template = PAGE_TEMPLATE
else:
template = Template(
open(os.path.join(TEMPLATE_DIR, t + '.j2')).read())
content = f.read()
atts['content'] = mistune.markdown(content)
new_filename = os.path.splitext(x)[0] + '.html'
open(os.path.join(OUTPUT_DIR, new_filename),
'w').write(template.render(atts))
parser = argparse.ArgumentParser(description = 'Download all the PDF/HTML links into README.md')
parser.add_argument('-d', action="store", dest="directory")
parser.add_argument('--no-html', action="store_true", dest="nohtml", default = False)
parser.add_argument('--overwrite', action="store_true", default = False)
results = parser.parse_args()
output_directory = 'pdfs' if results.directory is None else results.directory
forbidden_extensions = ['html', 'htm'] if results.nohtml else []
if results.overwrite and os.path.exists(output_directory):
shutil.rmtree(output_directory)
with open('README.md') as readme:
readme_html = mistune.markdown(readme.read())
readme_soup = BeautifulSoup.BeautifulSoup(readme_html, "html.parser")
point = readme_soup.find_all('h1')[1]
failures = []
while point is not None:
if point.name:
if re.search('h[1-2]', point.name):
if point.name == 'h1':
h1_directory = os.path.join(output_directory, clean_text(point.text))
current_directory = h1_directory
elif point.name == 'h2':
current_directory = os.path.join(h1_directory, clean_text(point.text))
if not os.path.exists(current_directory):
os.makedirs(current_directory)
articles[key]['translations'][result[2]] = filename
# Prepare a template for articles
with open('templates/article.html') as file:
article_template = Template(file.read())
# Create translated articles
translated_articles = []
for key, article in articles.items():
with open(article['original']) as file:
original = file.read()
original_metadata, original = md_parse(original)
article['metadata'] = original_metadata
original_html = mistune.markdown(original, escape=False,
hard_wrap=True).replace(
'<br>', '</p><p>')
print('Building: {}'.format(article['metadata'].get('title', key)))
for locale, filename in article['translations'].items():
with open(filename) as file:
translation = file.read()
translation_metadata, translation = md_parse(translation)
translation_html_filename = 'translations/' + os.path.splitext(
os.path.basename(filename))[0] + '.html'
translation_html = mistune.markdown(translation,
escape=False,
hard_wrap=True).replace(
'<br>', '</p><p>')
def generate_html(self):
if self.text:
html = mistune.markdown(self.text)
self.text_html = html
def parse_readme(directory):
# Make sure there is a readme file
readme = os.path.join(root, "README.md")
if not os.path.isfile(readme):
return
path_to_root = os.path.relpath('.', directory)
# Open an output file
f = open(os.path.join(directory, "index.html"), 'w+')
# Write a header
f.write("<!DOCTYPE HTML><html><head>")
# Title
f.write("<title>RSS Team " + TEAM_NUMBER + "</title>")
# Stylesheet
f.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"")
f.write(os.path.join(path_to_root, "css/style.css"))
f.write("\">")
# Add mathjax
f.write("<script type=\"text/x-mathjax-config\">")
f.write("MathJax.Hub.Config({tex2jax: {")
f.write("inlineMath: [['$','$']],")
f.write("displayMath: [['$$','$$']],")
f.write(
"skipTags: [\"script\",\"noscript\",\"style\",\"textarea\",\"code\"]")
f.write("},")
f.write("TeX: {equationNumbers: {autoNumber: \"AMS\"}}});")
f.write("</script>")
f.write("<script type=\"text/javascript\" async ")
f.write(
"src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-MML-AM_CHTML\">"
)
f.write("</script>")
# End Header
f.write("</head>")
# Begin the body
f.write("<body>")
# Add a bar at the top with the team name
f.write("<div id=\"main\">")
f.write("<div id=\"logo\">")
f.write("<div id=\"logo_text\">")
f.write("<h1><a href=\"" + path_to_root +
"\"><span class=\"logo_colour\">")
f.write("RSS Team " + str(TEAM_NUMBER))
f.write("</span></a></h1>")
f.write("<h2>MIT Spring 2019</h2>")
f.write("</div></div>")
# Add a menu bar
f.write("<div id=\"header\">")
f.write("<div id=\"menubar\">")
f.write("<ul id=\"menu\">")
# Select the write page
if "labs" in directory:
lab_class = "selected"
home_class = ""
else:
home_class = "selected"
lab_class = ""
f.write("<li class=\"" + home_class + "\"><a href=\"" + path_to_root +
"\">Home</a></li>")
f.write("<li class=\"" + lab_class + "\"><a href=\"" +
os.path.join(path_to_root, "labs") + "\">Labs</a></li>")
f.write("<li><a href=\"" + VIDEO_LINK + "\">Video</a></li>")
f.write("<li><a href=\"https://github.mit.edu/rss2019-" +
str(TEAM_NUMBER) + "\">Github</a></li>")
f.write("</ul></div></div>")
f.write("<div id=\"site_content\">")
f.write("<div id=\"content\">")
# Add the text from the README
readme = os.path.join(directory, "README.md")
with open(readme, 'r') as readme_f:
prev_markdown = "."
for line in readme_f:
markdown = str(mistune.markdown(line))
# Get rid of paragraph markers
markdown = markdown.replace('<p>', '')
markdown = markdown.replace('</p>', '')
f.write(markdown)
if prev_markdown == "" and markdown == "":
# Add a new line
f.write("<br/><br/>")
prev_markdown = markdown
# Write the footer
f.write("</div></div></div>")
f.write("</body>")
f.write("</html>")
# Close the file
f.close()
def normalize_post_text(text, charmap=None):
# footer
# normalized = re.sub(r'(.+)\n---\n.+', r'\1', text, flags=re.DOTALL)
# quotes
normalized = re.sub(r'^>.+\n?.+\n\n', '', text, flags=re.M)
# derived from https://stackoverflow.com/a/761847
# stripping markdown syntax
html = mistune.markdown(normalized)
normalized = ''.join(BeautifulSoup(html,
"html5lib").findAll(text=True)).strip()
# whitespace html entity
normalized = re.sub(r' ', ' ', normalized)
# ^() syntax
normalized = re.sub(r'\^\(([^\^]+)\)', r'\1', normalized)
normalized = re.sub(r'(^|\s|\w+|[/,|])\^(\s|\w+|[/,|])', r'\1\2',
normalized)
normalized = re.sub(r'[\u2018\u2019]', "'",
normalized.replace("\u2605", "*"))
normalized = re.sub(r'[\u201C\u201D]', '"', normalized)
# consecutive whitespaces
normalized = re.sub(r'\s+', ' ', normalized)
# users and subreddit mentions
normalized = re.sub(r'/u/([\w\-]+)', 'RedditUser', normalized)
normalized = re.sub(r'(^|\s)[@+]([\w\-]+(\s|:|$))', 'RedditUser',
normalized)
normalized = re.sub(r'(^|\s)/?\s?r\s?/([a-zA-Z0-9]\w+)(\s|:|\.|,|\?|$)',
r'\1@url$\3', normalized)
normalized = re.sub(r'(\w|\.|:)((http|ftp)s?://)', r'\1 \2', normalized)
url_regex = r'\b((http|ftp)s?://)?(www\.)?[\w\-@:%_+~#=]+(\.[a-zA-Z]{2,3})+(/.*)?(\?.*)?\b'
normalized = re.sub(url_regex, '@url$', normalized)
normalized = strip_emojis_and_emoticons(normalized).strip()
normalized = re.sub('´', "'", normalized)
normalized = re.sub(r'-LRB-', '(', normalized, re.IGNORECASE)
normalized = re.sub(r'-RRB-', ')', normalized, re.IGNORECASE)
normalized = re.sub(
r'[\u2500-\u25ff\u2800-\u28ff\u2700-\u27bf\u2000-\u204A]', '',
normalized)
normalized = re.sub(r'[\u0C81-\u0CF2\-][_.][\u0C81-\u0CF2\-]', '',
normalized)
normalized = re.sub(
r'[\u0E81-\u0EDF\u30A0-\u30FF\u0300-\u0362\uFF5F-\uFFEE\u0C81-\u0CF2\u00AF-\u00B0'
r'\u0275-\u027B\u0292-\u0296\u02AC-\u02AF\u0298\u029A]', '',
normalized)
normalized = re.sub(
r'[\u0D00-\u0D7F\u0E00-\u0E7F\u1780-\u17FF\u1400-\u167F]', '',
normalized)
s1, e1 = u"\U0001F100", u"\U0001F9FF"
s2, e2 = u"\U00010000", u"\U0001342E"
normalized = re.sub(r'[{}-{}{}-{}]'.format(s1, e1, s2, e2), '', normalized)
normalized = re.sub(r"[\uFEFF\u2060\u2E18]", '', normalized)
normalized = re.sub(r'(\.\.\.\s){2,}', '... ', normalized)
normalized = re.sub(r'(([\^.></_|])\s){2,}', '', normalized)
normalized = re.sub(r'(\"){2,}', '"', normalized)
normalized = re.sub(r'(_){2,}', '', normalized)
normalized = re.sub(r'!{2,}', '!!', normalized)
if charmap is not None:
for ch in charmap:
if ch in normalized:
try:
normalized = re.sub(ch, charmap[ch], normalized)
except sre_constants.error:
pass
normalized = re.sub(r'\([_\-.\u2200-\u22FF\s]+\)', '', normalized)
normalized = re.sub(r'\[[_\-.\u2200-\u22FF\s]+\]', '', normalized)
normalized = re.sub(r'\\[_\-.\u2200-\u22FF\s]+/', '', normalized)
normalized = re.sub('¿', '', normalized)
return normalized.strip()
def body_html(self):
return markdown(self.body)
def save(self, *args, **kwargs):
self.slug = slugify(self.name)
self.notes_html = mistune.markdown(self.notes)
super(Talk, self).save(*args, **kwargs)
def test_skip_style():
ret = mistune.markdown('foo\n<style>body{color:red}</style>',
skip_style=True)
assert ret == '<p>foo</p>\n'
def test_not_escape_inline_tags():
text = '<a name="top"></a>'
assert text in mistune.markdown(text, escape=False)
def test_block_html():
ret = mistune.markdown('<div ></div>', escape=False)
assert '<div ></div>' in ret
def test_not_escape_block_tags():
text = '<h1>heading</h1> text'
assert text in mistune.markdown(text, escape=False)
def test_escape():
ret = mistune.markdown('<div>**foo**</div>', escape=True)
assert '>' in ret
ret = mistune.markdown('this **foo** is <b>bold</b>', escape=True)
assert '>' in ret
