def create_issue(self, data, **kwargs): """ Creates the issue on the remote service and returns an issue ID. """ project_id = data.get('project') if project_id is None: raise ValueError('Azure DevOps expects project') client = self.get_client() title = data['title'] description = data['description'] try: created_item = client.create_work_item( instance=self.instance, project=project_id, title=title, # Decriptions cannot easily be seen. So, a comment will be added as well. description=markdown(description), comment=markdown(description) ) except Exception as e: self.raise_error(e) project_name = created_item['fields']['System.AreaPath'] return { 'key': six.text_type(created_item['id']), 'title': title, 'description': description, 'metadata': { 'display_name': '%s#%s' % (project_name, created_item['id']), } }
def import_file(cls,path=None): ''' 导入博客文件,自动跳过已导入的文件。 ''' path=abspath(path or '~/OneDrive/notes') note_files=[] for root,paths,files in os.walk(path): note_files.extend([join(root,fi) for fi in files if fi.endswith('.md')]) for fl in cls.find({'filename':{"$in":note_files}}, {'mtime':1,'filename':1}): if fl['mtime']>=dt_tz(os.path.getmtime(fl['filename'])): print('文件:%s 已导入,跳过'%(fl['filename'])) note_files.remove(fl['filename']) renderer=ConvertRenderer() for filename in note_files: body='\n'.join(read_file(filename)) renderer.clear() mistune.markdown(body,renderer=renderer) cls.update_one({'filename':filename}, {'$set': {'title':renderer.title, 'abbr':renderer.abbr, 'meta':renderer.meta, 'mtime':dt_tz(os.path.getmtime(filename)), 'ctime':dt_tz(os.path.getctime(filename)), 'body':body}}, True)
def _pre_put_hook(self): if not self.original_name: self.original_name = self.name self.name = clean_name(self.name) self.category = strip_string(self.category) self.badge = strip_string(self.badge) self.barcode = strip_string(self.barcode) self.brand = strip_string(self.brand) self.country = strip_string(self.country) self.material = strip_string(self.material) self.description_md = self.description_md.strip() if self.description_md: self.description_html = mistune.markdown(self.description_md) self.description = re.sub( r'(<!--.*?-->|<[^>]*>)', '', self.description_html) else: self.description = self.description.strip() self.description_md = self.description self.description_html = mistune.markdown(self.description) if self.description: self.short_description = u' '.join(self.description.split()[:5]) else: self.short_description = '' self.meta_keywords = self.clear_name.replace(' ', ', ')
def test_parse_block_html(): ret = mistune.markdown( '<div>**foo**</div>', parse_block_html=True, escape=False ) assert '<div><strong>' in ret ret = mistune.markdown( '<span>**foo**</span>', parse_block_html=True, escape=False ) assert '<strong>' not in ret
def test_use_xhtml(): ret = mistune.markdown('foo\n\n----\n\nbar') assert '<hr>' in ret ret = mistune.markdown('foo\n\n----\n\nbar', use_xhtml=True) assert '<hr />' in ret ret = mistune.markdown('foo \nbar', use_xhtml=True) assert '<br />' in ret ret = mistune.markdown('![foo](bar "title")', use_xhtml=True) assert '<img src="bar" alt="foo" title="title" />' in ret
def add_post_post(): post = Word( title = request.form["title"], content = mistune.markdown(request.form["content"]), category = mistune.markdown(request.form["category"]), level = mistune.markdown(request.form["level"]), author = current_user ) session.add(post) session.commit() return redirect(url_for("words"))
def __init__(self, proto): self.__proto = proto self.name = proto.name self.documentation = mistune.markdown(proto.documentation) self.example_documentation = mistune.markdown(proto.example_documentation) self.signature = self._get_signature(proto) self.attributes = [] for attribute in proto.attribute: self.attributes.append(Attribute(attribute)) self.outputs = [] for output in proto.output: self.outputs.append(Output(output))
def test_parse_nested_html(): ret = mistune.markdown( '<div><a href="http://example.org">**foo**</a></div>', parse_block_html=True, escape=False ) assert '<div><a href="http://example.org">' in ret assert '<strong>' not in ret ret = mistune.markdown( '<div><a href="http://example.org">**foo**</a></div>', parse_block_html=True, parse_inline_html=True, escape=False ) assert '<div><a href="http://example.org"><strong>' in ret
def _grab_blocks(self): """ Parse `self.source` and return a dict mapping block names to their raw content. """ r = BlockRecordRenderer() mistune.markdown(self.source, renderer=r) blocks = {} for block in r.blocks: name, content = self._parse_raw_block(block) blocks[name] = content self.blocks = blocks
def index(): dirs = _get_dirs() main_index = dict(standard="1377", issue="1990", org="BS", title="BS1377:1990", parts=[]) print dirs for d in dirs: files = _get_md_files(d) print "d=", d #print "==========", ROOT_PATH + d + "/README.md" txt = _read_file(ROOT_PATH + d + "/README.md") html = mistune.markdown(txt) soup = BeautifulSoup(html) dic = {} dic['title'] = str(soup.find("h1").text).strip() dic['part'] = dic['title'][4:].strip().split(" ")[0].strip() #dic['sections'] = [] idx = [] for f in files: #print f txt = _read_file(ROOT_PATH + f) html = mistune.markdown(txt) #print html soup = BeautifulSoup(html) print "------------------------------" title = str(soup.find("h1").text).strip() toc = [] for h2 in soup.findAll("h2"): toc.append( dict(title=str(h2.text).strip()) ) #print title, toc #'toc.append( dict(file=f, title=title, toc=toc) ) #fff = f #print f, fff num = f.split("/")[-1][:-3] print "NO = ", num, type(num) data = dict(title=title, clauses=toc, file=f, part= str(d)[1:], org="BS", section=num) idx.append( data ) dic['sections'] = idx _write_yaml("%s/%s/index.yaml" % (ROOT_PATH, d), dic ) main_index['parts'].append(dic) #print idx #break _write_yaml("%s/index.yaml" % (ROOT_PATH), main_index )
def test_parse_inline_html(): ret = mistune.markdown( '<div>**foo**</div>', parse_inline_html=True, escape=False ) assert '<strong>' not in ret ret = mistune.markdown( '<span>**foo**</span>', parse_inline_html=True, escape=False ) assert '<span><strong>' in ret ret = mistune.markdown( '<a>http://lepture.com</a>', parse_inline_html=True, escape=False ) assert 'href' not in ret
def get_html(self): if self.text_type == 'rst': from docutils.core import publish_parts return publish_parts(self.text, writer_name='html')['html_body'] else: import mistune return mistune.markdown(self.text)
def edit_post_post(post_id): post=session.query(Post).get(post_id) post.title = request.form["title"] post.content = mistune.markdown(request.form["content"]) session.commit() return redirect(url_for("posts"))
def create(cls, author, raw_comment, parent): """ Create a new comment instance. If the parent is submisison update comment_count field and save it. If parent is comment post it as child comment :param author: RedditUser instance :type author: RedditUser :param raw_comment: Raw comment text :type raw_comment: str :param parent: Comment or Submission that this comment is child of :type parent: Comment | Submission :return: New Comment instance :rtype: Comment """ html_comment = mistune.markdown(raw_comment) # todo: any exceptions possible? comment = cls(author=author, author_name=author.user.username, raw_comment=raw_comment, html_comment=html_comment) if isinstance(parent, Submission): submission = parent comment.submission = submission elif isinstance(parent, Comment): submission = parent.submission comment.submission = submission comment.parent = parent else: return submission.comment_count += 1 submission.save() return comment
def add_post_post(): members = session.query(User).all() post = Post( title=request.form["title"], content=mistune.markdown(request.form["content"]), description = request.form["description"], author=current_user ) session.add(post) session.commit() all_members = [] for member in members: all_members.append(member.email) all_members = [str(i) for i in all_members] describe = request.form["description"] +".... View full message on TeamForum!" mail=Mail(app) with mail.connect() as conn: for email in all_members: message = Message(subject="A new forum has been created by "+ current_user.name, body= describe, sender=("TeamForum", "*****@*****.**"), recipients=[email] ) conn.send(message) flash("You have created a new forum. Team members have been notified.", "info") return redirect(url_for("posts"))
def main(): md = urllib2.urlopen('https://raw.githubusercontent.com/syhw/wer_are_we/master/README.md').read() bs = BeautifulSoup(mistune.markdown(md)) wer_data_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/wer.py")) file_output = "# The file was autogenerated by ../scrapers/wer.py\n\nfrom datetime import date\n\nfrom data.acoustics import speech_recognition, swb_hub_500\nfrom scales import *\n\n" wer_metrics = [] for table, header in zip(bs.findAll('table'), bs.findAll('h3')): header = header.getText() rows = table.findAll('tr') metric_data = get_metrics(header, rows[0].findAll('th')[:-3], file_output) metric_names = metric_data[0] wer_metrics += metric_names table_data = [] for row in rows: if row.findAll('td') == []: continue measure_data, targets, target_source = add_measures(metric_names, row) if not targets: table_data += measure_data elif not measure_data: metric_data = get_metrics(header, rows[0].findAll('th')[:-3], file_output, targets = targets, target_source = target_source) file_output = metric_data[1] file_output += "".join(sorted(table_data)) file_output = file_output + "\n\nwer_metrics=[" + ", ".join(wer_metrics) + "]" with open(wer_data_file, 'wb') as f: f.write(file_output)
def description(self): """Extract description from a release.""" if self.release.get('body'): return markdown(self.release['body']) elif self.repository.get('description'): return self.repository['description'] return 'No description provided.'
def edit_post_post(id=1): post = session.query(Post) post = post.filter(Post.id == id).first() post.title=request.form["title"] post.content=mistune.markdown(request.form["content"]) session.commit() return redirect(url_for("posts"))
def preview_page(): title = request.form.get('page_title', '') content_markdown = request.form.get('new_page_content', '') content_html = mistune.markdown(content_markdown) preview = {'title': title, 'content': content_html} return jsonify(preview)
def markdownDeck(deck): fl = open("decords/{}".format(deck), "r") xml = mistune.markdown(fl.read()) fl.close() del (fl) xml = etree.fromstring(xml) pass
def generate_post(self, text, out_file_name): html = mistune.markdown(text, escape=False) result = self.blog_template.format( content=html, blog_name=self.blog_name) with open(out_file_name, "w+") as f: f.write(result)
def job(job_id): ''' View a single job (any status) with AJAX callback to update elements, e.g. - STDOUT/STDERR - File outputs (figures, etc.), intelligent render handling - Download link for all files :param job_id: :return: ''' # Get the job object from the database job = Job.query.get(job_id) script = job.script display = {} cwd = os.path.join(job.path, 'output') # Excution path of the job if os.path.isdir(cwd): # Execution has begun/finished files = job.get_output_files() display = build_display_objects(files) documentation = script.load_docs() if documentation: documentation = mistune.markdown(documentation) return render_template("public/job.html", script=script, job=job, metadata=script.load_config(), display=display, documentation=documentation)
def edit_post_post(id): post = session.query(Post).get(id) post.title=request.form["title"] post.content=mistune.markdown(request.form["content"]) session.add(post) session.commit() return redirect(url_for("view_post", id=post.id))
def render(self, context): value = self.nodelist.render(context) try: return mark_safe(mistune.markdown(value)) except ImportError: raise template.TemplateSyntaxError("Error in `markdown` tag: " "The mistune library isn't installed.")
def render(file): """Render HTML from Markdown file content.""" fp = file.open() content = fp.read() result = mistune.markdown(content.decode('utf-8')) fp.close() return result
def stack_announcement(announcement_name): ANNMT = u'ANNOUNCEMENT_' + announcement_name hide = 'true' # This is javascript :-D message = getattr(config, ANNMT, u'').strip() prefix = getattr(config, ANNMT + u'_PREFIX', u'') if message: # NOTE: This is a non-breakable space, else it gets lost. message = mistune.markdown(prefix + (u' ' if prefix else u'') + message).strip() priority = getattr(config, ANNMT + u'_PRIORITY', u'').lower() if priority not in (u'', u'info', u'success', u'error'): priority = u'info' if priority in (u'info', u'error'): hide = 'false' start_date = getattr(config, ANNMT + u'_START', u'') end_date = getattr(config, ANNMT + u'_END', u'') if (not start_date or start_date <= dttoday) and ( not end_date or end_date >= dttoday): announcements.append((message, priority, hide))
def markdownify(): md = request.form.get("markdown") if md: html = mistune.markdown(md) return html else: return "no markdown supplied"
def _renderHtmlWithMistune(text): options= { "escape": True, "use_xhtml": True } renderer= _MermaidRenderer(**options) return mistune.markdown(text, renderer=renderer, **options), renderer.hasDiagram
def read_repo(self, git=None, ls=None): clone_path = path.join(self.dir_base, str(uuid.uuid4()) if git is not None else 'upstream') print('github url', git if git is not None else self.upstream) git_url = urlparse(git if git is not None else self.upstream) clone_url = None splitted_path = git_url.path.strip('/').split('/') if len(splitted_path) > 2: clone_url = git_url.scheme + '://' + git_url.netloc + '/' + splitted_path[0] + '/' + splitted_path[1] + '.git' else: clone_url = git if git is not None else self.upstream print('clone url', clone_url) # cloning the repository Gittle.clone(clone_url, clone_path) if len(splitted_path) > 2: readme_file = self.find_readme(clone_path, '/'.join(str(x) for x in splitted_path[2:])) else: readme_file = self.find_readme(clone_path) print(clone_url, readme_file) try: with open(path.join(clone_path, readme_file)) as f: soup = BeautifulSoup(mistune.markdown(f.read()), 'html.parser') self.save_md(soup, True if git is None else False, ls) self.stdout.write(self.style.SUCCESS('Successfully read the upstream')) except Exception as exp: print('An error happened while reading the repo.', exp)
def edit_entry_post(id): entry = session.query(Entry) entry = entry.filter(Entry.id == id).first() entry.title=request.form["title"] entry.content= mistune.markdown(request.form["content"]) session.commit() return redirect(url_for("entries"))
def save(self, *args, **kwargs): new_hash = xxhash.xxh64(self.content_raw).hexdigest() if new_hash != self.raw_content_hash or (not self.pk): self.content_rendered = mistune.markdown(self.content_raw) super(Appendix, self).save(*args, **kwargs) self.raw_content_hash = new_hash
def get_html_description(self, obj): # Parse markdown return mistune.markdown(obj.description)
def get_file(post_filename): blob = Blob(post_filename, get_storage()) blog_as_text = blob.download_as_string().decode() return mistune.markdown(blog_as_text, escape=True, hard_wrap=True)
def convert_to_confluence(markdown, metadata={}): renderer = ConfluenceRenderer() content_html = mistune.markdown(markdown, renderer=renderer) page_html = renderer.layout(content_html) return page_html, renderer.attachments
def test_linebreak(): ret = mistune.markdown('this **foo** \nis me') assert '<br>' not in ret ret = mistune.markdown('this **foo** \nis me', hard_wrap=True) assert '<br>' in ret
def save(self, *args, **kwargs): self.content_html = mistune.markdown(self.content) super().save(*args, **kwargs)
def clean_content(self): content = self.cleaned_data.get('content') if len(content) < 10: raise forms.ValidationError('内容长度太短') content = mistune.markdown(content) return content
from flask.json import JSONEncoder from mistune import markdown from .constants import OPTIONAL_FIELDS from .utils import extract_definitions from .utils import get_specs from .utils import get_schema_specs from .utils import parse_definition_docstring from .utils import get_vendor_extension_fields from .utils import validate from .utils import LazyString from . import __version__ NO_SANITIZER = lambda text: text # noqa BR_SANITIZER = lambda text: text.replace('\n', '<br/>' ) if text else text # noqa MK_SANITIZER = lambda text: Markup(markdown(text)) if text else text # noqa class APIDocsView(MethodView): """ The /apidocs """ def __init__(self, *args, **kwargs): view_args = kwargs.pop('view_args', {}) self.config = view_args.get('config') super(APIDocsView, self).__init__(*args, **kwargs) def get(self): """ The data under /apidocs json or Swagger UI
from pathlib import Path from shutil import copyfile import os import mistune scriptDir = os.path.dirname(os.path.abspath(__file__)) DEST_DIR = "{}/../build/doc/".format(scriptDir) template = open("{}/template.html".format(scriptDir), "r").read() files = ["{}/../README.md".format(scriptDir), "{}/../virtualKeyNames.md".format(scriptDir), "{}/../CHANGELOG.md".format(scriptDir)] for mdPath in files: mdFilename = Path(mdPath).name file = open(mdPath, "r") md = file.read() html = mistune.markdown(md) html = template.replace("${body}", html).replace("${title}", mdFilename.replace(".md", "")) # also replace local markdown links with local html links html = html.replace(".md", ".html") htmlFilename = DEST_DIR + mdFilename.replace(".md", ".html") parentDir = Path(htmlFilename).parent parentDir.mkdir(parents=True, exist_ok=True) with open(htmlFilename, "w") as htmlFile: htmlFile.write(html) # don't forget to copy the css file copyfile("{}/html.css".format(scriptDir), DEST_DIR + "/html.css")
def html(self): return markdown(self.text)
def render_type(self, f, depth): if f["name"] in self.rendered or f["name"] in self.redirects: return self.rendered.add(f["name"]) if "doc" not in f: f["doc"] = "" f["type"] = copy.deepcopy(f) f["doc"] = "" f = f["type"] if "doc" not in f: f["doc"] = "" def extendsfrom(item, ex): if "extends" in item: for e in aslist(item["extends"]): ex.insert(0, self.typemap[e]) extendsfrom(self.typemap[e], ex) ex = [f] extendsfrom(f, ex) enumDesc = {} if f["type"] == "enum" and isinstance(f["doc"], list): for e in ex: for i in e["doc"]: idx = i.find(":") if idx > -1: enumDesc[i[:idx]] = i[idx + 1:] e["doc"] = [ i for i in e["doc"] if i.find(":") == -1 or i.find(" ") < i.find(":") ] f["doc"] = fix_doc(f["doc"]) if f["type"] == "record": for field in f.get("fields", []): if "doc" not in field: field["doc"] = "" if f["type"] != "documentation": lines = [] for l in f["doc"].splitlines(): if len(l) > 0 and l[0] == "#": l = ("#" * depth) + l lines.append(l) f["doc"] = "\n".join(lines) _, frg = urlparse.urldefrag(f["name"]) num = self.toc.add_entry(depth, frg) doc = "## %s %s\n" % (num, frg) else: doc = "" if self.title is None and f["doc"]: self.title = f["doc"][0:f["doc"].index("\n")] if self.title.startswith('# '): self.title = self.title[2:] if f["type"] == "documentation": f["doc"] = number_headings(self.toc, f["doc"]) #if "extends" in f: # doc += "\n\nExtends " # doc += ", ".join([" %s" % linkto(ex) for ex in aslist(f["extends"])]) #if f["name"] in self.subs: # doc += "\n\nExtended by" # doc += ", ".join([" %s" % linkto(s) for s in self.subs[f["name"]]]) #if f["name"] in self.uses: # doc += "\n\nReferenced by" # doc += ", ".join([" [%s.%s](#%s)" % (s[0], s[1], to_id(s[0])) for s in self.uses[f["name"]]]) doc = doc + "\n\n" + f["doc"] doc = mistune.markdown(doc, renderer=MyRenderer()) if f["type"] == "record": doc += "<h3>Fields</h3>" doc += """<table class="table table-striped">""" doc += "<tr><th>field</th><th>type</th><th>required</th><th>description</th></tr>" required = [] optional = [] for i in f.get("fields", []): tp = i["type"] if isinstance( tp, list) and tp[0] == "https://w3id.org/cwl/salad#null": opt = False tp = tp[1:] else: opt = True desc = i["doc"] #if "inherited_from" in i: # desc = "%s _Inherited from %s_" % (desc, linkto(i["inherited_from"])) frg = schema.avro_name(i["name"]) tr = "<td><code>%s</code></td><td>%s</td><td>%s</td><td>%s</td>" % ( frg, self.typefmt( tp, self.redirects), opt, mistune.markdown(desc)) if opt: required.append(tr) else: optional.append(tr) for i in required + optional: doc += "<tr>" + i + "</tr>" doc += """</table>""" elif f["type"] == "enum": doc += "<h3>Symbols</h3>" doc += """<table class="table table-striped">""" doc += "<tr><th>symbol</th><th>description</th></tr>" for e in ex: for i in e.get("symbols", []): doc += "<tr>" frg = schema.avro_name(i) doc += "<td><code>%s</code></td><td>%s</td>" % ( frg, enumDesc.get(frg, "")) doc += "</tr>" doc += """</table>""" f["doc"] = doc self.typedoc.write(f["doc"]) subs = self.docParent.get(f["name"], []) + self.record_refs.get( f["name"], []) if len(subs) == 1: self.render_type(self.typemap[subs[0]], depth) else: for s in subs: self.render_type(self.typemap[s], depth + 1) for s in self.docAfter.get(f["name"], []): self.render_type(self.typemap[s], depth)
def html_body(self): """Format and return the markdown for the puzzle body.""" return mistune.markdown(self.get_body(), escape=False)
def view(map_identifier, topic_identifier): topic_store = get_topic_store() collaboration_mode = None if current_user.is_authenticated: # User is logged in is_map_owner = topic_store.is_topic_map_owner(map_identifier, current_user.id) if is_map_owner: topic_map = topic_store.get_topic_map(map_identifier, current_user.id) else: topic_map = topic_store.get_topic_map(map_identifier) if topic_map is None: current_app.logger.warning( f"Topic map not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}]" ) abort(404) collaboration_mode = topic_store.get_collaboration_mode( map_identifier, current_user.id) if topic_map.published: if not is_map_owner and topic_identifier == "home": flash( "You are accessing a published topic map of another user.", "primary") else: if not is_map_owner: # The map is private and doesn't belong to the user who is trying to access it if not collaboration_mode: # The user is not collaborating on the map abort(403) else: # User is not logged in topic_map = topic_store.get_topic_map(map_identifier) if topic_map is None: current_app.logger.warning( f"Topic map not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}]" ) abort(404) if not topic_map.published: # User is not logged in and the map is not published abort(403) # Determine if (active) scope filtering has been specified in the URL scope_filtered = request.args.get("filter", type=int) if scope_filtered is not None: session["scope_filter"] = scope_filtered if "scope_filter" in session: scope_filtered = session["scope_filter"] else: session["breadcrumbs"] = [] session["current_scope"] = UNIVERSAL_SCOPE session["scope_filter"] = 1 # If a context has been specified in the URL, then use that to set the context scope_identifier = request.args.get("context", type=str) if scope_identifier and topic_store.topic_exists(map_identifier, scope_identifier): session["current_scope"] = scope_identifier # Get topic if scope_filtered: topic = topic_store.get_topic( map_identifier, topic_identifier, scope=session["current_scope"], resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES, ) else: topic = topic_store.get_topic( map_identifier, topic_identifier, scope=session["current_scope"], resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES, ) if topic is None: current_app.logger.warning( f"Topic not found: user identifier: [{current_user.id}], topic map identifier: [{map_identifier}], topic identifier: [{topic_identifier}]" ) session["inexistent_topic_identifier"] = topic_identifier abort(404) else: session.pop("inexistent_topic_identifier", None) if scope_filtered: topic_occurrences = topic_store.get_topic_occurrences( map_identifier, topic_identifier, scope=session["current_scope"], inline_resource_data=RetrievalMode.INLINE_RESOURCE_DATA, resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES, ) else: topic_occurrences = topic_store.get_topic_occurrences( map_identifier, topic_identifier, inline_resource_data=RetrievalMode.INLINE_RESOURCE_DATA, resolve_attributes=RetrievalMode.RESOLVE_ATTRIBUTES, ) occurrences = { "text": None, "images": [], "3d-scenes": [], "files": [], "links": [], "videos": [], "notes": [], } for occurrence in topic_occurrences: if occurrence.instance_of == "text" and occurrence.scope == session[ "current_scope"]: if occurrence.resource_data: occurrences["text"] = mistune.markdown( occurrence.resource_data.decode()) elif occurrence.instance_of == "image": occurrences["images"].append({ "title": occurrence.get_attribute_by_name("title").value, "url": occurrence.resource_ref, }) elif occurrence.instance_of == "3d-scene": occurrences["3d-scenes"].append({ "title": occurrence.get_attribute_by_name("title").value, "url": occurrence.resource_ref, }) elif occurrence.instance_of == "file": occurrences["files"].append({ "title": occurrence.get_attribute_by_name("title").value, "url": occurrence.resource_ref, }) elif occurrence.instance_of == "url": occurrences["links"].append({ "title": occurrence.get_attribute_by_name("title").value, "url": occurrence.resource_ref, }) elif occurrence.instance_of == "video": occurrences["videos"].append({ "title": occurrence.get_attribute_by_name("title").value, "url": occurrence.resource_ref, }) elif occurrence.instance_of == "note": occurrences["notes"].append({ "identifier": occurrence.identifier, "title": occurrence.get_attribute_by_name("title").value, "timestamp": maya.parse( occurrence.get_attribute_by_name( "modification-timestamp").value), "text": mistune.markdown(occurrence.resource_data.decode()), }) if scope_filtered: associations = topic_store.get_association_groups( map_identifier, topic_identifier, scope=session["current_scope"]) else: associations = topic_store.get_association_groups( map_identifier, topic_identifier) creation_date = maya.parse( topic.get_attribute_by_name("creation-timestamp").value) modification_date_attribute = topic.get_attribute_by_name( "modification-timestamp") modification_date = maya.parse( modification_date_attribute.value ) if modification_date_attribute else "Undefined" # Breadcrumbs if "breadcrumbs" not in session: session["breadcrumbs"] = [] breadcrumbs = deque(session["breadcrumbs"], BREADCRUMBS_COUNT) if topic_identifier in breadcrumbs: breadcrumbs.remove(topic_identifier) breadcrumbs.append(topic_identifier) session["breadcrumbs"] = list(breadcrumbs) return render_template( "topic/view.html", topic_map=topic_map, topic=topic, occurrences=occurrences, associations=associations, creation_date=creation_date, modification_date=modification_date, breadcrumbs=breadcrumbs, collaboration_mode=collaboration_mode, )
def test_ExpectationSuitePageRenderer_render_expectation_suite_notes(): result = ExpectationSuitePageRenderer._render_expectation_suite_notes( ExpectationSuite(expectation_suite_name="test", meta={"notes": "*hi*"})) # print(RenderedContent.rendered_content_list_to_json(result.text)) assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', "*hi*" ] result = ExpectationSuitePageRenderer._render_expectation_suite_notes( ExpectationSuite(expectation_suite_name="test", meta={"notes": ["*alpha*", "_bravo_", "charlie"]})) # print(RenderedContent.rendered_content_list_to_json(result.text)) assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', "*alpha*", "_bravo_", "charlie" ] result = ExpectationSuitePageRenderer._render_expectation_suite_notes( ExpectationSuite(expectation_suite_name="test", meta={ "notes": { "format": "string", "content": ["*alpha*", "_bravo_", "charlie"] } })) # print(RenderedContent.rendered_content_list_to_json(result.text)) assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', "*alpha*", "_bravo_", "charlie" ] result = ExpectationSuitePageRenderer._render_expectation_suite_notes( ExpectationSuite( expectation_suite_name="test", meta={"notes": { "format": "markdown", "content": "*alpha*" }})) # print(RenderedContent.rendered_content_list_to_json(result.text)) try: mistune.markdown("*test*") assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', { 'content_block_type': 'markdown', 'styling': { 'parent': {} }, 'markdown': '*alpha*' } ] except OSError: assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', "*alpha*" ] result = ExpectationSuitePageRenderer._render_expectation_suite_notes( ExpectationSuite(expectation_suite_name="test", meta={ "notes": { "format": "markdown", "content": ["*alpha*", "_bravo_", "charlie"] } })) # print(RenderedContent.rendered_content_list_to_json(result.text)) try: mistune.markdown("*test*") assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', { 'content_block_type': 'markdown', 'styling': { 'parent': {} }, 'markdown': '*alpha*' }, { 'content_block_type': 'markdown', 'styling': { 'parent': {} }, 'markdown': '_bravo_' }, { 'content_block_type': 'markdown', 'styling': { 'parent': {} }, 'markdown': 'charlie' } ] except OSError: assert RenderedContent.rendered_content_list_to_json(result.text) == [ 'This Expectation suite currently contains 0 total Expectations across 0 columns.', "*alpha*", "_bravo_", "charlie" ]
def save(self, *args, **kwargs): if self.is_md: self.content_html = mistune.markdown(self.content) else: self.content_html = self.content super().save(*args, **kwargs)
def markdown(text): return Markup(md.markdown(text, escape=True))
#!/usr/bin/python import subprocess import os import mistune for root, dirs, files in os.walk("/home/karl/jgit/data"): for file in files: if file.lower().endswith(".md"): full_path = os.path.join(root, file) text = open(full_path).read() html = mistune.markdown(text, escape=False) renamed_path = full_path.replace("/", "@") with open("/tmp/mdtest/" + renamed_path + ".mdorig", "w") as f: f.write(text) with open("/tmp/mdtest/" + renamed_path + ".mistune", "w") as f: f.write(html) for root, dirs, files in os.walk("/tmp/mdtest"): for file in files: if file.lower().endswith(".mdorig"): full_path = os.path.join(root, file) old_path = full_path.replace(".mdorig", ".mistune") new_path = full_path.replace(".mdorig", ".net") p = subprocess.Popen(["diff", old_path, new_path], stdout=subprocess.PIPE) out, err = p.communicate() rc = p.wait() print "%s: rc=%d" % (full_path, rc) if rc != 0: print out
def generate_html(): for att in CONFIG: atts[att] = CONFIG[att] # Generate the HTML for posts and prepare variables for get_pages for x in get_posts(): atts['date'] = str(datetime.datetime.today().date()) atts['post_title'] = "Preview" with open(os.path.join(POST_DIR, x)) as f: if os.path.isfile( os.path.join(BASE_DIR, POST_DIR, x.split('.')[0] + ".json")): with open( os.path.join(BASE_DIR, POST_DIR, x.split('.')[0] + '.json')) as s: j = json.load(s) t = j['layout'] if t == 'default': template = POST_TEMPLATE atts['post_title'] = j['title'] atts['date'] = j['date'] else: template = Template(t.read()) atts['post_title'] = j['title'] atts['date'] = j['date'] else: template = POST_TEMPLATE atts['content'] = mistune.markdown(f.read()) new_filename = atts['date'] + '_' + os.path.splitext(x)[0] + '.html' open(os.path.join(OUTPUT_DIR, BLOG_DIR, new_filename), 'w').write(template.render(atts)) atts['blog'].append({ 'title': atts['date'] + ' / ' + atts['post_title'], 'link': 'blog/' + atts['date'] + '_' + atts['post_title'].replace(' ', '-') + '.html' }) atts['blog'] = sorted(atts['blog'], key=byTitle_key, reverse=True) # Generate the HTML for pages for x in get_pages(): with open(os.path.join(PAGE_DIR, x)) as f: if os.path.isfile( os.path.join(BASE_DIR, PAGE_DIR, x.split('.')[0] + ".json")): with (open( os.path.join(BASE_DIR, PAGE_DIR, x.split('.')[0] + '.json'))) as s: j = json.load(s) t = j['layout'] if t == 'default': template = PAGE_TEMPLATE else: template = Template( open(os.path.join(TEMPLATE_DIR, t + '.j2')).read()) content = f.read() atts['content'] = mistune.markdown(content) new_filename = os.path.splitext(x)[0] + '.html' open(os.path.join(OUTPUT_DIR, new_filename), 'w').write(template.render(atts))
parser = argparse.ArgumentParser(description = 'Download all the PDF/HTML links into README.md') parser.add_argument('-d', action="store", dest="directory") parser.add_argument('--no-html', action="store_true", dest="nohtml", default = False) parser.add_argument('--overwrite', action="store_true", default = False) results = parser.parse_args() output_directory = 'pdfs' if results.directory is None else results.directory forbidden_extensions = ['html', 'htm'] if results.nohtml else [] if results.overwrite and os.path.exists(output_directory): shutil.rmtree(output_directory) with open('README.md') as readme: readme_html = mistune.markdown(readme.read()) readme_soup = BeautifulSoup.BeautifulSoup(readme_html, "html.parser") point = readme_soup.find_all('h1')[1] failures = [] while point is not None: if point.name: if re.search('h[1-2]', point.name): if point.name == 'h1': h1_directory = os.path.join(output_directory, clean_text(point.text)) current_directory = h1_directory elif point.name == 'h2': current_directory = os.path.join(h1_directory, clean_text(point.text)) if not os.path.exists(current_directory): os.makedirs(current_directory)
articles[key]['translations'][result[2]] = filename # Prepare a template for articles with open('templates/article.html') as file: article_template = Template(file.read()) # Create translated articles translated_articles = [] for key, article in articles.items(): with open(article['original']) as file: original = file.read() original_metadata, original = md_parse(original) article['metadata'] = original_metadata original_html = mistune.markdown(original, escape=False, hard_wrap=True).replace( '<br>', '</p><p>') print('Building: {}'.format(article['metadata'].get('title', key))) for locale, filename in article['translations'].items(): with open(filename) as file: translation = file.read() translation_metadata, translation = md_parse(translation) translation_html_filename = 'translations/' + os.path.splitext( os.path.basename(filename))[0] + '.html' translation_html = mistune.markdown(translation, escape=False, hard_wrap=True).replace( '<br>', '</p><p>')
def generate_html(self): if self.text: html = mistune.markdown(self.text) self.text_html = html
def parse_readme(directory): # Make sure there is a readme file readme = os.path.join(root, "README.md") if not os.path.isfile(readme): return path_to_root = os.path.relpath('.', directory) # Open an output file f = open(os.path.join(directory, "index.html"), 'w+') # Write a header f.write("<!DOCTYPE HTML><html><head>") # Title f.write("<title>RSS Team " + TEAM_NUMBER + "</title>") # Stylesheet f.write("<link rel=\"stylesheet\" type=\"text/css\" href=\"") f.write(os.path.join(path_to_root, "css/style.css")) f.write("\">") # Add mathjax f.write("<script type=\"text/x-mathjax-config\">") f.write("MathJax.Hub.Config({tex2jax: {") f.write("inlineMath: [['$','$']],") f.write("displayMath: [['$$','$$']],") f.write( "skipTags: [\"script\",\"noscript\",\"style\",\"textarea\",\"code\"]") f.write("},") f.write("TeX: {equationNumbers: {autoNumber: \"AMS\"}}});") f.write("</script>") f.write("<script type=\"text/javascript\" async ") f.write( "src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-MML-AM_CHTML\">" ) f.write("</script>") # End Header f.write("</head>") # Begin the body f.write("<body>") # Add a bar at the top with the team name f.write("<div id=\"main\">") f.write("<div id=\"logo\">") f.write("<div id=\"logo_text\">") f.write("<h1><a href=\"" + path_to_root + "\"><span class=\"logo_colour\">") f.write("RSS Team " + str(TEAM_NUMBER)) f.write("</span></a></h1>") f.write("<h2>MIT Spring 2019</h2>") f.write("</div></div>") # Add a menu bar f.write("<div id=\"header\">") f.write("<div id=\"menubar\">") f.write("<ul id=\"menu\">") # Select the write page if "labs" in directory: lab_class = "selected" home_class = "" else: home_class = "selected" lab_class = "" f.write("<li class=\"" + home_class + "\"><a href=\"" + path_to_root + "\">Home</a></li>") f.write("<li class=\"" + lab_class + "\"><a href=\"" + os.path.join(path_to_root, "labs") + "\">Labs</a></li>") f.write("<li><a href=\"" + VIDEO_LINK + "\">Video</a></li>") f.write("<li><a href=\"https://github.mit.edu/rss2019-" + str(TEAM_NUMBER) + "\">Github</a></li>") f.write("</ul></div></div>") f.write("<div id=\"site_content\">") f.write("<div id=\"content\">") # Add the text from the README readme = os.path.join(directory, "README.md") with open(readme, 'r') as readme_f: prev_markdown = "." for line in readme_f: markdown = str(mistune.markdown(line)) # Get rid of paragraph markers markdown = markdown.replace('<p>', '') markdown = markdown.replace('</p>', '') f.write(markdown) if prev_markdown == "" and markdown == "": # Add a new line f.write("<br/><br/>") prev_markdown = markdown # Write the footer f.write("</div></div></div>") f.write("</body>") f.write("</html>") # Close the file f.close()
def normalize_post_text(text, charmap=None): # footer # normalized = re.sub(r'(.+)\n---\n.+', r'\1', text, flags=re.DOTALL) # quotes normalized = re.sub(r'^>.+\n?.+\n\n', '', text, flags=re.M) # derived from https://stackoverflow.com/a/761847 # stripping markdown syntax html = mistune.markdown(normalized) normalized = ''.join(BeautifulSoup(html, "html5lib").findAll(text=True)).strip() # whitespace html entity normalized = re.sub(r' ', ' ', normalized) # ^() syntax normalized = re.sub(r'\^\(([^\^]+)\)', r'\1', normalized) normalized = re.sub(r'(^|\s|\w+|[/,|])\^(\s|\w+|[/,|])', r'\1\2', normalized) normalized = re.sub(r'[\u2018\u2019]', "'", normalized.replace("\u2605", "*")) normalized = re.sub(r'[\u201C\u201D]', '"', normalized) # consecutive whitespaces normalized = re.sub(r'\s+', ' ', normalized) # users and subreddit mentions normalized = re.sub(r'/u/([\w\-]+)', 'RedditUser', normalized) normalized = re.sub(r'(^|\s)[@+]([\w\-]+(\s|:|$))', 'RedditUser', normalized) normalized = re.sub(r'(^|\s)/?\s?r\s?/([a-zA-Z0-9]\w+)(\s|:|\.|,|\?|$)', r'\1@url$\3', normalized) normalized = re.sub(r'(\w|\.|:)((http|ftp)s?://)', r'\1 \2', normalized) url_regex = r'\b((http|ftp)s?://)?(www\.)?[\w\-@:%_+~#=]+(\.[a-zA-Z]{2,3})+(/.*)?(\?.*)?\b' normalized = re.sub(url_regex, '@url$', normalized) normalized = strip_emojis_and_emoticons(normalized).strip() normalized = re.sub('´', "'", normalized) normalized = re.sub(r'-LRB-', '(', normalized, re.IGNORECASE) normalized = re.sub(r'-RRB-', ')', normalized, re.IGNORECASE) normalized = re.sub( r'[\u2500-\u25ff\u2800-\u28ff\u2700-\u27bf\u2000-\u204A]', '', normalized) normalized = re.sub(r'[\u0C81-\u0CF2\-][_.][\u0C81-\u0CF2\-]', '', normalized) normalized = re.sub( r'[\u0E81-\u0EDF\u30A0-\u30FF\u0300-\u0362\uFF5F-\uFFEE\u0C81-\u0CF2\u00AF-\u00B0' r'\u0275-\u027B\u0292-\u0296\u02AC-\u02AF\u0298\u029A]', '', normalized) normalized = re.sub( r'[\u0D00-\u0D7F\u0E00-\u0E7F\u1780-\u17FF\u1400-\u167F]', '', normalized) s1, e1 = u"\U0001F100", u"\U0001F9FF" s2, e2 = u"\U00010000", u"\U0001342E" normalized = re.sub(r'[{}-{}{}-{}]'.format(s1, e1, s2, e2), '', normalized) normalized = re.sub(r"[\uFEFF\u2060\u2E18]", '', normalized) normalized = re.sub(r'(\.\.\.\s){2,}', '... ', normalized) normalized = re.sub(r'(([\^.></_|])\s){2,}', '', normalized) normalized = re.sub(r'(\"){2,}', '"', normalized) normalized = re.sub(r'(_){2,}', '', normalized) normalized = re.sub(r'!{2,}', '!!', normalized) if charmap is not None: for ch in charmap: if ch in normalized: try: normalized = re.sub(ch, charmap[ch], normalized) except sre_constants.error: pass normalized = re.sub(r'\([_\-.\u2200-\u22FF\s]+\)', '', normalized) normalized = re.sub(r'\[[_\-.\u2200-\u22FF\s]+\]', '', normalized) normalized = re.sub(r'\\[_\-.\u2200-\u22FF\s]+/', '', normalized) normalized = re.sub('¿', '', normalized) return normalized.strip()
def body_html(self): return markdown(self.body)
def save(self, *args, **kwargs): self.slug = slugify(self.name) self.notes_html = mistune.markdown(self.notes) super(Talk, self).save(*args, **kwargs)
def test_skip_style(): ret = mistune.markdown('foo\n<style>body{color:red}</style>', skip_style=True) assert ret == '<p>foo</p>\n'
def test_not_escape_inline_tags(): text = '<a name="top"></a>' assert text in mistune.markdown(text, escape=False)
def test_block_html(): ret = mistune.markdown('<div ></div>', escape=False) assert '<div ></div>' in ret
def test_not_escape_block_tags(): text = '<h1>heading</h1> text' assert text in mistune.markdown(text, escape=False)
def test_escape(): ret = mistune.markdown('<div>**foo**</div>', escape=True) assert '>' in ret ret = mistune.markdown('this **foo** is <b>bold</b>', escape=True) assert '>' in ret