class MarkdownReader(Reader):
enabled = bool(Markdown)
file_extensions = ['md', 'markdown', 'mkd', 'mdown']
default_extensions = ['codehilite(css_class=highlight)', 'extra']
def __init__(self, *args, **kwargs):
super(MarkdownReader, self).__init__(*args, **kwargs)
self.extensions = self.settings.get('MD_EXTENSIONS',
self.default_extensions)
self.extensions.append('meta')
self._md = Markdown(extensions=self.extensions)
def _parse_metadata(self, meta):
"""Return the dict containing document metadata"""
output = {}
for name, value in meta.items():
name = name.lower()
if name == "summary":
summary_values = "\n".join(value)
# reset the markdown instance to clear any state
self._md.reset()
summary = self._md.convert(summary_values)
output[name] = self.process_metadata(name, summary)
else:
output[name] = self.process_metadata(name, value[0])
return output
def read(self, source_path):
"""Parse content and metadata of markdown files"""
with pelican_open(source_path) as text:
content = self._md.convert(text)
metadata = self._parse_metadata(self._md.Meta)
return content, metadata
class HtmlHeaderExtensionTest(unittest.TestCase):
DOCTYPE_AND_META = '^' + \
r"""<!DOCTYPE html>\n""" + \
r"""<meta charset=['"]utf-8['"](>|></meta>| */>)\s*"""
def setUp(self):
self.md = Markdown([HtmlHeaderExtension()])
def test_basic(self):
source = "Текст"
expectedRe = self.DOCTYPE_AND_META + "<p>Текст</p>$"
self.assertRegex(self.md.convert(source), expectedRe)
def test_title_is_inserted_for_1st_level_heading(self):
source = "# Заголовок"
expectedRe = self.DOCTYPE_AND_META + \
"<title>Заголовок</title>\s*" + \
"<h1>Заголовок</h1>$"
self.assertRegex(self.md.convert(source), expectedRe)
def test_title_not_inserted_if_several_headings(self):
source = "# Заголовок 1\n\n# Заголовок 2"
expectedRe = self.DOCTYPE_AND_META + \
"<h1>Заголовок 1</h1>\s*" + \
"<h1>Заголовок 2</h1>\s*$"
self.assertRegex(self.md.convert(source), expectedRe)
def test_title_tags_are_property_stripped(self):
source = "# Заголовок со [ссылкой](http://example.com/) внутри"
expectedRe = self.DOCTYPE_AND_META + "<title>Заголовок со ссылкой внутри</title>"
self.assertRegex(self.md.convert(source), expectedRe)
def __init__(self, info_yaml, basedir=''):
"""Import the solution's info.yaml file."""
f, url_parts = self._open(info_yaml, basedir)
solution_yaml = f.read().decode('utf-8')
self.basedir = urlunparse((url_parts.scheme, url_parts.netloc,
os.path.dirname(url_parts.path),
None, None, None))
# create a markdown converter and modify it to rebase image links
markdown = Markdown()
markdown.inlinePatterns['image_link'] = _RebasedImageLinkPattern(
self.basedir, IMAGE_LINK_RE, markdown)
markdown.inlinePatterns['image_reference'] = _RebasedImageRefPattern(
self.basedir, IMAGE_REFERENCE_RE, markdown)
# import the solution's metadata
info = yaml.load(solution_yaml)
self.id = hashlib.md5(solution_yaml.encode('utf-8')).hexdigest()
self.title = info['name']
self.release = str(info['release'])
if 'logo' in info:
self.logo = self._make_absolute_path(info.get('logo'),
self.basedir)[0]
# in all the following fields, newlines are suppressed because they
# are not rendered properly in Javascript strings by Django
self.short_description = \
markdown.convert(info['short_desc']).replace('\n', '')
self.long_description = \
markdown.convert(info['long_desc']).replace('\n', '')
self.architecture = \
markdown.convert(info['architecture']).replace('\n', '')
self.design_specs = info.get('design_specs', [])
self.heat_template = info['heat_template']
self.env_file = info.get('env_file') # environments are optional
class UrlizeExtensionTest(unittest.TestCase):
def setUp(self):
self.md = Markdown([UrlizeExtension()])
def test_links_in_angle_brackets_processed(self):
source = "<http://yandex.ru/>"
expectedRe = r"""^<p><a href=['"]http://yandex.ru/['"]>http://yandex.ru/</a></p>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_links_starting_with_http_processed(self):
source = "http://example.com/"
expectedRe = r"""^<p><a href=['"]http://example.com/['"]>http://example.com/</a></p>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_links_starting_with_www_processed(self):
source = "www.example.com"
expectedRe = r"""^<p><a href=['"]http://www.example.com['"]>www.example.com</a></p>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_markdown_links_not_messed_up(self):
source = "[](http://yandex.ru/)"
expectedRe = r""" href=['"]http://yandex.ru/['"]"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_manual_links_not_messed_up(self):
source = "<a href='http://yandex.ru/'>Яндекс</a>"
expected = "<p>%s</p>" % source
self.assertEqual(expected, self.md.convert(source))
def __init__(self, templatefile, *apifiles, **kwargs):
if len(apifiles) == 0:
raise ValueError("There must be at least one APIBlueprint file "
"specified")
include_comments = kwargs.pop("include_comments", True)
base_class = kwargs.pop("base_class", None)
if base_class is None:
base_class = "rest_framework.test.APITestCase"
self._use_html2text = kwargs.pop("html2text", False)
m = Markdown(extensions=["plueprint"])
m.set_output_format("apiblueprint")
with codecs.open(apifiles[0], "r", "utf-8") as fin:
self._api = m.convert(fin.read())
for f in apifiles[1:]:
with codecs.open(f, "r", "utf-8") as fin:
self._api.merge(m.convert(fin.read()))
env = Environment(loader=FileSystemLoader(templatefile),
autoescape=False, trim_blocks=True,
lstrip_blocks=True,
extensions=("jinja2.ext.loopcontrols",))
env.filters["symbolize"] = self._symbolize
env.filters["html2text"] = self._html2text
self._template = env.get_template("")
self._include_comments = include_comments
self._base_class = base_class[base_class.rfind('.') + 1:]
self._base_module = base_class[:-len(self._base_class) - 1]
self._counter = 1
class TableMarkupExtensionTest(unittest.TestCase):
def setUp(self):
self.originalMd = Markdown()
self.md = Markdown([TableMarkupExtension()])
def test_basic(self):
firstPart = "First part with two paragraphs\n\nThe second paragraph\n\n* A list"
secondPart = "Second part"
actual = self.md.convert(
firstPart + \
"\n\n-----\n\n" + \
secondPart)
expectedRe = \
r"^<tr>\s*" + \
r"<td>\s*" + re.escape(self.originalMd.convert(firstPart)) + "\s*</td>\s*" + \
r"<td>\s*" + re.escape(self.originalMd.convert(secondPart)) + "\s*</td>\s*" + \
r"</tr>$"""
self.assertRegex(actual, expectedRe)
def test_beginning_hr_is_ignored(self):
source = "-----\n\nТекст"
expectedRe = r"""^<tr>\s*<td>\s*<p>Текст</p>\s*</td>\s*</tr>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_ending_hr_is_ignored(self):
source = "Текст\n\n-----"
expectedRe = r"""^<tr>\s*<td>\s*<p>Текст</p>\s*</td>\s*</tr>$"""
self.assertRegex(self.md.convert(source), expectedRe)
class ItemTypeExtensionTest(unittest.TestCase):
def setUp(self):
self.md = Markdown([ItemTypeExtension()])
def test_beginning(self):
source = "* [deferred] элемент списка";
expectedRe = r"""^<ul>\s*<li class="type_deferred">элемент списка</li>\s*</ul>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_middle(self):
source = "* элемент [wait] списка";
expectedRe = r"""^<ul>\s*<li class="type_wait">элемент +списка</li>\s*</ul>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_end(self):
source = "* элемент списка [freetime]";
expectedRe = r"""^<ul>\s*<li class="type_freetime">элемент списка *</li>\s*</ul>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_unknown_is_not_replaced(self):
source = "* элемент [списка]";
expectedRe = r"""^<ul>\s*<li>элемент \[списка\]</li>\s*</ul>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_references_not_messed_up(self):
source = "Тестовая [freetime][deferred] ссылка\n\n[deferred]: http://example.com"
expectedRe = r"""^<p>Тестовая <a href=["']http://example.com["']>freetime</a> ссылка</p>$"""
self.assertRegex(self.md.convert(source), expectedRe)
def test_i18n_always_after_toc(self):
text = '# This is h1'
expected_toc = (
'<div class="toc">'
' <ul>'
' <li><a href="#esto-es-h1">Esto es h1</a></li>'
' </ul>'
'</div>'
)
self.catalog.add(
'This is h1',
'Esto es h1'
)
self.write_po()
md = Markdown(
extensions=['markdown.extensions.toc', 'markdown_i18n'],
extension_configs={
'markdown_i18n': {
'i18n_dir': self.dir,
'i18n_lang': 'es_ES'
}
}
)
md.convert(text)
toc = getattr(md, 'toc', '')
self.assertEqual(clean_xml(toc), clean_xml(expected_toc))
def get_feeds(feeds_file, max_age, max_feeds):
opml = lp.parse(feeds_file)
feeds = opml.feeds
feeds = feeds[:max_feeds]
md = Markdown()
filename = "rssdigest.html"
with open(filename, "w") as text_file:
text_file.write(md.convert("# Daily RSS Digest \n----"))
digeststring = "# Daily RSS Digest \n----\n\n"
number_of_feeds = len(feeds)
for index, feed in enumerate(feeds):
feed = feedparser.parse(feed.url)
feedstring = ""
addfeed = False
print("[" + str(index) + "/" + str(number_of_feeds) + "]")
if 'title' in feed.feed:
feedstring += "## " + feed.feed.title + "\n"
for entry in feed.entries:
localtime = time.localtime()
try:
publishedtime = entry.published_parsed
# age in days
age = (time.mktime(localtime) - time.mktime(publishedtime)) / 60 / 60 / 24
if age < max_age:
feedstring += "## ["+entry.title+"]("+entry.link+")\n\n"
if 'description' in entry:
if len(entry.description) < 500:
feedstring += entry.description + "\n\n"
addfeed = True
except:
pass
if not addfeed:
print(feedstring + "No new posts\n")
feedstring += "----\n"
if addfeed:
print(feedstring)
# Append to string
digeststring += feedstring
# Append to file
with open(filename, "a") as text_file:
feedhtml = md.convert(feedstring)
text_file.write(feedhtml)
digesthtml = md.convert(digeststring)
# print("Final: " + digesthtml)
return digesthtml
def getPostMeta(self, postFile):
if not postFile:
return "", {}, ""
postId = postFile.split('/')[-1].split('.')[0]
content = readFile(postFile)
md = Markdown(extensions=["meta"])
md.convert(content.decode('utf-8'))
meta = self.genMeta(md.Meta)
return postId, meta, content
def load_issue(md_source):
"""Reads an issue from a string-like object containing a Markdown document.
Returns the issue number, preamble and stories.
:param md_source: A string containing the Markdown source of the document.
:returns: Tuple of ``(issue number, preamble, stories)``, where the issue
number is an integer, the preamble is a Markdown string and stories is a
list of dictionaries with keys ``title``, ``body`` and ``link``, which are
strings which should be interpreted as Markdown, Markdown and a URL,
respectively."""
# Just parsing to get metadata
parser = Markdown(extensions=["markdown.extensions.meta"])
parser.convert(md_source)
try:
issue_no = int("".join(parser.Meta["issue"]))
except (ValueError, KeyError):
raise ParseError("No issue number present")
# Split document according to level 1 headings (only using # style at BOL).
# Heading group matches will be in elements 1, 3, 5, etc. (so 0, 2, 4, and
# so on are sections)
try:
_, no_meta = md_source.split("\n\n", 1)
except ValueError:
raise ParseError("No preamble after the metadata!")
heading_re = re.compile("\n#([^\n]+)\n")
sections = heading_re.split(no_meta)
if len(sections) <= 3:
raise ParseError("Need more than one section (preamble + at least one story)")
preamble = sections[0]
if not preamble.strip():
raise ParseError("Preamble is empty")
# Process stories
stories = []
for heading_md, story in zip(sections[1::2], sections[2::2]):
lines = story.strip().split("\n")
if not lines:
raise ParseError('Story with heading "{}" is empty'.format(heading_md))
# This is the body of the story
remainder = "\n".join(lines[:-1])
# "Learn more" link should be last line of the story. Last line is
# stripped during processing otherwise. This is a little magic (I
# probably should *enforce* a specific format), but whatever.
link_line = lines[-1]
parsed_ll = BeautifulSoup(markdown(link_line), "lxml")
elem = parsed_ll.select_one("a[href]")
if elem is None:
raise ParseError('Could not find "learn more" link for "{}"'.format(heading_md))
parsed_link = elem["href"]
stories.append({"title": heading_md, "body": remainder, "link": parsed_link})
return (issue_no, preamble, stories)
def POST(self):
web.header('Content-Type', 'application/json')
try:
param = json.loads(web.data())
try:
postId = str(int(param['postId']))
postContent = param['postContent'].encode('utf-8')
except:
logging.error("%s", traceback.format_exc())
return json.dumps(InputError)
if os.path.exists('%s/frontend/md/%s.md' % (ProjectPath, postId)):
oldStatus = 'public'
postId, oldMeta, tmp = self.getPostMeta('%s/frontend/md/%s.md' % (ProjectPath, postId))
else:
oldStatus = 'private'
if os.path.exists('%s/frontend/md/%s.md_' % (ProjectPath, postId)):
postId, oldMeta, tmp = self.getPostMeta('%s/frontend/md/%s.md_' % (ProjectPath, postId))
else:
oldMeta = {}
try:
md = Markdown(extensions=["meta"])
md.convert(postContent.decode('utf-8'))
meta = self.genMeta(md.Meta)
for key in ["category", "tags", "date", "title", "status"]:
if not meta.has_key(key) or not meta[key]:
raise ValueError("input error")
if key == "status" and meta[key] not in ["public", "private"]:
raise ValueError("input error")
except:
logging.error("%s", traceback.format_exc())
return json.dumps(MetaError)
status = meta['status'] if meta.has_key('status') else 'private'
os.system('rm -rf %s/frontend/md/%s.md*' % (ProjectPath, postId))
os.system('rm -rf %s/frontend/post/%s.html*' % (ProjectPath, postId))
isLatest = self.isLatestPost(postId, status)
writeFile(self.getFilename('md', postId, status), postContent)
writeFile(self.getFilename('post', postId, status), self.genPostPage(postId, postContent, meta, isLatest and status == 'public'))
if isLatest:
writeFile("%s/frontend/home.html" % (ProjectPath), self.genHomePage())
if oldStatus != status:
writeFile('%s/frontend/archive.html' % (ProjectPath), self.genArchivePage())
self.genSitemap()
if oldStatus != status or oldMeta != meta:
self.updateIndex(oldMeta,
self.getFilename('md', postId, oldStatus),
meta,
self.getFilename('md', postId, status))
return json.dumps(Success)
except:
logging.error("SavePost get exception: %s", traceback.format_exc())
return json.dumps(SaveError)
def _get_slide_io2012(self, contents=None, extensions=[]):
""" io-2012 style """
md = Markdown(extensions=extensions)
splits = [
{'horizon': '---', 'style': 'none'},
{'horizon': '___', 'style': 'smaller'},
{'horizon': '***', 'style': 'fill'}
]
styles = []
for split in splits:
styles.append(split['style'])
horizon = '\n' + split['horizon'] + '\n'
contents = contents.replace(horizon, '\n---\n' + split['style'] + '\n')
pages = contents.split('\n---\n')
# from pages to slides
slides = []
for page in pages:
sections = page.split('\n\n', 2)
slide = {}
if not sections[0] in styles:
if len(sections) > 2:
sections[1] += '\n\n' + sections[2]
sections.insert(0, 'none')
slide['style'] = sections[0]
if len(sections) > 1:
slide['hgroup'] = sections[1]
if len(sections) > 2:
slide['article'] = sections[2]
slides.append(slide)
# from slides to html
html = '\n'
for slide in slides:
html += '<slide>\n'
if slide.has_key('hgroup'):
html += '<hgroup>\n'
html += md.convert(slide['hgroup']) + '\n'
html += '</hgroup>\n'
if slide.has_key('article'):
html += '<article class="' + slide['style'] + '">\n'
html += md.convert(slide['article']) + '\n'
html += '</article>\n'
html += '</slide>\n\n'
# from comment out to presener note
html = html.replace('\n<!--\n', '\n<aside class="note">\n')
html = html.replace('\n-->\n', '\n</aside>\n')
return html
def render_view(t_path, view='static', values=None):
template_values={}
if view == 'logs':
md = Markdown()
for log in values[0]:
log.display = md.convert(log.content)
template_values = {
'logs': values[0],
'older': values[1],
'newer': values[2],
'spacer': values[1] and values[2],
}
if view == 'stufflist':
template_values = {
'stufflist':values,
}
if view == 'logentry':
template_values = {
'log':values,
}
if view == 'stuffentry':
md = Markdown()
for log in values[1][0]:
log.display = md.convert(log.content)
template_values = {
'stuff':values[0],
'logs': values[1][0],
'older': values[1][1],
'newer': values[1][2],
'spacer': values[1][1] and values[1][2],
}
if view == 'info':
md = Markdown()
conv_info = md.convert(values)
template_values = {'tt': conv_info }
if view == 'editlogs':
template_values = {'logs': values}
if view == 'editstuff':
template_values = {'stufflist': values}
if view == 'edittt':
template_values = {'tt': values}
if view == 'add':
template_values = {'numbers': values }
path = os.path.join(os.path.dirname(__file__),t_path)
template_values['admin'] = users.is_current_user_admin()
return template.render(path,template_values)
class StrikeExtensionTest(unittest.TestCase):
def setUp(self):
self.md = Markdown([StrikeExtension()])
def test_basic(self):
source = "--Вычеркнутый текст--"
expectedRe = "<s(trike)?>Вычеркнутый текст</s(trike)?>"
self.assertRegex(self.md.convert(source), expectedRe)
def test_spaces_are_preserved(self):
source = "-- Вычеркнутый текст --"
expectedRe = "<s(trike)?> Вычеркнутый текст </s(trike)?>"
self.assertRegex(self.md.convert(source), expectedRe)
class CodeNewlinesRemovalExtensionTest(unittest.TestCase):
def setUp(self):
self.md = Markdown([CodeNewlinesRemovalExtension()])
def test_basic(self):
source = '''<pre><code>\nНекоторый код, напечатанный на новой строке\n</code></pre>'''
expected = '''<pre><code>Некоторый код, напечатанный на новой строке</code></pre>'''
self.assertEqual(expected, self.md.convert(source))
def test_attributes_are_preserved(self):
source = '''<pre id='id'><code class='class'>\n...\n</code></pre>'''
expected = '''<pre id='id'><code class='class'>...</code></pre>'''
self.assertEqual(expected, self.md.convert(source))
class MDProgram(TemplatedProgram):
"""A program that renders an rst file into a template"""
default_template = 'md_display.html'
def __init__(self, context):
from markdown import Markdown
self.md = Markdown(
output_format = 'html5',
safe_mode = 'escape',
enable_attributes = True,
extensions = [
'smart_strong',
'fenced_code',
'footnotes',
'attr_list',
'def_list',
'tables',
'abbr',
'meta',
'headerid',
'codehilite(pygments_style=tango, css_class=syntax, guess_lang=True)'
]
)
self.contents = {}
TemplatedProgram.__init__(self, context)
def prepare(self):
with self.context.open_source_file() as f:
parsed = self.md.convert(f.read())
self.context.config = self.context.config.add_from_dict(self.md.Meta)
self.contents['fragment'] = parsed
self.contents['html_title'] = self.contents['title'] = self.context.title = u' '.join(self.md.Meta.get('title', u''))
self.contents['summary'] = self.context.summary = u' '.join(self.md.Meta.get('summary', u''))
def render(self, contents):
self.md.reset()
return self.md.convert(contents)
def render_contents(self):
return self.contents['fragment']
def get_template_context(self):
ctx = TemplatedProgram.get_template_context(self)
ctx['md'] = self.contents
return ctx
def main():
m = Markdown(extensions=["plueprint"])
m.set_output_format("apiblueprint")
tests_dir = os.path.join(os.path.dirname(__file__), "api-blueprint",
"examples")
skip_number = 0
index = 0
for doc in sorted(os.listdir(tests_dir)):
if os.path.splitext(doc)[1] != ".md" or doc == "README.md":
continue
index += 1
if index <= skip_number:
continue
with codecs.open(os.path.join(tests_dir, doc), "r", "utf-8") as fin:
txt = fin.read()
api = m.convert(txt)
print("-- %s --" % doc)
print(api)
try:
api[">"].print_resources()
except KeyError:
pass
print("Actions:")
for action in api["/"]:
print(action)
def load_stories(story_stuff):
meta = {
"title": (True, ("one",)),
"description": (True, ("one",)),
"image": (True, ("one",)),
"button": (True, ("one",)),
"link": (True, ("one",)),
"button2": (False, ("one",)),
"link2": (False, ("one",)),
"action": (False, ("one",)),
}
markdowner = Markdown(extensions=["meta"], output_format="html5")
stories = []
for story_file, filename in story_stuff:
story = {}
html = markdowner.convert(story_file.read()) # also loads metadata
story["body"] = Markup(html)
story["slug"] = os.path.splitext(filename)[0]
for field, (required, filters) in meta.items():
field_val = markdowner.Meta.get(field)
try:
val = apply_field_constraints(field_val, required, filters)
except MetaError as e:
e.apply_context(filename, field)
raise e
story[field] = val
stories.append(story)
markdowner.reset()
return stories
class AutoLinkTargetExtensionTest(unittest.TestCase):
BLANK_TARGET_RE = re.compile(r""" target=['"]_blank['"]""")
def assertNewWindow(self, source):
self.assertRegex(self.md.convert(source), self.BLANK_TARGET_RE)
def assertSameWindow(self, source):
self.assertNotRegex(self.md.convert(source), self.BLANK_TARGET_RE)
def setUp(self):
self.md = Markdown([AutoLinkTargetExtension()])
def test_absolute_links_in_new_window(self):
self.assertNewWindow("[](http://example.com/)")
def test_relative_links_in_same_window(self):
self.assertSameWindow("[](other_page.html)")
self.assertSameWindow("[](/index.jsp)")
def test_existing_target_not_changed(self):
ret = self.md.convert("[{@target=my_frame}](http://example.com/)")
self.assertRegex(ret, r"""target=['"]my_frame['"]""")
self.assertNotRegex(ret, self.BLANK_TARGET_RE)
@unittest.skip
def test_manual_markup_is_processed(self):
self.assertNewWindow("<a href='http://example.com/'/>")
def get_html_content(self):
wikitables = WikiTableExtension()
nofollow = NofollowExtension()
margin_notes = MarginNotesExtension()
toc = TocExtension([('title', 'Table of Contents')])
md = Markdown(
extensions=[
toc,
'outline',
'mathjax',
wikitables,
nofollow,
'def_list',
margin_notes,
],
output_format='html5',
safe_mode='escape'
)
markdowned_text = md.convert(self.content)
article = {
'html': markdowned_text,
'toc': getattr(md, 'toc', None),
}
return article
def html_content(self):
try:
return self._html_content
except AttributeError:
md = Markdown(extensions=['meta'])
self._html_content = md.convert(self.content)
return self._html_content
class MarkdownImportReader(MarkdownReader): """Reader for Markdown Import Files. Files require, at minimum the following metadata: Title: TITLE Source: RELATIVE_SOURCE Content is not read. Source metadata stripped. """ file_extensions = ['mdi'] def read(self, filename): """Parse content and metadata of markdown files""" self._md = Markdown(extensions=self.extensions) with pelican_open(filename) as mdi: with pelican_open(re.search(r"Source: ([\w/\\\.-]*)", mdi).group(1)) as md: md = re.sub(r'\(([^/()]*).md((?:#[-a-z]*)?)\)', r'(\1.html\2)', md) content = self._md.convert(mdi+"Display: mdimport\n"+md) metadata = self._parse_metadata(self._md.Meta) return content, metadata
def load_blog(blog_stuff):
meta = {
"title": (True, ("one",)),
"author": (True, ("one",)),
"date": (True, ("one", "iso-date")),
"photo": (False, ("one",)),
"photo_caption": (False, ("")),
}
markdowner = Markdown(extensions=["meta"], output_format="html5")
posts = []
for blog_file, filename in blog_stuff:
post = {}
html = markdowner.convert(blog_file.read()) # also loads metadata
post["content"] = Markup(html)
post["summary"] = Markup("\n".join(html.split("\n", 1)[:-1]))
post["slug"] = os.path.splitext(filename)[0]
for field, (required, filters) in meta.items():
field_val = markdowner.Meta.get(field)
try:
val = apply_field_constraints(field_val, required, filters)
except MetaError as e:
e.apply_context(filename, field)
raise e
post[field] = val
posts.append(post)
markdowner.reset()
return posts
def read(self, filename):
"""Parses the given file and returns a :class:`Page` object
:param filename: path to the file to read.
:type filename: str.
:returns: Page
"""
try:
text = open(filename, 'r', encoding='utf-8').read()
except UnicodeDecodeError:
print('wrong encoding: {0}'.format(filename))
raise
md = Markdown(extensions=set(self.extensions + ['meta']))
content = md.convert(text)
metadata = {}
for name, value in md.Meta.items():
name = name.lower()
metadata[name] = self.process_metadata(name, value[0])
return Page(filename.replace(self.path, ''),
content,
metadata.get('title'),
metadata.get('title_short'),
metadata.get('template'),
metadata.get('parent'),
metadata.get('sort'),
metadata.get('in_nav'))
class MarkdownReader(BaseReader):
"""Reader for Markdown files"""
enabled = bool(Markdown)
file_extensions = ['md', 'markdown', 'mkd', 'mdown']
def __init__(self, *args, **kwargs):
super(MarkdownReader, self).__init__(*args, **kwargs)
settings = self.settings['MARKDOWN']
settings.setdefault('extension_configs', {})
settings.setdefault('extensions', [])
for extension in settings['extension_configs'].keys():
if extension not in settings['extensions']:
settings['extensions'].append(extension)
if 'markdown.extensions.meta' not in settings['extensions']:
settings['extensions'].append('markdown.extensions.meta')
self._source_path = None
def _parse_metadata(self, meta):
"""Return the dict containing document metadata"""
formatted_fields = self.settings['FORMATTED_FIELDS']
output = {}
for name, value in meta.items():
name = name.lower()
if name in formatted_fields:
# formatted metadata is special case and join all list values
formatted_values = "\n".join(value)
# reset the markdown instance to clear any state
self._md.reset()
formatted = self._md.convert(formatted_values)
output[name] = self.process_metadata(name, formatted)
elif not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True):
if len(value) > 1:
logger.warning(
'Duplicate definition of `%s` '
'for %s. Using first one.',
name, self._source_path)
output[name] = self.process_metadata(name, value[0])
elif len(value) > 1:
# handle list metadata as list of string
output[name] = self.process_metadata(name, value)
else:
# otherwise, handle metadata as single string
output[name] = self.process_metadata(name, value[0])
return output
def read(self, source_path):
"""Parse content and metadata of markdown files"""
self._source_path = source_path
self._md = Markdown(**self.settings['MARKDOWN'])
with pelican_open(source_path) as text:
content = self._md.convert(text)
if hasattr(self._md, 'Meta'):
metadata = self._parse_metadata(self._md.Meta)
else:
metadata = {}
return content, metadata
def mk_manual():
import codecs
md = Markdown(extensions=['toc'])
content = md.convert(codecs.open('readme.md', encoding='utf8').read())
with open('summary/asema-manual.html','w') as out:
out.write(html_manual.substitute(content=content.encode('utf8')))
upload('summary/asema-manual.html')
class Markdown:
def __init__(self):
self.markdown = MarkdownParser(
output_format='html5',
extensions = ['tables'],
)
def parse(self, fp):
meta = {}
def read_file():
in_head = True
for line in fp:
if not in_head:
yield line
else:
if len(line) < 2:
in_head = False
yield line
else:
name, value = line.split(':')
meta[name.lower()] = value.strip()
content = self.markdown.convert(''.join(read_file()))
self.markdown.reset()
pagetype = meta.pop('type', 'templated') # Default Pagetype
return pagetype, meta, content
def __call__(self, stream): from jinja2 import Markup from markdown import Markdown # Markdown is not thread safe markdown = Markdown(**self._markdown_options) return Markup(markdown.convert(stream))
class Convert:
"""Ковертирование из markdown в html и получение заголовка статьи"""
def __init__(self):
self._md = Markdown(extensions = ['extra', 'meta', 'codehilite', 'nl2br'])
def read(self):
return open(self.fname, encoding="utf-8").read()
def test(self, pattern, tag):
return re.match(pattern, tag)
def go(self, fname):
self.fname = fname
content = self._md.convert(self.read())
meta = self._md.Meta
test = [None for i in ['date', 'title', 'slug'] if i not in meta.keys()]
if None in test:
return None
if self.test(r'^\d{4}-\d{2}-\d{2}$', meta['date'][0])\
and self.test(r'^[a-zA-Z0-9-]+$', meta['slug'][0]):
return {'date': meta['date'][0],
'title': meta['title'][0],
'slug': meta['slug'][0],
'content': content}
else:
return None
def get_md(self):
md = Markdown(
extensions=[
'markdown.extensions.extra',
'markdown.extensions.codehilite',
'markdown.extensions.toc',
]
)
md_body = md.convert(self.body)
# toc 是渲染后的目录
return md_body, md.toc
def read(self, filename):
"""Parse content and metadata of markdown files"""
text = open(filename)
md = Markdown(extensions=['meta', 'codehilite'])
content = md.convert(text)
metadata = {}
for name, value in md.Meta.items():
name = name.lower()
metadata[name] = _process_metadata(name, value[0])
return content, metadata
def generate_rich_content(value):
md = Markdown(extensions=[
'markdown.extensions.extra', 'markdown.extensions.codehilite',
TocExtension(slugify=slugify)
])
content = md.convert(value)
result = re.search(r'<ul>(.*)</ul>', md.toc, re.S)
if result:
toc = result.group(1)
else:
toc = ""
return {'content': content, 'toc': toc}
def read(self, filename):
"""Parse content and metadata of markdown files"""
text = open(filename)
md = Markdown(extensions=['meta', 'codehilite'])
content = md.convert(text)
metadatas = {}
for name, value in md.Meta.items():
name = name.lower()
metadatas[name] = _METADATAS_PROCESSORS.get(name,
lambda x: x)(value[0])
return content, metadatas
class TestNewTab(TestCase):
def setUp(self):
self.md = Markdown(extensions=[NewTabExtension()])
self.maxDiff = None
def assertEqualMarkdown(self, before, after):
self.assertEqual(self.md.convert(before), after)
def test_link(self):
self.assertEqualMarkdown(
"""\
[one](https://ddg.gg) \
[two](https://duck.co "test")""", """\
<p><a href="https://ddg.gg" target="_blank">one</a> \
<a href="https://duck.co" target="_blank" title="test">two</a></p>""")
def test_reference(self):
self.assertEqualMarkdown(
"""\
[one][un] \
[two][deux]
[un]: https://ddg.gg
[deux]: https://duck.co "test\"""", """\
<p><a href="https://ddg.gg" target="_blank">one</a> \
<a href="https://duck.co" target="_blank" title="test">two</a></p>""")
def test_short_reference(self):
self.assertEqualMarkdown(
"""\
[one], \
[two]
[one]: https://ddg.gg
[two]: https://duck.co "test\"""", """\
<p><a href="https://ddg.gg" target="_blank">one</a>, \
<a href="https://duck.co" target="_blank" title="test">two</a></p>""")
def test_autolink(self):
self.assertEqualMarkdown(
"""\
<https://ddg.gg>""", """\
<p><a href="https://ddg.gg" target="_blank">https://ddg.gg</a></p>""")
def test_automail(self):
self.assertEqualMarkdown(
"""\
<*****@*****.**>""", """\
<p><a href="mailto:add\
ress@example\
.com" target="_blank">addre\
ss@example.c\
om</a></p>""")
def tournament_display(idx):
from app.cron import check_if_deadline_passed
check_if_deadline_passed(app_cls.service)
#############################################
from app.models import Tournament
tournament = (Tournament.query.filter(Tournament.id == idx).first())
is_owner = False
is_joined = False
try:
auth_token = request.cookies.get('auth_token')
user = app.action.user.user_instance_from_token(app_cls, auth_token)
if user.id == tournament.owner_id:
is_owner = True
from app.models import Tier
already_joined = (Tier.query.filter(Tier.user_id == user.id).filter(
Tier.tournament_id == tournament.id).first())
if already_joined is not None:
is_joined = True
except:
pass
# @1 MARKDOWN
mk = Markdown()
html = mk.convert(tournament.readme)
# @2 LOCATION
location_geo = (0, 0)
location_map = False
if tournament.location:
from geopy.geocoders import Nominatim
geolocator = Nominatim(user_agent="python")
location = geolocator.geocode(tournament.location)
if location:
location_geo = (location.latitude, location.longitude)
location_map = True
# FIMXE: 404
is_ladder = False
if tournament.ladder:
is_ladder = True
js_teams = app.action.tournament.get_js_teams(app_cls, tournament)
js_results = app.action.tournament.get_js_results(app_cls, tournament)
return render_template('tournament_display.html',
tournament=tournament,
is_owner=is_owner,
is_joined=is_joined,
html=html,
location=location_geo,
location_map=location_map,
js_teams=js_teams,
js_results=js_results,
is_ladder=is_ladder)
def markdown(value):
try:
from markdown import Markdown
md = Markdown(extensions=['footnotes'])
except ImportError:
try:
from markdown2 import Markdown
md = Markdown(extras=['footnotes', 'code-friendly'])
except ImportError:
raise jinja2.TemplateError('Markdown is not installed!')
return jinja2.Markup(md.convert(value))
def markdown_to_html(body):
"""解析markdown"""
if not body:
return ''
md = Markdown(extensions=[
'fenced_code',
'codehilite(css_class=highlight,linenums=None)',
'admonition', 'tables', 'extra'])
content = md.convert(body)
return content
def __html__(self):
if len(self.markdown) == 0:
with open(self.path) as post_file:
for line in post_file:
self.markdown += line
md = Markdown()
months = [None, 'January', 'Febuary', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
date = self.date_posted
self.html = md.convert(self.markdown) + '<center><time>%s %d, %d</time></center>' % (months[date.month], date.day, date.year)
return self.html
def save(self, *args, **kwargs):
#判断是否存在摘要,如果不存在则处理
if not self.excerpt:
#生成markdown的实例对象,然后使用convert
md = Markdown(extensions=[
'markdown.extensions.extra', 'markdown.extensions.codehilite'
])
self.excerpt = strip_tags(md.convert(self.content))[:54]
#不管有没有都要调用save方法
super(Post, self).save(*args, **kwargs)
def test_autodoc_class_members():
md = Markdown(extensions=[AutoDocExtension()])
text = md.convert("""
# API reference
This is an API reference.
::: import_examples.ExampleClass
:docstring:
:members:
""")
assert text == """<h1>API reference</h1>
def test_markdown_mentions_dict(self):
"""
markdown mentions dict
"""
comment = u"@nitely, @esteban"
md = Markdown(output_formats='html5',
safe_mode='escape',
extensions=['spirit.utils.markdown.mention', ])
comment_md = md.convert(comment)
# mentions get dianmically added on MentionifyExtension
self.assertDictEqual(md.mentions, {'nitely': self.user,
'esteban': self.user2})
def get_info(self, query=""):
file = self.postdir + '/info.md'
with open(file, 'r', encoding='UTF-8') as content_file:
content = content_file.read()
md = Markdown(extensions=[
'full_yaml_metadata', 'codehilite', 'markdown.extensions.toc',
'mdx_math'
])
content = md.convert(content)
meta = md.Meta
if meta is not None:
return meta.get(query).split(",")
def get_content(filename, overrides=None):
markdown = Markdown(extensions=["fenced_code"])
content = ""
if os.path.isfile(filename):
with codecs.open(filename, 'r', 'utf-8') as f:
md_data = f.read()
md_data = md_data.replace(':ref:', '')
if overrides is not None:
v1, v2 = overrides
md_data = md_data.replace(v1, v2)
content = markdown.convert(md_data)
return content
class MarkdownReader(BaseReader):
"""Reader for Markdown files"""
enabled = bool(Markdown)
file_extensions = ['md', 'markdown', 'mkd', 'mdown']
def __init__(self, *args, **kwargs):
super(MarkdownReader, self).__init__(*args, **kwargs)
self.extensions = self.settings['MD_EXTENSIONS']
self.extensions.setdefault('markdown.extensions.meta', {})
self._source_path = None
def _parse_metadata(self, meta):
"""Return the dict containing document metadata"""
formatted_fields = self.settings['FORMATTED_FIELDS']
output = {}
for name, value in meta.items():
name = name.lower()
if name in formatted_fields:
# formatted metadata is special case and join all list values
formatted_values = "\n".join(value)
# reset the markdown instance to clear any state
self._md.reset()
formatted = self._md.convert(formatted_values)
output[name] = self.process_metadata(name, formatted)
elif name in METADATA_PROCESSORS:
if len(value) > 1:
logger.warning(
'Duplicate definition of `%s` '
'for %s. Using first one.',
name, self._source_path)
output[name] = self.process_metadata(name, value[0])
elif len(value) > 1:
# handle list metadata as list of string
output[name] = self.process_metadata(name, value)
else:
# otherwise, handle metadata as single string
output[name] = self.process_metadata(name, value[0])
return output
def read(self, source_path):
"""Parse content and metadata of markdown files"""
self._source_path = source_path
self._md = Markdown(extensions=self.extensions.keys(),
extension_configs=self.extensions)
with pelican_open(source_path) as text:
content = self._md.convert(text)
metadata = self._parse_metadata(self._md.Meta)
return content, metadata
def timeline(request):
timelines = TimeLine.objects.all()
md = Markdown(extensions=[
'markdown.extensions.extra',
'markdown.extensions.codehilite',
])
for obj in timelines:
obj.update_content = md.convert(obj.update_content)
obj.save()
context = {}
context['timelines'] = timelines
return render(request, 'timeline.html', context)
def test_callbacks(self):
def dont_linkify_txt_extension(attrs, new=False):
if attrs["_text"].endswith(".txt"):
return None
return attrs
md = Markdown(extensions=[LinkifyExtension()],
extension_configs={
"linkify": {
"linkify_callbacks":
[[dont_linkify_txt_extension], ""]
}
})
actual = md.convert("not_link.txt")
expected = '<p>not_link.txt</p>'
self.assertEqual(actual, expected)
actual = md.convert("example.com")
expected = '<p><a href="http://example.com">example.com</a></p>'
self.assertEqual(actual, expected)
def md2html(fpath, page):
"""``fpath`` is identified as a file containing markdown text. Read
the file content, gather metadata from the content (if specified),
translate content to HTML.
And return a tuple of (metadata, content). Content is HTML text."""
from markdown import Markdown
md = Markdown( extensions=['meta'],
output_format='html5', safe_mode='escape' )
content = md.convert( open(fpath).read() )
metadata = {name.lower() : value[0] for name, value in md.Meta.items()}
return metadata, content
def test1(self):
md = Markdown(extensions=[MarkdownImageExpander()])
source = """
"""
mustbe = """
<img alt="" src="example.png">
"""
self.assertEqual(
Bs(md.convert(textwrap.dedent(source)),
"html.parser").prettify().replace("\n", ""),
Bs(textwrap.dedent(mustbe),
"html.parser").prettify().replace("\n", ""))
class MarkdownParser(object):
"""Parser for Markdown files"""
def __init__(self):
self._extensions = [
'codehilite(css_class=highlight)', 'markdown.extensions.tables'
]
self._parser = Markdown(extensions=self._extensions)
def parse(self, content):
"""Parse content and metadata of markdown files"""
self._content = content
return self._parser.convert(self._content)
def news_detail(request, id):
finds = News.objects.filter(id=id)
if finds:
find = finds[0]
else:
find = None
markdowner = Markdown()
find.content = markdowner.convert(find.content)
context = {}
context['find'] = find
return render(request, 'web/news_detail.html', context)
def _parse_metadata(self, meta):
"""Return the dict containing document metadata"""
md = Markdown(extensions=set(self.extensions + ['meta']))
output = {}
for name, value in meta.items():
name = name.lower()
if name == "summary":
summary_values = "\n".join(str(item) for item in value)
summary = md.convert(summary_values)
output[name] = self.process_metadata(name, summary)
else:
output[name] = self.process_metadata(name, value[0])
return output
def read_markdown(filename):
# Use utf-8-sig codec to remove BOM if it is present
with codecs.open(filename, 'r', 'utf-8-sig') as f:
text = f.read()
md = Markdown(extensions=['meta'])
html = md.convert(text)
return {
'title': md.Meta.get('title', [''])[0],
'description': html,
'meta': md.Meta.copy()
}
def _markdown_to_html(md_text):
common_opts = dict(
output_format='xhtml',
safe_mode='replace',
extensions=['headerid(forceid=False, level=3)',
SkipTagsExtension()])
try:
md = Markdown(html_replacement_text='(could not show this)',
lazy_ol=False,
**common_opts)
except TypeError:
md = Markdown(**common_opts)
return md.convert(md_text)
def generate(packages, outdir):
markdown = Markdown()
if not exists(outdir):
makedirs(outdir)
appsdir = join(outdir, 'apps')
if not exists(appsdir):
makedirs(appsdir)
def package_item(item, in_apps_dir):
package = {k: item[k] for k in MANIFEST_KEYS if k in item}
package['shortDescription'] = item['manifest'].get(
'appDescription', None)
if in_apps_dir:
package[
'fullDescriptionUrl'] = f'{item["id"]}-full_description.html'
else:
package[
'fullDescriptionUrl'] = f'apps/{item["id"]}-full_description.html'
return package
items_per_page = 30
packages_length = len(packages)
max_page = math.ceil(packages_length / items_per_page)
for index, items in enumerate(
more_itertools.chunked(packages, items_per_page)):
page = index + 1
json_file = join(appsdir, '%d.json' %
page) if page > 1 else join(outdir, 'apps.json')
with open(json_file, 'w', encoding='utf-8') as f:
json.dump(
{
'paging': {
'page': page,
'count': len(items),
'maxPage': max_page,
'itemsTotal': packages_length,
},
'packages':
list(map(lambda x: package_item(x, page > 1), items))
},
f,
indent=2)
for item in items:
app_info = join(appsdir, f'{item["id"]}.json')
with open(app_info, 'w', encoding='utf-8') as f:
json.dump(package_item(item, True), f)
desc_html = join(appsdir, f'{item["id"]}-full_description.html')
with open(desc_html, 'w', encoding='utf-8') as f:
f.write(markdown.convert(item['description']))
print('Generated json data for %d packages.' % len(packages))
def test_callbacks(self):
def dont_linkify_net_tld(attrs, new=False):
if attrs["_text"].endswith(".net"):
return None
return attrs
# assert expected behavior WITHOUT our callback
actual = markdown("https://linked.net",
extensions=["mdx_linkify"])
expected = '<p><a href="https://linked.net" rel="nofollow">https://linked.net</a></p>'
self.assertEqual(actual, expected)
md = Markdown(
extensions=[
LinkifyExtension(
linker_options={
"callbacks": [dont_linkify_net_tld],
},
),
],
)
# assert .net no longer works
actual = md.convert("https://not-linked.net")
expected = '<p>https://not-linked.net</p>'
self.assertEqual(actual, expected)
# assert other links still work
actual = md.convert("example.com")
expected = '<p><a href="http://example.com">example.com</a></p>'
self.assertEqual(actual, expected)
# assert that configuration parameters can be over-ridden at run time
# https://python-markdown.github.io/extensions/api/#configsettings
expected = '<p><a href="https://should-be-linked.net" rel="nofollow">https://should-be-linked.net</a></p>'
actual = markdown("https://should-be-linked.net", extensions=["mdx_linkify"])
self.assertEqual(expected, actual)
class MarkdownParser(BaseParser):
widget_class = MarkdownWidget
def __init__(self):
self._parser = Markdown(safe_mode='escape')
def format(self, text, instance=None):
if instance and instance.pk:
text = self.format_attachments(
text, attachments=instance.attachments.all())
return smile_it(self._parser.convert(text))
def quote(self, text, username=''):
return '>' + text.replace('\n', '\n>').replace('\r', '\n>') + '\n'
def update_env(self, md: Markdown, config: dict) -> None:
"""
Update the Jinja environment.
Arguments:
md: The Markdown instance. Useful to add functions able to convert Markdown into the environment filters.
config: Configuration options for `mkdocs` and `mkdocstrings`, read from `mkdocs.yml`. See the source code
of [mkdocstrings.plugin.MkdocstringsPlugin.on_config][] to see what's in this dictionary.
"""
# Re-instantiate md: see https://github.com/tomchristie/mkautodoc/issues/14
md = Markdown(extensions=config["mdx"],
extensions_configs=config["mdx_configs"])
self.env.filters["convert_markdown"] = lambda text: do_mark_safe(
md.convert(text))
def post(path):
path = ROOT + "/mdposts/" + path + ".*"
content = ""
for file in glob.glob(path):
content = codecs.open(file, mode='r', encoding="UTF-8").read()
content = content.replace("$base", request.url_root)
md = Markdown(extensions=['meta'])
html = md.convert(content)
return render_template(templates_path + "post.html",
content=html,
metainfos=md.Meta,
img=md.Meta["img"][0])
def test_lorem_ipsum_extension(self):
md = Markdown(extensions=[
MarkdownYamlMetaExtension(),
MarkdownLoremIpsumExtension()
])
html = "<html>" + md.convert(TEST_LOREM_IPSUM_CONTENT) + "</html>"
tree = ET.fromstring(html)
p = tree.findall('./p')[0]
self.assertEqual(100, lipsum.count_words(p.text))
p = tree.findall('./p')[1]
self.assertEqual(5, lipsum.count_sentences(p.text))
self.assertEqual(3, len(tree.findall('./p')[2:]))
html = "<html>" + md.convert(
TEST_LOREM_IPSUM_SINGLE_INSTANCES) + "</html>"
tree = ET.fromstring(html)
p = tree.findall('./p')[0]
self.assertEqual(1, lipsum.count_words(p.text))
p = tree.findall('./p')[1]
self.assertEqual(1, lipsum.count_sentences(p.text))
self.assertEqual(1, len(tree.findall('./p')[2:]))