def get_markdown_ast(markdown_file):
try:
f = open(markdown_file, 'r')
return CommonMark.DocParser().parse(f.read())
except:
logging.error(
"Error: Can't open {0} for reading".format(markdown_file))
sys.exit(1)
finally:
f.close()
def commonmark(value):
"""
Runs CommonMark over a given value.
Syntax::
{{ value|commonmark }}
:type value: str
:rtype: str
"""
import CommonMark
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
ast = parser.parse(value)
return mark_safe(
force_text(renderer.render(ast))
)
def init_app(app):
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
app.extensions['markdown'] = UDataMarkdown(app, parser, renderer)
@app.template_filter()
def mdstrip(value, length=None):
'''
Truncate and strip tags from a markdown source
The markdown source is truncated at the excerpt if present and
smaller than the required length. Then, all html tags are stripped.
'''
if not value:
return ''
if EXCERPT_TOKEN in value:
value = value.split(EXCERPT_TOKEN, 1)[0]
if length > 0:
value = do_truncate(value, length, end='…')
rendered = md(value)
return do_striptags(rendered)
#!/home/wil/Documents/research_tutorials/env/bin/python
import argparse, sys
import CommonMark
parser = argparse.ArgumentParser(description="Process Markdown according to the CommonMark specification.")
if sys.version_info < (3, 0):
reload(sys)
sys.setdefaultencoding('utf-8')
parser.add_argument('infile', nargs="?", type=argparse.FileType('r'), default=sys.stdin, help="Input Markdown file to parse, defaults to STDIN")
parser.add_argument('-o', nargs="?", type=argparse.FileType('w'), default=sys.stdout, help="Output HTML/JSON file, defaults to STDOUT")
parser.add_argument('-a', action="store_true", help="Print formatted AST")
parser.add_argument('-aj', action="store_true", help="Output JSON AST")
args = parser.parse_args()
parser = CommonMark.DocParser()
f = args.infile
o = args.o
lines = []
for line in f:
lines.append(line)
data = "".join(lines)
ast = parser.parse(data)
if not args.a and not args.aj:
renderer = CommonMark.HTMLRenderer()
o.write(renderer.render(ast))
exit()
if args.a:
# print ast
CommonMark.dumpAST(ast)
exit()
#o.write(ast.to_JSON())
o.write(CommonMark.ASTtoJSON(ast))
def markdown_filter(data):
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
return renderer.render(parser.parse(data))
def markup(input):
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
ast = parser.parse(input)
return renderer.render(ast)
def generate(file=None, text=None, style='', custom=0):
# Open File
if file != None:
f = codecs.open(file, 'r', 'utf-8')
rawhtml = f.read()
f.close()
else:
rawhtml = text
# Parse
html = ''
txtlines = rawhtml.replace('\r\n', '\n').replace('\r', '\n').split('\n')
for t in txtlines:
# Ignore comments
html += t + '\n'
# Get Markdown (Common Mark)
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
md = renderer.render(parser.parse(html))
mhtml = ''
# Custom Markdown modifications
if custom:
md = md.replace(
'<li>[ ]',
'<li class="checkbox"><input type="checkbox" disabled="disabled">')
md = md.replace(
'<li>[]',
'<li class="checkbox"><input type="checkbox" disabled="disabled">')
md = md.replace(
'<li>[x]',
'<li class="checkbox checked"><input type="checkbox" checked="checked" disabled="disabled">'
)
md = md.replace(
'<li>[-]',
'<li class="cancelled"><input type="checkbox" checked="checked" disabled="disabled">'
)
md = md.replace(
'<li>[f]',
'<li class="future"><input type="checkbox" disabled="disabled"> FUTURE'
)
# Custom Style
style = '''
h1{border-bottom:2px solid gray;}
h2,h3,h4,h5,h6,h7 {border-bottom:1px solid gray;}
li.checkbox,li.cancelled,li.future {list-style-type: none;margin-left:-25px;}
li.cancelled {text-decoration:line-through;color:gray;}
li.future {font-style:italic;color:gray;}
li.checked {color:gray;}
pre,code {font-family:Courier New,mono,monospace;font-size:12px;}
img{max-width:100%;}
''' + style
# Add Style
if style != '':
mhtml += '<style>' + style + '</style>'
mhtml += md
return mhtml
def _parse_markdown(self, markdown):
parser = CommonMark.DocParser()
ast = parser.parse(markdown)
return ast
def run():
"""
Looks through the docs/ dir and parses each markdown document, looking for
sections to update from Python docstrings. Looks for section headers in
the format:
- ### `ClassName()` class
- ##### `.method_name()` method
- ##### `.attribute_name` attribute
- ### `function_name()` function
The markdown content following these section headers up until the next
section header will be replaced by new markdown generated from the Python
docstrings of the associated source files.
By default maps docs/{name}.md to {modulename}/{name}.py. Allows for
custom mapping via the MD_SOURCE_MAP variable.
"""
print('Updating API docs...')
md_files = []
for root, _, filenames in os.walk(docs_dir):
for filename in filenames:
if not filename.endswith('.md'):
continue
md_files.append(os.path.join(root, filename))
parser = CommonMark.DocParser()
for md_file in md_files:
md_file_relative = md_file[len(project_dir) + 1:]
if md_file_relative in MD_SOURCE_MAP:
py_files = MD_SOURCE_MAP[md_file_relative]
py_paths = [
os.path.join(project_dir, py_file) for py_file in py_files
]
else:
py_files = [os.path.basename(md_file).replace('.md', '.py')]
py_paths = [os.path.join(project_dir, module_name, py_files[0])]
if not os.path.exists(py_paths[0]):
continue
with open(md_file, 'rb') as f:
markdown = f.read().decode('utf-8')
original_markdown = markdown
md_lines = list(markdown.splitlines())
md_ast = parser.parse(markdown)
last_class = []
last = {}
sections = OrderedDict()
find_sections(md_ast, sections, last, last_class,
markdown.count("\n") + 1)
md_chunks = {}
for index, py_file in enumerate(py_files):
py_path = py_paths[index]
with open(os.path.join(py_path), 'rb') as f:
code = f.read().decode('utf-8')
module_ast = ast.parse(code, filename=py_file)
code_lines = list(code.splitlines())
for node in ast.iter_child_nodes(module_ast):
walk_ast(node, code_lines, sections, md_chunks)
added_lines = 0
def _replace_md(key, sections, md_chunk, md_lines, added_lines):
start, end = sections[key]
start -= 1
start += added_lines
end += added_lines
new_lines = md_chunk.split('\n')
added_lines += len(new_lines) - (end - start)
# Ensure a newline above each class header
if start > 0 and md_lines[start][0:4] == '### ' and md_lines[
start - 1][0:1] == '>':
added_lines += 1
new_lines.insert(0, '')
md_lines[start:end] = new_lines
return added_lines
for key in sections:
if key not in md_chunks:
raise ValueError('No documentation found for %s' % key[1])
added_lines = _replace_md(key, sections, md_chunks[key], md_lines,
added_lines)
markdown = '\n'.join(md_lines).strip() + '\n'
if original_markdown != markdown:
with open(md_file, 'wb') as f:
f.write(markdown.encode('utf-8'))
def markup(content):
parser = CommonMark.DocParser()
renderer = CommonMark.HTMLRenderer()
ast = parser.parse(content)
html = renderer.render(ast)
return html
def run():
print('Updating API docs...')
md_files = []
for root, _, filenames in os.walk(docs_dir):
for filename in filenames:
if not filename.endswith('.md'):
continue
md_files.append(os.path.join(root, filename))
parser = CommonMark.DocParser()
for md_file in md_files:
md_file_relative = md_file[len(project_dir) + 1:]
if md_file_relative in md_source_map:
py_files = md_source_map[md_file_relative]
py_paths = [
os.path.join(project_dir, py_file) for py_file in py_files
]
else:
py_files = [os.path.basename(md_file).replace('.md', '.py')]
py_paths = [os.path.join(project_dir, module_name, py_files[0])]
if not os.path.exists(py_paths[0]):
continue
with open(md_file, 'rb') as f:
markdown = f.read().decode('utf-8')
original_markdown = markdown
md_lines = list(markdown.splitlines())
md_ast = parser.parse(markdown)
last_class = []
last = {}
sections = OrderedDict()
find_sections(md_ast, sections, last, last_class,
markdown.count("\n") + 1)
md_chunks = {}
for index, py_file in enumerate(py_files):
py_path = py_paths[index]
with open(os.path.join(py_path), 'rb') as f:
code = f.read().decode('utf-8')
module_ast = ast.parse(code, filename=py_file)
code_lines = list(code.splitlines())
for node in ast.iter_child_nodes(module_ast):
walk_ast(node, code_lines, sections, md_chunks)
added_lines = 0
def _replace_md(key, sections, md_chunk, md_lines, added_lines):
start, end = sections[key]
start -= 1
start += added_lines
end += added_lines
new_lines = md_chunk.split('\n')
added_lines += len(new_lines) - (end - start)
# Ensure a newline above each class header
if start > 0 and md_lines[start][0:4] == '### ' and md_lines[
start - 1][0:1] == '>':
added_lines += 1
new_lines.insert(0, '')
md_lines[start:end] = new_lines
return added_lines
for key in sections:
if key not in md_chunks:
raise ValueError('No documentation found for %s' % key[1])
added_lines = _replace_md(key, sections, md_chunks[key], md_lines,
added_lines)
markdown = '\n'.join(md_lines).strip() + '\n'
if original_markdown != markdown:
with open(md_file, 'wb') as f:
f.write(markdown.encode('utf-8'))
def create(self, options):
self.parser = CommonMark.DocParser()
self.renderer = CommonMark.HTMLRenderer()
def __init__(self, *args, **kwargs):
super(CompileCommonMark, self).__init__(*args, **kwargs)
if CommonMark is not None:
self.parser = CommonMark.DocParser()
self.renderer = CommonMark.HTMLRenderer()
class BlogBuilder:
"Builds blog."
common_mark_parser = CommonMark.DocParser()
common_mark_renderer = CommonMark.HTMLRenderer()
def __init__(self, cursor, path):
self.cursor = cursor
self.path = path
def build(self):
"Build blog."
self.initialize_index()
self.page_size = self.cursor.get_option("blog.page_size")
self.theme_path = pathlib.Path(
__file__).parent / "themes" / self.cursor.get_option("blog.theme")
self.make_template_environment()
self.make_context()
self.build_index(self.index, self.path)
self.build_posts()
self.copy_static_files()
def initialize_index(self):
logging.info("Initializing index…")
self.index = Index(self.cursor)
posts = self.cursor.get_posts()
for post in posts:
self.index.append(post)
def build_index(self, entry, path, segments=()):
logging.info("Building index pages in `%s`…", path)
# Build pages at the current level.
pages = paginate(entry.posts, self.page_size)
for page, posts in enumerate(pages, 1):
page_path = path / str(page) if page != 1 else path
self.build_index_page(page, page == len(pages),
page_path / "index.html", segments, posts)
# Recursively build child index pages.
for segment, child in entry.children.items():
self.build_index(child, path / str(segment),
segments + (segment, ))
def build_index_page(self, page, is_last, path, segments, posts):
logging.info("Building index page `%s`: %d posts…", path, len(posts))
self.render(path,
"index.html",
current_page=page,
is_last_page=is_last,
posts=posts,
segments=segments)
def build_posts(self):
"Builds single post pages."
for post in self.index.posts:
self.build_post_page(post)
def build_post_page(self, post):
"Builds post page."
path = self.path / post.key / "index.html"
logging.info("Building post page `%s`…", path)
self.render(path, "post.html", post=post)
def make_context(self):
"Makes template context."
options = self.cursor.get_options()
for key, value in list(options.items()):
options[key.replace(".", "_")] = value
self.context = {"index": self.index, "options": options}
def make_template_environment(self):
self.env = jinja2.Environment(
loader=jinja2.PackageLoader("lje", str(self.theme_path)))
self.env.filters.update({
"markdown":
self.markdown,
"joinsegments":
lambda segments: "".join(map("/{0}".format, segments)),
"tags":
self.cursor.get_post_tags,
"timestamp":
datetime.datetime.utcfromtimestamp,
})
def markdown(self, text):
"Renders markdown using CommonMark."
ast = self.common_mark_parser.parse(text)
return self.common_mark_renderer.render(ast)
def render(self, path, template_name, **context):
"Renders template to the specified path."
if not path.parent.exists():
path.parent.mkdir(parents=True)
body = self.env.get_template(template_name).render(
dict(self.context, **context))
with open(str(path), "wt", encoding="utf-8") as fp:
fp.write(body)
def copy_static_files(self):
"Copies static files to build path."
logging.info("Copying static files…")
shutil.copy(str(self.theme_path / "theme.css"),
str(self.path / "theme.css"))
self.dump_option("blog.favicon.ico", self.path / "favicon.ico")
self.dump_option("blog.favicon.png", self.path / "favicon.png")
def dump_option(self, name, path):
"Dumps binary option into file."
value = self.cursor.get_option(name)
if value:
logging.info("Writing `%s`…", path)
with path.open("wb") as fp:
fp.write(value)
