def get_markdown_ast(markdown_file): try: f = open(markdown_file, 'r') return CommonMark.DocParser().parse(f.read()) except: logging.error( "Error: Can't open {0} for reading".format(markdown_file)) sys.exit(1) finally: f.close()
def commonmark(value): """ Runs CommonMark over a given value. Syntax:: {{ value|commonmark }} :type value: str :rtype: str """ import CommonMark parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() ast = parser.parse(value) return mark_safe( force_text(renderer.render(ast)) )
def init_app(app): parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() app.extensions['markdown'] = UDataMarkdown(app, parser, renderer) @app.template_filter() def mdstrip(value, length=None): ''' Truncate and strip tags from a markdown source The markdown source is truncated at the excerpt if present and smaller than the required length. Then, all html tags are stripped. ''' if not value: return '' if EXCERPT_TOKEN in value: value = value.split(EXCERPT_TOKEN, 1)[0] if length > 0: value = do_truncate(value, length, end='…') rendered = md(value) return do_striptags(rendered)
#!/home/wil/Documents/research_tutorials/env/bin/python import argparse, sys import CommonMark parser = argparse.ArgumentParser(description="Process Markdown according to the CommonMark specification.") if sys.version_info < (3, 0): reload(sys) sys.setdefaultencoding('utf-8') parser.add_argument('infile', nargs="?", type=argparse.FileType('r'), default=sys.stdin, help="Input Markdown file to parse, defaults to STDIN") parser.add_argument('-o', nargs="?", type=argparse.FileType('w'), default=sys.stdout, help="Output HTML/JSON file, defaults to STDOUT") parser.add_argument('-a', action="store_true", help="Print formatted AST") parser.add_argument('-aj', action="store_true", help="Output JSON AST") args = parser.parse_args() parser = CommonMark.DocParser() f = args.infile o = args.o lines = [] for line in f: lines.append(line) data = "".join(lines) ast = parser.parse(data) if not args.a and not args.aj: renderer = CommonMark.HTMLRenderer() o.write(renderer.render(ast)) exit() if args.a: # print ast CommonMark.dumpAST(ast) exit() #o.write(ast.to_JSON()) o.write(CommonMark.ASTtoJSON(ast))
def markdown_filter(data): parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() return renderer.render(parser.parse(data))
def markup(input): parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() ast = parser.parse(input) return renderer.render(ast)
def generate(file=None, text=None, style='', custom=0): # Open File if file != None: f = codecs.open(file, 'r', 'utf-8') rawhtml = f.read() f.close() else: rawhtml = text # Parse html = '' txtlines = rawhtml.replace('\r\n', '\n').replace('\r', '\n').split('\n') for t in txtlines: # Ignore comments html += t + '\n' # Get Markdown (Common Mark) parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() md = renderer.render(parser.parse(html)) mhtml = '' # Custom Markdown modifications if custom: md = md.replace( '<li>[ ]', '<li class="checkbox"><input type="checkbox" disabled="disabled">') md = md.replace( '<li>[]', '<li class="checkbox"><input type="checkbox" disabled="disabled">') md = md.replace( '<li>[x]', '<li class="checkbox checked"><input type="checkbox" checked="checked" disabled="disabled">' ) md = md.replace( '<li>[-]', '<li class="cancelled"><input type="checkbox" checked="checked" disabled="disabled">' ) md = md.replace( '<li>[f]', '<li class="future"><input type="checkbox" disabled="disabled"> FUTURE' ) # Custom Style style = ''' h1{border-bottom:2px solid gray;} h2,h3,h4,h5,h6,h7 {border-bottom:1px solid gray;} li.checkbox,li.cancelled,li.future {list-style-type: none;margin-left:-25px;} li.cancelled {text-decoration:line-through;color:gray;} li.future {font-style:italic;color:gray;} li.checked {color:gray;} pre,code {font-family:Courier New,mono,monospace;font-size:12px;} img{max-width:100%;} ''' + style # Add Style if style != '': mhtml += '<style>' + style + '</style>' mhtml += md return mhtml
def _parse_markdown(self, markdown): parser = CommonMark.DocParser() ast = parser.parse(markdown) return ast
def run(): """ Looks through the docs/ dir and parses each markdown document, looking for sections to update from Python docstrings. Looks for section headers in the format: - ### `ClassName()` class - ##### `.method_name()` method - ##### `.attribute_name` attribute - ### `function_name()` function The markdown content following these section headers up until the next section header will be replaced by new markdown generated from the Python docstrings of the associated source files. By default maps docs/{name}.md to {modulename}/{name}.py. Allows for custom mapping via the MD_SOURCE_MAP variable. """ print('Updating API docs...') md_files = [] for root, _, filenames in os.walk(docs_dir): for filename in filenames: if not filename.endswith('.md'): continue md_files.append(os.path.join(root, filename)) parser = CommonMark.DocParser() for md_file in md_files: md_file_relative = md_file[len(project_dir) + 1:] if md_file_relative in MD_SOURCE_MAP: py_files = MD_SOURCE_MAP[md_file_relative] py_paths = [ os.path.join(project_dir, py_file) for py_file in py_files ] else: py_files = [os.path.basename(md_file).replace('.md', '.py')] py_paths = [os.path.join(project_dir, module_name, py_files[0])] if not os.path.exists(py_paths[0]): continue with open(md_file, 'rb') as f: markdown = f.read().decode('utf-8') original_markdown = markdown md_lines = list(markdown.splitlines()) md_ast = parser.parse(markdown) last_class = [] last = {} sections = OrderedDict() find_sections(md_ast, sections, last, last_class, markdown.count("\n") + 1) md_chunks = {} for index, py_file in enumerate(py_files): py_path = py_paths[index] with open(os.path.join(py_path), 'rb') as f: code = f.read().decode('utf-8') module_ast = ast.parse(code, filename=py_file) code_lines = list(code.splitlines()) for node in ast.iter_child_nodes(module_ast): walk_ast(node, code_lines, sections, md_chunks) added_lines = 0 def _replace_md(key, sections, md_chunk, md_lines, added_lines): start, end = sections[key] start -= 1 start += added_lines end += added_lines new_lines = md_chunk.split('\n') added_lines += len(new_lines) - (end - start) # Ensure a newline above each class header if start > 0 and md_lines[start][0:4] == '### ' and md_lines[ start - 1][0:1] == '>': added_lines += 1 new_lines.insert(0, '') md_lines[start:end] = new_lines return added_lines for key in sections: if key not in md_chunks: raise ValueError('No documentation found for %s' % key[1]) added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines) markdown = '\n'.join(md_lines).strip() + '\n' if original_markdown != markdown: with open(md_file, 'wb') as f: f.write(markdown.encode('utf-8'))
def markup(content): parser = CommonMark.DocParser() renderer = CommonMark.HTMLRenderer() ast = parser.parse(content) html = renderer.render(ast) return html
def run(): print('Updating API docs...') md_files = [] for root, _, filenames in os.walk(docs_dir): for filename in filenames: if not filename.endswith('.md'): continue md_files.append(os.path.join(root, filename)) parser = CommonMark.DocParser() for md_file in md_files: md_file_relative = md_file[len(project_dir) + 1:] if md_file_relative in md_source_map: py_files = md_source_map[md_file_relative] py_paths = [ os.path.join(project_dir, py_file) for py_file in py_files ] else: py_files = [os.path.basename(md_file).replace('.md', '.py')] py_paths = [os.path.join(project_dir, module_name, py_files[0])] if not os.path.exists(py_paths[0]): continue with open(md_file, 'rb') as f: markdown = f.read().decode('utf-8') original_markdown = markdown md_lines = list(markdown.splitlines()) md_ast = parser.parse(markdown) last_class = [] last = {} sections = OrderedDict() find_sections(md_ast, sections, last, last_class, markdown.count("\n") + 1) md_chunks = {} for index, py_file in enumerate(py_files): py_path = py_paths[index] with open(os.path.join(py_path), 'rb') as f: code = f.read().decode('utf-8') module_ast = ast.parse(code, filename=py_file) code_lines = list(code.splitlines()) for node in ast.iter_child_nodes(module_ast): walk_ast(node, code_lines, sections, md_chunks) added_lines = 0 def _replace_md(key, sections, md_chunk, md_lines, added_lines): start, end = sections[key] start -= 1 start += added_lines end += added_lines new_lines = md_chunk.split('\n') added_lines += len(new_lines) - (end - start) # Ensure a newline above each class header if start > 0 and md_lines[start][0:4] == '### ' and md_lines[ start - 1][0:1] == '>': added_lines += 1 new_lines.insert(0, '') md_lines[start:end] = new_lines return added_lines for key in sections: if key not in md_chunks: raise ValueError('No documentation found for %s' % key[1]) added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines) markdown = '\n'.join(md_lines).strip() + '\n' if original_markdown != markdown: with open(md_file, 'wb') as f: f.write(markdown.encode('utf-8'))
def create(self, options): self.parser = CommonMark.DocParser() self.renderer = CommonMark.HTMLRenderer()
def __init__(self, *args, **kwargs): super(CompileCommonMark, self).__init__(*args, **kwargs) if CommonMark is not None: self.parser = CommonMark.DocParser() self.renderer = CommonMark.HTMLRenderer()
class BlogBuilder: "Builds blog." common_mark_parser = CommonMark.DocParser() common_mark_renderer = CommonMark.HTMLRenderer() def __init__(self, cursor, path): self.cursor = cursor self.path = path def build(self): "Build blog." self.initialize_index() self.page_size = self.cursor.get_option("blog.page_size") self.theme_path = pathlib.Path( __file__).parent / "themes" / self.cursor.get_option("blog.theme") self.make_template_environment() self.make_context() self.build_index(self.index, self.path) self.build_posts() self.copy_static_files() def initialize_index(self): logging.info("Initializing index…") self.index = Index(self.cursor) posts = self.cursor.get_posts() for post in posts: self.index.append(post) def build_index(self, entry, path, segments=()): logging.info("Building index pages in `%s`…", path) # Build pages at the current level. pages = paginate(entry.posts, self.page_size) for page, posts in enumerate(pages, 1): page_path = path / str(page) if page != 1 else path self.build_index_page(page, page == len(pages), page_path / "index.html", segments, posts) # Recursively build child index pages. for segment, child in entry.children.items(): self.build_index(child, path / str(segment), segments + (segment, )) def build_index_page(self, page, is_last, path, segments, posts): logging.info("Building index page `%s`: %d posts…", path, len(posts)) self.render(path, "index.html", current_page=page, is_last_page=is_last, posts=posts, segments=segments) def build_posts(self): "Builds single post pages." for post in self.index.posts: self.build_post_page(post) def build_post_page(self, post): "Builds post page." path = self.path / post.key / "index.html" logging.info("Building post page `%s`…", path) self.render(path, "post.html", post=post) def make_context(self): "Makes template context." options = self.cursor.get_options() for key, value in list(options.items()): options[key.replace(".", "_")] = value self.context = {"index": self.index, "options": options} def make_template_environment(self): self.env = jinja2.Environment( loader=jinja2.PackageLoader("lje", str(self.theme_path))) self.env.filters.update({ "markdown": self.markdown, "joinsegments": lambda segments: "".join(map("/{0}".format, segments)), "tags": self.cursor.get_post_tags, "timestamp": datetime.datetime.utcfromtimestamp, }) def markdown(self, text): "Renders markdown using CommonMark." ast = self.common_mark_parser.parse(text) return self.common_mark_renderer.render(ast) def render(self, path, template_name, **context): "Renders template to the specified path." if not path.parent.exists(): path.parent.mkdir(parents=True) body = self.env.get_template(template_name).render( dict(self.context, **context)) with open(str(path), "wt", encoding="utf-8") as fp: fp.write(body) def copy_static_files(self): "Copies static files to build path." logging.info("Copying static files…") shutil.copy(str(self.theme_path / "theme.css"), str(self.path / "theme.css")) self.dump_option("blog.favicon.ico", self.path / "favicon.ico") self.dump_option("blog.favicon.png", self.path / "favicon.png") def dump_option(self, name, path): "Dumps binary option into file." value = self.cursor.get_option(name) if value: logging.info("Writing `%s`…", path) with path.open("wb") as fp: fp.write(value)