def parse_protocol(protocol): protocol_parsed = mistune.Markdown(mistune.AstRenderer()).parse(protocol) h2_indices = np.where([ x['type'] == 'heading' and x['level'] == 2 for x in protocol_parsed ])[0] result = {'topics': []} for section in iterate_sections(protocol_parsed, h2_indices): section_name = section[0]['children'][0]['text'] if section_name != 'Tagesordnungspunkte': continue topic_indices = np.where( [x['type'] == 'heading' and x['level'] == 3 for x in section])[0] for topic in iterate_sections(section, topic_indices): title = topic[0]['children'][0]['text'].strip() if 'Vorstellungsrunde' in title: continue title_text = ' '.join([x['text'] for x in topic[0]['children']]) author_match = re.search(r'@(.+)?\]', title_text) if author_match: author = author_match.groups()[0] else: author = None was_discussed = 'Thema wurde nicht besprochen' not in str( topic[-1]) result['topics'].append({ 'title': title, 'author': author, 'was_discussed': was_discussed, }) return result
def test_code_blocks_all_present(): """ Test that all the code blocks in the docs (aries-cloud-agent-example.md) are present in the Aries test module (tests/test_examples/test_http_client_connection_to_aries_cloud_agent.py). """ markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) skill_doc_file = Path(ROOT_DIR, "docs", "aries-cloud-agent-example.md") doc = markdown_parser(skill_doc_file.read_text()) # get only code blocks offset = 1 code_blocks = list(filter(lambda x: x["type"] == "block_code", doc))[offset:] expected_code_path = Path( ROOT_DIR, "tests", "test_examples", "test_http_client_connection_to_aries_cloud_agent.py", ) expected_code = expected_code_path.read_text() # all code blocks must be present in the expected code for code_block in code_blocks: text = code_block["text"] if text.strip() not in expected_code: pytest.fail("The following code cannot be found in {}:\n{}".format( expected_code_path, text))
def test_strategy_consistency(): """ Test that the seller strategy specified in the documentation is the same we use in the tests. """ markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) skill_doc_file = Path(ROOT_DIR, "docs", "orm-integration.md") doc = markdown_parser(skill_doc_file.read_text()) # get only code blocks code_blocks = list(filter(lambda x: x["type"] == "block_code", doc)) python_code_blocks = list( filter( lambda x: x["info"] is not None and x["info"].strip() == "python", code_blocks, ) ) strategy_file_content = ORM_SELLER_STRATEGY_PATH.read_text() for python_code_block in python_code_blocks: if not python_code_block["text"] in strategy_file_content: pytest.fail( "Code block not present in strategy file:\n{}".format( python_code_block["text"] ) )
def setup_class(cls): """Set up the test.""" markdown_parser = mistune.create_markdown( renderer=mistune.AstRenderer()) cls.doc_path = cls.DOC_PATH cls.doc_content = cls.doc_path.read_text() cls.blocks = markdown_parser(cls.doc_content) cls.code_blocks = list(filter(block_code_filter, cls.blocks))
def setup_class(cls): """Set the test up.""" markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) ledger_doc_file = Path(ROOT_DIR, "docs", "ledger-integration.md") doc = markdown_parser(ledger_doc_file.read_text()) # get only code blocks cls.code_blocks = list(filter(lambda x: x["type"] == "block_code", doc))
def extract_code_blocks(filepath, filter_=None): """Extract code blocks from .md files.""" content = Path(filepath).read_text(encoding="utf-8") markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) blocks = markdown_parser(content) actual_type_filter = partial(type_filter, filter_) code_blocks = list(filter(block_code_filter, blocks)) bash_code_blocks = filter(actual_type_filter, code_blocks) return list(b["text"] for b in bash_code_blocks)
def setup_class(cls): """Test skill.md""" markdown_parser = mistune.create_markdown( renderer=mistune.AstRenderer()) skill_doc_file = Path(ROOT_DIR, "docs", "protocol.md") doc = markdown_parser(skill_doc_file.read_text()) # get only code blocks cls.code_blocks = list(filter(lambda x: x["type"] == "block_code", doc))
def get_script(): doc = open('script.md', 'r').read() markdown = mistune.create_markdown(renderer=mistune.AstRenderer()) markdown_ast = markdown(doc) doc_ast = {} sections = [make_doc_section(s) for s in get_sections(markdown_ast)] return sections
def _create_ast(self, lines: List[str]) -> List[AstNode]: # It is easier to (accuractly) detect a ListIndentError # using the ast and reasoning about that then using a regex # that matches any indented list create_ast = mistune.create_markdown( escape=False, renderer=mistune.AstRenderer(), # TODO: Add plugnis ) ast: List[AstNode] = create_ast('\n'.join(lines)) # type: ignore return ast
def build(ctx, out_file, template, template_settings): print(ctx.default_map) markdown = mistune.create_markdown(renderer=mistune.AstRenderer(), plugins=['table', ImageDirective(), BoxDirective(), TitleDirective(), TextStyleDirective(), SetStyleDirective(), SlideDirective()]) try: settings=read_template_settings(Path(template).with_suffix('.sls')) except FileNotFoundError: settings = DEFAULT_TEMPLATE_SETTINGS r = PPTXWriter(template, settings=settings) r.feed(markdown(sys.stdin.read())) r.save(out_file)
def parse_markdown(filename): mistune.scanner.Matcher.PARAGRAPH_END = re.compile( mistune.scanner.Matcher.PARAGRAPH_END.pattern + r'|(?:\n {0,3}!)') markdown = mistune.create_markdown( plugins=[plugin_equation, plugin_block_table, plugin_picture, plugin_misc, DirectiveBook()], renderer=mistune.AstRenderer() ) text_file = open(filename, mode='r', encoding="utf-8") text = text_file.read() text_file.close() parsed = markdown(text) return parsed
def validate_markdown(self, markdown): m = mistune.create_markdown(renderer=mistune.AstRenderer()) for block in m(markdown): if block['type'] == 'heading': # we dont want colon after section names text_children = [c for c in block['children'] if c['type'] == 'text'] for c in text_children: assert not c['text'].endswith(':') if c['text'] in self.required_sections: self.required_sections.remove(c['text']) try: assert len(self.required_sections) == 0 except AssertionError as e: print("Missing required sections: {}".format(self.required_sections)) raise e
def get_changelog_version(changelog_markdown_path: str) -> str: markdown = mistune.create_markdown(renderer=mistune.AstRenderer()) with open(changelog_markdown_path, 'r') as f: changelog_lines = f.read() changelog_ast = markdown(changelog_lines) first_heading_has_list = (changelog_ast[1]['type'] == 'list') if not first_heading_has_list: die(f'Latest changelog entry does not include a _list_ of what has changed' ) first_heading_str = find_child_text(changelog_ast[0]) if not first_heading_str: die(f'Could not find the latest changelog heading: {changelog_ast[0]}') print(f'Latest changelog heading: {first_heading_str}') return first_heading_str
def has_extra_formatting(cell): if 'source' not in cell.keys(): return False # gather the entire markdown cell source as one string markdown_source = "".join(cell['source']) # pass to the markdown parser to generate tokens markdown = mistune.create_markdown(renderer=mistune.AstRenderer()) markdown_tokens = markdown(markdown_source) # iterate through tokens to check for extra formatting for token in markdown_tokens: if is_extra_formatting(token): return True return False
def main(): subprocess.check_call(["git", "clone", WIKI_URL]) projects_per_user = collections.defaultdict(set) markdown = mistune.create_markdown(renderer=mistune.AstRenderer()) for fname in pathlib.Path("wiki.wiki").glob("**/*.md"): process(markdown, projects_per_user, fname) num_users = len(projects_per_user) colors_per_user = {} for n, user in enumerate(projects_per_user): colors_per_user[user] = optimal_color(n, num_users) dot = "graph { rankdir=LR; splines=ortho; ranksep=2; nodesep=0.1;\n" for k, v in projects_per_user.items(): for u in v: dot += f'"{k}"--"{u}" [color={colors_per_user[k]}]\n' dot += "}" rendered = subprocess.check_output(["dot", "-Tsvg"], input=dot.encode()) with open("media-w-wiki/kto-co-kontroluje.svg", "wb") as f: f.write(rendered) shutil.rmtree("wiki.wiki")
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import collections import mistune _parse_markdown = mistune.create_markdown(renderer=mistune.AstRenderer()) def _get_all_ast_nodes(ast_nodes): for node in ast_nodes: yield node if 'children' in node: # workaround for https://github.com/lepture/mistune/issues/269 if isinstance(node['children'], str): yield {'type': 'text', 'text': node['children']} else: yield from _get_all_ast_nodes(node['children']) def _get_text_from_ast(ast_nodes): def get_text_from_node(node): if node['type'] == 'text': return node['text']