def parse_protocol(protocol):
protocol_parsed = mistune.Markdown(mistune.AstRenderer()).parse(protocol)
h2_indices = np.where([
x['type'] == 'heading' and x['level'] == 2 for x in protocol_parsed
])[0]
result = {'topics': []}
for section in iterate_sections(protocol_parsed, h2_indices):
section_name = section[0]['children'][0]['text']
if section_name != 'Tagesordnungspunkte':
continue
topic_indices = np.where(
[x['type'] == 'heading' and x['level'] == 3 for x in section])[0]
for topic in iterate_sections(section, topic_indices):
title = topic[0]['children'][0]['text'].strip()
if 'Vorstellungsrunde' in title:
continue
title_text = ' '.join([x['text'] for x in topic[0]['children']])
author_match = re.search(r'@(.+)?\]', title_text)
if author_match:
author = author_match.groups()[0]
else:
author = None
was_discussed = 'Thema wurde nicht besprochen' not in str(
topic[-1])
result['topics'].append({
'title': title,
'author': author,
'was_discussed': was_discussed,
})
return result
def test_code_blocks_all_present():
"""
Test that all the code blocks in the docs (aries-cloud-agent-example.md)
are present in the Aries test module
(tests/test_examples/test_http_client_connection_to_aries_cloud_agent.py).
"""
markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer())
skill_doc_file = Path(ROOT_DIR, "docs", "aries-cloud-agent-example.md")
doc = markdown_parser(skill_doc_file.read_text())
# get only code blocks
offset = 1
code_blocks = list(filter(lambda x: x["type"] == "block_code",
doc))[offset:]
expected_code_path = Path(
ROOT_DIR,
"tests",
"test_examples",
"test_http_client_connection_to_aries_cloud_agent.py",
)
expected_code = expected_code_path.read_text()
# all code blocks must be present in the expected code
for code_block in code_blocks:
text = code_block["text"]
if text.strip() not in expected_code:
pytest.fail("The following code cannot be found in {}:\n{}".format(
expected_code_path, text))
def test_strategy_consistency():
"""
Test that the seller strategy specified in the documentation
is the same we use in the tests.
"""
markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer())
skill_doc_file = Path(ROOT_DIR, "docs", "orm-integration.md")
doc = markdown_parser(skill_doc_file.read_text())
# get only code blocks
code_blocks = list(filter(lambda x: x["type"] == "block_code", doc))
python_code_blocks = list(
filter(
lambda x: x["info"] is not None and x["info"].strip() == "python",
code_blocks,
)
)
strategy_file_content = ORM_SELLER_STRATEGY_PATH.read_text()
for python_code_block in python_code_blocks:
if not python_code_block["text"] in strategy_file_content:
pytest.fail(
"Code block not present in strategy file:\n{}".format(
python_code_block["text"]
)
)
def setup_class(cls):
"""Set up the test."""
markdown_parser = mistune.create_markdown(
renderer=mistune.AstRenderer())
cls.doc_path = cls.DOC_PATH
cls.doc_content = cls.doc_path.read_text()
cls.blocks = markdown_parser(cls.doc_content)
cls.code_blocks = list(filter(block_code_filter, cls.blocks))
def setup_class(cls):
"""Set the test up."""
markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer())
ledger_doc_file = Path(ROOT_DIR, "docs", "ledger-integration.md")
doc = markdown_parser(ledger_doc_file.read_text())
# get only code blocks
cls.code_blocks = list(filter(lambda x: x["type"] == "block_code", doc))
def extract_code_blocks(filepath, filter_=None):
"""Extract code blocks from .md files."""
content = Path(filepath).read_text(encoding="utf-8")
markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer())
blocks = markdown_parser(content)
actual_type_filter = partial(type_filter, filter_)
code_blocks = list(filter(block_code_filter, blocks))
bash_code_blocks = filter(actual_type_filter, code_blocks)
return list(b["text"] for b in bash_code_blocks)
def setup_class(cls):
"""Test skill.md"""
markdown_parser = mistune.create_markdown(
renderer=mistune.AstRenderer())
skill_doc_file = Path(ROOT_DIR, "docs", "protocol.md")
doc = markdown_parser(skill_doc_file.read_text())
# get only code blocks
cls.code_blocks = list(filter(lambda x: x["type"] == "block_code",
doc))
def get_script():
doc = open('script.md', 'r').read()
markdown = mistune.create_markdown(renderer=mistune.AstRenderer())
markdown_ast = markdown(doc)
doc_ast = {}
sections = [make_doc_section(s) for s in get_sections(markdown_ast)]
return sections
def _create_ast(self, lines: List[str]) -> List[AstNode]:
# It is easier to (accuractly) detect a ListIndentError
# using the ast and reasoning about that then using a regex
# that matches any indented list
create_ast = mistune.create_markdown(
escape=False, renderer=mistune.AstRenderer(),
# TODO: Add plugnis
)
ast: List[AstNode] = create_ast('\n'.join(lines)) # type: ignore
return ast
def build(ctx, out_file, template, template_settings):
print(ctx.default_map)
markdown = mistune.create_markdown(renderer=mistune.AstRenderer(),
plugins=['table', ImageDirective(), BoxDirective(),
TitleDirective(), TextStyleDirective(), SetStyleDirective(),
SlideDirective()])
try:
settings=read_template_settings(Path(template).with_suffix('.sls'))
except FileNotFoundError:
settings = DEFAULT_TEMPLATE_SETTINGS
r = PPTXWriter(template, settings=settings)
r.feed(markdown(sys.stdin.read()))
r.save(out_file)
def parse_markdown(filename):
mistune.scanner.Matcher.PARAGRAPH_END = re.compile(
mistune.scanner.Matcher.PARAGRAPH_END.pattern + r'|(?:\n {0,3}!)')
markdown = mistune.create_markdown(
plugins=[plugin_equation, plugin_block_table,
plugin_picture, plugin_misc, DirectiveBook()],
renderer=mistune.AstRenderer()
)
text_file = open(filename, mode='r', encoding="utf-8")
text = text_file.read()
text_file.close()
parsed = markdown(text)
return parsed
def validate_markdown(self, markdown):
m = mistune.create_markdown(renderer=mistune.AstRenderer())
for block in m(markdown):
if block['type'] == 'heading':
# we dont want colon after section names
text_children = [c for c in block['children'] if c['type'] == 'text']
for c in text_children:
assert not c['text'].endswith(':')
if c['text'] in self.required_sections:
self.required_sections.remove(c['text'])
try:
assert len(self.required_sections) == 0
except AssertionError as e:
print("Missing required sections: {}".format(self.required_sections))
raise e
def get_changelog_version(changelog_markdown_path: str) -> str:
markdown = mistune.create_markdown(renderer=mistune.AstRenderer())
with open(changelog_markdown_path, 'r') as f:
changelog_lines = f.read()
changelog_ast = markdown(changelog_lines)
first_heading_has_list = (changelog_ast[1]['type'] == 'list')
if not first_heading_has_list:
die(f'Latest changelog entry does not include a _list_ of what has changed'
)
first_heading_str = find_child_text(changelog_ast[0])
if not first_heading_str:
die(f'Could not find the latest changelog heading: {changelog_ast[0]}')
print(f'Latest changelog heading: {first_heading_str}')
return first_heading_str
def has_extra_formatting(cell):
if 'source' not in cell.keys():
return False
# gather the entire markdown cell source as one string
markdown_source = "".join(cell['source'])
# pass to the markdown parser to generate tokens
markdown = mistune.create_markdown(renderer=mistune.AstRenderer())
markdown_tokens = markdown(markdown_source)
# iterate through tokens to check for extra formatting
for token in markdown_tokens:
if is_extra_formatting(token):
return True
return False
def main():
subprocess.check_call(["git", "clone", WIKI_URL])
projects_per_user = collections.defaultdict(set)
markdown = mistune.create_markdown(renderer=mistune.AstRenderer())
for fname in pathlib.Path("wiki.wiki").glob("**/*.md"):
process(markdown, projects_per_user, fname)
num_users = len(projects_per_user)
colors_per_user = {}
for n, user in enumerate(projects_per_user):
colors_per_user[user] = optimal_color(n, num_users)
dot = "graph { rankdir=LR; splines=ortho; ranksep=2; nodesep=0.1;\n"
for k, v in projects_per_user.items():
for u in v:
dot += f'"{k}"--"{u}" [color={colors_per_user[k]}]\n'
dot += "}"
rendered = subprocess.check_output(["dot", "-Tsvg"], input=dot.encode())
with open("media-w-wiki/kto-co-kontroluje.svg", "wb") as f:
f.write(rendered)
shutil.rmtree("wiki.wiki")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import mistune
_parse_markdown = mistune.create_markdown(renderer=mistune.AstRenderer())
def _get_all_ast_nodes(ast_nodes):
for node in ast_nodes:
yield node
if 'children' in node:
# workaround for https://github.com/lepture/mistune/issues/269
if isinstance(node['children'], str):
yield {'type': 'text', 'text': node['children']}
else:
yield from _get_all_ast_nodes(node['children'])
def _get_text_from_ast(ast_nodes):
def get_text_from_node(node):
if node['type'] == 'text':
return node['text']
