def _markdown(checklist):
"""
Generate markdown for checklist
"""
checklist = json.load(open(checklist), object_pairs_hook=OrderedDict)
mdFile = MdUtils(file_name='Ikigai-Checklist', title='PDP 2019 Checklist')
mdFile.new_paragraph(checklist['overview'])
sections = sorted(checklist['checklist'].values(),
key=lambda s: int(s['no']))
for section in sections:
mdFile.new_header(level=1, title=section['title'])
for subject in section['subjects'].values():
mdFile.new_header(level=2, title=subject['title'])
mdFile.new_paragraph(subject['description'])
mdFile.new_paragraph("Reference: " + subject['provisions'])
mdFile.new_paragraph("Functions: " +
", ".join(subject['functions']))
mdFile.new_paragraph("Groups: " + ", ".join(subject['groups']))
actions = [['No', 'Description', 'Tags', 'Check']]
actions += [[a['no'], a['description'], ", ".join(a['tags']), '']
for a in subject['actions'].values()]
rows = len(actions)
actions = flatten(actions)
mdFile.new_table(columns=4,
rows=rows,
text=actions,
text_align='left')
mdFile.create_md_file()
def episodeToMarkdown(data, podInfo, auth):
""" Take in a podcast episode data from PodcastIndex and return Markdowns """
fileName = str(data['id']) + '.md'
published = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
time.gmtime(data['datePublished']))
mf = MdUtils(file_name=fileName, title=data['title'])
showTxt = data['title'] + ' - ' + data['enclosureType']
mf.new_line(mf.new_inline_link(link=data['enclosureUrl'], text=showTxt))
mf.new_line(mf.new_inline_link(link=data['link'], text='Show Notes Link'))
mf.new_line(f'Published: {published}')
if 'image' in data:
mf.new_paragraph(Html.image(path=data['image']))
elif 'feedImage':
mf.new_paragraph(Html.image(path=data['feedImage']))
showTitle = "Podcast: " + podInfo['feed']['title']
mf.new_header(1, showTitle)
feedUrl = podInfo['feed']['url']
podLink = f'[Original RSS Feed]({feedUrl})'
mf.new_paragraph(podLink)
mf.new_paragraph(data['description'])
mf.new_paragraph('PodcastIndex.org ID: ' + str(data['id']))
mf.read_md_file('postfooter.md')
mf.file_data_text = mf.file_data_text.replace('somethingtoreplacehere777',
auth)
print(f'Working on {showTxt}')
return mf, fileName
def to_markdown(self):
"""Generate markdown file."""
params = self._construct()
markdown = MdUtils(file_name=params._name)
markdown.new_header(level=1, title=str(params._name).capitalize())
markdown.new_header(level=2, title="Description")
markdown.new_paragraph(params._desc_feature_set)
markdown.new_line()
markdown.new_header(level=2, title="Feature Set Pipeline")
markdown.new_header(level=3, title="Source")
source = ["Reader", "Location"]
for r, l in params._source:
source.extend([r, l])
count_rows = len(source) // 2
markdown.new_table(columns=2,
rows=count_rows,
text=source,
text_align="center")
markdown.new_header(level=3, title="Sink")
sink = ["Writer"]
for w in params._sink:
sink.extend([w])
count_rows = len(sink)
markdown.new_table(columns=1,
rows=count_rows,
text=sink,
text_align="center")
markdown.new_header(level=3, title="Features")
features = ["Column name", "Data type", "Description"]
for c, desc in params._features:
features.extend([c["column_name"], str(c["type"]), desc])
count_rows = len(features) // 3
markdown.new_table(columns=3,
rows=count_rows,
text=features,
text_align="center")
if self.save:
markdown.create_md_file()
else:
return markdown.file_data_text
def write_requirements(self, root_dir='.', md_file_name=None, format='md'):
if not md_file_name:
if self._current_tag:
md_file_name = os.path.join(root_dir, self._current_tag, 'REQUIREMENTS.md')
else:
dev_or_stable = "dev" if self._dev else "stable"
raise NoAppropriateVersionFoundException("No suitable version for " + dev_or_stable + "release")
os.makedirs(os.path.dirname(md_file_name), exist_ok=True)
requirements_md = MdUtils(file_name=md_file_name, title="Requirements Summary")
for req_topic in self._requirements:
requirements_md.new_header(level=1, title=req_topic)
for req in self._requirements[req_topic]:
impacted = self._current_tag in self._requirements_tag_map[req['number']]['tags'] if req['number'] in self._requirements_tag_map else False
impacted_icon = ':boom:' if impacted else ''
title = f"{req['title']} ([#{req['number']}](https://github.com/{self._repo}/issues/{req['number']})) {impacted_icon}"
requirements_md.new_header(level=2, title=title)
if impacted:
issue_lines = {t : [] for t in Requirements.ISSUE_TYPES}
for n in self._requirements_tag_map[req['number']]['issues']:
issue = self._repo.issue(n)
bug_or_enhancement = Requirements._issue_is_bug_or_enhancement(issue)
issue_lines[bug_or_enhancement].append(f'{issue.title} ([#{n}](https://github.com/{self._repo}/issues/{n}))')
for issue_type, issue_list in issue_lines.items():
if len(issue_lines[issue_type]):
requirements_md.new_paragraph(f'The {issue_type}s which impact this requirements are:')
requirements_md.new_list(issue_list)
else:
requirements_md.new_paragraph('This requirement is not impacted by the current version')
requirements_md.create_md_file()
if format == 'md':
return md_file_name
if format == 'html':
html_file_name = md_file_name.replace('.md', '.html')
return md_to_html(md_file_name, html_file_name,
{'name': self._repo, 'description': self._repo.description, 'tag': self._current_tag})
else:
logger.error(f'output format {format} is not supported')
return ''
self._clean_previous_dev_requirements(root_dir)
def report(fails: Dict[str, List[Fail]]):
print('Start writing report')
start_time = time.time()
mdFile = MdUtils(file_name='report', title='OCR Recognition Report')
for fail_typ in fails.keys():
mdFile.new_header(level=2,
title=fail_typ.title,
add_table_of_contents='n')
mdFile.new_header(level=3,
title="Explanation",
add_table_of_contents='n')
mdFile.new_paragraph(fail_typ.explanation)
mdFile.new_line(
f'There were in total {len(fails[fail_typ])} of {fail_typ.title}')
mdFile.new_header(level=3, title="Fails", add_table_of_contents='n')
mdFile.new_line()
if fail_typ == LenFail:
md_text: List[str] = [
'Font name - size', 'Words in PDF', 'Recognized words',
'Levenshtein distance of total text'
]
for len_fail in fails[fail_typ]:
md_text.extend([
f'{len_fail.font_name} - {len_fail.font_size}',
str(len_fail.pdf_len),
str(len_fail.ocr_len),
str(len_fail.total_levenshtein)
])
mdFile.new_table(columns=4,
rows=int(len(md_text) / 4),
text=md_text,
text_align='center')
else:
for fail in fails[fail_typ]:
mdFile = fail.to_md(mdFile)
mdFile.create_md_file()
print(f'Finished reporting after {time.time() - start_time} sec')
def makeMarkdown(data, path):
# Creates the README file
global base_url
mdf = MdUtils(file_name=path + 'README', title='RBA TechRadar for Azure')
adopt_list = list()
trial_list = list()
assess_list = list()
hold_list = list()
reject_list = list()
# Create categories on status
for key in data:
status = data[key].get("status")
if status == "ADOPT": adopt_list.append(key)
if status == "TRIAL": trial_list.append(key)
if status == "ASSESS": assess_list.append(key)
if status == "HOLD": hold_list.append(key)
if status == "REJECT": reject_list.append(key)
mdf.new_header(level=1, title='Overview')
mdf.new_header(level=2, title='What is the purpose?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is a tool to inspire and "
"support engineering teams at Risk & Business Analytics to pick the best "
"technologies for new projects; it provides a platform to share knowledge "
"and experience in technologies, to reflect on technology decisions and "
"continuously evolve our technology landscape. Based on the pioneering "
"work at Thought Works, our radar sets out the changes in technologies "
"that are interesting in cloud development - changes that we think our "
"engineering teams should pay attention to and consider using in their "
"projects.")
mdf.new_header(level=2, title='How do we maintain it?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is maintained by the Cloud "
"Center of Excellence - an open group of senior RBA technologists committed "
"to devote time to this purpose. The CCoE self organizes to maintain these "
"documents, including this version. Assignment of technologies to rings is "
"the outcome of status change proposals, which are discussed and voted on "
"in CCoE meetings. The radar depends on active participation and input from "
"all engineering teams at RBA.")
mdf.new_header(level=2, title='What are the current ring assignments?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is a list of technologies, "
"complemented by an assesment result, called ring assignment. We use five "
"rings with the following semantics:")
# Handle the Adopt Section
mdf.new_header(level=3, title='Adopt')
mdf.new_paragraph(
"Technologies we have high confidence in to serve our "
"purpose, also at large scale. Technologies with a usage culture in the "
"RBA production environment, low risk, automated policy enforcement and "
"are recommended to be widely used.")
adopt_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
adopt_cnt = len(adopt_list) + 1
for key in adopt_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
adopt_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if adopt_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=adopt_cnt, text=adopt_tbl)
# Handle the Trial Section
mdf.new_header(level=3, title='Trial')
mdf.new_paragraph(
"Technologies that we have seen work with success in projects "
"to solve real problems; first serious usage experience that confirm benefits "
"and uncover limitations. TRIAL technologies are slightly more risky; some "
"engineers in our organization walked this path and will share knowledge and "
"experiences. This area can contain services that have been architecture and "
"security reviewed but do not contain automated policy managmeent.")
trial_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
trial_cnt = len(trial_list) + 1
for key in trial_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
trial_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if trial_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=trial_cnt, text=trial_tbl)
# Handle the Assess Section
mdf.new_header(level=3, title='Assess')
mdf.new_paragraph(
"Technologies that are promising and have clear potential "
"value-add for us; technologies worth investing some research and "
"prototyping efforts to see if it has impact. ASSESS technologies have "
"higher risks; they are often new to our organization and highly unproven "
"within RBA. You will find some engineers that have knowledge in the "
"technology and promote it, you may even find teams that have started "
"a prototyping effort. These technologies can also include services that "
"are currently in architecture or security review.")
assess_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
assess_cnt = len(assess_list) + 1
for key in assess_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
assess_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if assess_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=assess_cnt, text=assess_tbl)
# Handle the Hold Section
mdf.new_header(level=3, title='Hold')
mdf.new_paragraph(
"Technologies not recommended to be used for new projects. "
"Technologies that we think are not (yet) worth to (further) invest in. "
"HOLD technologies should not be used for new projects, but usually can be "
"continued for existing projects. These technologies may include services "
"that have yet to be evaluated by architecture and security due to a lack "
"of interest, time, or need.")
hold_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
hold_cnt = len(hold_list) + 1
for key in hold_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
#resourceName = "["+resourceName+"]("+resourceUrl+")"
hold_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if hold_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=hold_cnt, text=hold_tbl)
# Handle the Reject Section
mdf.new_header(level=3, title='Reject')
mdf.new_paragraph(
"Technologies not recommended to be used for any projects. "
"Technologies that have undergone architecture and security review but do "
"not meet company standards for use. REJECT technologies should never be "
"used on any project and should be considered deprecated for existing "
"projects.")
reject_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
reject_cnt = len(reject_list) + 1
for key in reject_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
#resourceName = "["+resourceName+"]("+resourceUrl+")"
reject_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if reject_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=reject_cnt, text=reject_tbl)
mdf.create_md_file()
def create_report(res_files: [], output: str, title="", description=""):
res = []
for f in res_files:
with open(f) as json_file:
res.append(json.load(json_file))
mdFile = MdUtils(file_name=output, title=title)
if description:
mdFile.new_paragraph(description, bold_italics_code='b')
mdFile.new_header(level=1, title='Platform')
platform = res[0]['platform']
list_of_strings = ["System", "Information"]
rows = 1
for key in platform.keys():
list_of_strings.extend([key, str(platform[key])])
rows += 1
mdFile.new_line()
mdFile.new_table(columns=2,
rows=rows,
text=list_of_strings,
text_align='left')
mdFile.new_header(level=1, title='Test results')
tests = res[0]['tests']
for test_key in tests.keys():
mdFile.new_header(level=2, title=test_key)
mdFile.new_line('~~~')
mdFile.new_line(tests[test_key]['command'])
mdFile.new_line('~~~')
test_res = tests[test_key]['result']
list_of_strings = ['Test'] + list(test_res.keys())
cols = len(list_of_strings)
rows = 1
avg = {}
avgc = {}
for r in res:
list_of_strings.extend(
[r['name'] + " (" + r['iteration'] + ")"] +
list(r['tests'][test_key]['result'].values()))
rows += 1
if r['name'] not in avg.keys():
avg[r['name']] = list(r['tests'][test_key]['result'].values())
avgc[r['name']] = 1
else:
avg[r['name']] = [
str(float(x) + float(y)) for x, y in zip(
avg[r['name']],
list(r['tests'][test_key]['result'].values()))
]
avgc[r['name']] += 1
for avg_key in avg.keys():
avg[avg_key] = [
str(round(float(x) / avgc[avg_key], 3)) for x in avg[avg_key]
]
list_of_strings.extend([avg_key + " (avg)"] + avg[avg_key])
rows += 1
mdFile.new_table(columns=cols,
rows=rows,
text=list_of_strings,
text_align='left')
mdFile.new_line()
mdFile.create_md_file()
print("{}.md succesfully created".format(output))