def _markdown(checklist):
"""
Generate markdown for checklist
"""
checklist = json.load(open(checklist), object_pairs_hook=OrderedDict)
mdFile = MdUtils(file_name='Ikigai-Checklist', title='PDP 2019 Checklist')
mdFile.new_paragraph(checklist['overview'])
sections = sorted(checklist['checklist'].values(),
key=lambda s: int(s['no']))
for section in sections:
mdFile.new_header(level=1, title=section['title'])
for subject in section['subjects'].values():
mdFile.new_header(level=2, title=subject['title'])
mdFile.new_paragraph(subject['description'])
mdFile.new_paragraph("Reference: " + subject['provisions'])
mdFile.new_paragraph("Functions: " +
", ".join(subject['functions']))
mdFile.new_paragraph("Groups: " + ", ".join(subject['groups']))
actions = [['No', 'Description', 'Tags', 'Check']]
actions += [[a['no'], a['description'], ", ".join(a['tags']), '']
for a in subject['actions'].values()]
rows = len(actions)
actions = flatten(actions)
mdFile.new_table(columns=4,
rows=rows,
text=actions,
text_align='left')
mdFile.create_md_file()
def write_table_to_md_file(md_file: MdUtils, table_rows_: List[List[str]]):
flat_rows = []
for row in table_rows_:
flat_rows += row
md_file.new_table(columns=len(table_rows_[0]),
rows=len(table_rows_),
text=flat_rows,
text_align='center')
def md_table_writer(name, rows):
if name == 'common':
mdfile = MdUtils(file_name='results_common')
mdfile.new_table(columns=5, rows=int(len(rows)/5), text=rows, text_align='center')
else:
mdfile = MdUtils(file_name='results_{}'.format(name))
mdfile.new_table(columns=6, rows=int(len(rows)/6), text=rows, text_align='center')
mdfile.create_md_file()
lines_delete('results_{}.md'.format(name))
def write_table(parts, keys, md: mdutils.MdUtils, sort_by='Comment'):
sorted_parts = sorted(parts, key=lambda i: i[sort_by])
tabulated = list(keys)
for part in sorted_parts:
tabulated.extend([part.get(key) for key in keys])
#print(part)
#print(len(keys), len(sorted_parts)+1, len(tabulated))
md.new_line()
md.new_table(columns=len(keys), rows=len(sorted_parts) + 1, text=tabulated)
def write_designators(parts, keys, md: mdutils.MdUtils, sort_by='designator'):
parts = sorted(parts, key=lambda i: i[sort_by])
groups = groupby(parts, lambda i: i[sort_by])
for designator, group in groups:
#print('Designator {}'.format(designator))
md.new_line()
md.new_header(level=2, title=designator)
sorted_parts = sorted(group, key=lambda i: i['id'])
tabulated = list(keys)
for part in sorted_parts:
tabulated.extend([part.get(key) for key in keys])
md.new_line()
md.new_table(columns=len(keys),
rows=len(sorted_parts) + 1,
text=tabulated)
def add_table(self, md_file: MdUtils, translations):
table_entries = ["Intention", "Command", "Msg Id"]
for translation in translations:
intent = translation.occurrences[0][
0] + ":" + translation.occurrences[0][1]
# print(translation.msgid, translation.msgstr)
table_entries.extend(
[intent, translation.msgstr, translation.msgid])
md_file.new_table(columns=3,
rows=len(translations) + 1,
text=table_entries,
text_align='left')
def write_md_file(herd, output_file_name, version):
from mdutils import MdUtils
software_summary_md = MdUtils(file_name=output_file_name,
title=f'Software Summary (build {version})')
table = get_table_columns_md()
n_columns = len(table)
for k, ch in herd.get_cattle_heads().items():
table.extend(ch.get_table_row(format='md'))
software_summary_md.new_table(columns=n_columns,
rows=herd.number_of_heads() + 1,
text=table,
text_align='center')
logger.info(f'Create file {output_file_name}.md')
software_summary_md.create_md_file()
def report(fails: Dict[str, List[Fail]]):
print('Start writing report')
start_time = time.time()
mdFile = MdUtils(file_name='report', title='OCR Recognition Report')
for fail_typ in fails.keys():
mdFile.new_header(level=2,
title=fail_typ.title,
add_table_of_contents='n')
mdFile.new_header(level=3,
title="Explanation",
add_table_of_contents='n')
mdFile.new_paragraph(fail_typ.explanation)
mdFile.new_line(
f'There were in total {len(fails[fail_typ])} of {fail_typ.title}')
mdFile.new_header(level=3, title="Fails", add_table_of_contents='n')
mdFile.new_line()
if fail_typ == LenFail:
md_text: List[str] = [
'Font name - size', 'Words in PDF', 'Recognized words',
'Levenshtein distance of total text'
]
for len_fail in fails[fail_typ]:
md_text.extend([
f'{len_fail.font_name} - {len_fail.font_size}',
str(len_fail.pdf_len),
str(len_fail.ocr_len),
str(len_fail.total_levenshtein)
])
mdFile.new_table(columns=4,
rows=int(len(md_text) / 4),
text=md_text,
text_align='center')
else:
for fail in fails[fail_typ]:
mdFile = fail.to_md(mdFile)
mdFile.create_md_file()
print(f'Finished reporting after {time.time() - start_time} sec')
def to_md(self, mdFile: MdUtils) -> MdUtils:
mdFile.new_line(f'{self.font_name} - {self.font_size}:',
bold_italics_code='b')
mdFile.new_line(
f'There were in total {len(self.error_words)} mismatches.')
mdFile.new_line(
f'The levenshtein distance of the complete text is {self.total_levenshtein}'
)
mdFile.new_line()
md_text: List[str] = [
'Word in PDF', 'Recognized Word', 'Levenshtein Distance'
]
for word_fail in self.error_words:
md_text.extend([
word_fail.pdf_word, word_fail.ocr_word,
str(word_fail.error_value)
])
mdFile.new_table(columns=3,
rows=len(self.error_words) + 1,
text=md_text,
text_align='center')
return mdFile
def to_markdown(self):
"""Generate markdown file."""
params = self._construct()
markdown = MdUtils(file_name=params._name)
markdown.new_header(level=1, title=str(params._name).capitalize())
markdown.new_header(level=2, title="Description")
markdown.new_paragraph(params._desc_feature_set)
markdown.new_line()
markdown.new_header(level=2, title="Feature Set Pipeline")
markdown.new_header(level=3, title="Source")
source = ["Reader", "Location"]
for r, l in params._source:
source.extend([r, l])
count_rows = len(source) // 2
markdown.new_table(columns=2,
rows=count_rows,
text=source,
text_align="center")
markdown.new_header(level=3, title="Sink")
sink = ["Writer"]
for w in params._sink:
sink.extend([w])
count_rows = len(sink)
markdown.new_table(columns=1,
rows=count_rows,
text=sink,
text_align="center")
markdown.new_header(level=3, title="Features")
features = ["Column name", "Data type", "Description"]
for c, desc in params._features:
features.extend([c["column_name"], str(c["type"]), desc])
count_rows = len(features) // 3
markdown.new_table(columns=3,
rows=count_rows,
text=features,
text_align="center")
if self.save:
markdown.create_md_file()
else:
return markdown.file_data_text
def run(self):
Responder.run(self)
caseNumber = self.get_param('data.caseId') #Friendly case number
caseId = self.get_param('data.id') #Raw case number
case_observables = self.api.get_case_observables(caseId).json()
title = self.get_param('data.title', None, 'title is missing')
description = self.get_param('data.description', None,
'description is missing')
tags = self.get_param('data.tags')
data = self.get_param('data')
tlp = self.getTLP(data['tlp'])
# Title
#mdFile = MdUtils(file_name=str(caseNumber),title=tlp[0] + ' Case #' + str(caseNumber) + ': ' + title)
mdFile = MdUtils(file_name=str(self.tmpPath) + str(caseNumber),
title=tlp[0] + ' Case #' + str(caseNumber) + ': ' +
title)
# Case Summary
caseSummary = self.getCaseSummary(data)
mdFile.new_header(level=1, title='Case Summary')
mdFile.new_line(str(tlp[1]))
mdFile.new_table(columns=2,
rows=int(caseSummary.__len__() / 2),
text=caseSummary,
text_align='left')
# Case Description
mdFile.new_line('<div style="page-break-after: always;"></div>')
mdFile.new_line(' ')
mdFile.new_header(level=1, title='Case Description')
mdFile.new_line(str(data['description']))
mdFile.new_line(' ')
# Task Log
allTaskIds = self.getCaseTasks(caseId)
allTaskIds_sorted = sorted(allTaskIds.items(),
key=lambda x: x[1]['createdAt'])
mdFile.new_header(level=1, title='Task Log Entries')
for task in allTaskIds_sorted:
title = str(task[1]['taskGroup'] + ' \: ' + task[1]['taskTitle'])
createdAt = time.strftime(
'%Y-%m-%dT%H:%M:%SZ',
time.localtime(
task[1]['createdAt'] /
1000)) #Convert epoch ms to sec then human readable
mdFile.new_header(level=2, title=title)
mdFile.new_line(str('**Created At:** ') + str(createdAt))
mdFile.new_line(
str('**Created By:** ') + str(task[1]['createdBy']))
mdFile.new_line(str('**Assigned To:** ') + str(task[1]['owner']))
mdFile.new_line(str('**Case Status:** ') + str(task[1]['status']))
mdFile.new_line(' ')
mdFile.new_line(str('**Description:** '))
mdFile.new_line(str(task[1]['description']))
mdFile.new_line(' ')
caseTaskLog = self.getCaseTaskLog(task[0])
caseTaskLogEntries = (json.loads(caseTaskLog))
caseTaskLogEntries_sorted = sorted(caseTaskLogEntries,
key=lambda k: k['createdAt'])
for caseTaskLogEntry in caseTaskLogEntries_sorted:
createdAt = time.strftime(
'%Y-%m-%dT%H:%M:%SZ',
time.localtime(
caseTaskLogEntry['createdAt'] /
1000)) #Convert epoch ms to sec then human readable
mdFile.new_line(
str(createdAt) + ' : ' + str(caseTaskLogEntry['message']))
# Case Observables
mdFile.new_header(level=1, title='Case Observables')
caseObservables = self.getCaseObservables(case_observables)
mdFile.new_table(columns=6,
rows=int(caseObservables.__len__() / 6),
text=caseObservables,
text_align='left')
# TLP Protocol description
mdFile.new_line('<div style="page-break-after: always;"></div>')
mdFile.new_line(' ')
mdFile.new_header(
level=1,
title='Traffic Light Protocol (TLP) Definitions and Usage')
tlpFooter = self.getTlpFooter()
mdFile.new_table(columns=3, rows=5, text=tlpFooter, text_align='left')
# Build TOC
mdFile.new_table_of_contents(table_title='Table of Contents', depth=2)
# Compile the report
mdFile.create_md_file()
# Add the report to the case
addTask = json.loads(self.addTask(caseId))
taskId = addTask['_id']
# Add the MD file to the task
addTaskLog = json.loads(
self.addTaskLog(taskId,
str(self.tmpPath) + str(caseNumber) + '.md'))
# Cleanup the MD file
os.remove(str(self.tmpPath) + str(caseNumber) + '.md')
self.report({'report': 'created'})
def makeMarkdown(data, path):
# Creates the README file
global base_url
mdf = MdUtils(file_name=path + 'README', title='RBA TechRadar for Azure')
adopt_list = list()
trial_list = list()
assess_list = list()
hold_list = list()
reject_list = list()
# Create categories on status
for key in data:
status = data[key].get("status")
if status == "ADOPT": adopt_list.append(key)
if status == "TRIAL": trial_list.append(key)
if status == "ASSESS": assess_list.append(key)
if status == "HOLD": hold_list.append(key)
if status == "REJECT": reject_list.append(key)
mdf.new_header(level=1, title='Overview')
mdf.new_header(level=2, title='What is the purpose?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is a tool to inspire and "
"support engineering teams at Risk & Business Analytics to pick the best "
"technologies for new projects; it provides a platform to share knowledge "
"and experience in technologies, to reflect on technology decisions and "
"continuously evolve our technology landscape. Based on the pioneering "
"work at Thought Works, our radar sets out the changes in technologies "
"that are interesting in cloud development - changes that we think our "
"engineering teams should pay attention to and consider using in their "
"projects.")
mdf.new_header(level=2, title='How do we maintain it?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is maintained by the Cloud "
"Center of Excellence - an open group of senior RBA technologists committed "
"to devote time to this purpose. The CCoE self organizes to maintain these "
"documents, including this version. Assignment of technologies to rings is "
"the outcome of status change proposals, which are discussed and voted on "
"in CCoE meetings. The radar depends on active participation and input from "
"all engineering teams at RBA.")
mdf.new_header(level=2, title='What are the current ring assignments?')
mdf.new_paragraph(
"The RBA TechRadar for Azure is a list of technologies, "
"complemented by an assesment result, called ring assignment. We use five "
"rings with the following semantics:")
# Handle the Adopt Section
mdf.new_header(level=3, title='Adopt')
mdf.new_paragraph(
"Technologies we have high confidence in to serve our "
"purpose, also at large scale. Technologies with a usage culture in the "
"RBA production environment, low risk, automated policy enforcement and "
"are recommended to be widely used.")
adopt_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
adopt_cnt = len(adopt_list) + 1
for key in adopt_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
adopt_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if adopt_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=adopt_cnt, text=adopt_tbl)
# Handle the Trial Section
mdf.new_header(level=3, title='Trial')
mdf.new_paragraph(
"Technologies that we have seen work with success in projects "
"to solve real problems; first serious usage experience that confirm benefits "
"and uncover limitations. TRIAL technologies are slightly more risky; some "
"engineers in our organization walked this path and will share knowledge and "
"experiences. This area can contain services that have been architecture and "
"security reviewed but do not contain automated policy managmeent.")
trial_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
trial_cnt = len(trial_list) + 1
for key in trial_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
trial_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if trial_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=trial_cnt, text=trial_tbl)
# Handle the Assess Section
mdf.new_header(level=3, title='Assess')
mdf.new_paragraph(
"Technologies that are promising and have clear potential "
"value-add for us; technologies worth investing some research and "
"prototyping efforts to see if it has impact. ASSESS technologies have "
"higher risks; they are often new to our organization and highly unproven "
"within RBA. You will find some engineers that have knowledge in the "
"technology and promote it, you may even find teams that have started "
"a prototyping effort. These technologies can also include services that "
"are currently in architecture or security review.")
assess_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
assess_cnt = len(assess_list) + 1
for key in assess_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")"
assess_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if assess_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=assess_cnt, text=assess_tbl)
# Handle the Hold Section
mdf.new_header(level=3, title='Hold')
mdf.new_paragraph(
"Technologies not recommended to be used for new projects. "
"Technologies that we think are not (yet) worth to (further) invest in. "
"HOLD technologies should not be used for new projects, but usually can be "
"continued for existing projects. These technologies may include services "
"that have yet to be evaluated by architecture and security due to a lack "
"of interest, time, or need.")
hold_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
hold_cnt = len(hold_list) + 1
for key in hold_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
#resourceName = "["+resourceName+"]("+resourceUrl+")"
hold_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if hold_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=hold_cnt, text=hold_tbl)
# Handle the Reject Section
mdf.new_header(level=3, title='Reject')
mdf.new_paragraph(
"Technologies not recommended to be used for any projects. "
"Technologies that have undergone architecture and security review but do "
"not meet company standards for use. REJECT technologies should never be "
"used on any project and should be considered deprecated for existing "
"projects.")
reject_tbl = [
"<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>",
"<sub>Status</sub>"
]
reject_cnt = len(reject_list) + 1
for key in reject_list:
resourceName = key
resourceDesc = data[key].get("description", "")
resourcePath = data[key].get("path", "")
resourceType = data[key].get("architecture review", "").get("type", "")
resourceUrl = data[key].get("url", "")
resourceStatus = data[key].get("status", "")
#resourceName = "["+resourceName+"]("+resourceUrl+")"
reject_tbl.extend([
'<sub>' + resourceName + '</sub>',
'<sub>' + resourceDesc + '</sub>',
'<sub>' + resourceType + '</sub>',
'<sub>' + resourceStatus + '</sub>'
])
if reject_cnt == 1:
mdf.new_line("")
mdf.new_line("There are currently no resources at this ring level.",
bold_italics_code='bi',
color='red')
else:
mdf.new_line("")
mdf.new_table(columns=4, rows=reject_cnt, text=reject_tbl)
mdf.create_md_file()
def generate_report():
def generate_phrase_image(width: int, font: ImageFont) -> Image:
result = Image.new(mode=GRAYSCALE_MODE,
size=(width, font.size),
color=WHITE)
result_draw = ImageDraw.Draw(im=result, mode=GRAYSCALE_MODE)
result_draw.text(xy=(0, 0),
text=PHRASE,
font=font,
fill=0,
anchor='lt')
return cut_empty_rows_and_cols(result)
def p_ass_for_table(p_ass):
result = []
for row in p_ass:
for element in row:
result.append(str(element))
return result
report = MdUtils(file_name=f'./report.md')
report.new_header(level=1, title='Классификация')
report.new_line(text='Выполнил Ахманов Алексей Б18-514')
report.new_line(text=f'Алфавит - {ALPHABET}')
report.new_line(text=f'Исходная фраза - {PHRASE}')
report.new_line(text=f'Размер шрифта - {FONT_SIZE}')
# Phrase
phrase_image = generate_phrase_image(
1200, ImageFont.truetype(font=FONT_PATH, size=FONT_SIZE))
phrase_image_small_font = generate_phrase_image(
1200, ImageFont.truetype(font=FONT_PATH, size=SMALL_FONT_SIZE))
# Proximity assessment
p_assessment = proximity_assessment(image=phrase_image,
diff_threshold=SYMBOLS_DIFF_THRESHOLD,
phrase=PHRASE)
p_assessment_small_font = proximity_assessment(
image=phrase_image_small_font,
diff_threshold=SYMBOLS_DIFF_THRESHOLD,
phrase=PHRASE)
p_assessment_for_table = p_ass_for_table(p_assessment)
p_assessment_small_font_for_table = p_ass_for_table(
p_assessment_small_font)
# Report
report.new_header(level=2,
title=f'Оценка близости для размера шрифта {FONT_SIZE}')
rows = len(p_assessment)
columns = len(p_assessment[0]) if len(p_assessment) > 0 else 0
report.new_table(columns=columns, rows=rows, text=p_assessment_for_table)
report.new_header(
level=2, title=f'Оценка близости для размера шрифта {SMALL_FONT_SIZE}')
rows = len(p_assessment_small_font)
columns = len(
p_assessment_small_font[0]) if len(p_assessment_small_font) > 0 else 0
report.new_table(columns=columns,
rows=rows,
text=p_assessment_small_font_for_table)
report.new_line(
text=
'Так как были использованы нормализованные параметры для оценки близости символов, размер шрифта не влияет на результат'
)
report.create_md_file()
def create_report(res_files: [], output: str, title="", description=""):
res = []
for f in res_files:
with open(f) as json_file:
res.append(json.load(json_file))
mdFile = MdUtils(file_name=output, title=title)
if description:
mdFile.new_paragraph(description, bold_italics_code='b')
mdFile.new_header(level=1, title='Platform')
platform = res[0]['platform']
list_of_strings = ["System", "Information"]
rows = 1
for key in platform.keys():
list_of_strings.extend([key, str(platform[key])])
rows += 1
mdFile.new_line()
mdFile.new_table(columns=2,
rows=rows,
text=list_of_strings,
text_align='left')
mdFile.new_header(level=1, title='Test results')
tests = res[0]['tests']
for test_key in tests.keys():
mdFile.new_header(level=2, title=test_key)
mdFile.new_line('~~~')
mdFile.new_line(tests[test_key]['command'])
mdFile.new_line('~~~')
test_res = tests[test_key]['result']
list_of_strings = ['Test'] + list(test_res.keys())
cols = len(list_of_strings)
rows = 1
avg = {}
avgc = {}
for r in res:
list_of_strings.extend(
[r['name'] + " (" + r['iteration'] + ")"] +
list(r['tests'][test_key]['result'].values()))
rows += 1
if r['name'] not in avg.keys():
avg[r['name']] = list(r['tests'][test_key]['result'].values())
avgc[r['name']] = 1
else:
avg[r['name']] = [
str(float(x) + float(y)) for x, y in zip(
avg[r['name']],
list(r['tests'][test_key]['result'].values()))
]
avgc[r['name']] += 1
for avg_key in avg.keys():
avg[avg_key] = [
str(round(float(x) / avgc[avg_key], 3)) for x in avg[avg_key]
]
list_of_strings.extend([avg_key + " (avg)"] + avg[avg_key])
rows += 1
mdFile.new_table(columns=cols,
rows=rows,
text=list_of_strings,
text_align='left')
mdFile.new_line()
mdFile.create_md_file()
print("{}.md succesfully created".format(output))
