def _markdown(checklist): """ Generate markdown for checklist """ checklist = json.load(open(checklist), object_pairs_hook=OrderedDict) mdFile = MdUtils(file_name='Ikigai-Checklist', title='PDP 2019 Checklist') mdFile.new_paragraph(checklist['overview']) sections = sorted(checklist['checklist'].values(), key=lambda s: int(s['no'])) for section in sections: mdFile.new_header(level=1, title=section['title']) for subject in section['subjects'].values(): mdFile.new_header(level=2, title=subject['title']) mdFile.new_paragraph(subject['description']) mdFile.new_paragraph("Reference: " + subject['provisions']) mdFile.new_paragraph("Functions: " + ", ".join(subject['functions'])) mdFile.new_paragraph("Groups: " + ", ".join(subject['groups'])) actions = [['No', 'Description', 'Tags', 'Check']] actions += [[a['no'], a['description'], ", ".join(a['tags']), ''] for a in subject['actions'].values()] rows = len(actions) actions = flatten(actions) mdFile.new_table(columns=4, rows=rows, text=actions, text_align='left') mdFile.create_md_file()
def write_table_to_md_file(md_file: MdUtils, table_rows_: List[List[str]]): flat_rows = [] for row in table_rows_: flat_rows += row md_file.new_table(columns=len(table_rows_[0]), rows=len(table_rows_), text=flat_rows, text_align='center')
def md_table_writer(name, rows): if name == 'common': mdfile = MdUtils(file_name='results_common') mdfile.new_table(columns=5, rows=int(len(rows)/5), text=rows, text_align='center') else: mdfile = MdUtils(file_name='results_{}'.format(name)) mdfile.new_table(columns=6, rows=int(len(rows)/6), text=rows, text_align='center') mdfile.create_md_file() lines_delete('results_{}.md'.format(name))
def write_table(parts, keys, md: mdutils.MdUtils, sort_by='Comment'): sorted_parts = sorted(parts, key=lambda i: i[sort_by]) tabulated = list(keys) for part in sorted_parts: tabulated.extend([part.get(key) for key in keys]) #print(part) #print(len(keys), len(sorted_parts)+1, len(tabulated)) md.new_line() md.new_table(columns=len(keys), rows=len(sorted_parts) + 1, text=tabulated)
def write_designators(parts, keys, md: mdutils.MdUtils, sort_by='designator'): parts = sorted(parts, key=lambda i: i[sort_by]) groups = groupby(parts, lambda i: i[sort_by]) for designator, group in groups: #print('Designator {}'.format(designator)) md.new_line() md.new_header(level=2, title=designator) sorted_parts = sorted(group, key=lambda i: i['id']) tabulated = list(keys) for part in sorted_parts: tabulated.extend([part.get(key) for key in keys]) md.new_line() md.new_table(columns=len(keys), rows=len(sorted_parts) + 1, text=tabulated)
def add_table(self, md_file: MdUtils, translations): table_entries = ["Intention", "Command", "Msg Id"] for translation in translations: intent = translation.occurrences[0][ 0] + ":" + translation.occurrences[0][1] # print(translation.msgid, translation.msgstr) table_entries.extend( [intent, translation.msgstr, translation.msgid]) md_file.new_table(columns=3, rows=len(translations) + 1, text=table_entries, text_align='left')
def write_md_file(herd, output_file_name, version): from mdutils import MdUtils software_summary_md = MdUtils(file_name=output_file_name, title=f'Software Summary (build {version})') table = get_table_columns_md() n_columns = len(table) for k, ch in herd.get_cattle_heads().items(): table.extend(ch.get_table_row(format='md')) software_summary_md.new_table(columns=n_columns, rows=herd.number_of_heads() + 1, text=table, text_align='center') logger.info(f'Create file {output_file_name}.md') software_summary_md.create_md_file()
def report(fails: Dict[str, List[Fail]]): print('Start writing report') start_time = time.time() mdFile = MdUtils(file_name='report', title='OCR Recognition Report') for fail_typ in fails.keys(): mdFile.new_header(level=2, title=fail_typ.title, add_table_of_contents='n') mdFile.new_header(level=3, title="Explanation", add_table_of_contents='n') mdFile.new_paragraph(fail_typ.explanation) mdFile.new_line( f'There were in total {len(fails[fail_typ])} of {fail_typ.title}') mdFile.new_header(level=3, title="Fails", add_table_of_contents='n') mdFile.new_line() if fail_typ == LenFail: md_text: List[str] = [ 'Font name - size', 'Words in PDF', 'Recognized words', 'Levenshtein distance of total text' ] for len_fail in fails[fail_typ]: md_text.extend([ f'{len_fail.font_name} - {len_fail.font_size}', str(len_fail.pdf_len), str(len_fail.ocr_len), str(len_fail.total_levenshtein) ]) mdFile.new_table(columns=4, rows=int(len(md_text) / 4), text=md_text, text_align='center') else: for fail in fails[fail_typ]: mdFile = fail.to_md(mdFile) mdFile.create_md_file() print(f'Finished reporting after {time.time() - start_time} sec')
def to_md(self, mdFile: MdUtils) -> MdUtils: mdFile.new_line(f'{self.font_name} - {self.font_size}:', bold_italics_code='b') mdFile.new_line( f'There were in total {len(self.error_words)} mismatches.') mdFile.new_line( f'The levenshtein distance of the complete text is {self.total_levenshtein}' ) mdFile.new_line() md_text: List[str] = [ 'Word in PDF', 'Recognized Word', 'Levenshtein Distance' ] for word_fail in self.error_words: md_text.extend([ word_fail.pdf_word, word_fail.ocr_word, str(word_fail.error_value) ]) mdFile.new_table(columns=3, rows=len(self.error_words) + 1, text=md_text, text_align='center') return mdFile
def to_markdown(self): """Generate markdown file.""" params = self._construct() markdown = MdUtils(file_name=params._name) markdown.new_header(level=1, title=str(params._name).capitalize()) markdown.new_header(level=2, title="Description") markdown.new_paragraph(params._desc_feature_set) markdown.new_line() markdown.new_header(level=2, title="Feature Set Pipeline") markdown.new_header(level=3, title="Source") source = ["Reader", "Location"] for r, l in params._source: source.extend([r, l]) count_rows = len(source) // 2 markdown.new_table(columns=2, rows=count_rows, text=source, text_align="center") markdown.new_header(level=3, title="Sink") sink = ["Writer"] for w in params._sink: sink.extend([w]) count_rows = len(sink) markdown.new_table(columns=1, rows=count_rows, text=sink, text_align="center") markdown.new_header(level=3, title="Features") features = ["Column name", "Data type", "Description"] for c, desc in params._features: features.extend([c["column_name"], str(c["type"]), desc]) count_rows = len(features) // 3 markdown.new_table(columns=3, rows=count_rows, text=features, text_align="center") if self.save: markdown.create_md_file() else: return markdown.file_data_text
def run(self): Responder.run(self) caseNumber = self.get_param('data.caseId') #Friendly case number caseId = self.get_param('data.id') #Raw case number case_observables = self.api.get_case_observables(caseId).json() title = self.get_param('data.title', None, 'title is missing') description = self.get_param('data.description', None, 'description is missing') tags = self.get_param('data.tags') data = self.get_param('data') tlp = self.getTLP(data['tlp']) # Title #mdFile = MdUtils(file_name=str(caseNumber),title=tlp[0] + ' Case #' + str(caseNumber) + ': ' + title) mdFile = MdUtils(file_name=str(self.tmpPath) + str(caseNumber), title=tlp[0] + ' Case #' + str(caseNumber) + ': ' + title) # Case Summary caseSummary = self.getCaseSummary(data) mdFile.new_header(level=1, title='Case Summary') mdFile.new_line(str(tlp[1])) mdFile.new_table(columns=2, rows=int(caseSummary.__len__() / 2), text=caseSummary, text_align='left') # Case Description mdFile.new_line('<div style="page-break-after: always;"></div>') mdFile.new_line(' ') mdFile.new_header(level=1, title='Case Description') mdFile.new_line(str(data['description'])) mdFile.new_line(' ') # Task Log allTaskIds = self.getCaseTasks(caseId) allTaskIds_sorted = sorted(allTaskIds.items(), key=lambda x: x[1]['createdAt']) mdFile.new_header(level=1, title='Task Log Entries') for task in allTaskIds_sorted: title = str(task[1]['taskGroup'] + ' \: ' + task[1]['taskTitle']) createdAt = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.localtime( task[1]['createdAt'] / 1000)) #Convert epoch ms to sec then human readable mdFile.new_header(level=2, title=title) mdFile.new_line(str('**Created At:** ') + str(createdAt)) mdFile.new_line( str('**Created By:** ') + str(task[1]['createdBy'])) mdFile.new_line(str('**Assigned To:** ') + str(task[1]['owner'])) mdFile.new_line(str('**Case Status:** ') + str(task[1]['status'])) mdFile.new_line(' ') mdFile.new_line(str('**Description:** ')) mdFile.new_line(str(task[1]['description'])) mdFile.new_line(' ') caseTaskLog = self.getCaseTaskLog(task[0]) caseTaskLogEntries = (json.loads(caseTaskLog)) caseTaskLogEntries_sorted = sorted(caseTaskLogEntries, key=lambda k: k['createdAt']) for caseTaskLogEntry in caseTaskLogEntries_sorted: createdAt = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.localtime( caseTaskLogEntry['createdAt'] / 1000)) #Convert epoch ms to sec then human readable mdFile.new_line( str(createdAt) + ' : ' + str(caseTaskLogEntry['message'])) # Case Observables mdFile.new_header(level=1, title='Case Observables') caseObservables = self.getCaseObservables(case_observables) mdFile.new_table(columns=6, rows=int(caseObservables.__len__() / 6), text=caseObservables, text_align='left') # TLP Protocol description mdFile.new_line('<div style="page-break-after: always;"></div>') mdFile.new_line(' ') mdFile.new_header( level=1, title='Traffic Light Protocol (TLP) Definitions and Usage') tlpFooter = self.getTlpFooter() mdFile.new_table(columns=3, rows=5, text=tlpFooter, text_align='left') # Build TOC mdFile.new_table_of_contents(table_title='Table of Contents', depth=2) # Compile the report mdFile.create_md_file() # Add the report to the case addTask = json.loads(self.addTask(caseId)) taskId = addTask['_id'] # Add the MD file to the task addTaskLog = json.loads( self.addTaskLog(taskId, str(self.tmpPath) + str(caseNumber) + '.md')) # Cleanup the MD file os.remove(str(self.tmpPath) + str(caseNumber) + '.md') self.report({'report': 'created'})
def makeMarkdown(data, path): # Creates the README file global base_url mdf = MdUtils(file_name=path + 'README', title='RBA TechRadar for Azure') adopt_list = list() trial_list = list() assess_list = list() hold_list = list() reject_list = list() # Create categories on status for key in data: status = data[key].get("status") if status == "ADOPT": adopt_list.append(key) if status == "TRIAL": trial_list.append(key) if status == "ASSESS": assess_list.append(key) if status == "HOLD": hold_list.append(key) if status == "REJECT": reject_list.append(key) mdf.new_header(level=1, title='Overview') mdf.new_header(level=2, title='What is the purpose?') mdf.new_paragraph( "The RBA TechRadar for Azure is a tool to inspire and " "support engineering teams at Risk & Business Analytics to pick the best " "technologies for new projects; it provides a platform to share knowledge " "and experience in technologies, to reflect on technology decisions and " "continuously evolve our technology landscape. Based on the pioneering " "work at Thought Works, our radar sets out the changes in technologies " "that are interesting in cloud development - changes that we think our " "engineering teams should pay attention to and consider using in their " "projects.") mdf.new_header(level=2, title='How do we maintain it?') mdf.new_paragraph( "The RBA TechRadar for Azure is maintained by the Cloud " "Center of Excellence - an open group of senior RBA technologists committed " "to devote time to this purpose. The CCoE self organizes to maintain these " "documents, including this version. Assignment of technologies to rings is " "the outcome of status change proposals, which are discussed and voted on " "in CCoE meetings. The radar depends on active participation and input from " "all engineering teams at RBA.") mdf.new_header(level=2, title='What are the current ring assignments?') mdf.new_paragraph( "The RBA TechRadar for Azure is a list of technologies, " "complemented by an assesment result, called ring assignment. We use five " "rings with the following semantics:") # Handle the Adopt Section mdf.new_header(level=3, title='Adopt') mdf.new_paragraph( "Technologies we have high confidence in to serve our " "purpose, also at large scale. Technologies with a usage culture in the " "RBA production environment, low risk, automated policy enforcement and " "are recommended to be widely used.") adopt_tbl = [ "<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>", "<sub>Status</sub>" ] adopt_cnt = len(adopt_list) + 1 for key in adopt_list: resourceName = key resourceDesc = data[key].get("description", "") resourcePath = data[key].get("path", "") resourceType = data[key].get("architecture review", "").get("type", "") resourceUrl = data[key].get("url", "") resourceStatus = data[key].get("status", "") resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")" adopt_tbl.extend([ '<sub>' + resourceName + '</sub>', '<sub>' + resourceDesc + '</sub>', '<sub>' + resourceType + '</sub>', '<sub>' + resourceStatus + '</sub>' ]) if adopt_cnt == 1: mdf.new_line("") mdf.new_line("There are currently no resources at this ring level.", bold_italics_code='bi', color='red') else: mdf.new_line("") mdf.new_table(columns=4, rows=adopt_cnt, text=adopt_tbl) # Handle the Trial Section mdf.new_header(level=3, title='Trial') mdf.new_paragraph( "Technologies that we have seen work with success in projects " "to solve real problems; first serious usage experience that confirm benefits " "and uncover limitations. TRIAL technologies are slightly more risky; some " "engineers in our organization walked this path and will share knowledge and " "experiences. This area can contain services that have been architecture and " "security reviewed but do not contain automated policy managmeent.") trial_tbl = [ "<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>", "<sub>Status</sub>" ] trial_cnt = len(trial_list) + 1 for key in trial_list: resourceName = key resourceDesc = data[key].get("description", "") resourcePath = data[key].get("path", "") resourceType = data[key].get("architecture review", "").get("type", "") resourceUrl = data[key].get("url", "") resourceStatus = data[key].get("status", "") resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")" trial_tbl.extend([ '<sub>' + resourceName + '</sub>', '<sub>' + resourceDesc + '</sub>', '<sub>' + resourceType + '</sub>', '<sub>' + resourceStatus + '</sub>' ]) if trial_cnt == 1: mdf.new_line("") mdf.new_line("There are currently no resources at this ring level.", bold_italics_code='bi', color='red') else: mdf.new_line("") mdf.new_table(columns=4, rows=trial_cnt, text=trial_tbl) # Handle the Assess Section mdf.new_header(level=3, title='Assess') mdf.new_paragraph( "Technologies that are promising and have clear potential " "value-add for us; technologies worth investing some research and " "prototyping efforts to see if it has impact. ASSESS technologies have " "higher risks; they are often new to our organization and highly unproven " "within RBA. You will find some engineers that have knowledge in the " "technology and promote it, you may even find teams that have started " "a prototyping effort. These technologies can also include services that " "are currently in architecture or security review.") assess_tbl = [ "<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>", "<sub>Status</sub>" ] assess_cnt = len(assess_list) + 1 for key in assess_list: resourceName = key resourceDesc = data[key].get("description", "") resourcePath = data[key].get("path", "") resourceType = data[key].get("architecture review", "").get("type", "") resourceUrl = data[key].get("url", "") resourceStatus = data[key].get("status", "") resourceName = "[" + resourceName + "](" + base_url + '/' + resourceUrl + ")" assess_tbl.extend([ '<sub>' + resourceName + '</sub>', '<sub>' + resourceDesc + '</sub>', '<sub>' + resourceType + '</sub>', '<sub>' + resourceStatus + '</sub>' ]) if assess_cnt == 1: mdf.new_line("") mdf.new_line("There are currently no resources at this ring level.", bold_italics_code='bi', color='red') else: mdf.new_line("") mdf.new_table(columns=4, rows=assess_cnt, text=assess_tbl) # Handle the Hold Section mdf.new_header(level=3, title='Hold') mdf.new_paragraph( "Technologies not recommended to be used for new projects. " "Technologies that we think are not (yet) worth to (further) invest in. " "HOLD technologies should not be used for new projects, but usually can be " "continued for existing projects. These technologies may include services " "that have yet to be evaluated by architecture and security due to a lack " "of interest, time, or need.") hold_tbl = [ "<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>", "<sub>Status</sub>" ] hold_cnt = len(hold_list) + 1 for key in hold_list: resourceName = key resourceDesc = data[key].get("description", "") resourcePath = data[key].get("path", "") resourceType = data[key].get("architecture review", "").get("type", "") resourceUrl = data[key].get("url", "") resourceStatus = data[key].get("status", "") #resourceName = "["+resourceName+"]("+resourceUrl+")" hold_tbl.extend([ '<sub>' + resourceName + '</sub>', '<sub>' + resourceDesc + '</sub>', '<sub>' + resourceType + '</sub>', '<sub>' + resourceStatus + '</sub>' ]) if hold_cnt == 1: mdf.new_line("") mdf.new_line("There are currently no resources at this ring level.", bold_italics_code='bi', color='red') else: mdf.new_line("") mdf.new_table(columns=4, rows=hold_cnt, text=hold_tbl) # Handle the Reject Section mdf.new_header(level=3, title='Reject') mdf.new_paragraph( "Technologies not recommended to be used for any projects. " "Technologies that have undergone architecture and security review but do " "not meet company standards for use. REJECT technologies should never be " "used on any project and should be considered deprecated for existing " "projects.") reject_tbl = [ "<sub>Resource</sub>", "<sub>Description</sub>", "<sub>Type</sub>", "<sub>Status</sub>" ] reject_cnt = len(reject_list) + 1 for key in reject_list: resourceName = key resourceDesc = data[key].get("description", "") resourcePath = data[key].get("path", "") resourceType = data[key].get("architecture review", "").get("type", "") resourceUrl = data[key].get("url", "") resourceStatus = data[key].get("status", "") #resourceName = "["+resourceName+"]("+resourceUrl+")" reject_tbl.extend([ '<sub>' + resourceName + '</sub>', '<sub>' + resourceDesc + '</sub>', '<sub>' + resourceType + '</sub>', '<sub>' + resourceStatus + '</sub>' ]) if reject_cnt == 1: mdf.new_line("") mdf.new_line("There are currently no resources at this ring level.", bold_italics_code='bi', color='red') else: mdf.new_line("") mdf.new_table(columns=4, rows=reject_cnt, text=reject_tbl) mdf.create_md_file()
def generate_report(): def generate_phrase_image(width: int, font: ImageFont) -> Image: result = Image.new(mode=GRAYSCALE_MODE, size=(width, font.size), color=WHITE) result_draw = ImageDraw.Draw(im=result, mode=GRAYSCALE_MODE) result_draw.text(xy=(0, 0), text=PHRASE, font=font, fill=0, anchor='lt') return cut_empty_rows_and_cols(result) def p_ass_for_table(p_ass): result = [] for row in p_ass: for element in row: result.append(str(element)) return result report = MdUtils(file_name=f'./report.md') report.new_header(level=1, title='Классификация') report.new_line(text='Выполнил Ахманов Алексей Б18-514') report.new_line(text=f'Алфавит - {ALPHABET}') report.new_line(text=f'Исходная фраза - {PHRASE}') report.new_line(text=f'Размер шрифта - {FONT_SIZE}') # Phrase phrase_image = generate_phrase_image( 1200, ImageFont.truetype(font=FONT_PATH, size=FONT_SIZE)) phrase_image_small_font = generate_phrase_image( 1200, ImageFont.truetype(font=FONT_PATH, size=SMALL_FONT_SIZE)) # Proximity assessment p_assessment = proximity_assessment(image=phrase_image, diff_threshold=SYMBOLS_DIFF_THRESHOLD, phrase=PHRASE) p_assessment_small_font = proximity_assessment( image=phrase_image_small_font, diff_threshold=SYMBOLS_DIFF_THRESHOLD, phrase=PHRASE) p_assessment_for_table = p_ass_for_table(p_assessment) p_assessment_small_font_for_table = p_ass_for_table( p_assessment_small_font) # Report report.new_header(level=2, title=f'Оценка близости для размера шрифта {FONT_SIZE}') rows = len(p_assessment) columns = len(p_assessment[0]) if len(p_assessment) > 0 else 0 report.new_table(columns=columns, rows=rows, text=p_assessment_for_table) report.new_header( level=2, title=f'Оценка близости для размера шрифта {SMALL_FONT_SIZE}') rows = len(p_assessment_small_font) columns = len( p_assessment_small_font[0]) if len(p_assessment_small_font) > 0 else 0 report.new_table(columns=columns, rows=rows, text=p_assessment_small_font_for_table) report.new_line( text= 'Так как были использованы нормализованные параметры для оценки близости символов, размер шрифта не влияет на результат' ) report.create_md_file()
def create_report(res_files: [], output: str, title="", description=""): res = [] for f in res_files: with open(f) as json_file: res.append(json.load(json_file)) mdFile = MdUtils(file_name=output, title=title) if description: mdFile.new_paragraph(description, bold_italics_code='b') mdFile.new_header(level=1, title='Platform') platform = res[0]['platform'] list_of_strings = ["System", "Information"] rows = 1 for key in platform.keys(): list_of_strings.extend([key, str(platform[key])]) rows += 1 mdFile.new_line() mdFile.new_table(columns=2, rows=rows, text=list_of_strings, text_align='left') mdFile.new_header(level=1, title='Test results') tests = res[0]['tests'] for test_key in tests.keys(): mdFile.new_header(level=2, title=test_key) mdFile.new_line('~~~') mdFile.new_line(tests[test_key]['command']) mdFile.new_line('~~~') test_res = tests[test_key]['result'] list_of_strings = ['Test'] + list(test_res.keys()) cols = len(list_of_strings) rows = 1 avg = {} avgc = {} for r in res: list_of_strings.extend( [r['name'] + " (" + r['iteration'] + ")"] + list(r['tests'][test_key]['result'].values())) rows += 1 if r['name'] not in avg.keys(): avg[r['name']] = list(r['tests'][test_key]['result'].values()) avgc[r['name']] = 1 else: avg[r['name']] = [ str(float(x) + float(y)) for x, y in zip( avg[r['name']], list(r['tests'][test_key]['result'].values())) ] avgc[r['name']] += 1 for avg_key in avg.keys(): avg[avg_key] = [ str(round(float(x) / avgc[avg_key], 3)) for x in avg[avg_key] ] list_of_strings.extend([avg_key + " (avg)"] + avg[avg_key]) rows += 1 mdFile.new_table(columns=cols, rows=rows, text=list_of_strings, text_align='left') mdFile.new_line() mdFile.create_md_file() print("{}.md succesfully created".format(output))