def write_summary(args : argparse.Namespace, options : Sequence[Tuple[str, str]], cur_commit : str, cur_date : datetime.datetime, individual_stats : List['ResultStats']) -> None: def report_header(tag : Any, doc : Doc, text : Text) -> None: header(tag, doc, text,report_css, report_js, "Proverbot Report") combined_stats = combine_file_results(individual_stats) doc, tag, text, line = Doc().ttl() with tag('html'): report_header(tag, doc, text) with tag('body'): with tag('h4'): text("{} files processed".format(len(args.filenames))) with tag('h5'): text("Commit: {}".format(cur_commit)) if args.message: with tag('h5'): text("Message: {}".format(args.message)) with tag('h5'): text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f"))) with tag('img', ('src', 'logo.png'), ('id', 'logo')): pass with tag('h2'): text("Overall Accuracy: {}% ({}/{})" .format(stringified_percent(combined_stats.num_correct, combined_stats.num_tactics), combined_stats.num_correct, combined_stats.num_tactics)) with tag('ul'): for k, v in options: if k == 'filenames': continue elif k == 'message': continue elif not v: continue with tag('li'): text("{}: {}".format(k, v)) with tag('table'): with tag('tr', klass="header"): line('th', 'Filename') line('th', 'Number of Tactics in File') line('th', '% Initially Correct') line('th', '% Top {}'.format(args.num_predictions)) line('th', '% Partial') line('th', '% Top {} Partial'.format(args.num_predictions)) line('th', 'Testing Loss') line('th', 'Details') sorted_rows = sorted(individual_stats, key=lambda fresult: fresult.num_tactics, reverse=True) for fresult in sorted_rows: if fresult.num_tactics == 0: continue with tag('tr'): line('td', fresult.filename) line('td', str(fresult.num_tactics)) line('td', stringified_percent(fresult.num_correct, fresult.num_tactics)) line('td', stringified_percent(fresult.num_topN, fresult.num_tactics)) line('td', stringified_percent(fresult.num_partial, fresult.num_tactics)) line('td', stringified_percent(fresult.num_topNPartial, fresult.num_tactics)) line('td', "{:10.2f}".format(fresult.loss)) with tag('td'): with tag('a', href=escape_filename(fresult.filename) + ".html"): text("Details") with tag('tr'): line('td', "Total"); line('td', str(combined_stats.num_tactics)) line('td', stringified_percent(combined_stats.num_correct, combined_stats.num_tactics)) line('td', stringified_percent(combined_stats.num_topN, combined_stats.num_tactics)) line('td', stringified_percent(combined_stats.num_partial, combined_stats.num_tactics)) line('td', stringified_percent(combined_stats.num_topNPartial, combined_stats.num_tactics)) line('td', "{:10.2f}".format(combined_stats.total_loss / combined_stats.num_tactics)) base = Path2(os.path.dirname(os.path.abspath(__file__))) for filename in extra_files: (base.parent / "reports" / filename).copyfile(args.output / filename) with open("{}/report.html".format(args.output), "w") as fout: fout.write(doc.getvalue())
def generate_evaluation_index(file_summary_results: List[FileSummary], unparsed_args: List[str], output_dir: Path2): doc, tag, text, line = Doc().ttl() with tag('html'): header(tag, doc, text, index_css, index_js, "Proverbot State Evaluation Report") with tag("body"): total_states = sum( [result.total for result in file_summary_results]) total_correct = sum( [result.correct for result in file_summary_results]) total_close = sum( [result.close for result in file_summary_results]) with tag('h2'): text("States Correctly Scored: {}% ({}/{})".format( stringified_percent(total_correct, total_states), total_correct, total_states)) with tag('img', ('src', 'logo.png'), ('id', 'logo')): pass with tag('h5'): cur_commit = subprocess.check_output( ["git show --oneline | head -n 1"], shell=True).decode('utf-8').strip() diff_path = output_dir / "diff.txt" subprocess.run([f"git diff HEAD > {str(diff_path)}"], shell=True) subprocess.run([f"git status >> {str(diff_path)}"], shell=True) with tag('a', href=str("diff.txt")): text('Commit: {}'.format(cur_commit)) with tag('h5'): cur_date = datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S.%f") text('Run on: {}'.format(cur_date)) # with tag('ul'): # for k, v in options: # if k == 'filenames': # continue # elif not v: # continue # with tag('li'): # text("{}: {}".format(k, v)) with tag('h4'): text("{} files processed".format(len(file_summary_results))) with tag('table'): with tag('tr', klass="header"): line('th', 'Filename') line('th', 'Number of States in File') line('th', 'Correct (e < 1)') line('th', 'Close (e < 5)') line('th', 'Details') sorted_files = sorted(file_summary_results, key=lambda fresult: fresult.total, reverse=True) for fresult in sorted_files: if fresult.total == 0: continue with tag('tr'): line('td', fresult.filename.name) line('td', str(fresult.total)) line( 'td', stringified_percent(fresult.correct, fresult.total) + "%") line( 'td', stringified_percent(fresult.close, fresult.total) + "%") with tag('td'): with tag('a', href=str( fresult.filename.with_suffix( ".html").name)): text("Details") with tag('tr'): line('td', "Total") line('td', str(total_states)) line('td', stringified_percent(total_correct, total_states)) line('td', stringified_percent(total_close, total_states)) text(f'Trained as: {" ".join(unparsed_args)}') doc.stag('br') text(f'Reported as: {" ".join(sys.argv)}') with (output_dir / "report.html").open("w") as fout: fout.write(doc.getvalue()) pass
def write_summary_html(filename : Path2, options : Sequence[Tuple[str, str]], unparsed_args : List[str], cur_commit : str, cur_date : datetime.datetime, weights_hash: str, individual_stats : List[ReportStats], combined_stats : ReportStats) -> None: def report_header(tag : Any, doc : Doc, text : Text) -> None: html_header(tag, doc, text,index_css, index_js, "Proverbot Report") doc, tag, text, line = Doc().ttl() with tag('html'): report_header(tag, doc, text) with tag('body'): with tag('h4'): text("{} files processed".format(len(individual_stats))) with tag('h5'): text("Commit: {}".format(cur_commit)) with tag('h5'): text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f"))) with tag('img', ('src', 'logo.png'), ('id', 'logo')): pass with tag('h2'): text("Proofs Completed: {}% ({}/{})" .format(stringified_percent(combined_stats.num_proofs_completed, combined_stats.num_proofs), combined_stats.num_proofs_completed, combined_stats.num_proofs)) with tag('ul'): for k, v in options: if k == 'filenames': continue elif not v: continue with tag('li'): text("{}: {}".format(k, v)) with tag('table'): with tag('tr', klass="header"): line('th', 'Filename') line('th', 'Number of Proofs in File') line('th', '% Proofs Completed') line('th', '% Proofs Incomplete') line('th', '% Proofs Failed') line('th', 'Details') sorted_rows = sorted(individual_stats, key=lambda fresult:fresult.num_proofs, reverse=True) for fresult in sorted_rows: if fresult.num_proofs == 0: continue with tag('tr'): line('td', fresult.filename) line('td', str(fresult.num_proofs)) line('td', stringified_percent(fresult.num_proofs_completed, fresult.num_proofs)) line('td', stringified_percent(fresult.num_proofs - (fresult.num_proofs_completed + fresult.num_proofs_failed), fresult.num_proofs)) line('td', stringified_percent(fresult.num_proofs_failed, fresult.num_proofs)) with tag('td'): with tag('a', href=escape_filename(fresult.filename) + ".html"): text("Details") with tag('tr'): line('td', "Total") line('td', str(combined_stats.num_proofs)) line('td', stringified_percent(combined_stats.num_proofs_completed, combined_stats.num_proofs)) line('td', stringified_percent(combined_stats.num_proofs - (combined_stats.num_proofs_completed + combined_stats.num_proofs_failed), combined_stats.num_proofs)) line('td', stringified_percent(combined_stats.num_proofs_failed, combined_stats.num_proofs)) text(f'Trained as: {unparsed_args}') doc.stag('br') text(f"Reported as: {sys.argv}") doc.stag('br') text(f"Weights hash: {weights_hash}") with filename.open("w") as fout: fout.write(doc.getvalue())