def write_summary(args : argparse.Namespace, options : Sequence[Tuple[str, str]],
cur_commit : str, cur_date : datetime.datetime,
individual_stats : List['ResultStats']) -> None:
def report_header(tag : Any, doc : Doc, text : Text) -> None:
header(tag, doc, text,report_css, report_js,
"Proverbot Report")
combined_stats = combine_file_results(individual_stats)
doc, tag, text, line = Doc().ttl()
with tag('html'):
report_header(tag, doc, text)
with tag('body'):
with tag('h4'):
text("{} files processed".format(len(args.filenames)))
with tag('h5'):
text("Commit: {}".format(cur_commit))
if args.message:
with tag('h5'):
text("Message: {}".format(args.message))
with tag('h5'):
text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f")))
with tag('img',
('src', 'logo.png'),
('id', 'logo')):
pass
with tag('h2'):
text("Overall Accuracy: {}% ({}/{})"
.format(stringified_percent(combined_stats.num_correct,
combined_stats.num_tactics),
combined_stats.num_correct, combined_stats.num_tactics))
with tag('ul'):
for k, v in options:
if k == 'filenames':
continue
elif k == 'message':
continue
elif not v:
continue
with tag('li'):
text("{}: {}".format(k, v))
with tag('table'):
with tag('tr', klass="header"):
line('th', 'Filename')
line('th', 'Number of Tactics in File')
line('th', '% Initially Correct')
line('th', '% Top {}'.format(args.num_predictions))
line('th', '% Partial')
line('th', '% Top {} Partial'.format(args.num_predictions))
line('th', 'Testing Loss')
line('th', 'Details')
sorted_rows = sorted(individual_stats,
key=lambda fresult: fresult.num_tactics,
reverse=True)
for fresult in sorted_rows:
if fresult.num_tactics == 0:
continue
with tag('tr'):
line('td', fresult.filename)
line('td', str(fresult.num_tactics))
line('td', stringified_percent(fresult.num_correct,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_topN,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_partial,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_topNPartial,
fresult.num_tactics))
line('td', "{:10.2f}".format(fresult.loss))
with tag('td'):
with tag('a', href=escape_filename(fresult.filename) + ".html"):
text("Details")
with tag('tr'):
line('td', "Total");
line('td', str(combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_correct,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_topN,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_partial,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_topNPartial,
combined_stats.num_tactics))
line('td', "{:10.2f}".format(combined_stats.total_loss /
combined_stats.num_tactics))
base = Path2(os.path.dirname(os.path.abspath(__file__)))
for filename in extra_files:
(base.parent / "reports" / filename).copyfile(args.output / filename)
with open("{}/report.html".format(args.output), "w") as fout:
fout.write(doc.getvalue())
def generate_evaluation_index(file_summary_results: List[FileSummary],
unparsed_args: List[str], output_dir: Path2):
doc, tag, text, line = Doc().ttl()
with tag('html'):
header(tag, doc, text, index_css, index_js,
"Proverbot State Evaluation Report")
with tag("body"):
total_states = sum(
[result.total for result in file_summary_results])
total_correct = sum(
[result.correct for result in file_summary_results])
total_close = sum(
[result.close for result in file_summary_results])
with tag('h2'):
text("States Correctly Scored: {}% ({}/{})".format(
stringified_percent(total_correct, total_states),
total_correct, total_states))
with tag('img', ('src', 'logo.png'), ('id', 'logo')):
pass
with tag('h5'):
cur_commit = subprocess.check_output(
["git show --oneline | head -n 1"],
shell=True).decode('utf-8').strip()
diff_path = output_dir / "diff.txt"
subprocess.run([f"git diff HEAD > {str(diff_path)}"],
shell=True)
subprocess.run([f"git status >> {str(diff_path)}"], shell=True)
with tag('a', href=str("diff.txt")):
text('Commit: {}'.format(cur_commit))
with tag('h5'):
cur_date = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S.%f")
text('Run on: {}'.format(cur_date))
# with tag('ul'):
# for k, v in options:
# if k == 'filenames':
# continue
# elif not v:
# continue
# with tag('li'):
# text("{}: {}".format(k, v))
with tag('h4'):
text("{} files processed".format(len(file_summary_results)))
with tag('table'):
with tag('tr', klass="header"):
line('th', 'Filename')
line('th', 'Number of States in File')
line('th', 'Correct (e < 1)')
line('th', 'Close (e < 5)')
line('th', 'Details')
sorted_files = sorted(file_summary_results,
key=lambda fresult: fresult.total,
reverse=True)
for fresult in sorted_files:
if fresult.total == 0:
continue
with tag('tr'):
line('td', fresult.filename.name)
line('td', str(fresult.total))
line(
'td',
stringified_percent(fresult.correct, fresult.total)
+ "%")
line(
'td',
stringified_percent(fresult.close, fresult.total) +
"%")
with tag('td'):
with tag('a',
href=str(
fresult.filename.with_suffix(
".html").name)):
text("Details")
with tag('tr'):
line('td', "Total")
line('td', str(total_states))
line('td', stringified_percent(total_correct,
total_states))
line('td', stringified_percent(total_close, total_states))
text(f'Trained as: {" ".join(unparsed_args)}')
doc.stag('br')
text(f'Reported as: {" ".join(sys.argv)}')
with (output_dir / "report.html").open("w") as fout:
fout.write(doc.getvalue())
pass
def write_summary_html(filename : Path2,
options : Sequence[Tuple[str, str]],
unparsed_args : List[str],
cur_commit : str, cur_date : datetime.datetime,
weights_hash: str,
individual_stats : List[ReportStats],
combined_stats : ReportStats) -> None:
def report_header(tag : Any, doc : Doc, text : Text) -> None:
html_header(tag, doc, text,index_css, index_js,
"Proverbot Report")
doc, tag, text, line = Doc().ttl()
with tag('html'):
report_header(tag, doc, text)
with tag('body'):
with tag('h4'):
text("{} files processed".format(len(individual_stats)))
with tag('h5'):
text("Commit: {}".format(cur_commit))
with tag('h5'):
text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f")))
with tag('img',
('src', 'logo.png'),
('id', 'logo')):
pass
with tag('h2'):
text("Proofs Completed: {}% ({}/{})"
.format(stringified_percent(combined_stats.num_proofs_completed,
combined_stats.num_proofs),
combined_stats.num_proofs_completed,
combined_stats.num_proofs))
with tag('ul'):
for k, v in options:
if k == 'filenames':
continue
elif not v:
continue
with tag('li'):
text("{}: {}".format(k, v))
with tag('table'):
with tag('tr', klass="header"):
line('th', 'Filename')
line('th', 'Number of Proofs in File')
line('th', '% Proofs Completed')
line('th', '% Proofs Incomplete')
line('th', '% Proofs Failed')
line('th', 'Details')
sorted_rows = sorted(individual_stats,
key=lambda fresult:fresult.num_proofs,
reverse=True)
for fresult in sorted_rows:
if fresult.num_proofs == 0:
continue
with tag('tr'):
line('td', fresult.filename)
line('td', str(fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs_completed,
fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs -
(fresult.num_proofs_completed +
fresult.num_proofs_failed),
fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs_failed,
fresult.num_proofs))
with tag('td'):
with tag('a',
href=escape_filename(fresult.filename) + ".html"):
text("Details")
with tag('tr'):
line('td', "Total")
line('td', str(combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs_completed,
combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs -
(combined_stats.num_proofs_completed +
combined_stats.num_proofs_failed),
combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs_failed,
combined_stats.num_proofs))
text(f'Trained as: {unparsed_args}')
doc.stag('br')
text(f"Reported as: {sys.argv}")
doc.stag('br')
text(f"Weights hash: {weights_hash}")
with filename.open("w") as fout:
fout.write(doc.getvalue())