def write_proof_summary_csv(output_dir : str, filenames : List[str]):
with open('{}/proofs.csv'.format(output_dir), 'w') as fout:
fout.write("lemma,status,prooflength\n")
for filename in filenames:
with open("{}/{}.csv".format(output_dir, escape_filename(filename)), 'r') \
as fin:
fout.writelines(fin)
def write_csv(output_dir: Path2, filename: Path2, args: argparse.Namespace,
command_results: List[CommandResult],
stats: 'ResultStats') -> None:
with (output_dir / escape_filename(str(filename))).with_suffix(".csv")\
.open(mode='w', newline='') \
as csvfile:
for k, v in vars(args).items():
csvfile.write("# {}: {}\n".format(k, v))
rowwriter = csv.writer(csvfile, lineterminator=os.linesep)
for row in command_results:
if len(row) == 1:
rowwriter.writerow([re.sub(r"\n", r"\\n", cast(str, row[0]))])
else:
# Type hack
command, hyps, goal, prediction_results = cast(
TacticResult, row)
rowwriter.writerow([
re.sub(r"\n", r"\\n", item)
for item in [command] + hyps + [goal] + [
item
for prediction, grade, certainty in prediction_results
for item in [prediction, grade]
]
])
def read_stats_from_csv(output_dir : str, vfilename : str) -> \
Tuple[argparse.Namespace, ReportStats]:
num_proofs = 0
num_proofs_failed = 0
num_proofs_completed = 0
with open("{}/{}.csv".format(output_dir, escape_filename(str(vfilename))),
'r', newline='') as csvfile:
saved_args, rest_iter = read_csv_options(csvfile)
rowreader = csv.reader(rest_iter, lineterminator=os.linesep)
for row in rowreader:
num_proofs += 1
if row[1] == "SUCCESS":
num_proofs_completed += 1
elif row[1] == "FAILURE":
num_proofs_failed += 1
else:
assert row[1] == "INCOMPLETE", row
return saved_args, ReportStats(str(vfilename),
num_proofs, num_proofs_failed, num_proofs_completed)
def write_html(output_dir : Path2, filename : Path2, command_results : List[CommandResult],
stats : 'ResultStats') -> None:
def details_header(tag : Any, doc : Doc, text : Text, filename : Path2) -> None:
header(tag, doc, text, details_css, details_javascript,
"Proverbot Detailed Report for {}".format(filename))
doc, tag, text, line = Doc().ttl()
with tag('html'):
details_header(tag, doc, text, filename)
with tag('div', id='overlay', onclick='event.stopPropagation();'):
with tag('div', id='predicted'):
pass
with tag('div', id='context'):
pass
with tag('div', id='stats'):
pass
pass
with tag('body', onclick='deselectTactic()',
onload='init()'), tag('pre'):
for region_idx, region in enumerate(split_into_regions(command_results)):
if len(region) > 1 and len(region[1]) == 1:
for cmd_idx, command_result in enumerate(region):
assert isinstance(command_result[0], str)
with tag('code', klass='plaincommand'):
text("\n" + command_result[0].strip('\n'))
else:
doc.stag("br")
with tag('button', klass='collapsible',
id='collapsible-{}'.format(region_idx)):
with tag('code', klass='buttontext'):
assert isinstance(region[0][0], str), region
text(region[0][0].strip("\n"))
num_unfiltered = count_region_unfiltered(region)
with tag('code', klass='numtacs ' +
('nonempty' if num_unfiltered > 3 else 'empty')):
text(num_unfiltered)
with tag('div', klass='region'):
for cmd_idx, command_result in enumerate(region[1:]):
if len(command_result) == 1:
assert isinstance(command_result[0], str)
with tag('code', klass='plaincommand'):
text("\n" + command_result[0].strip('\n'))
else:
command, hyps, goal, prediction_results = \
cast(TacticResult, command_result)
predictions : List[str]
grades : List[str]
certainties : List[float]
if len(prediction_results) > 0:
predictions, grades, certainties = zip(*prediction_results) # type: ignore
else:
predictions, grades, certainties = [], [], []
with tag('span',
('data-hyps',"\n".join(hyps)),
('data-goal',format_goal(goal)),
('data-num-total', str(stats.num_tactics)),
('data-predictions',
to_list_string(cast(List[str], predictions))),
('data-num-predicteds',
to_list_string([stats.predicted_tactic_frequency
.get(get_stem(prediction), 0)
for prediction in cast(List[str],
predictions)])),
('data-num-corrects',
to_list_string([stats.correctly_predicted_frequency
.get(get_stem(prediction), 0)
for prediction in
cast(List[str], predictions)])),
('data-certainties',
to_list_string(cast(List[float], certainties))),
('data-num-actual-corrects',
stats.correctly_predicted_frequency
.get(get_stem(command), 0)),
('data-num-actual-in-file',
stats.actual_tactic_frequency
.get(get_stem(command), 0)),
('data-actual-tactic',
strip_comments(command)),
('data-grades',
to_list_string(cast(List[str], grades))),
('data-search-idx', 0),
id='command-{}-{}'.format(region_idx, cmd_idx),
onmouseover='hoverTactic("{}-{}")'\
.format(region_idx, cmd_idx),
onmouseout='unhoverTactic()',
onclick='selectTactic("{}-{}"); event.stopPropagation();'
.format(region_idx, cmd_idx)):
doc.stag("br")
if len(grades) == 0:
with tag('code', klass="plaincommand"):
text(command.strip("\n"))
else:
with tag('code', klass=grades[0]):
text(command.strip("\n"))
for grade in grades[1:]:
with tag('span', klass=grade):
doc.asis(" ⬤")
with (output_dir / escape_filename(str(filename))).with_suffix(".html")\
.open(mode='w') as fout:
fout.write(doc.getvalue())
pass
def write_summary(args : argparse.Namespace, options : Sequence[Tuple[str, str]],
cur_commit : str, cur_date : datetime.datetime,
individual_stats : List['ResultStats']) -> None:
def report_header(tag : Any, doc : Doc, text : Text) -> None:
header(tag, doc, text,report_css, report_js,
"Proverbot Report")
combined_stats = combine_file_results(individual_stats)
doc, tag, text, line = Doc().ttl()
with tag('html'):
report_header(tag, doc, text)
with tag('body'):
with tag('h4'):
text("{} files processed".format(len(args.filenames)))
with tag('h5'):
text("Commit: {}".format(cur_commit))
if args.message:
with tag('h5'):
text("Message: {}".format(args.message))
with tag('h5'):
text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f")))
with tag('img',
('src', 'logo.png'),
('id', 'logo')):
pass
with tag('h2'):
text("Overall Accuracy: {}% ({}/{})"
.format(stringified_percent(combined_stats.num_correct,
combined_stats.num_tactics),
combined_stats.num_correct, combined_stats.num_tactics))
with tag('ul'):
for k, v in options:
if k == 'filenames':
continue
elif k == 'message':
continue
elif not v:
continue
with tag('li'):
text("{}: {}".format(k, v))
with tag('table'):
with tag('tr', klass="header"):
line('th', 'Filename')
line('th', 'Number of Tactics in File')
line('th', '% Initially Correct')
line('th', '% Top {}'.format(args.num_predictions))
line('th', '% Partial')
line('th', '% Top {} Partial'.format(args.num_predictions))
line('th', 'Testing Loss')
line('th', 'Details')
sorted_rows = sorted(individual_stats,
key=lambda fresult: fresult.num_tactics,
reverse=True)
for fresult in sorted_rows:
if fresult.num_tactics == 0:
continue
with tag('tr'):
line('td', fresult.filename)
line('td', str(fresult.num_tactics))
line('td', stringified_percent(fresult.num_correct,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_topN,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_partial,
fresult.num_tactics))
line('td', stringified_percent(fresult.num_topNPartial,
fresult.num_tactics))
line('td', "{:10.2f}".format(fresult.loss))
with tag('td'):
with tag('a', href=escape_filename(fresult.filename) + ".html"):
text("Details")
with tag('tr'):
line('td', "Total");
line('td', str(combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_correct,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_topN,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_partial,
combined_stats.num_tactics))
line('td', stringified_percent(combined_stats.num_topNPartial,
combined_stats.num_tactics))
line('td', "{:10.2f}".format(combined_stats.total_loss /
combined_stats.num_tactics))
base = Path2(os.path.dirname(os.path.abspath(__file__)))
for filename in extra_files:
(base.parent / "reports" / filename).copyfile(args.output / filename)
with open("{}/report.html".format(args.output), "w") as fout:
fout.write(doc.getvalue())
def write_summary_html(filename : Path2,
options : Sequence[Tuple[str, str]],
unparsed_args : List[str],
cur_commit : str, cur_date : datetime.datetime,
weights_hash: str,
individual_stats : List[ReportStats],
combined_stats : ReportStats) -> None:
def report_header(tag : Any, doc : Doc, text : Text) -> None:
html_header(tag, doc, text,index_css, index_js,
"Proverbot Report")
doc, tag, text, line = Doc().ttl()
with tag('html'):
report_header(tag, doc, text)
with tag('body'):
with tag('h4'):
text("{} files processed".format(len(individual_stats)))
with tag('h5'):
text("Commit: {}".format(cur_commit))
with tag('h5'):
text("Run on {}".format(cur_date.strftime("%Y-%m-%d %H:%M:%S.%f")))
with tag('img',
('src', 'logo.png'),
('id', 'logo')):
pass
with tag('h2'):
text("Proofs Completed: {}% ({}/{})"
.format(stringified_percent(combined_stats.num_proofs_completed,
combined_stats.num_proofs),
combined_stats.num_proofs_completed,
combined_stats.num_proofs))
with tag('ul'):
for k, v in options:
if k == 'filenames':
continue
elif not v:
continue
with tag('li'):
text("{}: {}".format(k, v))
with tag('table'):
with tag('tr', klass="header"):
line('th', 'Filename')
line('th', 'Number of Proofs in File')
line('th', '% Proofs Completed')
line('th', '% Proofs Incomplete')
line('th', '% Proofs Failed')
line('th', 'Details')
sorted_rows = sorted(individual_stats,
key=lambda fresult:fresult.num_proofs,
reverse=True)
for fresult in sorted_rows:
if fresult.num_proofs == 0:
continue
with tag('tr'):
line('td', fresult.filename)
line('td', str(fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs_completed,
fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs -
(fresult.num_proofs_completed +
fresult.num_proofs_failed),
fresult.num_proofs))
line('td', stringified_percent(fresult.num_proofs_failed,
fresult.num_proofs))
with tag('td'):
with tag('a',
href=escape_filename(fresult.filename) + ".html"):
text("Details")
with tag('tr'):
line('td', "Total")
line('td', str(combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs_completed,
combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs -
(combined_stats.num_proofs_completed +
combined_stats.num_proofs_failed),
combined_stats.num_proofs))
line('td', stringified_percent(combined_stats.num_proofs_failed,
combined_stats.num_proofs))
text(f'Trained as: {unparsed_args}')
doc.stag('br')
text(f"Reported as: {sys.argv}")
doc.stag('br')
text(f"Weights hash: {weights_hash}")
with filename.open("w") as fout:
fout.write(doc.getvalue())