def count_lengths(args: argparse.Namespace, filename: str):
print(f"Counting {filename}")
full_filename = args.prelude + "/" + filename
scraped_commands = list(
read_all_text_data(Path2(full_filename + ".scrape")))
scraped_iter = iter(scraped_commands)
if args.post_linear:
original_commands = serapi_instance.load_commands_preserve(
args, 0, full_filename + ".lin")
else:
original_commands = serapi_instance.load_commands_preserve(
args, 0, full_filename)
with open(full_filename + ".csv", 'w') as fout:
rowwriter = csv.writer(fout)
lemma_statement = ""
in_proof = False
cur_len = 0
for cmd in original_commands:
if not serapi_instance.possibly_starting_proof(
cmd) and not in_proof:
continue
if serapi_instance.possibly_starting_proof(cmd) and not in_proof:
normalized_command = norm(cmd)
cur_scraped = norm(next(scraped_iter))
while cur_scraped != normalized_command:
cur_scraped = norm(next(scraped_iter))
try:
next_after_start = next(scraped_iter)
except StopIteration:
next_after_start = ""
if isinstance(next_after_start, ScrapedTactic):
lemma_statement = norm(cmd)
in_proof = True
cur_len = 0
else:
scraped_iter = itertools.chain([next_after_start],
scraped_iter)
elif serapi_instance.ending_proof(cmd):
assert in_proof
rowwriter.writerow([lemma_statement.strip(), cur_len])
cur_len = -1
in_proof = False
elif in_proof:
assert cur_len >= 0
if re.match("[{}]|[*-+]*$", norm(cmd)):
continue
if re.match("Proof\.", norm(cmd)):
continue
cur_len += 1
if args.add_semis or args.post_linear:
cur_len += count_outside_matching("\{\|", "\|\}", ";",
norm(cmd))
return full_filename + ".csv"
def generate_evaluation_details(args: argparse.Namespace, idx: int,
filename: Path2,
evaluator: StateEvaluator) -> FileSummary:
scrape_path = args.prelude / filename.with_suffix(".v.scrape")
interactions = list(read_all_text_data(scrape_path))
context_filter = get_context_filter(args.context_filter)
json_rows: List[Dict[str, Any]] = []
num_points = 0
num_close = 0
num_correct = 0
num_proofs = 0
doc, tag, text, line = Doc().ttl()
def write_highlighted(vernac: str) -> None:
nonlocal text
nonlocal tag
substrings = syntax_highlight(vernac)
for substring in substrings:
if isinstance(substring, ColoredString):
with tag('span', style=f'color:{substring.color}'):
text(substring.contents)
else:
text(substring)
def write_vernac(block: VernacBlock):
nonlocal tag
for command in block.commands:
with tag('code', klass='plaincommand'):
write_highlighted(command)
def generate_proof_evaluation_details(block: ProofBlock, region_idx: int):
nonlocal num_proofs
nonlocal num_close
nonlocal num_correct
nonlocal json_rows
num_proofs += 1
nonlocal num_points
distanced_tactics = label_distances(block.proof_interactions)
proof_length = len(distanced_tactics)
num_points += proof_length
with tag('div', klass='region'):
nonlocal evaluator
for idx, (interaction,
distance_from_end) in enumerate(distanced_tactics, 1):
if interaction.tactic.strip() in [
"Proof.", "Qed.", "Defined."
]:
with tag('code', klass='plaincommand'):
write_highlighted(interaction.tactic.strip("\n"))
doc.stag('br')
else:
predicted_distance_from_end = evaluator.scoreState(
interaction.context_before)
grade = grade_prediction(distance_from_end,
predicted_distance_from_end)
if grade == "goodcommand":
num_correct += 1
elif grade == "okaycommand":
num_close += 1
num_points += 1
json_rows.append({
"lemma": block.lemma_statement,
"hyps": interaction.context_before.hypotheses,
"goal": interaction.context_before.goal,
"actual-distance": distance_from_end,
"predicted-distance": predicted_distance_from_end,
"grade": grade
})
with tag('span',
('data-hyps', "\n".join(interaction.context_before.hypotheses)),
('data-goal', interaction.context_before.goal),
('data-actual-distance', str(distance_from_end)),
('data-predicted-distance', str(predicted_distance_from_end)),
('data-region', region_idx),
('data-index', idx),
klass='tactic'), \
tag('code', klass=grade):
text(interaction.tactic)
doc.stag('br')
def write_lemma_button(lemma_statement: str, region_idx: int):
nonlocal tag
nonlocal text
lemma_name = \
serapi_instance.lemma_name_from_statement(lemma_statement)
with tag('button', klass='collapsible',
id=f'collapsible-{region_idx}'):
with tag('code', klass='buttontext'):
write_highlighted(lemma_statement.strip())
def grade_prediction(correct_number: int, predicted_number: float) -> str:
distance = abs(correct_number - predicted_number)
if distance < 1:
return "goodcommand"
elif distance < 5:
return "okaycommand"
else:
return "badcommand"
with tag('html'):
header(tag, doc, text, details_css, details_javascript,
"Proverbot9001 Report")
with tag('body', onload='init()'), tag('pre'):
for idx, block in enumerate(get_blocks(interactions)):
if isinstance(block, VernacBlock):
write_vernac(block)
else:
assert isinstance(block, ProofBlock)
write_lemma_button(block.lemma_statement, idx)
generate_proof_evaluation_details(block, idx)
base = Path2(os.path.dirname(os.path.abspath(__file__)))
for extra_filename in extra_files:
(base.parent / "reports" / extra_filename).copyfile(args.output /
extra_filename)
with (args.output /
filename.with_suffix(".html").name).open(mode='w') as fout:
fout.write(doc.getvalue())
with (args.output /
filename.with_suffix(".json").name).open(mode='w') as fout:
for row in json_rows:
fout.write(json.dumps(row))
fout.write("\n")
return FileSummary(filename, num_close, num_correct, num_points,
num_proofs)
def count_proofs(args : argparse.Namespace, filename : str) \
-> Tuple[int, int]:
eprint(f"Counting {filename}", guard=args.debug)
scrapefile = args.prelude + "/" + filename + ".scrape"
interactions = list(
read_all_text_data(args.prelude + "/" + filename + ".scrape"))
filter_func = get_context_filter(args.context_filter)
count = 0
total_count = 0
cur_proof_counts = False
cur_lemma_stmt = ""
extended_interactions : List[Optional[ScrapedCommand]] = \
cast(List[Optional[ScrapedCommand]], interactions[1:]) + [None]
for inter, next_inter in zip(interactions, extended_interactions):
if isinstance(inter, ScrapedTactic):
goal_before = inter.goal
hyps_before = inter.hypotheses
command = inter.tactic
else:
goal_before = ""
hyps_before = []
command = inter
if next_inter and isinstance(next_inter, ScrapedTactic):
goal_after = next_inter.goal
hyps_after = next_inter.hypotheses
else:
goal_after = ""
hyps_after = []
entering_proof = bool((not goal_before) and goal_after)
exiting_proof = bool(goal_before and not goal_after)
if entering_proof:
cur_lemma_stmt = next_inter.prev_tactics[0]
cur_proof_counts = False if args.some else True
continue
if cur_lemma_stmt:
if filter_func(
{
"goal": format_goal(goal_before),
"hyps": hyps_before
}, command, {
"goal": format_goal(goal_after),
"hyps": goal_after
}, args):
if args.some and not cur_proof_counts:
cur_proof_counts = True
else:
if args.all and cur_proof_counts:
cur_lemma_name = serapi_instance.lemma_name_from_statement(
cur_lemma_stmt)
eprint(
f"Eliminating proof {cur_lemma_name} "
f"because tactic {command.strip()} doesn't match",
guard=args.debug)
cur_proof_counts = False
if exiting_proof:
if cur_proof_counts:
cur_lemma_name = serapi_instance.lemma_name_from_statement(
cur_lemma_stmt)
if args.print_name:
print(cur_lemma_name)
if args.print_stmt:
print(re.sub("\n", "\\n", cur_lemma_stmt))
eprint(f"Proof of {cur_lemma_name} counts", guard=args.debug)
count += 1
total_count += 1
cur_lemma_stmt = ""
if not args.print_name and not args.print_stmt:
print(f"{filename}: {count}/{total_count} "
f"({stringified_percent(count, total_count)}%)")
return count, total_count