def check_vfile(vfile_path: Path2, includes: str,
args: argparse.Namespace) -> None:
html_path = vfile_path.with_suffix(".html")
src_path = Path2(unescape_filename(vfile_path.stem))
if args.skip_incomplete:
if not html_path.exists():
print(f"Skipping {src_path}")
return
else:
assert html_path.exists(), \
f"Couldn't find HTML file for {src_path}. "\
f"Are you sure the report is completed?"
src_f = src_path.with_suffix("")
src_ext = src_path.suffix
new_filename_path = Path2(str(src_f) + "_solution" + src_ext)
vfile_path.copyfile(args.prelude / new_filename_path)
result = subprocess.run(["coqc"] + includes.split() +
[str(new_filename_path)],
cwd=args.prelude,
capture_output=True,
encoding='utf8')
assert result.returncode == 0, \
f"Returned a non zero errorcode {result.returncode}! \n"\
f"{result.stderr}"
print(f"Checked {src_path}")
if args.print_stdout:
print(f"Output:\n{result.stdout}", end="")
def resume(resume_file: Path2, weights: Path2,
q_estimator: QEstimator) -> \
Tuple[List[LabeledTransition],
List[Job],
List[Tuple[str, ReinforceGraph]]]:
eprint("Looks like there was a session in progress for these weights! "
"Resuming")
q_estimator_name, *saved = \
torch.load(str(weights))
q_estimator.load_saved_state(*saved)
replay_memory = []
with resume_file.open('r') as f:
num_samples = sum(1 for _ in f)
if num_samples > args.buffer_max_size:
samples_to_use = random.sample(range(num_samples),
args.buffer_max_size)
else:
samples_to_use = None
with resume_file.open('r') as f:
for (idx, line) in enumerate(f, start=1):
if num_samples > args.buffer_max_size and \
idx not in samples_to_use:
continue
try:
replay_memory.append(LabeledTransition.from_dict(
json.loads(line)))
except json.decoder.JSONDecodeError:
eprint(f"Problem loading line {idx}: {line}")
raise
already_done = []
graphs_done = []
with weights.with_suffix('.done').open('r') as f:
for line in f:
next_done = json.loads(line)
already_done.append((Path2(next_done[0]), next_done[1],
next_done[2]))
graphpath = (args.graphs_dir / next_done[1])\
.with_suffix(".png")
graph = ReinforceGraph.load(graphpath + ".json")
graphs_done.append((graphpath, graph))
return replay_memory, already_done, graphs_done
def generate_evaluation_details(args: argparse.Namespace, idx: int,
filename: Path2,
evaluator: StateEvaluator) -> FileSummary:
scrape_path = args.prelude / filename.with_suffix(".v.scrape")
interactions = list(read_all_text_data(scrape_path))
context_filter = get_context_filter(args.context_filter)
json_rows: List[Dict[str, Any]] = []
num_points = 0
num_close = 0
num_correct = 0
num_proofs = 0
doc, tag, text, line = Doc().ttl()
def write_highlighted(vernac: str) -> None:
nonlocal text
nonlocal tag
substrings = syntax_highlight(vernac)
for substring in substrings:
if isinstance(substring, ColoredString):
with tag('span', style=f'color:{substring.color}'):
text(substring.contents)
else:
text(substring)
def write_vernac(block: VernacBlock):
nonlocal tag
for command in block.commands:
with tag('code', klass='plaincommand'):
write_highlighted(command)
def generate_proof_evaluation_details(block: ProofBlock, region_idx: int):
nonlocal num_proofs
nonlocal num_close
nonlocal num_correct
nonlocal json_rows
num_proofs += 1
nonlocal num_points
distanced_tactics = label_distances(block.proof_interactions)
proof_length = len(distanced_tactics)
num_points += proof_length
with tag('div', klass='region'):
nonlocal evaluator
for idx, (interaction,
distance_from_end) in enumerate(distanced_tactics, 1):
if interaction.tactic.strip() in [
"Proof.", "Qed.", "Defined."
]:
with tag('code', klass='plaincommand'):
write_highlighted(interaction.tactic.strip("\n"))
doc.stag('br')
else:
predicted_distance_from_end = evaluator.scoreState(
interaction.context_before)
grade = grade_prediction(distance_from_end,
predicted_distance_from_end)
if grade == "goodcommand":
num_correct += 1
elif grade == "okaycommand":
num_close += 1
num_points += 1
json_rows.append({
"lemma": block.lemma_statement,
"hyps": interaction.context_before.hypotheses,
"goal": interaction.context_before.goal,
"actual-distance": distance_from_end,
"predicted-distance": predicted_distance_from_end,
"grade": grade
})
with tag('span',
('data-hyps', "\n".join(interaction.context_before.hypotheses)),
('data-goal', interaction.context_before.goal),
('data-actual-distance', str(distance_from_end)),
('data-predicted-distance', str(predicted_distance_from_end)),
('data-region', region_idx),
('data-index', idx),
klass='tactic'), \
tag('code', klass=grade):
text(interaction.tactic)
doc.stag('br')
def write_lemma_button(lemma_statement: str, region_idx: int):
nonlocal tag
nonlocal text
lemma_name = \
serapi_instance.lemma_name_from_statement(lemma_statement)
with tag('button', klass='collapsible',
id=f'collapsible-{region_idx}'):
with tag('code', klass='buttontext'):
write_highlighted(lemma_statement.strip())
def grade_prediction(correct_number: int, predicted_number: float) -> str:
distance = abs(correct_number - predicted_number)
if distance < 1:
return "goodcommand"
elif distance < 5:
return "okaycommand"
else:
return "badcommand"
with tag('html'):
header(tag, doc, text, details_css, details_javascript,
"Proverbot9001 Report")
with tag('body', onload='init()'), tag('pre'):
for idx, block in enumerate(get_blocks(interactions)):
if isinstance(block, VernacBlock):
write_vernac(block)
else:
assert isinstance(block, ProofBlock)
write_lemma_button(block.lemma_statement, idx)
generate_proof_evaluation_details(block, idx)
base = Path2(os.path.dirname(os.path.abspath(__file__)))
for extra_filename in extra_files:
(base.parent / "reports" / extra_filename).copyfile(args.output /
extra_filename)
with (args.output /
filename.with_suffix(".html").name).open(mode='w') as fout:
fout.write(doc.getvalue())
with (args.output /
filename.with_suffix(".json").name).open(mode='w') as fout:
for row in json_rows:
fout.write(json.dumps(row))
fout.write("\n")
return FileSummary(filename, num_close, num_correct, num_points,
num_proofs)
def report_file(args : argparse.Namespace,
training_args : argparse.Namespace,
context_filter_str : str,
filename : Path2) -> Optional['ResultStats']:
def make_predictions(num_predictions : int,
tactic_interactions : List[ScrapedTactic]) -> \
Tuple[Iterable[Tuple[ScrapedTactic, List[Prediction]]], float]:
if len(tactic_interactions) == 0:
return [], 0
chunk_size = args.chunk_size
total_loss = 0.
for tactic_interaction in tactic_interactions:
assert isinstance(tactic_interaction.goal, str)
inputs = [TacticContext(tactic_interaction.prev_tactics,
tactic_interaction.hypotheses,
format_goal(tactic_interaction.goal))
for tactic_interaction in tactic_interactions]
corrects = [tactic_interaction.tactic
for tactic_interaction in tactic_interactions]
predictions : List[List[Prediction]] = []
for inputs_chunk, corrects_chunk in zip(chunks(inputs, chunk_size),
chunks(corrects, chunk_size)):
predictions_chunk, loss = predictor.predictKTacticsWithLoss_batch(
inputs_chunk, args.num_predictions, corrects_chunk)
predictions += predictions_chunk
total_loss += loss
del inputs
del corrects
return list(zip(tactic_interactions, predictions)), \
total_loss / math.ceil(len(tactic_interactions) / chunk_size)
def merge_indexed(lic : Sequence[Tuple[int, T1]], lib : Sequence[Tuple[int,T2]]) \
-> Iterable[Union[T1, T2]]:
lic = list(reversed(lic))
lib = list(reversed(lib))
while lic and lib:
lst : List[Tuple[int, Any]] = (lic if lic[-1][0] < lib[-1][0] else lib) # type: ignore
yield lst.pop()[1]
yield from list(reversed([c for _, c in lic]))
yield from list(reversed([b for _, b in lib]))
def get_should_filter(data : MixedDataset) -> Iterable[Tuple[ScrapedCommand, bool]]:
list_data : List[ScrapedCommand] = list(data)
extended_list : List[Optional[ScrapedCommand]] = \
cast(List[Optional[ScrapedCommand]], list_data[1:]) + [None]
for point, nextpoint in zip(list_data, extended_list):
if isinstance(point, ScrapedTactic) \
and not re.match("\s*[{}]\s*", point.tactic):
if isinstance(nextpoint, ScrapedTactic):
yield(point, not context_filter({"goal":format_goal(point.goal),
"hyps":point.hypotheses},
point.tactic,
{"goal":format_goal(nextpoint.goal),
"hyps":nextpoint.hypotheses},
training_args))
else:
yield(point, not context_filter({"goal":format_goal(point.goal),
"hyps":point.hypotheses},
point.tactic,
{"goal":"",
"hyps":""},
training_args))
else:
yield (point, True)
try:
scrape_path = args.prelude / filename.with_suffix(".v.scrape")
interactions = list(read_text_data_singlethreaded(scrape_path))
print("Loaded {} interactions for file {}".format(len(interactions), filename))
except FileNotFoundError:
print("Couldn't find file {}, skipping...".format(scrape_path))
return None
context_filter = get_context_filter(context_filter_str)
command_results : List[CommandResult] = []
stats = ResultStats(str(filename))
indexed_filter_aware_interactions = list(enumerate(get_should_filter(interactions)))
for idx, (interaction, should_filter) in indexed_filter_aware_interactions:
assert isinstance(idx, int)
if not should_filter:
assert isinstance(interaction, ScrapedTactic), interaction
indexed_filter_aware_prediction_contexts, indexed_filter_aware_pass_through = \
multipartition(indexed_filter_aware_interactions,
lambda indexed_filter_aware_interaction:
indexed_filter_aware_interaction[1][1])
indexed_prediction_contexts: List[Tuple[int, ScrapedTactic]] = \
[(idx, cast(ScrapedTactic, obj)) for (idx, (obj, filtered))
in indexed_filter_aware_prediction_contexts]
indexed_pass_through = [(idx, cast(Union[ScrapedTactic, str], obj))
for (idx, (obj, filtered))
in indexed_filter_aware_pass_through]
for idx, prediction_context in indexed_prediction_contexts:
assert isinstance(idx, int)
assert isinstance(prediction_context, ScrapedTactic)
prediction_interactions, loss = \
make_predictions(args.num_predictions,
[prediction_context for idx, prediction_context
in indexed_prediction_contexts])
indexed_prediction_interactions = \
[(idx, prediction_interaction)
for (idx, prediction_context), prediction_interaction
in zip(indexed_prediction_contexts, prediction_interactions)]
interactions_with_predictions = \
list(merge_indexed(indexed_prediction_interactions, indexed_pass_through))
for inter in interactions_with_predictions:
if isinstance(inter, tuple) and not isinstance(inter, ScrapedTactic):
assert len(inter) == 2, inter
scraped, predictions_and_certainties \
= inter #cast(Tuple[ScrapedTactic, List[Prediction]], inter)
(prev_tactics, hyps, goal, correct_tactic) = scraped
prediction_results = [PredictionResult(prediction,
grade_prediction(scraped,
prediction),
certainty)
for prediction, certainty in
predictions_and_certainties]
command_results.append(TacticResult(correct_tactic, hyps, goal,
prediction_results))
stats.add_tactic(prediction_results,
correct_tactic)
elif isinstance(inter, ScrapedTactic):
command_results.append(TacticResult(inter.tactic,inter.hypotheses, inter.goal, []))
else:
command_results.append((inter,))
stats.set_loss(loss)
print("Finished grading file {}".format(filename))
write_html(args.output, filename, command_results, stats)
write_csv(args.output, filename, args, command_results, stats)
print("Finished output for file {}".format(filename))
return stats
