def check_vfile(vfile_path: Path2, includes: str, args: argparse.Namespace) -> None: html_path = vfile_path.with_suffix(".html") src_path = Path2(unescape_filename(vfile_path.stem)) if args.skip_incomplete: if not html_path.exists(): print(f"Skipping {src_path}") return else: assert html_path.exists(), \ f"Couldn't find HTML file for {src_path}. "\ f"Are you sure the report is completed?" src_f = src_path.with_suffix("") src_ext = src_path.suffix new_filename_path = Path2(str(src_f) + "_solution" + src_ext) vfile_path.copyfile(args.prelude / new_filename_path) result = subprocess.run(["coqc"] + includes.split() + [str(new_filename_path)], cwd=args.prelude, capture_output=True, encoding='utf8') assert result.returncode == 0, \ f"Returned a non zero errorcode {result.returncode}! \n"\ f"{result.stderr}" print(f"Checked {src_path}") if args.print_stdout: print(f"Output:\n{result.stdout}", end="")
def resume(resume_file: Path2, weights: Path2, q_estimator: QEstimator) -> \ Tuple[List[LabeledTransition], List[Job], List[Tuple[str, ReinforceGraph]]]: eprint("Looks like there was a session in progress for these weights! " "Resuming") q_estimator_name, *saved = \ torch.load(str(weights)) q_estimator.load_saved_state(*saved) replay_memory = [] with resume_file.open('r') as f: num_samples = sum(1 for _ in f) if num_samples > args.buffer_max_size: samples_to_use = random.sample(range(num_samples), args.buffer_max_size) else: samples_to_use = None with resume_file.open('r') as f: for (idx, line) in enumerate(f, start=1): if num_samples > args.buffer_max_size and \ idx not in samples_to_use: continue try: replay_memory.append(LabeledTransition.from_dict( json.loads(line))) except json.decoder.JSONDecodeError: eprint(f"Problem loading line {idx}: {line}") raise already_done = [] graphs_done = [] with weights.with_suffix('.done').open('r') as f: for line in f: next_done = json.loads(line) already_done.append((Path2(next_done[0]), next_done[1], next_done[2])) graphpath = (args.graphs_dir / next_done[1])\ .with_suffix(".png") graph = ReinforceGraph.load(graphpath + ".json") graphs_done.append((graphpath, graph)) return replay_memory, already_done, graphs_done
def generate_evaluation_details(args: argparse.Namespace, idx: int, filename: Path2, evaluator: StateEvaluator) -> FileSummary: scrape_path = args.prelude / filename.with_suffix(".v.scrape") interactions = list(read_all_text_data(scrape_path)) context_filter = get_context_filter(args.context_filter) json_rows: List[Dict[str, Any]] = [] num_points = 0 num_close = 0 num_correct = 0 num_proofs = 0 doc, tag, text, line = Doc().ttl() def write_highlighted(vernac: str) -> None: nonlocal text nonlocal tag substrings = syntax_highlight(vernac) for substring in substrings: if isinstance(substring, ColoredString): with tag('span', style=f'color:{substring.color}'): text(substring.contents) else: text(substring) def write_vernac(block: VernacBlock): nonlocal tag for command in block.commands: with tag('code', klass='plaincommand'): write_highlighted(command) def generate_proof_evaluation_details(block: ProofBlock, region_idx: int): nonlocal num_proofs nonlocal num_close nonlocal num_correct nonlocal json_rows num_proofs += 1 nonlocal num_points distanced_tactics = label_distances(block.proof_interactions) proof_length = len(distanced_tactics) num_points += proof_length with tag('div', klass='region'): nonlocal evaluator for idx, (interaction, distance_from_end) in enumerate(distanced_tactics, 1): if interaction.tactic.strip() in [ "Proof.", "Qed.", "Defined." ]: with tag('code', klass='plaincommand'): write_highlighted(interaction.tactic.strip("\n")) doc.stag('br') else: predicted_distance_from_end = evaluator.scoreState( interaction.context_before) grade = grade_prediction(distance_from_end, predicted_distance_from_end) if grade == "goodcommand": num_correct += 1 elif grade == "okaycommand": num_close += 1 num_points += 1 json_rows.append({ "lemma": block.lemma_statement, "hyps": interaction.context_before.hypotheses, "goal": interaction.context_before.goal, "actual-distance": distance_from_end, "predicted-distance": predicted_distance_from_end, "grade": grade }) with tag('span', ('data-hyps', "\n".join(interaction.context_before.hypotheses)), ('data-goal', interaction.context_before.goal), ('data-actual-distance', str(distance_from_end)), ('data-predicted-distance', str(predicted_distance_from_end)), ('data-region', region_idx), ('data-index', idx), klass='tactic'), \ tag('code', klass=grade): text(interaction.tactic) doc.stag('br') def write_lemma_button(lemma_statement: str, region_idx: int): nonlocal tag nonlocal text lemma_name = \ serapi_instance.lemma_name_from_statement(lemma_statement) with tag('button', klass='collapsible', id=f'collapsible-{region_idx}'): with tag('code', klass='buttontext'): write_highlighted(lemma_statement.strip()) def grade_prediction(correct_number: int, predicted_number: float) -> str: distance = abs(correct_number - predicted_number) if distance < 1: return "goodcommand" elif distance < 5: return "okaycommand" else: return "badcommand" with tag('html'): header(tag, doc, text, details_css, details_javascript, "Proverbot9001 Report") with tag('body', onload='init()'), tag('pre'): for idx, block in enumerate(get_blocks(interactions)): if isinstance(block, VernacBlock): write_vernac(block) else: assert isinstance(block, ProofBlock) write_lemma_button(block.lemma_statement, idx) generate_proof_evaluation_details(block, idx) base = Path2(os.path.dirname(os.path.abspath(__file__))) for extra_filename in extra_files: (base.parent / "reports" / extra_filename).copyfile(args.output / extra_filename) with (args.output / filename.with_suffix(".html").name).open(mode='w') as fout: fout.write(doc.getvalue()) with (args.output / filename.with_suffix(".json").name).open(mode='w') as fout: for row in json_rows: fout.write(json.dumps(row)) fout.write("\n") return FileSummary(filename, num_close, num_correct, num_points, num_proofs)
def report_file(args : argparse.Namespace, training_args : argparse.Namespace, context_filter_str : str, filename : Path2) -> Optional['ResultStats']: def make_predictions(num_predictions : int, tactic_interactions : List[ScrapedTactic]) -> \ Tuple[Iterable[Tuple[ScrapedTactic, List[Prediction]]], float]: if len(tactic_interactions) == 0: return [], 0 chunk_size = args.chunk_size total_loss = 0. for tactic_interaction in tactic_interactions: assert isinstance(tactic_interaction.goal, str) inputs = [TacticContext(tactic_interaction.prev_tactics, tactic_interaction.hypotheses, format_goal(tactic_interaction.goal)) for tactic_interaction in tactic_interactions] corrects = [tactic_interaction.tactic for tactic_interaction in tactic_interactions] predictions : List[List[Prediction]] = [] for inputs_chunk, corrects_chunk in zip(chunks(inputs, chunk_size), chunks(corrects, chunk_size)): predictions_chunk, loss = predictor.predictKTacticsWithLoss_batch( inputs_chunk, args.num_predictions, corrects_chunk) predictions += predictions_chunk total_loss += loss del inputs del corrects return list(zip(tactic_interactions, predictions)), \ total_loss / math.ceil(len(tactic_interactions) / chunk_size) def merge_indexed(lic : Sequence[Tuple[int, T1]], lib : Sequence[Tuple[int,T2]]) \ -> Iterable[Union[T1, T2]]: lic = list(reversed(lic)) lib = list(reversed(lib)) while lic and lib: lst : List[Tuple[int, Any]] = (lic if lic[-1][0] < lib[-1][0] else lib) # type: ignore yield lst.pop()[1] yield from list(reversed([c for _, c in lic])) yield from list(reversed([b for _, b in lib])) def get_should_filter(data : MixedDataset) -> Iterable[Tuple[ScrapedCommand, bool]]: list_data : List[ScrapedCommand] = list(data) extended_list : List[Optional[ScrapedCommand]] = \ cast(List[Optional[ScrapedCommand]], list_data[1:]) + [None] for point, nextpoint in zip(list_data, extended_list): if isinstance(point, ScrapedTactic) \ and not re.match("\s*[{}]\s*", point.tactic): if isinstance(nextpoint, ScrapedTactic): yield(point, not context_filter({"goal":format_goal(point.goal), "hyps":point.hypotheses}, point.tactic, {"goal":format_goal(nextpoint.goal), "hyps":nextpoint.hypotheses}, training_args)) else: yield(point, not context_filter({"goal":format_goal(point.goal), "hyps":point.hypotheses}, point.tactic, {"goal":"", "hyps":""}, training_args)) else: yield (point, True) try: scrape_path = args.prelude / filename.with_suffix(".v.scrape") interactions = list(read_text_data_singlethreaded(scrape_path)) print("Loaded {} interactions for file {}".format(len(interactions), filename)) except FileNotFoundError: print("Couldn't find file {}, skipping...".format(scrape_path)) return None context_filter = get_context_filter(context_filter_str) command_results : List[CommandResult] = [] stats = ResultStats(str(filename)) indexed_filter_aware_interactions = list(enumerate(get_should_filter(interactions))) for idx, (interaction, should_filter) in indexed_filter_aware_interactions: assert isinstance(idx, int) if not should_filter: assert isinstance(interaction, ScrapedTactic), interaction indexed_filter_aware_prediction_contexts, indexed_filter_aware_pass_through = \ multipartition(indexed_filter_aware_interactions, lambda indexed_filter_aware_interaction: indexed_filter_aware_interaction[1][1]) indexed_prediction_contexts: List[Tuple[int, ScrapedTactic]] = \ [(idx, cast(ScrapedTactic, obj)) for (idx, (obj, filtered)) in indexed_filter_aware_prediction_contexts] indexed_pass_through = [(idx, cast(Union[ScrapedTactic, str], obj)) for (idx, (obj, filtered)) in indexed_filter_aware_pass_through] for idx, prediction_context in indexed_prediction_contexts: assert isinstance(idx, int) assert isinstance(prediction_context, ScrapedTactic) prediction_interactions, loss = \ make_predictions(args.num_predictions, [prediction_context for idx, prediction_context in indexed_prediction_contexts]) indexed_prediction_interactions = \ [(idx, prediction_interaction) for (idx, prediction_context), prediction_interaction in zip(indexed_prediction_contexts, prediction_interactions)] interactions_with_predictions = \ list(merge_indexed(indexed_prediction_interactions, indexed_pass_through)) for inter in interactions_with_predictions: if isinstance(inter, tuple) and not isinstance(inter, ScrapedTactic): assert len(inter) == 2, inter scraped, predictions_and_certainties \ = inter #cast(Tuple[ScrapedTactic, List[Prediction]], inter) (prev_tactics, hyps, goal, correct_tactic) = scraped prediction_results = [PredictionResult(prediction, grade_prediction(scraped, prediction), certainty) for prediction, certainty in predictions_and_certainties] command_results.append(TacticResult(correct_tactic, hyps, goal, prediction_results)) stats.add_tactic(prediction_results, correct_tactic) elif isinstance(inter, ScrapedTactic): command_results.append(TacticResult(inter.tactic,inter.hypotheses, inter.goal, [])) else: command_results.append((inter,)) stats.set_loss(loss) print("Finished grading file {}".format(filename)) write_html(args.output, filename, command_results, stats) write_csv(args.output, filename, args, command_results, stats) print("Finished output for file {}".format(filename)) return stats