def build_graph(lemma_cmd, proof_cmds): lemma_name = lemma_name_from_statement(getAddBody(lemma_cmd)) graph = ProofGraph(getAddBody(lemma_cmd)) next_node_id = 1 states = [(get_id(lemma_cmd), graph.start_node)] state_map = {} for cmd in proof_cmds: if isAdd(cmd): most_recent_state, most_recent_node = states[-1] if shouldFilterCommand(cmd): state_map[get_id(cmd)] = most_recent_state continue states.append((get_id(cmd), graph.mkNode(sanitizeTactic(getAddBody(cmd)), datetime.fromtimestamp(get_time(cmd)), most_recent_node))) next_node_id += 1 if isFinishingProofCmd(getAddBody(cmd)): for statenum, node in states: graph.setNodeColor(node, "blue") if isFailed(cmd): most_recent_state, most_recent_node = states[-1] graph.setNodeColor(most_recent_node, "red") if isCancel(cmd): cancel_dest = getCancelDest(cmd) cancel_dest = state_map.get(cancel_dest, cancel_dest) while len(states) > 0 and states[-1][0] != cancel_dest: states.pop() assert len(states) > 0 if not os.path.exists("graphs"): os.mkdir("graphs") graph_filename = "graphs/" + lemma_name + ".svg" graph.draw(graph_filename) return graph
def pop_proof(cmds): states = [] proof_cmds = [] while len(cmds) > 0: cmd = cmds.pop(0) proof_cmds.append(cmd) if isAdd(cmd): states.append(get_id(cmd)) if isEndingProofCmd(getAddBody(cmd)): return proof_cmds if isCancel(cmd): cancel_dest = getCancelDest(cmd) while len(states) > 0 and states[-1] != cancel_dest: state_num = states.pop() if len(states) == 0: return proof_cmds return proof_cmds
def get_cancel_lengths(cmds): states = [] cancel_lengths = collections.Counter() for dat in cmds: if isAdd(dat): states.append(get_id(dat)-1) if isCancel(dat): cancel_length = 0 cancel_dest = getCancelDest(dat) if len(states) == 0: return collections.Counter() while states[-1] != cancel_dest: cancel_length += 1 states.pop() if len(states) == 0: return collections.Counter() cancel_lengths[cancel_length] += 1 return cancel_lengths
def get_stats(proof_cmds): all_tactics = collections.Counter() cancelled_tactics = collections.Counter() failed_tactics = collections.Counter() num_tactics = 0 num_cancellations = 0 num_failures = 0 history_stack = [] for cmd in proof_cmds: if isAdd(cmd): tactic = getAddBody(cmd) history_stack.append((get_id(cmd), tactic)) if isVernacCmd(tactic): continue if isGoalPunctuation(tactic): continue num_tactics += 1 stem = get_stem(tactic) all_tactics[stem] += 1 if isCancel(cmd): cancel_dest = getCancelDest(cmd) cancellation_size = 0 while len( history_stack) > 0 and history_stack[-1][0] != cancel_dest: state_num, tactic = history_stack.pop() if isVernacCmd(tactic): continue if isGoalPunctuation(tactic): continue if isFailed(cmd): failed_tactics[get_stem(tactic)] += 1 cancelled_tactics[get_stem(tactic)] += 1 cancellation_size += 1 if isFailed(cmd): num_failures += cancellation_size num_cancellations += cancellation_size assert len(history_stack) > 0 return ProofMetadata(1, num_tactics, num_cancellations, num_failures, all_tactics, cancelled_tactics, failed_tactics)
def main(): total_stats = ProofMetadata(0, 0, 0, 0, collections.Counter(), collections.Counter(), collections.Counter()) num_backtracked_proofs = 0 num_other_noninteractive_proofs = 0 num_changes_add_semi = 0 add_semi_times = Timestat() num_changes_remove_semi = 0 remove_semi_times = Timestat() num_changes_change_args = 0 change_args_times = Timestat() num_changes_lookup_args = 0 lookup_args_times = Timestat() num_changes_change_tactic = 0 change_tactic_times = Timestat() user_proof_session_counts = collections.Counter() with open("users.txt", 'r') as usersfile: profiles = loads(usersfile.read()) for user in get_users(logdir): for session in get_sessions(logdir, user): cmds = get_commands(logdir, user, session) is_interactive_session = \ sublist_contained(cmds, [isCancel, lambda entry: not isCancel(entry)]) if not is_interactive_session: continue preprocessed_cmds = preprocess_the_works(profiles, cmds) while len(preprocessed_cmds) > 0: lemma_cmd = pop_to_next_proof(preprocessed_cmds) if lemma_cmd == None: break proof_cmds = pop_proof(preprocessed_cmds) if not is_interactive_proof(proof_cmds): num_other_noninteractive_proofs += 1 continue user_proof_session_counts[user] += 1 if is_backtracked_proof(proof_cmds): num_backtracked_proofs += 1 continue # if not isFinishingProofCmd(getAddBody(getLastNonObserve(proof_cmds))): # continue graph = build_graph(lemma_cmd, proof_cmds) for node in graph.nodes: if len(node.children) > 1: if any([ child.command.strip() != node.children[0].command.strip() for child in node.children ]): solution = node.children[-1].command for attempt in node.children[:-1]: if attempt.command.strip() == solution.strip(): continue if re.fullmatch("\s*[*+-{}]\s*", attempt.command): continue attempt_time = node.children[ -1].timestamp - attempt.timestamp period_match = \ re.match("(.*)\.", attempt.command, re.DOTALL) attempt_minus_period = period_match.group( 1) if period_match else attempt.command solution_period_match = \ re.match("(.*)\.", solution, re.DOTALL) solution_minus_period = solution_period_match.group(1)\ if solution_period_match else solution if re.match( f"{escape_as_re(attempt_minus_period)}\s*;.*", solution): num_changes_add_semi += 1 print( f"Added semicolon: {attempt.command} -> {solution}" ) print(f"Took {attempt_time}") add_semi_times.add_time(attempt_time) elif re.match( f"{escape_as_re(solution_minus_period)}\s*;.*", attempt.command): num_changes_remove_semi += 1 print( f"Removed semicolon: {attempt.command} -> {solution}" ) print(f"Took {attempt_time}") remove_semi_times.add_time(attempt_time) elif re.match( f"{get_stem(attempt.command)}[^;]*\.", solution): num_changes_change_args += 1 print( f"Change args: {attempt.command} -> {solution}" ) print(f"Took {attempt_time}") change_args_times.add_time(attempt_time) elif re.match(f"\s*(Search|Check).*", solution) and \ re.match(f"{get_stem(attempt.command)}[^;]*\.", node.children[-1].children[-1].command): num_changes_lookup_args += 1 print( f"Lookup, then change args: {attempt.command} -> {solution} {node.children[-1].children[-1].command}" ) print(f"Took {attempt_time}") lookup_args_times.add_time(attempt_time) else: print( f"Other change: {attempt.command} -> {solution}" ) print(f"Took {attempt_time}") num_changes_change_tactic += 1 change_tactic_times.add_time(attempt_time) # print(lemma_name_from_statement(getAddBody(lemma_cmd)), # [node.command for node in node.children]) proof_stats = get_stats(proof_cmds) total_stats = add_stats(total_stats, proof_stats) add_semi_times.pp("add semi") remove_semi_times.pp("remove semi") change_args_times.pp("change args") lookup_args_times.pp("lookup args") change_tactic_times.pp("change tactic") print( f"Number of changes which add a semicolon after: {num_changes_add_semi}" ) print(f"Number of changes which remove a semicolon clause after: " f"{num_changes_remove_semi}") print(f"Number of changes which change the arguments to a tactic: " f"{num_changes_change_args}") print( f"Number of changes which change the arguments to a tactic, after lookup: " f"{num_changes_lookup_args}") print(f"Number of other changes: {num_changes_change_tactic}") print(f"{num_backtracked_proofs} backtracked proofs") print(f"{num_other_noninteractive_proofs} other non-interactive proofs") print( f"User interactive proof sessions count: {user_proof_session_counts}") print_proof_metadata(total_stats)
def is_backtracked_proof(proof_cmds): if isCancel(proof_cmds[-1]): return True return False
def main(): def addToTacPairCount(table, tac1, tac2): if re.match("[-+*\{\}]", tac1) or re.match("[-+*\{\}]", tac2): return if isVernacCmd(tac1) or isVernacCmd(tac2): return stem1 = get_stem(tac1) stem2 = get_stem(tac2) table[(stem1, stem2)] += 1 def addToTacticCount(table, tactic): if re.match("[-+*\{\}]", tactic): return if isVernacCmd(tactic): return stem = get_stem(tactic) table[stem] += 1 with open("users.txt", 'r') as usersfile: profiles = loads(usersfile.read()) all_tactics_count = collections.Counter() prev_count = collections.Counter() failed_prevs_count = collections.Counter() next_count = collections.Counter() cancelled_replaced_pairs = collections.Counter() cancel_lengths = collections.Counter() previous_tactic = '' just_cancelled = False just_failed = False total_cancels = 0 total_failures = 0 full_matching_cancels = 0 matching_stem_cancels = 0 for user in get_users(logdir): for session in get_sessions(logdir, user): cmds = get_commands(logdir, user, session) is_interactive = sublist_contained(cmds, [isCancel, lambda entry: not isCancel(entry)]) if not is_interactive: continue preprocessed_cmds = preprocess_vernac_backtrack( preprocess_failures(profiles, cmds)) cancel_lengths += get_cancel_lengths(preprocessed_cmds) history = [] for dat in preprocessed_cmds: if isAdd(dat): added_tactic = getAddBody(dat) if not isVernacCmd(added_tactic) and \ not isGoalPunctuation(added_tactic): stem = get_stem(added_tactic) assert not isVernacKeyword(stem), added_tactic assert stem.strip() != "", added_tactic all_tactics_count[stem] += 1 if just_cancelled: addToTacticCount(next_count, added_tactic) addToTacPairCount(cancelled_replaced_pairs, previous_tactic, added_tactic) if previous_tactic.strip() == added_tactic.strip(): full_matching_cancels += 1 try: if get_stem(previous_tactic) == get_stem(added_tactic): matching_stem_cancels += 1 except: pass just_cancelled = False previous_tactic = added_tactic history.append((get_id(dat)-1, added_tactic)) if isCancel(dat): total_cancels += 1 just_cancelled = True cancel_dest = getCancelDest(dat) while len(history) > 0 and history[-1][0] != cancel_dest: state_num, tactic = history.pop() addToTacticCount(prev_count, tactic) if isFailed(dat): total_failures += 1 addToTacticCount(failed_prevs_count, previous_tactic) print(cancel_lengths) print(f"Of {total_cancels} cancels, {total_failures} were failures ({100 * total_failures / total_cancels:3.2f}%)") print(f"Of those cancels, {full_matching_cancels} " f"({100 * full_matching_cancels / total_cancels:3.2f}%) " f"were replaced with the exact same tactic, " f"and {matching_stem_cancels} " f"({100 * matching_stem_cancels / total_cancels:3.2f}%) " f"were replaced by a tactic with the same stem") print("All tactics:") for tactic, count in all_tactics_count.most_common(50): print(f"{tactic}: {count} occurances") print("Tactics before cancel:") for tactic, count in prev_count.most_common(25): print(f"{tactic}: {count} cancels, {failed_prevs_count[tactic]} failures ({100 * failed_prevs_count[tactic] / count:3.2f}%)") print("Tactics after cancel:") for tactic, count in next_count.most_common(25): print(f"{tactic}: {count} run after cancel") print(cancelled_replaced_pairs.most_common(25))