def build_graph(lemma_cmd, proof_cmds):
lemma_name = lemma_name_from_statement(getAddBody(lemma_cmd))
graph = ProofGraph(getAddBody(lemma_cmd))
next_node_id = 1
states = [(get_id(lemma_cmd), graph.start_node)]
state_map = {}
for cmd in proof_cmds:
if isAdd(cmd):
most_recent_state, most_recent_node = states[-1]
if shouldFilterCommand(cmd):
state_map[get_id(cmd)] = most_recent_state
continue
states.append((get_id(cmd),
graph.mkNode(sanitizeTactic(getAddBody(cmd)),
datetime.fromtimestamp(get_time(cmd)),
most_recent_node)))
next_node_id += 1
if isFinishingProofCmd(getAddBody(cmd)):
for statenum, node in states:
graph.setNodeColor(node, "blue")
if isFailed(cmd):
most_recent_state, most_recent_node = states[-1]
graph.setNodeColor(most_recent_node, "red")
if isCancel(cmd):
cancel_dest = getCancelDest(cmd)
cancel_dest = state_map.get(cancel_dest, cancel_dest)
while len(states) > 0 and states[-1][0] != cancel_dest:
states.pop()
assert len(states) > 0
if not os.path.exists("graphs"):
os.mkdir("graphs")
graph_filename = "graphs/" + lemma_name + ".svg"
graph.draw(graph_filename)
return graph
def pop_proof(cmds):
states = []
proof_cmds = []
while len(cmds) > 0:
cmd = cmds.pop(0)
proof_cmds.append(cmd)
if isAdd(cmd):
states.append(get_id(cmd))
if isEndingProofCmd(getAddBody(cmd)):
return proof_cmds
if isCancel(cmd):
cancel_dest = getCancelDest(cmd)
while len(states) > 0 and states[-1] != cancel_dest:
state_num = states.pop()
if len(states) == 0:
return proof_cmds
return proof_cmds
def get_cancel_lengths(cmds):
states = []
cancel_lengths = collections.Counter()
for dat in cmds:
if isAdd(dat):
states.append(get_id(dat)-1)
if isCancel(dat):
cancel_length = 0
cancel_dest = getCancelDest(dat)
if len(states) == 0:
return collections.Counter()
while states[-1] != cancel_dest:
cancel_length += 1
states.pop()
if len(states) == 0:
return collections.Counter()
cancel_lengths[cancel_length] += 1
return cancel_lengths
def get_stats(proof_cmds):
all_tactics = collections.Counter()
cancelled_tactics = collections.Counter()
failed_tactics = collections.Counter()
num_tactics = 0
num_cancellations = 0
num_failures = 0
history_stack = []
for cmd in proof_cmds:
if isAdd(cmd):
tactic = getAddBody(cmd)
history_stack.append((get_id(cmd), tactic))
if isVernacCmd(tactic):
continue
if isGoalPunctuation(tactic):
continue
num_tactics += 1
stem = get_stem(tactic)
all_tactics[stem] += 1
if isCancel(cmd):
cancel_dest = getCancelDest(cmd)
cancellation_size = 0
while len(
history_stack) > 0 and history_stack[-1][0] != cancel_dest:
state_num, tactic = history_stack.pop()
if isVernacCmd(tactic):
continue
if isGoalPunctuation(tactic):
continue
if isFailed(cmd):
failed_tactics[get_stem(tactic)] += 1
cancelled_tactics[get_stem(tactic)] += 1
cancellation_size += 1
if isFailed(cmd):
num_failures += cancellation_size
num_cancellations += cancellation_size
assert len(history_stack) > 0
return ProofMetadata(1, num_tactics, num_cancellations, num_failures,
all_tactics, cancelled_tactics, failed_tactics)
def main():
total_stats = ProofMetadata(0, 0, 0, 0, collections.Counter(),
collections.Counter(), collections.Counter())
num_backtracked_proofs = 0
num_other_noninteractive_proofs = 0
num_changes_add_semi = 0
add_semi_times = Timestat()
num_changes_remove_semi = 0
remove_semi_times = Timestat()
num_changes_change_args = 0
change_args_times = Timestat()
num_changes_lookup_args = 0
lookup_args_times = Timestat()
num_changes_change_tactic = 0
change_tactic_times = Timestat()
user_proof_session_counts = collections.Counter()
with open("users.txt", 'r') as usersfile:
profiles = loads(usersfile.read())
for user in get_users(logdir):
for session in get_sessions(logdir, user):
cmds = get_commands(logdir, user, session)
is_interactive_session = \
sublist_contained(cmds, [isCancel, lambda entry: not isCancel(entry)])
if not is_interactive_session:
continue
preprocessed_cmds = preprocess_the_works(profiles, cmds)
while len(preprocessed_cmds) > 0:
lemma_cmd = pop_to_next_proof(preprocessed_cmds)
if lemma_cmd == None:
break
proof_cmds = pop_proof(preprocessed_cmds)
if not is_interactive_proof(proof_cmds):
num_other_noninteractive_proofs += 1
continue
user_proof_session_counts[user] += 1
if is_backtracked_proof(proof_cmds):
num_backtracked_proofs += 1
continue
# if not isFinishingProofCmd(getAddBody(getLastNonObserve(proof_cmds))):
# continue
graph = build_graph(lemma_cmd, proof_cmds)
for node in graph.nodes:
if len(node.children) > 1:
if any([
child.command.strip() !=
node.children[0].command.strip()
for child in node.children
]):
solution = node.children[-1].command
for attempt in node.children[:-1]:
if attempt.command.strip() == solution.strip():
continue
if re.fullmatch("\s*[*+-{}]\s*",
attempt.command):
continue
attempt_time = node.children[
-1].timestamp - attempt.timestamp
period_match = \
re.match("(.*)\.", attempt.command, re.DOTALL)
attempt_minus_period = period_match.group(
1) if period_match else attempt.command
solution_period_match = \
re.match("(.*)\.", solution, re.DOTALL)
solution_minus_period = solution_period_match.group(1)\
if solution_period_match else solution
if re.match(
f"{escape_as_re(attempt_minus_period)}\s*;.*",
solution):
num_changes_add_semi += 1
print(
f"Added semicolon: {attempt.command} -> {solution}"
)
print(f"Took {attempt_time}")
add_semi_times.add_time(attempt_time)
elif re.match(
f"{escape_as_re(solution_minus_period)}\s*;.*",
attempt.command):
num_changes_remove_semi += 1
print(
f"Removed semicolon: {attempt.command} -> {solution}"
)
print(f"Took {attempt_time}")
remove_semi_times.add_time(attempt_time)
elif re.match(
f"{get_stem(attempt.command)}[^;]*\.",
solution):
num_changes_change_args += 1
print(
f"Change args: {attempt.command} -> {solution}"
)
print(f"Took {attempt_time}")
change_args_times.add_time(attempt_time)
elif re.match(f"\s*(Search|Check).*", solution) and \
re.match(f"{get_stem(attempt.command)}[^;]*\.",
node.children[-1].children[-1].command):
num_changes_lookup_args += 1
print(
f"Lookup, then change args: {attempt.command} -> {solution} {node.children[-1].children[-1].command}"
)
print(f"Took {attempt_time}")
lookup_args_times.add_time(attempt_time)
else:
print(
f"Other change: {attempt.command} -> {solution}"
)
print(f"Took {attempt_time}")
num_changes_change_tactic += 1
change_tactic_times.add_time(attempt_time)
# print(lemma_name_from_statement(getAddBody(lemma_cmd)),
# [node.command for node in node.children])
proof_stats = get_stats(proof_cmds)
total_stats = add_stats(total_stats, proof_stats)
add_semi_times.pp("add semi")
remove_semi_times.pp("remove semi")
change_args_times.pp("change args")
lookup_args_times.pp("lookup args")
change_tactic_times.pp("change tactic")
print(
f"Number of changes which add a semicolon after: {num_changes_add_semi}"
)
print(f"Number of changes which remove a semicolon clause after: "
f"{num_changes_remove_semi}")
print(f"Number of changes which change the arguments to a tactic: "
f"{num_changes_change_args}")
print(
f"Number of changes which change the arguments to a tactic, after lookup: "
f"{num_changes_lookup_args}")
print(f"Number of other changes: {num_changes_change_tactic}")
print(f"{num_backtracked_proofs} backtracked proofs")
print(f"{num_other_noninteractive_proofs} other non-interactive proofs")
print(
f"User interactive proof sessions count: {user_proof_session_counts}")
print_proof_metadata(total_stats)
def is_backtracked_proof(proof_cmds):
if isCancel(proof_cmds[-1]):
return True
return False
def main():
def addToTacPairCount(table, tac1, tac2):
if re.match("[-+*\{\}]", tac1) or re.match("[-+*\{\}]", tac2):
return
if isVernacCmd(tac1) or isVernacCmd(tac2):
return
stem1 = get_stem(tac1)
stem2 = get_stem(tac2)
table[(stem1, stem2)] += 1
def addToTacticCount(table, tactic):
if re.match("[-+*\{\}]", tactic):
return
if isVernacCmd(tactic):
return
stem = get_stem(tactic)
table[stem] += 1
with open("users.txt", 'r') as usersfile:
profiles = loads(usersfile.read())
all_tactics_count = collections.Counter()
prev_count = collections.Counter()
failed_prevs_count = collections.Counter()
next_count = collections.Counter()
cancelled_replaced_pairs = collections.Counter()
cancel_lengths = collections.Counter()
previous_tactic = ''
just_cancelled = False
just_failed = False
total_cancels = 0
total_failures = 0
full_matching_cancels = 0
matching_stem_cancels = 0
for user in get_users(logdir):
for session in get_sessions(logdir, user):
cmds = get_commands(logdir, user, session)
is_interactive = sublist_contained(cmds, [isCancel, lambda entry: not isCancel(entry)])
if not is_interactive:
continue
preprocessed_cmds = preprocess_vernac_backtrack(
preprocess_failures(profiles, cmds))
cancel_lengths += get_cancel_lengths(preprocessed_cmds)
history = []
for dat in preprocessed_cmds:
if isAdd(dat):
added_tactic = getAddBody(dat)
if not isVernacCmd(added_tactic) and \
not isGoalPunctuation(added_tactic):
stem = get_stem(added_tactic)
assert not isVernacKeyword(stem), added_tactic
assert stem.strip() != "", added_tactic
all_tactics_count[stem] += 1
if just_cancelled:
addToTacticCount(next_count, added_tactic)
addToTacPairCount(cancelled_replaced_pairs,
previous_tactic, added_tactic)
if previous_tactic.strip() == added_tactic.strip():
full_matching_cancels += 1
try:
if get_stem(previous_tactic) == get_stem(added_tactic):
matching_stem_cancels += 1
except:
pass
just_cancelled = False
previous_tactic = added_tactic
history.append((get_id(dat)-1, added_tactic))
if isCancel(dat):
total_cancels += 1
just_cancelled = True
cancel_dest = getCancelDest(dat)
while len(history) > 0 and history[-1][0] != cancel_dest:
state_num, tactic = history.pop()
addToTacticCount(prev_count, tactic)
if isFailed(dat):
total_failures += 1
addToTacticCount(failed_prevs_count, previous_tactic)
print(cancel_lengths)
print(f"Of {total_cancels} cancels, {total_failures} were failures ({100 * total_failures / total_cancels:3.2f}%)")
print(f"Of those cancels, {full_matching_cancels} "
f"({100 * full_matching_cancels / total_cancels:3.2f}%) "
f"were replaced with the exact same tactic, "
f"and {matching_stem_cancels} "
f"({100 * matching_stem_cancels / total_cancels:3.2f}%) "
f"were replaced by a tactic with the same stem")
print("All tactics:")
for tactic, count in all_tactics_count.most_common(50):
print(f"{tactic}: {count} occurances")
print("Tactics before cancel:")
for tactic, count in prev_count.most_common(25):
print(f"{tactic}: {count} cancels, {failed_prevs_count[tactic]} failures ({100 * failed_prevs_count[tactic] / count:3.2f}%)")
print("Tactics after cancel:")
for tactic, count in next_count.most_common(25):
print(f"{tactic}: {count} run after cancel")
print(cancelled_replaced_pairs.most_common(25))