def count_lengths(args : argparse.Namespace, filename : str):
print(f"Counting {filename}")
full_filename = args.prelude + "/" + filename
scraped_commands = list(read_all_text_data(Path2(full_filename + ".scrape")))
scraped_iter = iter(scraped_commands)
if args.post_linear:
original_commands = serapi_instance.load_commands_preserve(args, 0,
full_filename + ".lin")
else:
original_commands = serapi_instance.load_commands_preserve(args, 0, full_filename)
with open(full_filename + ".csv", 'w') as fout:
rowwriter = csv.writer(fout)
lemma_statement = ""
in_proof = False
cur_len = 0
for cmd in original_commands:
if not serapi_instance.possibly_starting_proof(cmd) and not in_proof:
continue
if serapi_instance.possibly_starting_proof(cmd) and not in_proof:
normalized_command = norm(cmd)
cur_scraped = norm(next(scraped_iter))
while cur_scraped != normalized_command:
cur_scraped = norm(next(scraped_iter))
try:
next_after_start = next(scraped_iter)
except StopIteration:
next_after_start = ""
if isinstance(next_after_start, ScrapedTactic):
lemma_statement = norm(cmd)
in_proof = True
cur_len = 0
else:
scraped_iter = itertools.chain([next_after_start], scraped_iter)
elif serapi_instance.ending_proof(cmd):
assert in_proof
rowwriter.writerow([lemma_statement.strip(),
cur_len])
cur_len = -1
in_proof=False
elif in_proof:
assert cur_len >= 0
if re.match("[{}]|[*-+]*$", norm(cmd)):
continue
if re.match("Proof\.", norm(cmd)):
continue
cur_len += 1
if args.add_semis or args.post_linear:
cur_len += count_outside_matching("\{\|", "\|\}", ";", norm(cmd))
return full_filename + ".csv"
def main():
parser = argparse.ArgumentParser(description="linearize a set of files")
parser.add_argument('--prelude', default=".")
parser.add_argument('--hardfail', default=False,
const=True, action='store_const')
parser.add_argument('-v', '--verbose', action='count', default=0)
parser.add_argument('--skip-nochange-tac', default=False, const=True,
action='store_const',
dest='skip_nochange_tac')
parser.add_argument("--progress",
action='store_const', const=True, default=False)
parser.add_argument("--linearizer-timeout",
type=int, default=(60 * 60 * 2))
parser.add_argument('filenames', nargs="+", help="proof file name (*.v)")
arg_values = parser.parse_args()
coqargs = ["sertop", "--implicit"]
for filename in arg_values.filenames:
if arg_values.verbose:
eprint("Linearizing {}".format(filename))
local_filename = arg_values.prelude + "/" + filename
original_commands = serapi_instance.load_commands_preserve(
arg_values, 0, arg_values.prelude + "/" + filename)
try:
fresh_commands = preprocess_file_commands(
arg_values, 0, original_commands,
coqargs, arg_values.prelude,
local_filename, filename, False)
serapi_instance.save_lin(fresh_commands, local_filename)
except CoqAnomaly:
serapi_instance.save_lin(original_commands, local_filename)
def get_linearized(args: argparse.Namespace, coqargs: List[str],
bar_idx: int, filename: str) -> List[str]:
local_filename = args.prelude + "/" + filename
loaded_commands = try_load_lin(
args, bar_idx, local_filename)
if loaded_commands is None:
original_commands = \
serapi_instance.load_commands_preserve(
args, bar_idx, args.prelude + "/" + filename)
try:
if args.linearizer_timeout:
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(args.linearizer_timeout)
fresh_commands = preprocess_file_commands(
args, bar_idx,
original_commands,
coqargs, args.prelude,
local_filename, filename, False)
signal.alarm(0)
except LinearizerTimeoutException:
fresh_commands = original_commands
except (CoqAnomaly, CoqExn):
fresh_commands = original_commands
save_lin(fresh_commands, local_filename)
return fresh_commands
else:
return loaded_commands
def get_proofs(args: argparse.Namespace,
t: Tuple[int, str]) -> List[Tuple[str, str, str]]:
idx, filename = t
with util.silent():
cmds = serapi_instance.load_commands_preserve(
args, idx, args.prelude / filename)
return [(filename, module, cmd) for module, cmd in
serapi_instance.lemmas_in_file(
filename, cmds, args.include_proof_relevant)]
def get_file_commands(args: argparse.Namespace, file_idx: int,
filename: str) -> List[str]:
local_filename = args.prelude + "/" + filename
loaded_commands = linearize_semicolons.try_load_lin(
args, file_idx, local_filename)
if loaded_commands is None:
print("Warning: this version of the reports can't linearize files! "
"Using original commands.")
return load_commands_preserve(args, file_idx, local_filename)
else:
return loaded_commands
def get_commands(self, args: argparse.Namespace, file_idx: int,
filename: str) -> List[str]:
local_filename = self.prelude + "/" + filename
loaded_commands = linearize_semicolons.try_load_lin(
args, file_idx, local_filename)
if loaded_commands is None:
fresh_commands = linearize_semicolons.preprocess_file_commands(
args, file_idx,
serapi_instance.load_commands_preserve(
args, file_idx, self.prelude + "/" + filename),
self.coqargs, self.includes, filename, local_filename,
self.skip_nochange_tac)
serapi_instance.save_lin(fresh_commands, local_filename)
return fresh_commands
else:
return loaded_commands
def scrape_file(coqargs: List[str], args: argparse.Namespace,
includes: str,
file_tuple: Tuple[int, str]) -> Optional[str]:
sys.setrecursionlimit(4500)
file_idx, filename = file_tuple
full_filename = args.prelude + "/" + filename
result_file = full_filename + ".scrape"
temp_file = full_filename + ".scrape.partial"
if args.cont:
with contextlib.suppress(FileNotFoundError):
with open(result_file, 'r') as f:
if args.verbose:
eprint(f"Found existing scrape at {result_file}! Using it")
return result_file
try:
if args.linearize:
commands = linearize_semicolons.get_linearized(args, coqargs, file_idx, filename)
else:
commands = serapi_instance.load_commands_preserve(
args, file_idx, str(full_filename))
with serapi_instance.SerapiContext(
coqargs,
serapi_instance.get_module_from_filename(filename),
args.prelude, args.relevant_lemmas == "hammer") as coq:
coq.verbose = args.verbose
try:
with open(temp_file, 'w') as f:
for command in tqdm(commands, file=sys.stdout,
disable=(not args.progress),
position=file_idx * 2,
desc="Scraping file", leave=False,
dynamic_ncols=True,
bar_format=mybarfmt):
process_statement(args, coq, command, f)
shutil.move(temp_file, result_file)
return result_file
except serapi_instance.TimeoutError:
eprint("Command in {} timed out.".format(filename))
return temp_file
except Exception as e:
eprint("FAILED: In file {}:".format(filename))
eprint(e)
if args.hardfail or len(args.inputs) == 1 or args.hardfail_scrape:
raise e
return None
def reinforce_worker(worker_idx: int,
args: argparse.Namespace,
lock: Lock,
namespace: multiprocessing.managers.Namespace,
samples: Queue[LabeledTransition],
jobs: Queue[Tuple[Job, Optional[Demonstration]]],
done: Queue[Tuple[Job,
Optional[Tuple[str,
ReinforceGraph]]]]):
sys.setrecursionlimit(100000)
failing_lemma = ""
try:
(next_file, next_module, next_lemma), demonstration = jobs.get_nowait()
except queue.Empty:
return
with util.silent():
all_commands = serapi_instance.load_commands_preserve(
args, worker_idx + 1, args.prelude / next_file)
rest_commands = all_commands
while rest_commands:
with serapi_instance.SerapiContext(["sertop", "--implicit"],
serapi_instance.
get_module_from_filename(str(next_file)),
str(args.prelude)) as coq:
coq.quiet = True
coq.verbose = args.verbose
while next_lemma:
try:
rest_commands, run_commands = coq.run_into_next_proof(
rest_commands)
if not rest_commands:
eprint(f"Couldn't find lemma {next_lemma}!")
break
except serapi_instance.CoqAnomaly:
with util.silent():
all_commands = serapi_instance.\
load_commands_preserve(
args, 0, args.prelude / next_file)
rest_commands = all_commands
break
except serapi_instance.SerapiException:
eprint(f"Failed getting to before: {next_lemma}")
eprint(f"In file {next_file}")
raise
lemma_statement = run_commands[-1]
if lemma_statement == next_lemma:
try:
graph_job = \
reinforce_lemma_multithreaded(args, coq,
lock, namespace,
worker_idx,
samples,
next_lemma,
next_module,
demonstration)
graphpath, graph = graph_job
graph.draw(graphpath + ".json")
except serapi_instance.CoqAnomaly:
if args.hardfail:
raise
if failing_lemma == lemma_statement:
eprint("Hit the same anomaly twice! Skipping",
guard=args.verbose >= 1)
done.put(((next_file, next_module, next_lemma),
None))
try:
(new_file, next_module, next_lemma), \
demonstration = jobs.get_nowait()
except queue.Empty:
return
if new_file != next_file:
next_file = new_file
with util.silent():
all_commands = serapi_instance.\
load_commands_preserve(
args, 0, args.prelude / next_file)
rest_commands = all_commands
break
else:
rest_commands = all_commands
else:
rest_commands = all_commands
failing_lemma = lemma_statement
break
except Exception as e:
if worker_idx == 0:
eprint(
f"FAILED in file {next_file}, "
f"lemma {next_lemma}")
eprint(e)
raise
serapi_instance.admit_proof(coq, lemma_statement)
while not serapi_instance.ending_proof(rest_commands[0]):
rest_commands = rest_commands[1:]
rest_commands = rest_commands[1:]
done.put(((next_file, next_module, next_lemma),
graph_job))
try:
(new_file, next_module, next_lemma), demonstration = \
jobs.get_nowait()
except queue.Empty:
return
if new_file != next_file:
next_file = new_file
with util.silent():
all_commands = serapi_instance.\
load_commands_preserve(
args, 0,
args.prelude / next_file)
rest_commands = all_commands
break
else:
proof_relevant = False
for cmd in rest_commands:
if serapi_instance.ending_proof(cmd):
if cmd.strip() == "Defined.":
proof_relevant = True
break
proof_relevant = proof_relevant or \
bool(re.match(
r"\s*Derive",
serapi_instance.kill_comments(lemma_statement))
) or\
bool(re.match(
r"\s*Let",
serapi_instance.kill_comments(lemma_statement))
) or\
bool(re.match(
r"\s*Equations",
serapi_instance.kill_comments(lemma_statement))
) or\
args.careful
if proof_relevant:
rest_commands, run_commands = coq.finish_proof(
rest_commands)
else:
try:
serapi_instance.admit_proof(coq, lemma_statement)
except serapi_instance.SerapiException:
next_lemma_name = \
serapi_instance.\
lemma_name_from_statement(next_lemma)
eprint(
f"{next_file}: Failed to admit proof "
f"{next_lemma_name}")
raise
while not serapi_instance.ending_proof(
rest_commands[0]):
rest_commands = rest_commands[1:]
rest_commands = rest_commands[1:]