def create_testbook(self, formula):
testbook = []
if not self.args.keep_mutants:
testcase = "%s/%s.smt2" % (self.args.scratchfolder, self.args.name)
else:
testcase = "%s/%s-%s-%s.smt2" % (self.args.scratchfolder,
escape(self.currentseeds),
self.args.name, random_string())
with open(testcase, 'w') as testcase_writer:
testcase_writer.write(formula.__str__())
for cli in self.args.SOLVER_CLIS:
if self.args.optfuzz != None:
if not self.args.keep_mutants:
testcase = "%s/%s-%s" % (self.args.scratchfolder,
plain(cli), self.args.name)
else:
testcase = "%s/%s-%s-%s-%s.smt2" % (
self.args.scratchfolder, plain(cli),
escape(self.currentseeds), self.args.name,
random_string())
with open(testcase, 'w') as testcase_writer:
testcase_writer.write(
self.args.optfuzz.generate(cli) + formula.__str__())
testbook.append((cli, testcase))
return testbook
def report(self, trigger, bugtype, cli, output, report_id, reference=None, bad_result=None):
plain_cli = plain(cli)
#<solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2
report = "%s/%s-%s-%s-%s.smt2" %(self.args.bugsfolder, bugtype, plain_cli, escape(self.currentseeds), report_id)
try:
shutil.copy(trigger, report)
except Exception as e:
print(e)
exit(0)
logpath = "%s/%s-%s-%s-%s.output" %(self.args.bugsfolder, bugtype, plain_cli, escape(self.currentseeds), report_id)
log_str = ""
if self.runforever:
with open(logpath, 'w') as log:
log.write("stderr:\n")
log.write((output.stderr).decode("utf-8"))
log.write("stdout:\n")
log.write((output.stdout).decode("utf-8"))
else:
log_str = "\nBUG (%s): %s %s \n" %(bugtype.upper(), cli, report)
if bugtype == "incorrect":
log_str +="*** REF results\n"
log_str += reference.__str__()+"\n"
log_str +="*** BAD results\n"
log_str += bad_result.__str__()+"\n"
else:
log_str += output.stdout.decode("utf-8")
log_str += "\n"
log_str += output.stderr.decode("utf-8")
return report_id, log_str
def report_diff(self, scratchfile, bugtype, ref_cli, ref_stdout,
ref_stderr, sol_cli, sol_stdout, sol_stderr, report_id):
plain_cli = plain(sol_cli)
#format: <solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2
report = "%s/%s-%s-%s-%s.smt2" % (self.args.bugsfolder, bugtype,
plain_cli, escape(
self.currentseeds), report_id)
try:
shutil.copy(scratchfile, report)
except Exception as e:
print(e)
exit(0)
logpath = "%s/%s-%s-%s-%s.output" % (
self.args.bugsfolder, bugtype, plain_cli, escape(
self.currentseeds), report_id)
with open(logpath, 'w') as log:
log.write("*** REFERENCE \n")
log.write("command: " + ref_cli + "\n")
log.write("stderr:\n")
log.write(ref_stderr)
log.write("stdout:\n")
log.write(ref_stdout)
log.write("\n\n*** INCORRECT \n")
log.write("command: " + sol_cli + "\n")
log.write("stderr:\n")
log.write(sol_stderr)
log.write("stdout:\n")
log.write(sol_stdout)
return report_id
def report(self, trigger, bugtype, cli, output, report_id):
plain_cli = plain(cli)
#<solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2
report = "%s/%s-%s-%s-%s.smt2" % (self.args.bugsfolder, bugtype,
plain_cli, escape(
self.currentseeds), report_id)
try:
shutil.copy(trigger, report)
except Exception as e:
print(e)
exit(0)
logpath = "%s/%s-%s-%s-%s.output" % (
self.args.bugsfolder, bugtype, plain_cli, escape(
self.currentseeds), report_id)
with open(logpath, 'w') as log:
log.write("stderr:\n")
log.write((output.stderr).decode("utf-8"))
log.write("stdout:\n")
log.write((output.stdout).decode("utf-8"))
return report_id
def validate(self, formula):
ret = True
log_str = ""
if (self.args.oracle == "unknown"):
oracle = SolverResult(SolverResultType.UNKNOWN)
elif (self.args.oracle == "sat"):
oracle = SolverResult(SolverResultType.SAT)
elif (self.args.oracle == "unsat"):
oracle = SolverResult(SolverResultType.UNSAT)
else: assert(False)
testbook = []
for cli in self.args.SOLVER_CLIS:
if not self.args.keep_mutants:
testcase = "%s/%s.smt2" % (self.args.scratchfolder, self.args.name)
else:
testcase = "%s/%s-%s-%s-%s.smt2" % (self.args.scratchfolder,\
plain(cli),\
escape(self.currentseeds),\
self.args.name,\
random_string())
with open(testcase, 'w') as testcase_writer:
if self.args.optfuzz != None:
testcase_writer.write(self.args.optfuzz.generate(cli) + formula.__str__())
else:
testcase_writer.write(formula.__str__())
testbook.append((cli,testcase))
reference = ("", "", "")
for testitem in testbook:
solver = Solver(testitem[0])
result, output = solver.solve(testitem[1], self.args.timeout)
# print("result", result)
# print("output", output)
if result.equals(SolverResultType.IGNORED):
#ignored
self.statistic.ignored += 1
elif result.equals(SolverResultType.UNKNOWN):
#unknown
pass
elif result.equals(SolverResultType.TIMEOUT):
#timeout
self.statistic.timeout += 1
elif result.equals(SolverResultType.CRASH):
#crash
self.statistic.error += 1
_,log_str = self.report(testitem[1], "crash", testitem[0], output, random_string())
ret = False
elif result.equals(SolverResultType.FAIL):
print("\nPlease check your solver command-line interfaces.")
exit(1)
else:
if oracle.equals(SolverResultType.UNKNOWN):
oracle = result
reference = (testitem[0], testitem[1], output)
elif oracle.equals(result):
#correct
pass
else:
#incorrect
self.statistic.soundness += 1
report_id, log_str = self.report(testitem[1], "incorrect", testitem[0], output, random_string(),oracle,result)
if reference != ("", "", ""):
_ ,log_str = self.report(reference[1], "incorrect", reference[0], reference[2], report_id, oracle, result)
ret = False
if not self.args.keep_mutants:
pass
# print("testcase", testcase)
# os.remove(testcase)
return ret, log_str
def validate(self, fn):
ret = True
if (self.args.oracle == "unknown"):
oracle = SolverResult(SolverResultType.UNKNOWN)
elif (self.args.oracle == "sat"):
oracle = SolverResult(SolverResultType.SAT)
elif (self.args.oracle == "unsat"):
oracle = SolverResult(SolverResultType.UNSAT)
else:
assert (False)
testbook = []
for cli in self.args.SOLVER_CLIS:
testcase = fn
if self.args.optfuzz != None:
with open(fn, 'r') as generated_test:
pure_formula = generated_test.read()
testcase = "%s/%s-%s" % (self.args.scratchfolder, plain(cli),
fn.split('/')[-1])
with open(testcase, 'w') as testcase_writer:
testcase_writer.write(
self.args.optfuzz.generate(cli) + pure_formula)
testbook.append((cli, testcase))
reference = ("", "", "")
for testitem in testbook:
solver = Solver(testitem[0])
result, output = solver.solve(testitem[1], self.args.timeout)
if result.equals(SolverResultType.IGNORED):
#ignored
self.statistic.ignored += 1
elif result.equals(SolverResultType.UNKNOWN):
#unknown
pass
elif result.equals(SolverResultType.TIMEOUT):
#timeout
self.statistic.timeout += 1
elif result.equals(SolverResultType.CRASH):
#crash
self.statistic.error += 1
self.report(testitem[1], "crash", testitem[0], output,
random_string())
ret = False
elif result.equals(SolverResultType.FAIL):
print("\nPlease check your solver command-line interfaces.")
else:
if oracle.equals(SolverResultType.UNKNOWN):
oracle = result
reference = (testitem[0], testitem[1], output)
elif oracle.equals(result):
#correct
pass
else:
#incorrect
self.statistic.soundness += 1
report_id = self.report(testitem[1],
"incorrect", testitem[0], output,
random_string())
if reference != ("", "", ""):
self.report(reference[1], "incorrect", reference[0],
reference[2], report_id)
ret = False
return ret