def create_testbook(self, formula): testbook = [] if not self.args.keep_mutants: testcase = "%s/%s.smt2" % (self.args.scratchfolder, self.args.name) else: testcase = "%s/%s-%s-%s.smt2" % (self.args.scratchfolder, escape(self.currentseeds), self.args.name, random_string()) with open(testcase, 'w') as testcase_writer: testcase_writer.write(formula.__str__()) for cli in self.args.SOLVER_CLIS: if self.args.optfuzz != None: if not self.args.keep_mutants: testcase = "%s/%s-%s" % (self.args.scratchfolder, plain(cli), self.args.name) else: testcase = "%s/%s-%s-%s-%s.smt2" % ( self.args.scratchfolder, plain(cli), escape(self.currentseeds), self.args.name, random_string()) with open(testcase, 'w') as testcase_writer: testcase_writer.write( self.args.optfuzz.generate(cli) + formula.__str__()) testbook.append((cli, testcase)) return testbook
def report(self, trigger, bugtype, cli, output, report_id, reference=None, bad_result=None): plain_cli = plain(cli) #<solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2 report = "%s/%s-%s-%s-%s.smt2" %(self.args.bugsfolder, bugtype, plain_cli, escape(self.currentseeds), report_id) try: shutil.copy(trigger, report) except Exception as e: print(e) exit(0) logpath = "%s/%s-%s-%s-%s.output" %(self.args.bugsfolder, bugtype, plain_cli, escape(self.currentseeds), report_id) log_str = "" if self.runforever: with open(logpath, 'w') as log: log.write("stderr:\n") log.write((output.stderr).decode("utf-8")) log.write("stdout:\n") log.write((output.stdout).decode("utf-8")) else: log_str = "\nBUG (%s): %s %s \n" %(bugtype.upper(), cli, report) if bugtype == "incorrect": log_str +="*** REF results\n" log_str += reference.__str__()+"\n" log_str +="*** BAD results\n" log_str += bad_result.__str__()+"\n" else: log_str += output.stdout.decode("utf-8") log_str += "\n" log_str += output.stderr.decode("utf-8") return report_id, log_str
def report_diff(self, scratchfile, bugtype, ref_cli, ref_stdout, ref_stderr, sol_cli, sol_stdout, sol_stderr, report_id): plain_cli = plain(sol_cli) #format: <solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2 report = "%s/%s-%s-%s-%s.smt2" % (self.args.bugsfolder, bugtype, plain_cli, escape( self.currentseeds), report_id) try: shutil.copy(scratchfile, report) except Exception as e: print(e) exit(0) logpath = "%s/%s-%s-%s-%s.output" % ( self.args.bugsfolder, bugtype, plain_cli, escape( self.currentseeds), report_id) with open(logpath, 'w') as log: log.write("*** REFERENCE \n") log.write("command: " + ref_cli + "\n") log.write("stderr:\n") log.write(ref_stderr) log.write("stdout:\n") log.write(ref_stdout) log.write("\n\n*** INCORRECT \n") log.write("command: " + sol_cli + "\n") log.write("stderr:\n") log.write(sol_stderr) log.write("stdout:\n") log.write(sol_stdout) return report_id
def report(self, trigger, bugtype, cli, output, report_id): plain_cli = plain(cli) #<solver><{crash,wrong,invalid_model}><seed-name>.<random-string>.smt2 report = "%s/%s-%s-%s-%s.smt2" % (self.args.bugsfolder, bugtype, plain_cli, escape( self.currentseeds), report_id) try: shutil.copy(trigger, report) except Exception as e: print(e) exit(0) logpath = "%s/%s-%s-%s-%s.output" % ( self.args.bugsfolder, bugtype, plain_cli, escape( self.currentseeds), report_id) with open(logpath, 'w') as log: log.write("stderr:\n") log.write((output.stderr).decode("utf-8")) log.write("stdout:\n") log.write((output.stdout).decode("utf-8")) return report_id
def validate(self, formula): ret = True log_str = "" if (self.args.oracle == "unknown"): oracle = SolverResult(SolverResultType.UNKNOWN) elif (self.args.oracle == "sat"): oracle = SolverResult(SolverResultType.SAT) elif (self.args.oracle == "unsat"): oracle = SolverResult(SolverResultType.UNSAT) else: assert(False) testbook = [] for cli in self.args.SOLVER_CLIS: if not self.args.keep_mutants: testcase = "%s/%s.smt2" % (self.args.scratchfolder, self.args.name) else: testcase = "%s/%s-%s-%s-%s.smt2" % (self.args.scratchfolder,\ plain(cli),\ escape(self.currentseeds),\ self.args.name,\ random_string()) with open(testcase, 'w') as testcase_writer: if self.args.optfuzz != None: testcase_writer.write(self.args.optfuzz.generate(cli) + formula.__str__()) else: testcase_writer.write(formula.__str__()) testbook.append((cli,testcase)) reference = ("", "", "") for testitem in testbook: solver = Solver(testitem[0]) result, output = solver.solve(testitem[1], self.args.timeout) # print("result", result) # print("output", output) if result.equals(SolverResultType.IGNORED): #ignored self.statistic.ignored += 1 elif result.equals(SolverResultType.UNKNOWN): #unknown pass elif result.equals(SolverResultType.TIMEOUT): #timeout self.statistic.timeout += 1 elif result.equals(SolverResultType.CRASH): #crash self.statistic.error += 1 _,log_str = self.report(testitem[1], "crash", testitem[0], output, random_string()) ret = False elif result.equals(SolverResultType.FAIL): print("\nPlease check your solver command-line interfaces.") exit(1) else: if oracle.equals(SolverResultType.UNKNOWN): oracle = result reference = (testitem[0], testitem[1], output) elif oracle.equals(result): #correct pass else: #incorrect self.statistic.soundness += 1 report_id, log_str = self.report(testitem[1], "incorrect", testitem[0], output, random_string(),oracle,result) if reference != ("", "", ""): _ ,log_str = self.report(reference[1], "incorrect", reference[0], reference[2], report_id, oracle, result) ret = False if not self.args.keep_mutants: pass # print("testcase", testcase) # os.remove(testcase) return ret, log_str
def validate(self, fn): ret = True if (self.args.oracle == "unknown"): oracle = SolverResult(SolverResultType.UNKNOWN) elif (self.args.oracle == "sat"): oracle = SolverResult(SolverResultType.SAT) elif (self.args.oracle == "unsat"): oracle = SolverResult(SolverResultType.UNSAT) else: assert (False) testbook = [] for cli in self.args.SOLVER_CLIS: testcase = fn if self.args.optfuzz != None: with open(fn, 'r') as generated_test: pure_formula = generated_test.read() testcase = "%s/%s-%s" % (self.args.scratchfolder, plain(cli), fn.split('/')[-1]) with open(testcase, 'w') as testcase_writer: testcase_writer.write( self.args.optfuzz.generate(cli) + pure_formula) testbook.append((cli, testcase)) reference = ("", "", "") for testitem in testbook: solver = Solver(testitem[0]) result, output = solver.solve(testitem[1], self.args.timeout) if result.equals(SolverResultType.IGNORED): #ignored self.statistic.ignored += 1 elif result.equals(SolverResultType.UNKNOWN): #unknown pass elif result.equals(SolverResultType.TIMEOUT): #timeout self.statistic.timeout += 1 elif result.equals(SolverResultType.CRASH): #crash self.statistic.error += 1 self.report(testitem[1], "crash", testitem[0], output, random_string()) ret = False elif result.equals(SolverResultType.FAIL): print("\nPlease check your solver command-line interfaces.") else: if oracle.equals(SolverResultType.UNKNOWN): oracle = result reference = (testitem[0], testitem[1], output) elif oracle.equals(result): #correct pass else: #incorrect self.statistic.soundness += 1 report_id = self.report(testitem[1], "incorrect", testitem[0], output, random_string()) if reference != ("", "", ""): self.report(reference[1], "incorrect", reference[0], reference[2], report_id) ret = False return ret