def main(): global start,config,sut,R,nonerror,error,file_name,num num = 0 file_name = 'failurefile' parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) sut = SUT.sut() R = random.Random(config.seed) start = time.time() elapsed = time.time() - start states = [sut.state()] nonerror = [] error = [] news = None while(time.time() < start + config.timeout): for s in states: sut.restart() sut.backtrack(s) for w in xrange(0, config.width): for d in xrange(0, config.depth): ok = check_action() news = sut.newStatements() if not ok: break if((len(news)>0) and (not ((news in error) or (news in nonerror)))): states.insert(0,sut.state()) nonerror.append(sut.currStatements()) if config.coverage: sut.internalReport() print "Bugs ",num
def main(argv): global sut, rgen, ntest, nbug, coverageCount, start, selectStatements, threshold parse_options(argv) # GLOBAL VARIABLES sut = sut.sut() rgen = random.Random(opts.seed) ntest = 0 nbug = 0 coverageCount = {} selectStatements = {} threshold = 0 start = time.time() # PHASES phaseLimit = 0 if opts.timeout < 30: phaseLimit = int(round(opts.timeout/3*2)) else: phaseLimit = int(round(opts.timeout/10)) while (time.time() - start) < (0.95 * opts.timeout): randomPool = phase1(phaseLimit) selectPool = filterPool(randomPool) phase2(randomPool, selectPool) print "FULL POOL:",len(randomPool),", SELECT POOL:",len(selectPool) print "EXECUTED:",ntest,"tests" print "BUGS:",nbug print "TOTAL ELAPSED:",round(time.time() - start,5),"seconds" if (opts.coverage): sut.internalReport()
def main(): global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() R = random.Random(seed) states = [sut.state()] print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: break checkAlg() if coverage: sut.internalReport() print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def test_tstl_regressions(): sut = SUT.sut() failCount = 0 passCount = 0 for f in glob.glob("tstl_tests/*.test"): print() print("*" * 50) print() t = sut.loadTest(f) failed = sut.failsCheck(t) if failed: print("TEST", f, "FAILS:") print() sut.verbose(True) sut.replay(t, checkProp=True, catchUncaught=True) failure = sut.failure() print(failure) traceback.print_tb(failure[2], file=sys.stdout) sut.verbose(False) failCount += 1 else: print("TEST", f, "PASSES") passCount += 1 print() print("*" * 50) print() print("TSTL REGRESSIONS RESULT:") print(passCount, "TESTS PASSED,", failCount, "TESTS FAILED") assert failCount == 0
def main(): global timeout parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Generalizing using config={}'.format(config))) sut = SUT.sut() timeout = config.timeout if not config.coverage: try: sut.stopCoverage() except BaseException: pass t = sut.loadTest(config.infile) f = None if config.matchException: print("RUNNING TO OBTAIN FAILURE FOR EXCEPTION MATCHING...") assert sut.fails(t) f = sut.failure() print("ERROR:", f) print("TRACEBACK:") traceback.print_tb(f[2], file=sys.stdout) if not config.sandbox: pred = (lambda x: sut.failsCheck(x, failure=f)) if not config.noCheck: pred = (lambda x: sut.fails(x, failure=f)) else: pred = sandboxReplay if config.coverage: print("EXECUTING TEST TO OBTAIN COVERAGE...") sut.replay(t, checkPropcheckProp=not config.noCheck, catchUncaught=config.uncaught) b = set(sut.currBranches()) s = set(sut.currStatements()) print("PRESERVING", len(b), "BRANCHES AND", len(s), "STATEMENTS") pred = sut.coversAll(s, b, checkProp=not config.noCheck, catchUncaught=config.uncaught) print("GENERALIZING...") start = time.time() sut.generalize(t, pred, verbose=config.verbose, fresh=not config.noFresh, keepLast=config.keepLast) print("GENERALIZED IN", time.time() - start, "SECONDS")
def __init__(self, pid): self.pid = pid # unique id for this pool self.sut = SUT.sut() # SUT object self.nseqs = [[]] # list of non-error sequences self.eseqs = [] # list of error sequences self.covered = dict() # dictionary, key: coverage by this pool, value: frequency of coverage by this pool self.time = 0.0 # how many seconds this pool is used in feedback() self.count = 0 # how many times this pool is selected in select_pool() self.score = 0.0 # score of this pool self.uniqueness = 0.0 # uniqueness of this pool
def main(): global config, R, sut, bugs, actCount, branchCount, statementCount, ntest, start parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start print "Starting Test" while elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() elapsed = time.time() - start print "Finishing Test" if time.time() - start > config.timeout: print "[TIMEOUT]" printCoverage() if config.coverage: sut.internalReport() print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
def main(): global sut, rgen, nbugs, start, CXPB, MUTPB, NGEN sut = sut.sut() rgen = random.Random(opts.seed) CXPB, MUTPB, NGEN = 0.5, 0.2, 1 best, timeclock = [], [] nbugs, g, prev_best = 0, 0, 1 init() start = time.time() while True: print("Building population") pop = toolbox.population(n=100) print(" n=%d" % len(pop)) print("Start of evolution") fitnesses = list(map(toolbox.evaluate, pop)) for ind, fit in zip(pop, fitnesses): ind.fitness.values = fit while True: g += 1 generation(g, pop) best_test = tools.selBest(pop, 1)[0] elapsed = time.time()-start if 0.02 > (best_test.fitness.values[0] - prev_best) / prev_best or elapsed > (0.95 * opts.timeout): break else: prev_best = max(best_test.fitness.values[0], prev_best) best_test = tools.selBest(pop, 1)[0] print("Best test has %s lines covered" % int(best_test.fitness.values[0])) best.append(best_test.fitness.values[0]) elapsed = time.time()-start timeclock.append(elapsed) if not timeclock else timeclock.append(elapsed - timeclock[-1]) average_time = sum(timeclock) / len(timeclock) print("elapsed: %f seconds out of %f seconds" % (elapsed, opts.timeout)) if (elapsed + average_time) > opts.timeout: break overall_best = max(best) print("Overall best test has %s lines covered" % int(overall_best)) if (opts.coverage): sut.internalReport()
def main(): global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() R = random.Random() R.seed(seed) states = [sut.state()] print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: error.append(sut.currStatements()) break checkAlg() #if not error ==[]: # print "SHOW ERROR SEQUENCE" # print error # f = open(("error" + str(actCount) + ".test"), 'w') # f.write(str(error)) # f.close() #else: # print "Data in noerror sequence" # print noerror if coverage: sut.internalReport() print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global swarm, showActions, noCheck, alwaysSave, noSave, sut if "--help" in sys.argv: print( "Usage: tstl_afl [--noCheck] [--swarm] [--verbose] [--showActions] [--noSave] [--alwaysSave]" ) print("Options:") print(" --noCheck: do not run property checks") print( " --swarm use first four bytes to determine a swarm configuration" ) print(" --verbose: make actions verbose") print(" --showActions: show actions in test") print( " --noSave: don't save failing tests as standard TSTL tests") print(" --alwaysSave: save even non-failing tests") sys.exit(0) sut = SUT.sut() try: sut.stopCoverage() except BaseException: pass sut.restart() swarm = "--swarm" in sys.argv showActions = "--showActions" in sys.argv if "--verbose" in sys.argv: sut.verbose(True) noSave = "--noSave" in sys.argv alwaysSave = "--alwaysSave" in sys.argv noCheck = "--noCheck" in sys.argv persist = "--persist" in sys.argv if not persist: afl.init() runTest() else: while afl.loop(): runTest() sut.restart() os._exit(0)
def main(): if "--help" in sys.argv: print("Usage: tstl_toafl <outputdir> <files>") sys.exit(0) sut = SUT.sut() outputDir = sys.argv[1] files = sys.argv[2:] acts = sut.actions() i = 0 for f in files: t = sut.loadTest(f) sut.saveTest(t,outputDir+"/"+os.path.basename(f)+".afl",afl=True) i += 1
def main(): global start,sut,R,noerror,error,reduceTime,actCount, bugs,ntest,newseq,currseq,states, beforeReduceS, beforeReduceB,quickCount actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() reduceTime = 0.0 R = random.Random() R.seed(seed) beforeReduceS = set(sut.allStatements()) beforeReduceB = set(sut.allBranches()) states = [sut.state()] quickCount=0 print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: break checkAlg() if coverage: sut.internalReport() if quickTests==1: #if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])): handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True) print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): if "--help" in sys.argv: print("Usage: tstl_fromafl <outputdir> <files> [--aflswarm]") sys.exit(0) sut = SUT.sut() aflswarm = "--aflswarm" in sys.argv if aflswarm: sys.argv.remove("--aflswarm") outputDir = sys.argv[1] files = sys.argv[2:] i = 0 for f in files: t = sut.loadTest(f, afl=True, swarm=aflswarm) sut.saveTest(t, outputDir + "/" + os.path.basename(f) + ".test") i += 1
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Triaging using config={}'.format(config))) sut = SUT.sut() try: sut.stopCoverage() except: pass signatures = {} for fn in glob.glob(config.testglob): t = sut.loadTest(fn) if not config.noCheck: fails = sut.failsAny(t) else: fails = sut.fails(t) if fails: e = repr(sut.failure()) e = e[:e.find("<traceback object at 0x")] + ")" l = t[-1][0] sig = (noDigits(e),noDigits(l)) if (sig not in signatures): signatures[sig] = (fn,t,sut.failure(),sut.prettyName(t[-1][0]),1) elif (len(t) < signatures[sig][1]): signatures[sig] = (fn,t,sut.failure(),sut.prettyName(t[-1][0]),signatures[sig][4]+1) for sig in sorted(signatures.keys(),key=lambda x:signatures[x][4],reverse=True): print ("="*80) print ("TEST:",signatures[sig][0],"LENGTH:",len(signatures[sig][1])) print ("OPERATION:",signatures[sig][3]) print ("FAILURE:") traceback.print_tb(signatures[sig][2][2],file=sys.stdout) print ("COUNT:",signatures[sig][4]) if config.showTests: sut.prettyPrintTest(signatures[sig][1])
def main(): R = random.Random() repeats = 0 pid = str(os.getpid()) depth = int(sys.argv[1]) sut = SUT.sut() taken = defaultdict(lambda: 0) takenClass = defaultdict(lambda: 0) allTaken = defaultdict(lambda: 0) allTakenClass = defaultdict(lambda: 0) # takenPath = defaultdict(lambda: 0) takenFull = {} start = time.time() lastEpoch = 0 count = 0 sut.standardSwarm(R) print(sut.swarmConfig()) totalPaths = int(math.pow(len(sut.actions()), depth)) print(totalPaths, "UPPER BOUND ON POSSIBLE TESTS") while count < totalPaths: sut.restart() path = [] i = 0 count += 1 while len(path) < depth: possible = sut.actions() possible = sorted(possible, key=lambda act: (allTakenClass[sut.actionClass( act)], allTaken[act[0]], takenClass[ (sut.actionClass(act), i)], taken[ (act[0], i)])) a = None for act in possible: if act[1](): a = act break a = sut.randomEnabled(R) if a is None: print("DEADLOCK!") break path.append(a) # takenPath[sut.captureReplay(path)] += 1 taken[(a[0], i)] += 1 takenClass[(sut.actionClass(a), i)] += 1 allTaken[(a[0])] += 1 allTakenClass[sut.actionClass(a)] += 1 ok = sut.safely(a) if not ok: print("FALIURE IN TEST", count) sut.saveTest(path, "failure.exhaust." + pid + ".test") sut.prettyPrintTest(path) print(sut.failure()) sys.exit(255) if not "--noCheck" not in sys.argv: okCheck = sut.check() if not okCheck: print("PROPERTY VIOLATION IN TEST", count) sut.saveTest(path, "failure.exhaust." + pid + ".test") sut.prettyPrintTest(path) print(sut.failure()) sys.exit(255) i += 1 if "--ultraVerbose" in sys.argv: print("=" * 25) sut.prettyPrintTest(path) p = sut.captureReplay(path) if p in takenFull: repeats += 1 takenFull[p] = True epoch = int((time.time() - start) / 2) if epoch > lastEpoch: lastEpoch = epoch print(time.time() - start, "ELAPSED", count, "TESTS", repeats, "REPEATS", end=' ') if "--noCover" not in sys.argv: print("[", len(sut.allStatements()), "stmts", len(sut.allBranches()), "branches ]", end=' ') print() if "--verbose" in sys.argv: print("*" * 50) print("PATH #", count) sut.prettyPrintTest(path) print() print("COUNTS:") print("=" * 20) for c in sorted(list(allTakenClass.keys()), key=lambda ac: allTakenClass[ac]): print(c, allTakenClass[c]) print("=" * 20) for a in sorted(list(allTaken.keys()), key=lambda act: allTaken[act]): print(a, allTaken[a]) print(repeats, "TOTAL REPEATED TESTS", len(takenFull), "DISTINCT TESTS")
def main(): global failCount, sut, config, reduceTime, quickCount, repeatCount, failures, cloudFailures, R, opTime, checkTime, guardTime, restartTime, nops, ntests global fullPool, activePool, branchCoverageCount, statementCoverageCount, localizeSFail, localizeBFail parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time() - start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] if config.gendepth != None: failCloud = {} allClouds = {} if config.exploit != None: fullPool = [] activePool = [] if config.quickAnalysis or (config.exploit != None): branchCoverageCount = {} statementCoverageCount = {} if config.uniqueValuesAnalysis: handledValues = {} uniquef = open("unique.corpus", 'w') allUniquePaths = [] sut = SUT.sut() if config.relax: sut.relax() if config.verboseActions: sut.verbose(True) if config.readQuick: print "REPLAYING QUICK TESTS" sqrtime = time.time() for f in glob.glob("quick*.test"): fn = int(f.split("quick")[1].split(".")[0]) if fn >= quickCount: quickCount = fn + 1 t = sut.loadTest(f) sut.replay(t, catchUncaught=True, checkProp=(not config.ignoreProps)) print "EXECUTION TIME:", time.time() - sqrtime print "BRANCH COVERAGE:", len(sut.allBranches()) print "STATEMENT COVERAGE:", len(sut.allStatements()) if config.logging != None: sut.setLog(config.logging) if config.profile: profileTime = {} profileCount = {} for a in set(map(sut.actionClass, sut.actions())): profileTime[a] = 0.0 profileCount[a] = 0 if config.markov != None: nactions = len(sut.actions()) mprobs = {} prefix = [] probs = [] inProbs = False readSize = False for l in open(config.markov): if not readSize: markovN = int(l) readSize = True elif "START CLASS" in l: if (prefix != []): mprobs[tuple(prefix)] = probs prefix = [] probs = [] inProbs = False elif inProbs: ls = l.split("%%%%") prob = float(ls[0]) ac = ls[1][1:-1] probs.append((prob, ac)) elif "END CLASS" in l: inProbs = True else: prefix.append(l[:-1]) tacts = sut.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 checkResult = True if config.total: fulltest = open("fulltest.test", 'w') if config.localize: localizeSFail = {} localizeSPass = {} localizeBFail = {} localizeBPass = {} testsPassed = 0 if config.computeFeatureStats: featureStatsB = {} featureStatsS = {} featureStatsA = {} if config.quickAnalysis: quickcf = open("quick.corpus", 'w') quickCorpus = [] quickDoneB = {} quickDoneS = {} quickAnalysisTotal = 0 quickAnalysisBCounts = {} quickAnalysisSCounts = {} quickAnalysisCounts = {} quickClassCounts = {} quickAnalysisRawCounts = {} for c in set(map(sut.actionClass, sut.actions())): quickAnalysisCounts[c] = 0 quickClassCounts[c] = 0 quickAnalysisRawCounts[c] = 0 quickAnalysisReducedB = {} quickAnalysisReducedS = {} if config.timedProgress: lastInterval = 0 if config.verbose: print "ABOUT TO START TESTING" sys.stdout.flush() while (config.maxtests == -1) or (ntests < config.maxtests): if config.progress: printStatus(elapsed) if config.verbose: print "STARTING TEST", ntests sys.stdout.flush() ntests += 1 if config.swarm: sut.standardSwarm(R, file=config.swarmProbs, P=config.swarmP) if config.progress: print "CONFIG:", (sut.swarmConfig()) if config.highLowSwarm != None: classP = sut.highLowSwarm(R, file=config.swarmProbs, highProb=config.highLowSwarm) if config.swarmSwitch != None: lastSwitch = 0 switches = [] for i in xrange(0, config.swarmSwitch): switch = R.randrange(lastSwitch + 1, config.depth - ((config.swarmSwitch - i))) switches.append(switch) lastSwitch = switch startRestart = time.time() sut.restart() restartTime += time.time() - startRestart if config.total: fulltest.write("<<RESTART>>\n") if config.replayable: currtest = open("currtest.test", 'w') if (config.exploit != None) and ( (time.time() - start) > config.startExploit): tryExploit() testFailed = False for s in xrange(0, config.depth): if config.verbose: print "GENERATING STEP", s, sys.stdout.flush() if (config.swarmSwitch != None) and (s in switches): if config.highLowSwarm == None: sut.standardSwarm(R, file=config.swarmProbs, P=config.swarmP) else: classP = sut.highLowSwarm(R, file=config.swarmProbs, highProb=config.highLowSwarm) if (config.swarmLength != None) and (((s + 1) % config.swarmLength) == 0): if config.highLowSwarm == None: sut.standardSwarm(R, file=config.swarmProbs, P=config.swarmP) else: classP = sut.highLowSwarm(R, file=config.swarmProbs, highProb=config.highLowSwarm) startGuard = time.time() tryStutter = (a != None) and (a[1]()) and ((config.stutter != None) or config.greedyStutter) if tryStutter: if (config.stutter == None) or (R.random() > config.stutter): tryStutter = False if (config.greedyStutter) and sawNew: print "TRYING TO STUTTER DUE TO COVERAGE GAIN" tryStutter = True else: if config.markov == None: if config.highLowSwarm == None: a = sut.randomEnabled(R) else: a = sut.randomEnabledClassProbs(R, classP) else: prefix = tuple(map(sut.actionClass, sut.test()[-markovN:])) if prefix not in mprobs: a = sut.randomEnabled(R) else: a = sut.randomEnabledClassProbs(R, mprobs[prefix]) if a == None: a = sut.randomEnabled(R) if a == None: #sut.prettyPrintTest(sut.test()) print "WARNING: DEADLOCK (NO ENABLED ACTIONS)" guardTime += time.time() - startGuard elapsed = time.time() - start if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len( sut.test()) break if config.timedProgress != -1: thisInterval = int(elapsed / config.timedProgress) if thisInterval > lastInterval: printStatus(elapsed) lastInterval = thisInterval sys.stdout.flush() if a == None: print "TERMINATING TEST DUE TO NO ENABLED ACTIONS" break if tryStutter: print "STUTTERING WITH", a[0] if config.replayable: currtest.write(a[0] + "\n") currtest.flush() if config.total: fulltest.write(a[0] + "\n") fulltest.flush() if config.verbose: print "ACTION:", sut.prettyName(a[0]) sys.stdout.flush() startOp = time.time() if config.quickAnalysis: quickClassCounts[sut.actionClass(a)] += 1 if config.showActions: print "STEP #" + str(s), sut.prettyName(a[0]) stepOk = sut.safely(a) thisOpTime = time.time() - startOp nops += 1 if config.profile: profileTime[sut.actionClass(a)] += thisOpTime profileCount[sut.actionClass(a)] += 1 opTime += thisOpTime if sut.warning() != None: print "SUT WARNING:", sut.warning() if tryStutter: print "DONE STUTTERING" if (stepOk or config.uncaught) and config.ignoreProps and ( config.exploit != None): collectExploitable() if (not config.uncaught) and (not stepOk): testFailed = True handle_failure(sut.test(), "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break startCheck = time.time() if not config.ignoreProps: checkResult = sut.check() checkTime += time.time() - startCheck if checkResult and (stepOk or config.uncaught) and ( config.exploit != None): collectExploitable() if not checkResult: testFailed = True handle_failure(sut.test(), "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): print "ACTION:", sut.prettyName(a[0]) for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b sys.stdout.flush() sawNew = True else: sawNew = False if sut.newStatements() != set([]): print "ACTION:", sut.prettyName(a[0]) for s in sut.newStatements(): print elapsed, len( sut.allStatements()), "New statement", s sys.stdout.flush() sawNew = True else: sawNew = False if config.uniqueValuesAnalysis: uvals = sut.uniqueVals() olds = sut.state() currTest = list(sut.test()) for u in uvals: if u not in handledValues: print "ANALYZING NEW UNIQUE VALUE:", u else: continue r = sut.reduce(currTest, sut.coversUnique(u), keepLast=False) rc = map(sut.actionClass, r) sut.replay(r) for ru in sut.uniqueVals(): handledValues[ru] = True if rc not in allUniquePaths: print "NEW PATH DISCOVERED" allUniquePaths.append(rc) for s in rc: uniquef.write(s + "\n") uniquef.write(("=" * 50) + "\n") uniquef.flush() sut.backtrack(olds) if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len( sut.test()) break if (config.computeFeatureStats): for act in sut.swarmConfig(): if act in featureStatsA: featureStatsA[act] += 1 else: featureStatsA[act] = 1 for b in sut.currBranches(): if b not in featureStatsB: featureStatsB[b] = [1, {}] else: featureStatsB[b][0] += 1 for act in sut.swarmConfig(): if act not in featureStatsB[b][1]: featureStatsB[b][1][act] = 1 else: featureStatsB[b][1][act] += 1 for s in sut.currStatements(): if s not in featureStatsS: featureStatsS[s] = [1, {}] else: featureStatsS[s][0] += 1 for act in sut.swarmConfig(): if act not in featureStatsS[s][1]: featureStatsS[s][1][act] = 1 else: featureStatsS[s][1][act] += 1 if (config.exploit != None) and (not config.quickAnalysis): for b in sut.currBranches(): if b not in branchCoverageCount: branchCoverageCount[b] = 1 else: branchCoverageCount[b] += 1 for s in sut.currStatements(): if s not in statementCoverageCount: statementCoverageCount[s] = 1 else: statementCoverageCount[s] += 1 if config.localize and not testFailed: testsPassed += 1 for s in sut.currStatements(): if s not in localizeSPass: localizeSPass[s] = 0 localizeSPass[s] += 1 for b in sut.currBranches(): if b not in localizeBPass: localizeBPass[b] = 0 localizeBPass[b] += 1 if config.quickAnalysis: currTest = list(sut.test()) sut.replay(currTest) currB = sut.currBranches() currS = sut.currStatements() clen = len(currTest) #print "GATHERING QUICK ANALYSIS DATA FOR",len(currB),"BRANCHES" for b in currB: if config.fastQuickAnalysis and (b in quickDoneB): continue print "ANALYZING BRANCH", b if b not in branchCoverageCount: branchCoverageCount[b] = 0 quickAnalysisReducedB[b] = 0 branchCoverageCount[b] += 1 r = sut.reduce(currTest, sut.coversBranches([b]), keepLast=False) print "REDUCED FROM", clen, "TO", len(r) sys.stdout.flush() sut.replay(r) for br in sut.currBranches(): quickDoneB[br] = True for sr in sut.currStatements(): quickDoneS[sr] = True rc = map(sut.actionClass, r) if rc not in quickCorpus: quickCorpus.append(rc) for s in rc: quickcf.write(s + "\n") quickcf.write(("=" * 50) + "\n") quickcf.flush() sut.replay(r) for b2 in sut.currBranches(): if b2 not in quickAnalysisReducedB: quickAnalysisReducedB[b2] = 0 quickAnalysisReducedB[b2] += 1 for s2 in sut.currStatements(): if s2 not in quickAnalysisReducedS: quickAnalysisReducedS[s2] = 0 quickAnalysisReducedS[s2] += 1 if b not in quickAnalysisBCounts: quickAnalysisBCounts[b] = {} quickAnalysisTotal += 1 for c in map(sut.actionClass, r): quickAnalysisRawCounts[c] += 1 for c in set(map(sut.actionClass, r)): quickAnalysisCounts[c] += 1 if c not in quickAnalysisBCounts[b]: quickAnalysisBCounts[b][c] = 0 quickAnalysisBCounts[b][c] += 1 #print "GATHERING QUICK ANALYSIS DATA FOR",len(currS),"STATEMENTS" for s in currS: if config.fastQuickAnalysis and (s in quickDoneS): continue if s not in statementCoverageCount: statementCoverageCount[s] = 0 quickAnalysisReducedS[s] = 0 statementCoverageCount[s] += 1 print "ANALYZING STATEMENT", s r = sut.reduce(currTest, sut.coversStatements([s]), keepLast=False) print "REDUCED FROM", clen, "TO", len(r) sys.stdout.flush() sut.replay(r) for br in sut.currBranches(): quickDoneB[br] = True for sr in sut.currStatements(): quickDoneS[sr] = True rc = map(sut.actionClass, r) if rc not in quickCorpus: quickCorpus.append(rc) sut.replay(r) for b2 in sut.currBranches(): if b2 not in quickAnalysisReducedB: quickAnalysisReducedB[b2] = 0 quickAnalysisReducedB[b2] += 1 for s2 in sut.currStatements(): quickAnalysisReducedS[s2] += 1 if s not in quickAnalysisSCounts: quickAnalysisSCounts[s] = {} quickAnalysisTotal += 1 for c in map(sut.actionClass, r): quickAnalysisRawCounts[c] += 1 for c in set(map(sut.actionClass, r)): quickAnalysisCounts[c] += 1 if c not in quickAnalysisSCounts[s]: quickAnalysisSCounts[s][c] = 0 quickAnalysisSCounts[s][c] += 1 if config.throughput: print "ACTION THROUGHPUT:", nops / (time.time() - start) if config.replayable: currtest.close() if config.quickTests: if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])): handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True) if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if config.total: fulltest.close() if not config.nocover: sut.restart() print sut.report(config.coverfile), "PERCENT COVERED" if config.internal: sut.internalReport() if config.html: sut.htmlReport(config.html) if config.uniqueValuesAnalysis: uniquef.close() if config.computeFeatureStats: fstatsf = open( "feature.stats." + str(os.getpid()) + str(R.randrange(1000, 10000)), 'w') fstatsf.write("TESTS:" + str(ntests) + "\n") for act in featureStatsA: fstatsf.write(act + " %%ACTCOUNT%% " + str(featureStatsA[act]) + "\n") for b in sut.allBranches(): fstatsf.write("BRANCH:" + str(b) + "\n") count = featureStatsB[b][0] fstatsf.write("COUNT:" + str(count) + "\n") for act in featureStatsB[b][1]: fstatsf.write(act + " %%%% " + str(featureStatsB[b][1][act]) + "\n") for s in sut.allStatements(): fstatsf.write("STATEMENT:" + str(s) + "\n") count = featureStatsS[s][0] fstatsf.write("COUNT:" + str(count) + "\n") for act in featureStatsS[s][1]: fstatsf.write(act + " %%%% " + str(featureStatsS[s][1][act]) + "\n") fstatsf.close() if config.quickAnalysis: quickcf.close() print "*" * 70 print "QUICK ANALYSIS RESULTS:" print "*" * 70 print "TEST PATTERNS:" for rc in quickCorpus: print "=" * 50 for s in rc: print s print "*" * 70 print "OVERALL ACTION ANALYSIS:" totalTaken = sum(quickClassCounts.values()) actSort = sorted(quickAnalysisRawCounts.keys(), key=lambda x: quickAnalysisCounts.get(x, 0), reverse=True) for a in actSort: print "=" * 50 print "ACTION CLASS:" print a print "APPEARS", quickClassCounts[a], "TIMES IN TESTS" print "APPEARS", quickAnalysisRawCounts[ a], "TIMES IN REDUCED TESTS" print "APPEARS IN", quickAnalysisCounts[ a], "REDUCED TESTS (" + str( round((quickAnalysisCounts[a] / (quickAnalysisTotal * 1.0)) * 100, 2)) + "%)" #baselineRate = quickClassCounts[a]/(totalTaken*1.0) #reducedRate = quickAnalysisRawCounts[a]/(quickAnalysisTotal*1.0) #if reducedRate > 0.0: # print "RATIO:",(baselineRate/reducedRate) #else: # print "RATIO: INFINITE" print "*" * 70 print "DETAILED BRANCH ANALYSIS" branchCoverageCountSort = sorted(branchCoverageCount.keys(), key=lambda x: branchCoverageCount[x]) for b in branchCoverageCountSort: print "=" * 50 print "BRANCH:", b baselineRate = branchCoverageCount[b] / (ntests * 1.0) print "IN", str(round(baselineRate * 100, 2)) + "% OF TESTS (" + str( branchCoverageCount[b]) + " TESTS)" reducedRate = quickAnalysisReducedB[b] / (quickAnalysisTotal * 1.0) print "IN", str(round(reducedRate * 100, 2)) + "% OF REDUCED TESTS" if reducedRate > 0.0: print "RATIO:", (baselineRate / reducedRate) print "REDUCED TEST ACTION ANALYSIS:" sortAs = sorted(quickAnalysisBCounts[b].keys(), key=lambda x: quickAnalysisBCounts[b][x], reverse=True) print branchCoverageCount[b], "TESTS" for a in sortAs: print a, str( round( quickAnalysisBCounts[b][a] / (branchCoverageCount[b] * 1.0) * 100, 2)) + "%" print "*" * 70 print "DETAILED STATEMENT ANALYSIS" statementCoverageCountSort = sorted( statementCoverageCount.keys(), key=lambda x: statementCoverageCount[x]) for s in statementCoverageCountSort: print "=" * 50 print "STATEMENT:", s baselineRate = statementCoverageCount[s] / (ntests * 1.0) print "IN", str(round(baselineRate * 100, 2)) + "% OF TESTS" reducedRate = quickAnalysisReducedS[s] / (quickAnalysisTotal * 1.0) print "IN", str(round(reducedRate * 100, 2)) + "% OF REDUCED TESTS" if reducedRate > 0.0: print "RATIO:", (baselineRate / reducedRate) print "REDUCED TEST ACTION ANALYSIS:" print statementCoverageCount[s], "TESTS" sortAs = sorted(quickAnalysisSCounts[s].keys(), key=lambda x: quickAnalysisSCounts[s][x], reverse=True) for a in sortAs: print a, str( round( quickAnalysisSCounts[s][a] / (statementCoverageCount[s] * 1.0) * 100, 2)) + "%" print time.time() - start, "TOTAL RUNTIME" print ntests, "EXECUTED" print nops, "TOTAL TEST OPERATIONS" print opTime, "TIME SPENT EXECUTING TEST OPERATIONS" print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS" if not config.ignoreProps: print checkTime, "TIME SPENT CHECKING PROPERTIES" print(opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT" print restartTime, "TIME SPENT RESTARTING" print reduceTime, "TIME SPENT REDUCING TEST CASES" if config.multiple: print failCount, "FAILED" print repeatCount, "REPEATS OF FAILURES" print len(failures), "ACTUAL DISTINCT FAILURES" print n = 0 for (test, err) in failures: print "FAILURE", n sut.prettyPrintTest(test) n += 1 if err != None: print "ERROR:", err print "TRACEBACK:" traceback.print_tb(err[2]) i = -1 if config.compareFails: # Comparison feature normally not useful, just for researching normalization for test1 in failures: i += 1 j = -1 for test2 in failures: j += 1 if (j > i): print "COMPARING FAILURE", i, "AND FAILURE", j for k in xrange(0, max(len(test1), len(test2))): if k >= len(test1): print "STEP", k, "-->", test2[k][0] elif k >= len(test2): print "STEP", k, test1[k][0], "-->" elif test1[k] != test2[k]: print "STEP", k, test1[k][0], "-->", test2[k][ 0] if config.profile: print "ACTION PROFILE:" averages = [] for a in profileTime: if profileCount[a] != 0: averages.append((a, profileTime[a] / profileCount[a])) averages = sorted(averages, key=lambda x: x[1]) maxAvg = averages[-1][1] minAvg = averages[0][1] sumAvg = sum(map(lambda x: x[1], averages)) for (a, t) in averages: print a, profileCount[a], t, round(t / maxAvg, 2), round( t / minAvg, 2), round(t / sumAvg, 2) if config.localize and failCount > 0: scoresS = {} scoresB = {} for s in sut.allStatements(): if s not in localizeSPass: localizeSPass[s] = 0 if s not in localizeSFail: localizeSFail[s] = 0 denom = math.sqrt(failCount * (localizeSFail[s] + localizeSPass[s])) if denom == 0.0: continue scoresS[s] = localizeSFail[s] / denom for b in sut.allBranches(): if b not in localizeBPass: localizeBPass[b] = 0 if b not in localizeBFail: localizeBFail[b] = 0 denom = math.sqrt(failCount * (localizeBFail[b] + localizeBPass[b])) if denom == 0.0: continue scoresB[b] = localizeBFail[b] / denom sortedS = sorted(scoresS.keys(), key=lambda x: scoresS[x]) sortedB = sorted(scoresB.keys(), key=lambda x: scoresB[x]) print "FAULT LOCALIZATION RESULTS:" for s in sortedS: if scoresS[s] > 0.0: print s, scoresS[s] for b in sortedB: if scoresB[b] > 0.0: print b, scoresB[b] if not config.nocover: print len(sut.allBranches()), "BRANCHES COVERED" print len(sut.allStatements()), "STATEMENTS COVERED"
def main(): global config, R, sut, fails, actCount parsed_args, parser = parse_args() config = make_config(parsed_args, parser) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() fails = 0 actCount = 0 if config.faults: test_file = open("failure1.test", "w") start = time.time() elapsed = time.time() - start for i in xrange(0, config.depth): sut.restart() for j in xrange(0, config.width): act = None act = sut.randomEnabled(R) actCount += 1 if act == None: print "No enabled actions" elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break ok = sut.safely(act) propok = sut.check() if ((not ok) or (not propok)): fails += 1 if config.faults == 1: test_file.write(str(sut.failure()) + "\n") R = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() sut.restart() if config.faults: test_file.close() if config.coverage: sut.restart() sut.report("coverage.out") print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", fails, "times" print "Total actions", actCount print "Total runtime", time.time() - start
def main(): global failCount,t,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] if config.gendepth != None: failCloud = {} allClouds = {} t = SUT.sut() if config.logging != None: t.setLog(config.logging) tacts = t.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 checkResult = True if config.total: fulltest = open("fulltest.txt",'w') if config.verbose: print "ABOUT TO START TESTING" sys.stdout.flush() while (config.maxtests == -1) or (ntests < config.maxtests): if config.verbose: print "STARTING TEST",ntests sys.stdout.flush() ntests += 1 startRestart = time.time() t.restart() restartTime += time.time() - startRestart test = [] if config.total: fulltest.write("<<RESTART>>\n") if config.replayable: currtest = open("currtest.txt",'w') for s in xrange(0,config.depth): if config.verbose: print "GENERATING STEP",s startGuard = time.time() acts = tacts while True: elapsed = time.time() - start if elapsed > config.timeout: break tryStutter = (a != None) if tryStutter: if (config.stutter == None) and (not config.greedyStutter): tryStutter = False if tryStutter: if (config.stutter == None) or (R.random() > config.stutter): tryStutter = False if (config.greedyStutter) and sawNew: print "TRYING TO STUTTER DUE TO COVERAGE GAIN" tryStutter = True if not tryStutter: if len(acts) == 1: p = 0 else: p = R.randint(0,len(acts)-1) a = acts[p] if a[1](): break else: a = None acts = acts[:p] + acts[p+1:] guardTime += time.time()-startGuard elapsed = time.time() - start if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break if tryStutter: print "STUTTERING WITH",a[0] test.append(a) nops += 1 if config.replayable: currtest.write(a[0] + "\n") currtest.flush() if config.total: fulltest.write(a[0] + "\n") fulltest.flush() if config.verbose: print "ACTION:",t.prettyName(a[0]) startOp = time.time() stepOk = t.safely(a) opTime += (time.time()-startOp) if tryStutter: print "DONE STUTTERING" if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break startCheck = time.time() if not config.ignoreprops: checkResult = t.check() checkTime += time.time()-startCheck if not checkResult: handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if config.running: if t.newBranches() != set([]): print "ACTION:",a[0],tryStutter for b in t.newBranches(): print elapsed,len(t.allBranches()),"New branch",b sawNew = True else: sawNew = False if t.newStatements() != set([]): print "ACTION:",a[0],tryStutter for s in t.newStatements(): print elapsed,len(t.allStatements()),"New statement",s sawNew = True else: sawNew = False if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break if config.replayable: currtest.close() if config.quickTests: if (t.newCurrBranches() != set([])) or (t.newCurrStatements() != set([])): handle_failure(test, "NEW COVERAGE", False, newCov=True) if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if config.total: fulltest.close() if not config.nocover: t.restart() print t.report(config.coverfile),"PERCENT COVERED" if config.internal: t.internalReport() if config.html: t.htmlReport(config.html) print time.time()-start, "TOTAL RUNTIME" print ntests, "EXECUTED" print nops, "TOTAL TEST OPERATIONS" print opTime, "TIME SPENT EXECUTING TEST OPERATIONS" print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS" if not config.ignoreprops: print checkTime, "TIME SPENT CHECKING PROPERTIES" print (opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT" print restartTime, "TIME SPENT RESTARTING" print reduceTime, "TIME SPENT REDUCING TEST CASES" if config.multiple: print failCount,"FAILED" print repeatCount,"REPEATS OF FAILURES" print len(failures),"ACTUAL DISTINCT FAILURES" print n = 0 for (test, err) in failures: print "FAILURE",n t.prettyPrintTest(test) n += 1 if err != None: print "ERROR:", err print "TRACEBACK:" traceback.print_tb(err[2]) i = -1 if False: # Comparison feature normally not useful, just for researching normalization for test1 in failures: i += 1 j = -1 for test2 in failures: j += 1 if (j > i): print "COMPARING FAILURE",i,"AND FAILURE",j for k in xrange(0,max(len(test1),len(test2))): if k >= len(test1): print "STEP",k,"-->",test2[k][0] elif k >= len(test2): print "STEP",k,test1[k][0],"-->" elif test1[k] != test2[k]: print "STEP",k,test1[k][0],"-->",test2[k][0] if not config.nocover: print len(t.allBranches()),"BRANCHES COVERED" print len(t.allStatements()),"STATEMENTS COVERED"
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random variation of beam search using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time() - start failCount = 0 t = SUT.sut() if config.logging != None: t.setLog(config.logging) tacts = t.actions() ntests = 0 while (config.maxtests == -1) or (ntests < config.maxtests): ntests += 1 t.restart() test = [] for s in xrange(0, config.depth): count = 0 newCover = False acts = tacts old = t.state() while (count <= config.width): count += 1 while True: p = R.randint(0, len(acts) - 1) a = acts[p] if a[1](): break acts = acts[:p] + acts[p + 1:] test.append(a) stepOk = t.safely(a) if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break if (not config.ignoreprops) and (not t.check()): handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if t.newBranches() != set([]): if config.running: print "ACTION:", a[0] for b in t.newBranches(): print elapsed, len( t.allBranches()), "New branch", b break # Continue with this choice if new branch exposed if elapsed > config.timeout: break test = test[:-1] t.backtrack(old) if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len( test) break if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if not config.nocover: print t.report(config.coverfile), "PERCENT COVERED" if config.html: t.htmlReport(config.html) print ntests, "EXECUTED" if config.multiple: print failCount, "FAILED" if not config.nocover: print len(t.allBranches()), "BRANCHES COVERED" print len(t.allStatements()), "STATEMENTS COVERED"
savedTest = None failureCount = 0 actioncount = 0 testsCovered = [] option = parser.parse_args() # a object named pointer is made for this SEED = option.SEED TIMEOUT = option.TIMEOUT DEPTH = option.DEPTH WIDTH = option.WIDTH FAULT_CHECK = option.FAULT_CHECK COVERAGE_REPORT = option.COVERAGE_REPORT DETAIL_OF_RUNNING = option.DETAIL_OF_RUNNING #INITIAL_POP = option.INITIAL_POP rgen = random.Random() rgen.seed(SEED) sut = sut.sut() #this will reset the system state start = time.time() #this will start the system time if len(sut.newStatements()) > 0: savedTest = sut.state() storedTest = True print "FOUND NEW STATEMENTS",sut.newStatements() while time.time()-start < TIMEOUT: for f in xrange(0,DEPTH): action = sut.randomEnabled(rgen) for f in xrange(0,WIDTH): action = sut.randomEnabled(rgen)
def main(): if "--help" in sys.argv: print("Usage: tstl_replay <test file> [--noCheck] [--logging loglevel] [--verbose] [--showActions] [--coverage] [--internal] [--html directory] [--delay secs] [--trace] [--afl] [--aflswarm]") print("Options:") print("--noCheck: do not run property checks") print("--logging: set the logging level for the test") print("--verbose: run with verbose action output") print("--hideOpaque: hide opaque values in verbose actions") print("--showActions: show all actions") print("--coverage: report code coverage") print("--internal: report detailed code coverage information") print("--html: produce HTML report on coverage") print("--delay: delay to inject between steps") print("--trace: trace lines executed (does not work with SUTs compiled with coverage)") print("--afl: test is in afl format") print("--aflswarm: test is in afl swarm format") sys.exit(0) sut = SUT.sut() if not (("--coverage" in sys.argv) or ("--internal" in sys.argv)): try: sut.stopCoverage() except: pass if ("--trace" in sys.argv): goodToTrace = False try: sut.stopCoverage() except: goodToTrace = True if not goodToTrace: print("CANNOT TRACE WHEN SUT IS COMPILED WITH COVERAGE. REBUILD WITH --noCover") sys.exit(1) rout = open("replay.out",'w') file = sys.argv[1] nocheck = "--noCheck" in sys.argv verbose = "--verbose" in sys.argv logLevel = None if "--logging" in sys.argv: lastWasLogging = False for l in sys.argv: if lastWasLogging: logLevel = int(l) if l == "--logging": lastWasLogging = True else: lastWasLogging = False delay = None if "--delay" in sys.argv: lastWasDelay = False for l in sys.argv: if lastWasDelay: delay = float(l) if l == "--delay": lastWasDelay = True else: lastWasDelay = False htmlOut = None lastWasHtml = False for f in sys.argv[1:]: if lastWasHtml: htmlOut = f lastWasHtml = False elif f == "--html": lastWasHtml = True else: lastWasHtml = False sut.restart() if logLevel != None: sut.setLog(logLevel) i = 0 if verbose: sut.verbose(True) if "--hideOpaque" in sys.argv: sut.verboseOpaque(False) if not "--afl" in sys.argv: with open(file,'r') as f: theTest = f.readlines() else: readTest = sut.loadTest(file,afl=True,swarm=("--aflswarm" in sys.argv)) theTest = map(lambda x:x[0]+"\n",readTest) for l in theTest: name = l[:-1] if name == "<<RESTART>>": if "--showActions" in sys.argv: print("<<RESTART>>") #print "RESTART" rout.write("<<RESTART>>\n") rout.flush() sut.restart() else: if verbose: print("STEP #"+str(i)+":", end=' ') rout.write(l) rout.flush() action = sut.playable(name) if "--showActions" in sys.argv: print(sut.prettyName(action[0])) if action[1](): # check the guard if "--trace" in sys.argv: sys.settrace(trace_calls) stepOk = sut.safely(action) if "--trace" in sys.argv: sys.settrace(None) if not stepOk: print("FAILED STEP") print(sut.failure()) traceback.print_tb(sut.failure()[2],file=sys.stdout) if "--internal" in sys.argv: sut.internalReport() if "--coverage" in sys.argv: print(sut.report("coverage.out"),"PERCENT COVERED") if htmlOut != None: sut.htmlReport(htmlOut) sys.exit(255) if not nocheck: checkResult = sut.check() if not checkResult: print("FAILED PROPERTY") print(sut.failure()) traceback.print_tb(sut.failure()[2],file=sys.stdout) if "--internal" in sys.argv: sut.internalReport() if "--coverage" in sys.argv: print(sut.report("coverage.out"),"PERCENT COVERED") if htmlOut != None: sut.htmlReport(htmlOut) sys.exit(255) if delay != None: time.sleep(delay) i += 1 rout.write("TEST REPLAYED SUCCESSFULLY\n") rout.close() if "--internal" in sys.argv: sut.internalReport() if "--coverage" in sys.argv: print(sut.report("coverage.out"),"PERCENT COVERED") if htmlOut != None: sut.htmlReport(htmlOut) sys.exit(0)
def main(): global failCount,t,config,quickCount,repeatCount,failures,R,opTime,checkTime,guardTime,restartTime,nops,lastact,parts , pt , ptlast parsed_args, parser = parse_args() config = make_config(parsed_args, parser) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 failures = [] t = SUT.sut() tacts = t.actions() a = None sawNew = False nops = 0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 test = [] while time.time() - start < config.timeout: t.restart() lastact = 0 pt = 0 ptlast = 0 for s in xrange(0,config.depth): acts = tacts while True: if len(acts) > 10 : partsize = len(acts)/3 pt = R.randint(0,2) while pt == ptlast: pt = R.randint(0,2) else: pt = 0 partsize = len(acts) p = R.randint(pt*partsize,(pt+1)*partsize-1) a = acts[p] lastact = p ptlast = pt if a[1](): break else: a = None acts = acts[:p] + acts[p+1:] test.append(a) nops += 1 stepOk = t.safely(a) if not stepOk: print "STOPPING TESTING DUE TO FAILED TEST" failCount+=1 if config.faults == 1: f = "failure" + str(failCount) + ".test" t.saveTest(t.test(),f) break if config.running: if t.newBranches() != set([]): print "ACTION:",a[0] for b in t.newBranches(): print elapsed,len(t.allBranches()),"New branch",b sawNew = True else: sawNew = False if t.newStatements() != set([]): print "ACTION:",a[0] for s in t.newStatements(): print elapsed,len(t.allStatements()),"New statement",s sawNew = True else: sawNew = False print time.time()-start, "TOTAL RUNTIME" print nops, "TOTAL TEST OPERATIONS" if config.coverage: t.restart() print t.report("coverage.out"),"PERCENT COVERED" t.internalReport()
def main(): if "--help" in sys.argv: print "Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover] [--verbose] [--running]" print "Options:" print " --noCheck: do not run property checks" print " --html: output an HTML coverage report to the chosen directory" print " --noCover: do not compute code coverage" print " --verbose: make actions verbose" print " --running: give a running report on code coverage" print " --keepGoing: don't stop on failed test" sys.exit(0) sut = sut.sut() nocover = False verbose = False running = False keepGoing = False ignoreProps = False lastWasHtml = False files = [] htmlOut = None for f in sys.argv[1:]: if lastWasHtml: htmlOut = f lastWasHtml = False elif "--" not in f: files.append(f) elif f == "--html": lastWasHtml = True if "--noCover" in sys.argv: nocover = True if "--verbose" in sys.argv: verbose = True if "--noCheck" in sys.argv: ignoreProps = True if "--running" in sys.argv: running = True if "--keepGoing" in sys.argv: keepGoing = True if verbose: sut.verbose(True) anyFailed = False stime = time.time() for f in files: print "RUNNING TEST", f t = sut.loadTest(f) ok = False try: ok = sut.replay(t, checkProp=(not ignoreProps)) except Exception as e: print "EXCEPTION RAISED:", e if running: if sut.newBranches() != set([]): for b in sut.newBranches(): print "New branch:", b if sut.newStatements() != set([]): for s in sut.newStatements(): print "New statement:", s if not ok: print "TEST", f, "FAILED:" print sut.failure() anyFailed = True if not keepGoing: sys.exit(255) print time.time() - stime, "ELAPSED" if not nocover: print "STATEMENTS:", len(sut.allStatements()), "BRANCHES:", len( sut.allBranches()) if not nocover: sut.internalReport() print sut.report("coverage.out"), "PERCENT COVERED" if htmlOut != None: sut.htmlReport(htmlOut) if not anyFailed: print "ALL TESTS SUCCESSFUL"
def main(): if "--help" in sys.argv: print( "Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover]", "[--verbose] [--running] [--afl] [--aflswarm]") print("Options:") print( " --glob: <test files> are glob expressions (used when too many files to use shell expansion)" ) print(" --noCheck: do not run property checks") print( " --html: output an HTML coverage report to the chosen directory" ) print(" --noCover: do not compute code coverage") print(" --verbose: make actions verbose") print(" --running: give a running report on code coverage") print(" --keepGoing: don't stop on failed test") print(" --afl: tests are in afl format") print(" --aflswarm: tests are in afl swarm format") sys.exit(0) sut = SUT.sut() nocover = False verbose = False running = False keepGoing = False afl = False aflswarm = False ignoreProps = False lastWasHtml = False files = [] htmlOut = None for f in sys.argv[1:]: if lastWasHtml: htmlOut = f lastWasHtml = False elif "--" not in f: files.append(f) elif f == "--html": lastWasHtml = True else: lastWasHtml = False if "--noCover" in sys.argv: nocover = True try: sut.stopCoverage() except BaseException: pass if "--glob" in sys.argv: newFiles = [] for f in files: newFiles.extend(glob.glob(f)) files = newFiles if "--verbose" in sys.argv: verbose = True if "--noCheck" in sys.argv: ignoreProps = True if "--running" in sys.argv: running = True if "--keepGoing" in sys.argv: keepGoing = True if "--afl" in sys.argv: afl = True if "--aflswarm" in sys.argv: aflswarm = True if verbose: sut.verbose(True) failedTests = [] anyFailed = False noNewCover = [] newCover = [] totalTests = 0 invalidTests = [] stime = time.time() for f in files: totalTests += 1 print("RUNNING TEST", f) try: t = sut.loadTest(f, afl=afl, swarm=aflswarm) except KeyError: print("INVALID TEST, SKIPPING...") invalidTests.append(f) continue ok = False try: ok = sut.replay(t, checkProp=(not ignoreProps)) except Exception as e: print("EXCEPTION RAISED:", e) if not nocover: if (len(sut.newCurrBranches()) == 0) and (len( sut.newCurrStatements()) == 0): noNewCover.append(f) else: newCover.append(f) if running: if sut.newCurrBranches() != set([]): for b in sut.newCurrBranches(): print("New branch:", b) if sut.newCurrStatements() != set([]): for s in sut.newCurrStatements(): print("New statement:", s) if not ok: print("TEST", f, "FAILED:") print(sut.failure()) failedTests.append(f) anyFailed = True if not keepGoing: sys.exit(255) print(time.time() - stime, "ELAPSED") if not nocover: print("STATEMENTS:", len(sut.allStatements()), "BRANCHES:", len(sut.allBranches())) if f not in noNewCover: print("NEW STATEMENTS:", len(sut.newCurrStatements()), "BRANCHES:", len(sut.newCurrBranches())) sys.stdout.flush() if not nocover: sut.internalReport() print(sut.report("coverage.out"), "PERCENT COVERED") if htmlOut is not None: sut.htmlReport(htmlOut) if (not nocover) and (len(noNewCover) > 0): for f in noNewCover: print("TEST", f, "REDUNDANT WITH RESPECT TO COVERAGE") print() print(len(newCover), "TESTS NEEDED FOR FULL COVERAGE:", ", ".join(newCover)) print() print("EXECUTED", totalTests, "TESTS") if len(invalidTests) > 0: print(len(invalidTests), "INVALID TESTS:") for f in invalidTests: print(f, end=' ') print() if not anyFailed: print("ALL TESTS SUCCESSFUL") else: print(len(failedTests), "FAILED TESTS:") for f in failedTests: print(f, end=' ') print()
def main(): if "--help" in sys.argv: print( "Usage: tstl_afl [--noCheck] [--swarm] [--verbose] [--showActions] [--noSave] [--alwaysSave]" ) print("Options:") print(" --noCheck: do not run property checks") print( " --swarm use first four bytes to determine a swarm configuration" ) print(" --verbose: make actions verbose") print(" --showActions: show actions in test") print( " --noSave: don't save failing tests as standard TSTL tests") print(" --alwaysSave: save even non-failing tests") sys.exit(0) sut = SUT.sut() try: sut.stopCoverage() except: pass sut.restart() if "--swarm" in sys.argv: swarm = True R = random.Random() else: swarm = False showActions = "--showActions" in sys.argv if "--verbose" in sys.argv: sut.verbose(True) noSave = "--noSave" in sys.argv alwaysSave = "--alwaysSave" in sys.argv noCheck = "--noCheck" in sys.argv afl.init() bytesin = sys.stdin.read() if len(bytesin) < 4: os._exit(0) if swarm: R.seed(struct.unpack("<L", bytesin[0:4])[0]) sut.standardSwarm(R) bytesin = bytesin[4:] alen = len(sut.actions()) test = sut.bytesToTest(bytesin) for a in test: if a[1](): if showActions: print(sut.prettyName(a[0])) ok = sut.safely(a) if (not noSave) and not ok: i = 0 saveFile = "aflfail." + str( os.getpid()) + "." + str(i) + ".test" while os.path.exists(saveFile): i += 1 saveFile = "aflfail." + str( os.getpid()) + "." + str(i) + ".test" sut.saveTest(sut.test(), saveFile) assert (ok) if not noCheck: checkResult = sut.check() if (not noSave) and not checkResult: i = 0 saveFile = "aflfail." + str( os.getpid()) + "." + str(i) + ".test" while os.path.exists(saveFile): i += 1 saveFile = "aflfail." + str( os.getpid()) + "." + str(i) + ".test" sut.saveTest(sut.test(), saveFile) assert (checkResult) if alwaysSave: i = 0 saveFile = "afltest." + str(os.getpid()) + "." + str(i) + ".test" while os.path.exists(saveFile): i += 1 saveFile = "afltest." + str(os.getpid()) + "." + str(i) + ".test" sut.saveTest(sut.test(), saveFile) os._exit(0)
def __init__(self): #initiialize lists self.errorSeqs = [] self.nonErrorSeqs = [] self.sut = sut.sut()
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Generating swarm probabilties using config={}'.format(config))) sut = SUT.sut() verbose = False try: sut.stopCoverage() except BaseException: pass numTests = 0 mode = "CONFIG" entryClasses = [] entryBranches = [] entryStatements = [] count = {} branches = set([]) statements = set([]) hits = {} print("READING DATA...") for line in open(config.swarmData): if "::::" in line: if mode == "CONFIG": ac = line.split("::::")[1][:-1] entryClasses.append(ac) if ac in count: count[ac] += 1 else: count[ac] = 1 elif mode == "BRANCHES": b = eval(line.split("::::")[1][:-1]) entryBranches.append(b) if b in count: count[b] += 1 else: count[b] = 1 hits[b] = [] branches.add(b) elif mode == "STATEMENTS": s = eval(line.split("::::")[1][:-1]) entryStatements.append(s) if s in count: count[s] += 1 else: count[s] = 1 hits[s] = [] statements.add(s) elif "CONFIG:" in line: if entryClasses != []: numTests += 1 for b in entryBranches: hits[b].append(entryClasses) for s in entryStatements: hits[s].append(entryClasses) entryClasses = [] entryBranches = [] entryStatements = [] mode = "CONFIG" elif "BRANCHES:" in line: mode = "BRANCHES" elif "STATEMENTS:" in line: mode = "STATEMENTS" numTests += 1 for b in entryBranches: hits[b].append(entryClasses) for s in entryStatements: hits[s].append(entryClasses) print("DONE") eqClasses = {} targets = branches.union(statements) print("ANALYZING", len(targets), "TARGETS") analyzed = 0 for target in targets: if float(count[target]) / numTests >= config.cutoff: analyzed += 1 continue triggers = [] suppressors = [] hitT = hits[target] if len(hitT) < config.minHits: analyzed += 1 continue for ac in sut.actionClasses(): rate = float(count[ac]) / numTests if rate == 1.0: # Neither a suppressor nor trigger if present in every test! continue chits = 0 for t in hitT: if ac in t: chits += 1 low, high = proportion_confint(chits, len(hitT), method='wilson', alpha=1 - config.confidence) if low > rate: triggers.append(ac) if high < rate: suppressors.append(ac) signature = (tuple(sorted(triggers)), tuple(sorted(suppressors))) data = (target, count[target]) analyzed += 1 if signature not in eqClasses: if verbose: print("NEW EQUIVALENCE CLASS AFTER ANALYZING", analyzed, "TARGETS") print("NOW", len(eqClasses), "EQUIVALENCE CLASSES") print("TRIGGERS:", triggers) print("SUPPRESSORS:", suppressors) print("TARGET:", target) print("FREQUENCY:", count[target]) eqClasses[signature] = (triggers, suppressors, [data]) else: eqClasses[signature][2].append(data) pcount = 0 with open(config.prefix + ".classes", 'w') as cfile: for c in sorted( eqClasses.keys(), key=lambda x: min(map(lambda y: y[1], eqClasses[x][2]))): triggers, suppressors, targets = eqClasses[c] if (triggers == []) and (suppressors == []): continue # Ignore the no-data class print("=" * 80) print("# TARGETS:", len(targets)) print("MINIMUM FREQUENCY:", min(map(lambda x: x[1], targets))) print("TRIGGERS:", triggers) print("SUPPRESSORS:", suppressors) cfile.write("TRIGGERS:\n") cp = {} for t in triggers: cfile.write(t + "\n") cp[t] = 1.0 cfile.write("SUPPRESSORS:\n") for s in suppressors: cfile.write(s + "\n") cp[s] = 0.0 cfile.write("TARGETS:\n") for t in targets: cfile.write(repr(t[0]) + " :::: " + str(t[1]) + "\n") cfile.write("FILE: " + config.prefix + "." + str(pcount) + ".prob\n") sut.writeProbFile(config.prefix + "." + str(pcount) + ".prob", cp) pcount += 1
def main(): global failCount, t, config, reduceTime, quickCount, repeatCount, failures, cloudFailures, R, opTime, checkTime, guardTime, restartTime, nops, ntests # region define the config object by parsing the command-line arguments parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) # endregion R = random.Random(config.seed) start = time.time() elapsed = time.time() - start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] if config.gendepth != None: failCloud = {} allClouds = {} # create an instance of object SUT t = SUT.sut() if config.logging != None: t.setLog(config.logging) tacts = t.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 checkResult = True if config.total: fulltest = open("fulltest.txt", 'w') if config.verbose: print "ABOUT TO START TESTING" sys.stdout.flush() while (config.maxtests == -1) or (ntests < config.maxtests): if config.verbose: print "STARTING TEST", ntests sys.stdout.flush() ntests += 1 startRestart = time.time() t.restart() restartTime += time.time() - startRestart test = [] if config.total: fulltest.write("<<RESTART>>\n") if config.replayable: currtest = open("currtest.txt", 'w') for s in xrange(0, config.depth): if config.verbose: print "GENERATING STEP", s startGuard = time.time() acts = tacts while True: elapsed = time.time() - start if elapsed > config.timeout: break tryStutter = (a != None) if tryStutter: if (config.stutter == None) and (not config.greedyStutter): tryStutter = False if tryStutter: if (config.stutter == None) or (R.random() > config.stutter): tryStutter = False if (config.greedyStutter) and sawNew: print "TRYING TO STUTTER DUE TO COVERAGE GAIN" tryStutter = True if not tryStutter: if len(acts) == 1: p = 0 else: p = R.randint(0, len(acts) - 1) a = acts[p] if a[1](): break else: a = None acts = acts[:p] + acts[p + 1:] guardTime += time.time() - startGuard elapsed = time.time() - start if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len( test) break if tryStutter: print "STUTTERING WITH", a[0] test.append(a) nops += 1 if config.replayable: currtest.write(a[0] + "\n") currtest.flush() if config.total: fulltest.write(a[0] + "\n") fulltest.flush() if config.verbose: print "ACTION:", t.prettyName(a[0]) startOp = time.time() stepOk = t.safely(a) if t.warning() != None: print "SUT WARNING:", t.warning() opTime += (time.time() - startOp) if tryStutter: print "DONE STUTTERING" if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break startCheck = time.time() if not config.ignoreprops: checkResult = t.check() checkTime += time.time() - startCheck if not checkResult: handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if config.running: if t.newBranches() != set([]): print "ACTION:", a[0], tryStutter for b in t.newBranches(): print elapsed, len(t.allBranches()), "New branch", b sawNew = True else: sawNew = False if t.newStatements() != set([]): print "ACTION:", a[0], tryStutter for s in t.newStatements(): print elapsed, len( t.allStatements()), "New statement", s sawNew = True else: sawNew = False if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len( test) break if config.replayable: currtest.close() if config.quickTests: if (t.newCurrBranches() != set([])) or (t.newCurrStatements() != set([])): handle_failure(test, "NEW COVERAGE", False, newCov=True) if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if config.total: fulltest.close() if not config.nocover: t.restart() print t.report(config.coverfile), "PERCENT COVERED" if config.internal: t.internalReport() if config.html: t.htmlReport(config.html) print time.time() - start, "TOTAL RUNTIME" print ntests, "EXECUTED" print nops, "TOTAL TEST OPERATIONS" print opTime, "TIME SPENT EXECUTING TEST OPERATIONS" print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS" if not config.ignoreprops: print checkTime, "TIME SPENT CHECKING PROPERTIES" print(opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT" print restartTime, "TIME SPENT RESTARTING" print reduceTime, "TIME SPENT REDUCING TEST CASES" if config.multiple: print failCount, "FAILED" print repeatCount, "REPEATS OF FAILURES" print len(failures), "ACTUAL DISTINCT FAILURES" print n = 0 for (test, err) in failures: print "FAILURE", n t.prettyPrintTest(test) n += 1 if err != None: print "ERROR:", err print "TRACEBACK:" traceback.print_tb(err[2]) i = -1 if False: # Comparison feature normally not useful, just for researching normalization for test1 in failures: i += 1 j = -1 for test2 in failures: j += 1 if (j > i): print "COMPARING FAILURE", i, "AND FAILURE", j for k in xrange(0, max(len(test1), len(test2))): if k >= len(test1): print "STEP", k, "-->", test2[k][0] elif k >= len(test2): print "STEP", k, test1[k][0], "-->" elif test1[k] != test2[k]: print "STEP", k, test1[k][0], "-->", test2[k][ 0] if not config.nocover: print len(t.allBranches()), "BRANCHES COVERED" print len(t.allStatements()), "STATEMENTS COVERED"
def main(): global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} if config.faults: test_file = open("failure1.test", "w") start = time.time() elapsed = time.time() - start for i in xrange(0, config.depth): sut.restart() for j in xrange(0, config.width): if not randomAct(): print "Function called randomAct() occurs error" break elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() if config.faults: test_file.close() if config.coverage: sut.restart() sut.report("coverage.out") #printCoverage() sut.internalReport() #print "Covered", len(sut.allBranches()), "branches" #print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total actions", actCount print "Total runtime", time.time() - start
def main(): mytester = simplerRandomTester(sys.argv[1:]) mysut = SUT.sut() mysut.testWith(mytester)
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Generating swarm probabilties using config={}'.format(config))) sut = SUT.sut() try: sut.stopCoverage() except BaseException: pass target = eval(config.target) tests = [] mode = "CONFIG" entryClasses = [] entryBranches = [] entryStatements = [] for line in open(config.swarmData): if "::::" in line: if mode == "CONFIG": ac = line.split("::::")[1][:-1] entryClasses.append(ac) elif mode == "BRANCHES": b = eval(line.split("::::")[1][:-1]) entryBranches.append(b) elif mode == "STATEMENTS": s = eval(line.split("::::")[1][:-1]) entryStatements.append(s) elif "CONFIG:" in line: if entryClasses != []: tests.append((entryClasses, entryBranches, entryStatements)) entryClasses = [] entryBranches = [] entryStatements = [] mode = "CONFIG" elif "BRANCHES:" in line: mode = "BRANCHES" elif "STATEMENTS:" in line: mode = "STATEMENTS" count = {} for t in tests: _, branches, statements = t for branch in branches: if branch not in count: count[branch] = 1 else: count[branch] += 1 for stmt in statements: if stmt not in count: count[stmt] = 1 else: count[stmt] += 1 rates = {} for ac in sut.actionClasses(): acCount = 0 for t in tests: if ac in t[0]: acCount += 1 rates[ac] = acCount / float(len(tests)) triggers = [] suppressors = [] hitT = [] for t in tests: if (target in t[1]) or (target in t[2]): hitT.append(t) for ac in sut.actionClasses(): if rates[ac] == 1.0: # Neither a suppressor nor trigger if present in every test! continue rate = rates[ac] hits = 0 for t in hitT: if ac in t[0]: hits += 1 low, high = proportion_confint(hits, len(hitT), method='wilson', alpha=1 - config.confidence) if low > rate: triggers.append(ac) if high < rate: suppressors.append(ac) print("=" * 80) sratio = float(count[target]) / len(tests) print(target, count[target], "HITS", sratio, "RATIO") print("TRIGGERS:") for tr in triggers: print(tr, rates[tr]) print() print("SUPPRESSORS:") for sp in suppressors: print(sp, rates[sp]) print() cp = {} for ac in sut.actionClasses(): if ac in triggers: cp[ac] = 1.0 elif ac in suppressors: cp[ac] = 0.0 else: if config.mode == "halfswarm": cp[ac] = 0.5 elif config.mode == "triggers-only": cp[ac] = 0.0 else: cp[ac] = 1.0 sut.writeProbFile(config.probFile, cp)
import os import sys current_working_dir = os.getcwd() sys.path.append(current_working_dir) from collections import namedtuple import sut as SUT import random import time import traceback import argparse sut = SUT.sut() def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('timeout', type=int, default=60, help='Timeout in seconds. (60 default)') parser.add_argument('seed', type=int, default=None, help='Random seed. (default = None)') parser.add_argument('depth', type=int, default=100, help='Maximum search depth. (100 default)') parser.add_argument('width', type=int, default=10000, help='Maximum memory. (10000 default)') parser.add_argument('faults', type=int, default=0, choices=[0, 1], help='Check for faults or not. 1 for check, 0 for do not check (0 default)') parser.add_argument('coverage', type=int, default=0, choices=[0, 1] ,help='report coverage or not. 1 for report, 0 for do not report(0 default)') parser.add_argument('running', type=int, default=0, choices=[0, 1], help='Produce running branch coverage report.') parsed_args = parser.parse_args(sys.argv[1:]) return (parsed_args, parser)
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Calibrating using config={}'.format(config))) calibFile = open(".tstl_calibration", 'w') sut = SUT.sut() R = random.Random() if config.seed: R.seed(config.seed) print("=" * 80) print("CALIBRATING COST OF COVERAGE...\n") oldOut = sys.stdout fnull = open(os.devnull, 'w') sys.stdout = fnull sut.stopCoverage() oldOut.write( "GENERATING 30 TESTS (OR 30+s of TESTS) WITHOUT COVERAGE...\n") oldOut.flush() start = time.time() noCovCount = 0 for i in range(0, 30): oldOut.write(str(i) + "...") oldOut.flush() (t, ok) = sut.makeTest(100, R, stopFail=False) noCovCount += 1 if (time.time() - start) > 30: oldOut.write("TIMEOUT AT " + str(time.time() - start)) break noCovTime = time.time() - start sut.startCoverage() oldOut.write( "\n\nGENERATING 30 TESTS (OR 30+s of TESTS) WITH COVERAGE...\n") start = time.time() covCount = 0 for i in range(0, 30): oldOut.write(str(i) + "...") oldOut.flush() (t, ok) = sut.makeTest(100, R, stopFail=False) covCount += 1 if (time.time() - start) > 30: oldOut.write("TIMEOUT AT " + str(time.time() - start)) break covTime = time.time() - start sys.stdout = oldOut print() print() sys.stdout.flush() covR = (covCount * 100) / covTime noCovR = (noCovCount * 100) / noCovTime if covR < noCovR: print("WITH COVERAGE:", covR, "ACTIONS/s") print("WITHOUT COVERAGE:", noCovR, "ACTIONS/s") overhead = (noCovR - covR) / noCovR print("COVERAGE OVERHEAD:", str(round(overhead * 100, 2)) + "%") else: print("NO DETECTABLE COVERAGE OVERHEAD") calibFile.write("COVERAGE OVERHEAD: " + str(overhead)) print("=" * 80) print("ESTIMATING LINES OF CODE IN ACTIONS...\n") subprocess.call([ "tstl_rt --timeout 180 --generateLOC .tstl_calibration_loc --noCover" ], shell=True, stdout=fnull, stderr=fnull) classLOCVals = {} for c in sut.actionClasses(): classLOCVals[c] = 0.0 totalLOCs = 0.0 num0 = 0.0 with open(".tstl_calibration_loc", 'r') as cf: for l in cf: ls = l.split(" %%%% ") c = ls[0] loc = float(ls[1]) totalLOCs += loc classLOCVals[c] = loc classP = {} for c in sut.actionClasses(): if classLOCVals[c] == 0.0: num0 += 1 for c in sut.actionClasses(): if classLOCVals[c] == 0.0: classP[c] = (0.20 / num0) else: classP[c] = (classLOCVals[c] / totalLOCs) sortLOC = sorted(classLOCVals.keys(), key=lambda x: classP[x], reverse=True) print("HIGHEST LOC-BASED PROBABILITY ACTIONS:") for c in sortLOC[:3]: print(" ", c, round(classP[c], 5)) print("\nLOWEST LOC-BASED PROBABILITY ACTIONS:") for c in sortLOC[-3:]: print(" ", c, round(classP[c], 5)) calibFile.close()
def main(): if "--help" in sys.argv: print( "Usage: tstl_aflcorpus <outputdir> <length> <time> [--noCheck] [--noReduce] [--noCover] [--swarm] [--skipFails]" ) print("Options:") print(" --noCheck: do not check properties") print(" --noReduce: do not reduce inputs by coverage") print(" --noCover: do not check for new coverage") print(" --swarm: use swarm format, generate tests using swarm") print(" --skipFails: just pass over failures, don't try to fix") sys.exit(0) sut = SUT.sut() pid = str(os.getpid()) outputDir = sys.argv[1] length = int(sys.argv[2]) timeout = int(sys.argv[3]) checkProp = not "--noCheck" in sys.argv noReduce = "--noReduce" in sys.argv noCover = "--noCover" in sys.argv swarm = "--swarm" in sys.argv skipFails = "--skipFails" in sys.argv if noCover: sut.stopCoverage() R = random.Random() Rswarm = random.Random() acts = sut.actions() i = 0 stime = time.time() while (time.time() - stime) < timeout: i += 1 if swarm: seed = R.randint(0, 2**32) Rswarm.seed(seed) sut.standardSwarm(Rswarm) (t, ok) = sut.makeTest(length, R, stopFail=True, checkProp=checkProp) if (not noCover) and (len(sut.newCurrBranches()) == 0): continue else: print("INPUT", i, "GENERATED", end=" ") if not noCover: print("NEW BRANCHES:", len(sut.newCurrBranches()), end=" ") type = "branch" + str(len(sut.newCurrBranches())) if ok: # failing tests won't work with afl if (not noCover) and (not noReduce): b = set(sut.currBranches()) s = set(sut.currStatements()) pred = sut.coversAll(s, b, checkProp=True, catchUncaught=False) r = sut.reduce(t, pred, verbose=False) else: r = t elif skipFails: print("SKIPPING FAILED TEST...") continue else: type = "nearfail" print("SAVING ALL BUT LAST STEP OF FAILED TEST") r = t[:-1] # always alpha convert to make actions clearer to afl, easier to splice r = sut.alphaConvert(r) if ok and (not noReduce): print("REDUCED LENGTH:", len(r)) sut.prettyPrintTest(r) print("=" * 80) if not swarm: sut.saveTest(r, outputDir + "/tstl." + type + "." + pid + "." + str(i) + ".afl", afl=True) else: bytes = sut.testToBytes(r) with open( outputDir + "/tstl." + type + "." + pid + "." + str(i) + ".afl", 'wb') as f: f.write(struct.pack("<L", seed)) f.write(bytes)
def main(): global timeout if "--help" in sys.argv: print "Usage: tstl_reduce <test file> <output test file> [--noCheck] [--noReduce] [--noAlpha] [--noNormalized] [--verbose verbosity] [--sandbox] [--quietSandbox] [--timeout secs]" print "Options:" print " --noCheck: do not run property checks" print " --noReduce do not reduce test (useful for normalizing an already reduced test)" print " --noAlpha do not alpha convert test" print " --noNormalize after reducing, do not also normalize" print " --verbose: set verbosity level for reduction/normalization (defaults to silent reduction/normalization)" print " --sandbox: run tests in a subprocess sandbox, for tests that crash Python interpreter;" print " due to common difficulties, sandbox is by default very verbose!" print " WARNING: if not needed, sandbox mode is VERY SLOW" print " --quietSandbox: run sandbox in a quiet mode" print " --timeout: if tests are run in a sandbox, consider tests running longer than this to have failed" sys.exit(0) sut = SUT.sut() vLevel = False if "--verbose" in sys.argv: lastWasVerbose = False for l in sys.argv: if lastWasVerbose: vLevel = l if l == "--verbose": lastWasVerbose = True if vLevel == "True": vLevel = True if vLevel == "False": vLevel = False timeout = None if "--timeout" in sys.argv: lastWasTimeout = False for l in sys.argv: if lastWasTimeout: timeout = l if l == "--timeout": lastWasTimeout = True if not "--sandbox" in sys.argv: pred = sut.failsCheck if "--noCheck" is sys.argv: pred = sut.fails else: pred = sandboxReplay r = sut.loadTest(sys.argv[1]) print "STARTING WITH TEST OF LENGTH", len(r) if not ("--noReduce" in sys.argv): start = time.time() print "REDUCING..." r = sut.reduce(r, pred, verbose=vLevel) print "REDUCED IN", time.time() - start, "SECONDS" print "NEW LENGTH", len(r) if not ("--noAlpha" in sys.argv): print "ALPHA CONVERTING..." r = sut.alphaConvert(r) if not ("--noNormalize" in sys.argv): start = time.time() print "NORMALIZING..." r = sut.normalize(r, pred, verbose=vLevel) print "NORMALIZED IN", time.time() - start, "SECONDS" print "NEW LENGTH", len(r) sut.saveTest(r, sys.argv[2]) print "TEST WRITTEN TO", sys.argv[2]
def main(): if "--help" in sys.argv: print( "Usage: tstl_standalone <test file> <output Python file> [<sut file>] [--noCheck] [--noRefs] [--regression] [--verbose] [--afl] [--aflswarm]" ) print(" default for <sut file> is sut.py") print("Options:") print(" --noCheck: do not include property checks") print(" --noRefs: do not include reference actions") print( " --regression: produce a regression test that captures values") print( " --verbose: produce a verbose test that shows actions taken") print(" --afl: test is in afl format") print(" --aflswarm: test is in afl swarm format") sys.exit(0) testFile = sys.argv[1] outFile = sys.argv[2] if (len(sys.argv) > 3) and (".py" in sys.argv[3]): sutFile = sys.argv[3] else: sutFile = "sut.py" checkProps = not "--noCheck" in sys.argv checkRefs = not "--noRefs" in sys.argv makeRegression = "--regression" in sys.argv verbose = "--verbose" in sys.argv afl = "--afl" in sys.argv aflswarm = "--aflswarm" in sys.argv t = SUT.sut() def globalCheck(str): if " # CHECK POOL INIT" not in str: return str newStr = str.replace("(", '("') newStr = newStr.replace(" != None", '" in globals()') return newStr.replace("# CHECK POOL INIT", "") print("STR=", str) condBegin = str.find("(") theCheck = str[condBegin + 1:str.find(" != None):")] print(theCheck) strNew = str[:condBegin + 1] + '"' + theCheck + '"' + str[str.find(" != None):"):] print(strNew) strNew = strNew.replace(" != None", " in globals()") print(strNew) strNew = strNew.replace("# CHECK POOL INIT", "") print(strNew) return strNew outf = open(outFile, 'w') startPrelim = False startCheck = False startReload = False startInit = False reloadCode = [] initCode = [] outf.write("from __future__ import print_function\n\n") for l in open(sutFile): if "END STANDALONE CODE" in l: startPrelim = False if "END CHECK CODE" in l: startCheck = False if "END RELOAD CODE" in l: startReload = False if "END INITIALIZATION CODE" in l: startInit = False if startPrelim: outf.write(l) if startCheck: outf.write( globalCheck(t.prettyName(l.replace("# GLOBAL ", "global ")))) if startReload: reloadCode.append(l.lstrip()) if startInit: initCode.append(l.lstrip()) if "BEGIN STANDALONE CODE" in l: startPrelim = True if checkProps and ("BEGIN CHECK CODE" in l): startCheck = True outf.write("\n\ndef check():\n") if "BEGIN RELOAD CODE" in l: startReload = True if "BEGIN INITIALIZATION CODE" in l: startInit = True outf.write("\n\n") for i in initCode: outf.write(t.prettyName(i)) outf.write("\n\n") for l in open(testFile): if l == "<<RESTART>>": if makeRegression: t.restart() outf.write("\n# RESTART\n\n") for r in reloadCode: outf.write(r) for i in initCode: outf.write(i) name = l[:-1] if t.getPreCode(name) != None: for p in t.getPreCode(name): outf.write(t.prettyName(p) + "\n") if makeRegression: t.safely(t.playable(name)) if verbose: outf.write("print ('''" + t.prettyName(name) + "''')\n") if t.getOkExceptions(name) == "": outf.write(t.prettyName(name) + "\n") else: outf.write("try:\n") outf.write(" " + t.prettyName(name) + "\n") outf.write("except (" + t.getOkExceptions(name) + "):\n") outf.write(" pass\n") if t.getPropCode(name) != None: outf.write("assert " + t.prettyName(t.getPropCode(name)) + "\n") if checkRefs: if t.getRefCode(name) != None: for r in t.getRefCode(name): outf.write("try:\n") outf.write(" " + t.prettyName(r) + "\n") outf.write("except:\n") outf.write(" pass\n") if checkProps: outf.write("check()\n") if makeRegression: v = t.shallowState() for (p, vals) in v: if t.prettyName(p) not in name: continue if p in t.opaque(): continue if t.abstraction(p) != None: absFun = t.abstraction(p) else: absFun = "" for i in vals: if vals[i] != None: pname = t.prettyName(p + "[" + str(i) + "]") #outf.write("print (repr("+pname+"))\n") outf.write("assert (repr(" + absFun + "(" + pname + ')) == (' + repr(repr(vals[i])) + '))\n') outf.write('\n\nprint ("TEST COMPLETED SUCCESSFULLY")\n') outf.close()
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('BFS exploration using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time() - start failCount = 0 maxDepth = 0 maxQueue = 0 t = SUT.sut() if config.logging != None: t.setLog(config.logging) queue = [] visited = [] test = [] t.restart() queue.append((t.state(), test)) while (queue != []): if len(queue) > maxQueue: maxQueue = len(queue) (s, test) = queue[0] queue = queue[1:] if len(test) > maxDepth: maxDepth = len(test) print "REACHED DEPTH", maxDepth, "QUEUE SIZE", len(queue) + 1 if len(test) == config.depth: continue t.backtrack(s) shuffleActs = t.enabled() if not config.deterministic: R.shuffle(shuffleActs) for c in shuffleActs: stepOk = t.safely(c) test.append(c) thisBug = False if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" thisBug = True if (not config.ignoreprops) and (not t.check()): handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" thisBug = True ns = t.state() if not thisBug: if config.novisited or (ns not in visited): if (random.random() < config.forget) and (not (queue == [])): break if len(queue) >= config.breadth: break if not config.novisited: visited.append(s) if config.verbose: print len(visited), "NEW STATE:" print s queue.append((ns, test)) elif not config.multiple: break elapsed = time.time() - start if config.running: if t.newBranches() != (set([])): print "ACTION:", action for b in t.newBranches(): print elapsed, len(t.allBranches()), "New branch", b if elapsed > config.timeout: print "STOPPING EXPLORATION DUE TO TIMEOUT, TERMINATED AT LENGTH", len( test) break t.backtrack(s) test = test[:-1] if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if not config.nocover: print t.report(config.coverfile), "PERCENT COVERED" if config.html: t.htmlReport(config.html) print len(visited), "STATES VISITED" print maxDepth, "MAX SEARCH DEPTH" print maxQueue, "MAX QUEUE SIZE" if config.multiple: print failCount, "FAILED" if not config.nocover: print len(t.allBranches()), "BRANCHES COVERED" print len(t.allStatements()), "STATEMENTS COVERED"
def main(): global failCount,t,config,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,lastact,parts , pt , ptlast parsed_args, parser = parse_args() config = make_config(parsed_args, parser) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] t = SUT.sut() tacts = t.actions() a = None sawNew = False nops = 0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 lastact = 0 parts = R.randint(2,10) pt = 0 ptlast = 0 startRestart = time.time() t.restart() restartTime += time.time() - startRestart test = [] for s in xrange(0,config.depth): acts = tacts while True: elapsed = time.time() - start partsize = len(acts)/parts pt = R.randint(0,parts-1) while pt == ptlast: pt = R.randint(0,parts-1) p = R.randint(pt*partsize,(pt+1)*partsize-1) a = acts[p] lastact = p ptlast = pt if a[1](): break else: a = None acts = acts[:p] + acts[p+1:] test.append(a) nops += 1 stepOk = t.safely(a) if not stepOk: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break stepOk = t.safely(a) checkResult = t.check() if not checkResult: print "STOPPING TESTING DUE TO FAILED TEST" break if config.running: if t.newBranches() != set([]): print "ACTION:",a[0] for b in t.newBranches(): print elapsed,len(t.allBranches()),"New branch",b sawNew = True else: sawNew = False if t.newStatements() != set([]): print "ACTION:",a[0] for s in t.newStatements(): print elapsed,len(t.allStatements()),"New statement",s sawNew = True else: sawNew = False if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break print time.time()-start, "TOTAL RUNTIME" print nops, "TOTAL TEST OPERATIONS" if config.coverage: t.restart() print t.report("coverage.out"),"PERCENT COVERED" t.internalReport()
def main(): global start,config,sut,R,nonerror,error,file_name,num,actioncount,Z num = 0 actioncount = 0 file_name = 'failurefile' parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing={}'.format(config)) sut = SUT.sut() R = random.Random(config.seed) start = time.time() elapsed = time.time() - start states = [sut.state()] nonerror = [] error = [] news = None i = 0 while(time.time() < start + config.timeout): for s in states: i += 1 if (time.time() > start + config.timeout): break sut.restart() #the sut is restarted sut.backtrack(s) for w in xrange(0, config.width): for d in xrange(0, config.depth): if (time.time() > start + config.timeout): break ok = check_action() news = sut.newStatements() if not ok: error.append(sut.currStatements()) break if((len(news)>0) and (not ((news in error) or (news in nonerror)))): states.insert(i-1,sut.state()) nonerror.append(sut.currStatements()) #if (config.fault): #if fault is found # for (s_reduces, _, _) in Z: # steps_reduce = "# STEP " + str(j) # print >> f, sut.prettyName(s_reduces).ljust(80 - len(steps_reduce), ' '), steps_reduce # j += 1 # f.close() if config.coverage: sut.internalReport() #this will print the internal report print "####### mytester.py ######" print "Running time: ", time.time() - start print "Bugs Found: ",num print "No. of actions: ", actioncount print "Random count is",(randint(0,actioncount)) print "Actual count", actioncount/(randint(0,actioncount)) print "Complex count", num/actioncount/(randint(0,actioncount))
#print n,co.co_filename,"ALREADY ADDED" return traceLOC lastFuncs[(n, co.co_filename)] = True if co.co_filename == sut.__file__.replace(".pyc", ".py"): #print n,"FROM SUT" return traceLOC loc = len(inspect.getsourcelines(co)[0]) print "==>", n, co.co_filename, loc lastLOCs += loc if not onlyFirst: return traceLOC else: return None SUT = sut.sut() R = random.Random() try: SUT.stopCoverage() except: pass actLOCs = {} N = len(SUT.actions()) numClasses = len(SUT.actionClasses()) print "CONSTRUCTING ESTIMATE FOR", numClasses, "CLASSES" start = time.time() TIMEOUT = int(sys.argv[1]) onlyFirst = "--onlyFirst" in sys.argv
def main(): global failCount,sut,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests global fullPool,activePool,branchCoverageCount,statementCoverageCount,localizeSFail,localizeBFail parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] if config.gendepth != None: failCloud = {} allClouds = {} if config.exploit != None: fullPool = [] activePool = [] if config.quickAnalysis or (config.exploit != None): branchCoverageCount = {} statementCoverageCount = {} if config.uniqueValuesAnalysis: handledValues = {} uniquef = open("unique.corpus",'w') allUniquePaths = [] sut = SUT.sut() if config.relax: sut.relax() if config.readQuick: print "REPLAYING QUICK TESTS" sqrtime = time.time() for f in glob.glob("quicktest.*"): fn = int(f.split("quicktest.")[1]) if fn >= quickCount: quickCount = fn + 1 t = sut.loadTest(f) sut.replay(t,catchUncaught=True,checkProp=(not config.ignoreprops)) print "EXECUTION TIME:",time.time()-sqrtime print "BRANCH COVERAGE:",len(sut.allBranches()) print "STATEMENT COVERAGE:",len(sut.allStatements()) if config.logging != None: sut.setLog(config.logging) if config.profile: profileTime = {} profileCount = {} for a in set(map(sut.actionClass, sut.actions())): profileTime[a] = 0.0 profileCount[a] = 0 if config.markov != None: nactions = len(sut.actions()) mprobs = {} prefix = [] probs = [] inProbs = False readSize = False for l in open(config.markov): if not readSize: markovN = int(l) readSize = True elif "START CLASS" in l: if (prefix != []): mprobs[tuple(prefix)] = probs prefix = [] probs = [] inProbs = False elif inProbs: ls = l.split("%%%%") prob = float(ls[0]) ac = ls[1][1:-1] probs.append((prob,ac)) elif "END CLASS" in l: inProbs = True else: prefix.append(l[:-1]) tacts = sut.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 checkResult = True if config.total: fulltest = open("fulltest.txt",'w') if config.localize: localizeSFail = {} localizeSPass = {} localizeBFail = {} localizeBPass = {} testsPassed = 0 if config.quickAnalysis: quickcf = open("quick.corpus",'w') quickCorpus = [] quickDoneB = {} quickDoneS = {} quickAnalysisTotal = 0 quickAnalysisBCounts = {} quickAnalysisSCounts = {} quickAnalysisCounts = {} quickClassCounts = {} quickAnalysisRawCounts = {} for c in set(map(sut.actionClass,sut.actions())): quickAnalysisCounts[c] = 0 quickClassCounts[c] = 0 quickAnalysisRawCounts[c] = 0 quickAnalysisReducedB = {} quickAnalysisReducedS = {} if config.verbose: print "ABOUT TO START TESTING" sys.stdout.flush() while (config.maxtests == -1) or (ntests < config.maxtests): if config.verbose: print "STARTING TEST",ntests sys.stdout.flush() ntests += 1 if config.swarm: sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP) #print "CONFIG:",(sut.swarmConfig()) if config.highLowSwarm != None: classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm) if config.swarmSwitch != None: lastSwitch = 0 switches = [] for i in xrange(0,config.swarmSwitch): switch = R.randrange(lastSwitch+1,config.depth-((config.swarmSwitch-i))) switches.append(switch) lastSwitch = switch startRestart = time.time() sut.restart() restartTime += time.time() - startRestart if config.total: fulltest.write("<<RESTART>>\n") if config.replayable: currtest = open("currtest.txt",'w') if (config.exploit != None) and ((time.time() - start) > config.startExploit): tryExploit() testFailed = False for s in xrange(0,config.depth): if config.verbose: print "GENERATING STEP",s if (config.swarmSwitch != None) and (s in switches): if config.highLowSwarm == None: sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP) else: classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm) if (config.swarmLength != None) and (((s + 1) % config.swarmLength) == 0): if config.highLowSwarm == None: sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP) else: classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm) startGuard = time.time() tryStutter = (a != None) and (a[1]()) and ((config.stutter != None) or config.greedyStutter) if tryStutter: if (config.stutter == None) or (R.random() > config.stutter): tryStutter = False if (config.greedyStutter) and sawNew: print "TRYING TO STUTTER DUE TO COVERAGE GAIN" tryStutter = True else: if config.markov == None: if config.highLowSwarm == None: a = sut.randomEnabled(R) else: a = sut.randomEnabledClassProbs(R,classP) else: prefix = tuple(map(sut.actionClass,sut.test()[-markovN:])) if prefix not in mprobs: a = sut.randomEnabled(R) else: a = sut.randomEnabledClassProbs(R,mprobs[prefix]) if a == None: print "WARNING: DEADLOCK (NO ENABLED ACTIONS)" guardTime += time.time()-startGuard elapsed = time.time() - start if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test()) break if a == None: print "TERMINATING TEST DUE TO NO ENABLED ACTIONS" break if tryStutter: print "STUTTERING WITH",a[0] if config.replayable: currtest.write(a[0] + "\n") currtest.flush() if config.total: fulltest.write(a[0] + "\n") fulltest.flush() if config.verbose: print "ACTION:",sut.prettyName(a[0]) startOp = time.time() if config.quickAnalysis: quickClassCounts[sut.actionClass(a)] += 1 stepOk = sut.safely(a) thisOpTime = time.time()-startOp nops += 1 if config.profile: profileTime[sut.actionClass(a)] += thisOpTime profileCount[sut.actionClass(a)] += 1 opTime += thisOpTime if sut.warning() != None: print "SUT WARNING:",sut.warning() if tryStutter: print "DONE STUTTERING" if (stepOk or config.uncaught) and config.ignoreprops and (config.exploit != None): collectExploitable() if (not config.uncaught) and (not stepOk): testFailed = True handle_failure(sut.test(), "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break startCheck = time.time() if not config.ignoreprops: checkResult = sut.check() checkTime += time.time()-startCheck if checkResult and (stepOk or config.uncaught) and (config.exploit != None): collectExploitable() if not checkResult: testFailed = True handle_failure(sut.test(), "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): print "ACTION:",sut.prettyName(a[0]) for b in sut.newBranches(): print elapsed,len(sut.allBranches()),"New branch",b sys.stdout.flush() sawNew = True else: sawNew = False if sut.newStatements() != set([]): print "ACTION:",sut.prettyName(a[0]) for s in sut.newStatements(): print elapsed,len(sut.allStatements()),"New statement",s sys.stdout.flush() sawNew = True else: sawNew = False if config.uniqueValuesAnalysis: uvals = sut.uniqueVals() olds = sut.state() currTest = list(sut.test()) for u in uvals: if u not in handledValues: print "ANALYZING NEW UNIQUE VALUE:",u else: continue r = sut.reduce(currTest, sut.coversUnique(u), keepLast=False) rc = map(sut.actionClass, r) sut.replay(r) for ru in sut.uniqueVals(): handledValues[ru] = True if rc not in allUniquePaths: print "NEW PATH DISCOVERED" allUniquePaths.append(rc) for s in rc: uniquef.write(s+"\n") uniquef.write(("="*50)+"\n") uniquef.flush() sut.backtrack(olds) if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test()) break if (config.exploit != None) and (not config.quickAnalysis): for b in sut.currBranches(): if b not in branchCoverageCount: branchCoverageCount[b] = 1 else: branchCoverageCount[b] += 1 for s in sut.currStatements(): if s not in statementCoverageCount: statementCoverageCount[s] = 1 else: statementCoverageCount[s] += 1 if config.localize and not testFailed: testsPassed += 1 for s in sut.currStatements(): if s not in localizeSPass: localizeSPass[s] = 0 localizeSPass[s] += 1 for b in sut.currBranches(): if b not in localizeBPass: localizeBPass[b] = 0 localizeBPass[b] += 1 if config.quickAnalysis: currTest = list(sut.test()) sut.replay(currTest) currB = sut.currBranches() currS = sut.currStatements() clen = len(currTest) #print "GATHERING QUICK ANALYSIS DATA FOR",len(currB),"BRANCHES" for b in currB: if config.fastQuickAnalysis and (b in quickDoneB): continue print "ANALYZING BRANCH",b if b not in branchCoverageCount: branchCoverageCount[b] = 0 quickAnalysisReducedB[b] = 0 branchCoverageCount[b] += 1 r = sut.reduce(currTest,sut.coversBranches([b]),keepLast=False) print "REDUCED FROM",clen,"TO",len(r) sys.stdout.flush() sut.replay(r) for br in sut.currBranches(): quickDoneB[br] = True for sr in sut.currStatements(): quickDoneS[sr] = True rc = map(sut.actionClass,r) if rc not in quickCorpus: quickCorpus.append(rc) for s in rc: quickcf.write(s+"\n") quickcf.write(("="*50)+"\n") quickcf.flush() sut.replay(r) for b2 in sut.currBranches(): if b2 not in quickAnalysisReducedB: quickAnalysisReducedB[b2] = 0 quickAnalysisReducedB[b2] += 1 for s2 in sut.currStatements(): if s2 not in quickAnalysisReducedS: quickAnalysisReducedS[s2] = 0 quickAnalysisReducedS[s2] += 1 if b not in quickAnalysisBCounts: quickAnalysisBCounts[b] = {} quickAnalysisTotal += 1 for c in map(sut.actionClass,r): quickAnalysisRawCounts[c] += 1 for c in set(map(sut.actionClass,r)): quickAnalysisCounts[c] += 1 if c not in quickAnalysisBCounts[b]: quickAnalysisBCounts[b][c] = 0 quickAnalysisBCounts[b][c] += 1 #print "GATHERING QUICK ANALYSIS DATA FOR",len(currS),"STATEMENTS" for s in currS: if config.fastQuickAnalysis and (s in quickDoneS): continue if s not in statementCoverageCount: statementCoverageCount[s] = 0 quickAnalysisReducedS[s] = 0 statementCoverageCount[s] += 1 print "ANALYZING STATEMENT",s r = sut.reduce(currTest,sut.coversStatements([s]),keepLast=False) print "REDUCED FROM",clen,"TO",len(r) sys.stdout.flush() sut.replay(r) for br in sut.currBranches(): quickDoneB[br] = True for sr in sut.currStatements(): quickDoneS[sr] = True rc = map(sut.actionClass,r) if rc not in quickCorpus: quickCorpus.append(rc) sut.replay(r) for b2 in sut.currBranches(): if b2 not in quickAnalysisReducedB: quickAnalysisReducedB[b2] = 0 quickAnalysisReducedB[b2] += 1 for s2 in sut.currStatements(): quickAnalysisReducedS[s2] += 1 if s not in quickAnalysisSCounts: quickAnalysisSCounts[s] = {} quickAnalysisTotal += 1 for c in map(sut.actionClass,r): quickAnalysisRawCounts[c] += 1 for c in set(map(sut.actionClass,r)): quickAnalysisCounts[c] += 1 if c not in quickAnalysisSCounts[s]: quickAnalysisSCounts[s][c] = 0 quickAnalysisSCounts[s][c] += 1 if config.throughput: print "ACTION THROUGHPUT:",nops/(time.time()-start) if config.replayable: currtest.close() if config.quickTests: if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])): handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True) if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if config.total: fulltest.close() if not config.nocover: sut.restart() print sut.report(config.coverfile),"PERCENT COVERED" if config.internal: sut.internalReport() if config.html: sut.htmlReport(config.html) if config.uniqueValuesAnalysis: uniquef.close() if config.quickAnalysis: quickcf.close() print "*" * 70 print "QUICK ANALYSIS RESULTS:" print "*" * 70 print "TEST PATTERNS:" for rc in quickCorpus: print "="*50 for s in rc: print s print "*" * 70 print "OVERALL ACTION ANALYSIS:" totalTaken = sum(quickClassCounts.values()) actSort = sorted(quickAnalysisRawCounts.keys(),key=lambda x: quickAnalysisCounts.get(x,0), reverse=True) for a in actSort: print "="*50 print "ACTION CLASS:" print a print "APPEARS",quickClassCounts[a],"TIMES IN TESTS" print "APPEARS",quickAnalysisRawCounts[a],"TIMES IN REDUCED TESTS" print "APPEARS IN",quickAnalysisCounts[a],"REDUCED TESTS ("+str(round((quickAnalysisCounts[a]/(quickAnalysisTotal*1.0))*100,2)) + "%)" #baselineRate = quickClassCounts[a]/(totalTaken*1.0) #reducedRate = quickAnalysisRawCounts[a]/(quickAnalysisTotal*1.0) #if reducedRate > 0.0: # print "RATIO:",(baselineRate/reducedRate) #else: # print "RATIO: INFINITE" print "*" * 70 print "DETAILED BRANCH ANALYSIS" branchCoverageCountSort = sorted(branchCoverageCount.keys(), key = lambda x: branchCoverageCount[x]) for b in branchCoverageCountSort: print "="*50 print "BRANCH:",b baselineRate = branchCoverageCount[b]/(ntests*1.0) print "IN",str(round(baselineRate*100,2))+"% OF TESTS ("+str(branchCoverageCount[b])+" TESTS)" reducedRate = quickAnalysisReducedB[b]/(quickAnalysisTotal*1.0) print "IN",str(round(reducedRate*100,2))+"% OF REDUCED TESTS" if reducedRate > 0.0: print "RATIO:",(baselineRate/reducedRate) print "REDUCED TEST ACTION ANALYSIS:" sortAs = sorted(quickAnalysisBCounts[b].keys(),key=lambda x: quickAnalysisBCounts[b][x],reverse=True) print branchCoverageCount[b],"TESTS" for a in sortAs: print a,str(round(quickAnalysisBCounts[b][a]/(branchCoverageCount[b]*1.0)*100,2))+"%" print "*" * 70 print "DETAILED STATEMENT ANALYSIS" statementCoverageCountSort = sorted(statementCoverageCount.keys(), key = lambda x: statementCoverageCount[x]) for s in statementCoverageCountSort: print "="*50 print "STATEMENT:",s baselineRate = statementCoverageCount[s]/(ntests*1.0) print "IN",str(round(baselineRate*100,2))+"% OF TESTS" reducedRate = quickAnalysisReducedS[s]/(quickAnalysisTotal*1.0) print "IN",str(round(reducedRate*100,2))+"% OF REDUCED TESTS" if reducedRate > 0.0: print "RATIO:",(baselineRate/reducedRate) print "REDUCED TEST ACTION ANALYSIS:" print statementCoverageCount[s],"TESTS" sortAs = sorted(quickAnalysisSCounts[s].keys(),key=lambda x: quickAnalysisSCounts[s][x],reverse=True) for a in sortAs: print a,str(round(quickAnalysisSCounts[s][a]/(statementCoverageCount[s]*1.0)*100,2))+"%" print time.time()-start, "TOTAL RUNTIME" print ntests, "EXECUTED" print nops, "TOTAL TEST OPERATIONS" print opTime, "TIME SPENT EXECUTING TEST OPERATIONS" print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS" if not config.ignoreprops: print checkTime, "TIME SPENT CHECKING PROPERTIES" print (opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT" print restartTime, "TIME SPENT RESTARTING" print reduceTime, "TIME SPENT REDUCING TEST CASES" if config.multiple: print failCount,"FAILED" print repeatCount,"REPEATS OF FAILURES" print len(failures),"ACTUAL DISTINCT FAILURES" print n = 0 for (test, err) in failures: print "FAILURE",n sut.prettyPrintTest(test) n += 1 if err != None: print "ERROR:", err print "TRACEBACK:" traceback.print_tb(err[2]) i = -1 if config.compareFails: # Comparison feature normally not useful, just for researching normalization for test1 in failures: i += 1 j = -1 for test2 in failures: j += 1 if (j > i): print "COMPARING FAILURE",i,"AND FAILURE",j for k in xrange(0,max(len(test1),len(test2))): if k >= len(test1): print "STEP",k,"-->",test2[k][0] elif k >= len(test2): print "STEP",k,test1[k][0],"-->" elif test1[k] != test2[k]: print "STEP",k,test1[k][0],"-->",test2[k][0] if config.profile: print "ACTION PROFILE:" averages = [] for a in profileTime: if profileCount[a] != 0: averages.append((a,profileTime[a]/profileCount[a])) averages = sorted(averages, key = lambda x: x[1]) maxAvg = averages[-1][1] minAvg = averages[0][1] sumAvg = sum(map(lambda x: x[1], averages)) for (a,t) in averages: print a,profileCount[a],t,round(t/maxAvg,2),round(t/minAvg,2),round(t/sumAvg,2) if config.localize and failCount > 0: scoresS = {} scoresB = {} for s in sut.allStatements(): if s not in localizeSPass: localizeSPass[s] = 0 if s not in localizeSFail: localizeSFail[s] = 0 denom = math.sqrt(failCount*(localizeSFail[s]+localizeSPass[s])) if denom == 0.0: continue scoresS[s] = localizeSFail[s]/denom for b in sut.allBranches(): if b not in localizeBPass: localizeBPass[b] = 0 if b not in localizeBFail: localizeBFail[b] = 0 denom = math.sqrt(failCount*(localizeBFail[b]+localizeBPass[b])) if denom == 0.0: continue scoresB[b] = localizeBFail[b]/denom sortedS = sorted(scoresS.keys(),key = lambda x:scoresS[x]) sortedB = sorted(scoresB.keys(),key = lambda x:scoresB[x]) print "FAULT LOCALIZATION RESULTS:" for s in sortedS: if scoresS[s] > 0.0: print s, scoresS[s] for b in sortedB: if scoresB[b] > 0.0: print b, scoresB[b] if not config.nocover: print len(sut.allBranches()),"BRANCHES COVERED" print len(sut.allStatements()),"STATEMENTS COVERED"
def main(): global timeout parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Reducing using config={}'.format(config))) sut = SUT.sut() timeout = config.timeout if not ((config.coverage) or (config.coverMore) or (config.decompose)): try: sut.stopCoverage() except BaseException: pass R = None if config.random: R = random.Random() if config.seed is not None: R.seed(config.seed) r = sut.loadTest(config.infile, afl=config.afl, swarm=config.aflswarm) f = None if config.matchException: print("EXECUTING TEST TO OBTAIN FAILURE FOR EXCEPTION MATCHING...") assert sut.fails(r) f = sut.failure() print("ERROR:", f) print("TRACEBACK:") traceback.print_tb(f[2], file=sys.stdout) if not config.sandbox: pred = (lambda x: sut.failsCheck(x, failure=f)) if config.noCheck: pred = (lambda x: sut.fails(x, failure=f)) else: pred = sandboxReplay if config.coverage or config.coverMore or (config.decompose and not config.noNormalize): print("EXECUTING TEST TO OBTAIN COVERAGE FOR CAUSE REDUCTION...") sut.replay(r, checkProp=not config.noCheck, catchUncaught=config.uncaught) # Replay it twice, due to odd coverage on startup sometimes sut.replay(r, checkProp=not config.noCheck, catchUncaught=config.uncaught) b = set(sut.currBranches()) s = set(sut.currStatements()) print("PRESERVING", len(b), "BRANCHES AND", len(s), "STATEMENTS") if config.coverMore: pred = sut.coversMore(s, b, checkProp=not config.noCheck, catchUncaught=config.uncaught) else: pred = sut.coversAll(s, b, checkProp=not config.noCheck, catchUncaught=config.uncaught) if config.checkDeterminism: pred = ( lambda t: sut.nondeterministic(t, delay=config.determinismDelay, tries=config.determinismTries, delay0=config.determinismDelay0)) if config.checkStepDeterminism: pred = (lambda t: sut.stepNondeterministic( t, delay=config.determinismDelay, tries=config.determinismTries, delay0=config.determinismDelay0)) if config.checkProcessDeterminism: pred = (lambda t: sut.processNondeterministic( t, delay=config.determinismDelay, tries=config.determinismTries, iterate=config.iterate)) if config.probability is not None: Ppred = pred pred = (lambda t: sut.forceP(t, Ppred, P=config.probability, samples=config.samples, replications=config.replications)) print("STARTING WITH TEST OF LENGTH", len(r)) if not config.noReduce: start = time.time() print("REDUCING...") if (not config.multiple) and (not config.decompose): r = sut.reduce(r, pred, verbose=config.verbose, tryFast=not config.ddmin, keepLast=config.keepLast, pruneGuards=not config.noPruneGuards, rgen=R) elif config.multiple: rs = sut.reductions(r, pred, verbose=config.verbose, recursive=config.recursive, limit=config.limit, keepLast=config.keepLast, pruneGuards=not config.noPruneGuards, tryFast=not config.ddmin) elif config.decompose: print("DECOMPOSING...") rs = sut.coverDecompose(r, verbose=config.verbose, checkProp=not config.noCheck, catchUncaught=config.uncaught) print("REDUCED IN", time.time() - start, "SECONDS") if (not config.multiple) and (not config.decompose): print("NEW LENGTH", len(r)) else: print("NEW LENGTHS", list(map(len, rs))) if not config.noAlpha: print("ALPHA CONVERTING...") if (not config.multiple) and (not config.decompose): r = sut.alphaConvert(r) else: rs = list(map(sut.alphaConvert, rs)) if not config.noNormalize: start = time.time() print("NORMALIZING...") if (not config.multiple) and (not config.decompose): r = sut.normalize(r, pred, verbose=config.verbose, keepLast=config.keepLast, checkEnabled=not config.noCheckEnabled, pruneGuards=not config.noPruneGuards, tryFast=not config.ddmin, speed=config.speed) else: newrs = [] for r in rs: f = None if config.matchException: print( "EXECUTING TEST TO OBTAIN FAILURE FOR EXCEPTION MATCHING..." ) assert sut.fails(r) f = sut.failure() print("ERROR:", f) print("TRACEBACK:") traceback.print_tb(f[2], file=sys.stdout) if not config.sandbox: pred = (lambda x: sut.failsCheck(x, failure=f)) if not config.noCheck: pred = (lambda x: sut.fails(x, failure=f)) else: pred = sandboxReplay if config.coverage or config.coverMore or ( config.decompose and not config.noNormalize): print( "EXECUTING TEST TO OBTAIN COVERAGE FOR CAUSE REDUCTION..." ) sut.replay(r, checkProp=not config.noCheck, catchUncaught=config.uncaught) b = set(sut.currBranches()) s = set(sut.currStatements()) print("PRESERVING", len(b), "BRANCHES AND", len(s), "STATEMENTS") if config.coverMore: pred = sut.coversMore(s, b, checkProp=not config.noCheck, catchUncaught=config.uncaught) else: pred = sut.coversAll(s, b, checkProp=not config.noCheck, catchUncaught=config.uncaught) newrs.append( sut.normalize(r, pred, verbose=config.verbose, keepLast=config.keepLast, pruneGuards=not config.noPruneGuards, tryFast=not config.ddmin)) rs = newrs print("NORMALIZED IN", time.time() - start, "SECONDS") if (not config.multiple) and (not config.decompose): print("NEW LENGTH", len(r)) else: print("NEW LENGTHS", list(map(len, rs))) if (not config.multiple) and (not config.decompose): sut.saveTest(r, config.outfile, afl=config.writeafl) sut.prettyPrintTest(r) print() print("TEST WRITTEN TO", config.outfile) else: i = 0 for r in rs: print("TEST #" + str(i) + ":") sut.saveTest(r, config.outfile + "." + str(i) + ".test", afl=config.writeafl) sut.prettyPrintTest(r) print() print("TEST WRITTEN TO", config.outfile + "." + str(i)) print() i += 1
def create_new_pool(): return [SUT.sut(), [[]], [], 0.0, dict(), 0.0]
def main(): if "--help" in sys.argv: print("Usage: tstl_markov <outfile> <prefix size> <test files> [--notRaw]") print("Options:") print(" --notRaw: corpus files are not raw TSTL tests, but action classes") sys.exit(0) sut = SUT.sut() classes = [] nacts = len(sut.actions()) for a in sut.actions(): if sut.actionClass(a) not in classes: classes.append(sut.actionClass(a)) #print len(classes) n = int(sys.argv[2]) outfile = sys.argv[1] corpfiles = sys.argv[3:] corpfiles = [x for x in corpfiles if x != "--notRaw"] tests = [] test = [] for f in corpfiles: for l in open(f): if "--notRaw" in sys.argv: test.append(l[:-1]) else: test.append(sut.actionClass(sut.playable(l[:-1]))) if test != []: tests.append(test) test = [] chains = {} for t in tests: for pos in range(n+1,len(t)): prefix = tuple(t[pos-n:pos]) #print prefix,"==>",t[pos] if prefix not in chains: chains[prefix] = [] chains[prefix].append(t[pos]) mout = open(outfile,'w') mout.write(str(n)+"\n") for c in chains: print("PREFIX:",c) mout.write("START CLASS\n") for ac in c: mout.write(ac+"\n") mout.write("END CLASS\n") counts = {} total = 0.0 for suffix in chains[c]: total += 1 if suffix not in counts: counts[suffix] = 0 counts[suffix] += 1 for suffix in counts: print(suffix,counts[suffix]/total) mout.write(str(counts[suffix]/total) + " %%%% "+suffix+"\n") mout.close()
def main(): global start,config,sut,R,nonErrorSeq,ErrorSeq,bug,act parsed_args, parser = parse_args() config = make_config(parsed_args, parser) nonErrorSeq = [] ErrorSeq = [] newSeq = None sut = SUT.sut() R = random.Random(config.seed) start = time.time() elapsed = time.time() - start states = [sut.state()] n = -1 bug = 0 act = 0 while(time.time() < start + config.timeout): for s in states: if (time.time() > start + config.timeout): break n+=1 sut.restart() sut.backtrack(s) for i in xrange(0, config.width): for j in xrange(0, config.depth): if (time.time() > start + config.timeout): break action = sut.randomEnabled(R) act+=1 ok = sut.safely(action) elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()),"New branch",b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s if elapsed > config.timeout: print "Timeout: Stop testing." break if not ok: bug += 1 print "Bug" , bug startReduce = time.time() r = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(r) print(sut.failure()) if config.fault: filename = 'failure%d.test' % bug sut.saveTest(r, filename) ErrorSeq.append(sut.currStatements()) newSeq = sut.newStatements() if not ok: break if not (newSeq in nonErrorSeq): if (len(newSeq)>0) : states.insert(n,sut.state()) nonErrorSeq.append(sut.currStatements()) else: continue else: states.insert(n,sut.state()) nonErrorSeq.append(sut.currStatements()) if config.coverage: sut.internalReport() print "Total Bugs: ",bug print "Total actions: ", act print "Total Running time: ", time.time() - start
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random variation of beam search using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 t = SUT.sut() if config.logging != None: t.setLog(config.logging) tacts = t.actions() ntests = 0 while (config.maxtests == -1) or (ntests < config.maxtests): ntests += 1 t.restart() test = [] for s in xrange(0,config.depth): count = 0 newCover = False acts = tacts old = t.state() while (count <= config.width): count += 1 while True: p = R.randint(0,len(acts)-1) a = acts[p] if a[1](): break acts = acts[:p] + acts[p+1:] test.append(a) stepOk = t.safely(a) if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break if (not config.ignoreprops) and (not t.check()): handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" break elapsed = time.time() - start if t.newBranches() != set([]): if config.running: print "ACTION:",a[0] for b in t.newBranches(): print elapsed,len(t.allBranches()),"New branch",b break # Continue with this choice if new branch exposed if elapsed > config.timeout: break test = test[:-1] t.backtrack(old) if elapsed > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if not config.nocover: print t.report(config.coverfile),"PERCENT COVERED" if config.html: t.htmlReport(config.html) print ntests, "EXECUTED" if config.multiple: print failCount,"FAILED" if not config.nocover: print len(t.allBranches()),"BRANCHES COVERED" print len(t.allStatements()),"STATEMENTS COVERED"
from __future__ import print_function import sys import sut import glob sut = sut.sut() def abstraction(s): if len(s) <= 3: return (sut.actionClass(s), ) else: return (sut.actionClass(s), s[3]) seedFiles = sys.argv[1] test = sys.argv[2] newTestName = sys.argv[3] abstract = "--abstract" in sys.argv seeds = [] for f in glob.glob(seedFiles): t = sut.loadTest(f) if abstract: t = list(map(abstraction, t)) seeds.append((t, f)) original = sut.loadTest(test) t = list(original)
def main(): global failCount,sut,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests rand = random.Random(seed) start = time.time() #elapsed = time.time()-start runningtime = time.time()-startprog sut = SUT.sut() tacts = sut.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0 bugs = 0 checkResult = True naction = 10 maxaction = 107 actiontime = 1 while runningtime < timeout: sut.restart() elapsed = time.time()-startprog for d in xrange(0,depth): elapsed = time.time()-startprog if elapsed > timeout: break startaction = time.time() for w in xrange(0,width): elapsed = time.time()-startprog #a = sut.randomEnabled(R) a = sut.randomEnableds(rand, naction) actionelapsed = time.time() - startaction for i in xrange(0,naction): if a[0] == None: print "WARNING: DEADLOCK (NO ENABLED ACTIONS)" if elapsed > timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test()) break if a[0] == None: print "TERMINATING TEST DUE TO NO ENABLED ACTIONS" break nops += 1 startOp = time.time() stepOk = sut.safely(a[0]) propok = sut.check() if sut.warning() != None: print "SUT WARNING:",sut.warning() opTime += (time.time()-startOp) if (not propok) or (not stepOk): bugs += 1 print "TEST FAILED" print "REDUCING..." R = sut.reduce(sut.test(), lambda x: sut.fails(x) or sut.failsCheck(x)) sut.prettyPrintTest(R) print sut.failure() if fault: filename='failure'+str(bugs)+'.test' for i in range(len(R)): with open(filename,'w') as f: f.write('\n'+'Bug no.'+str(bugs)+'\n') f.write(str(sut.failure())+'\n') f.write(str(R)+'\n') #print "NORMALIZING..." #N = sut.normalize(R, lambda x: sut.fails(x) or sut.failsCheck(x)) #sut.prettyPrintTest(N) #print "GENERALIZING..." #sut.generalize(N, lambda x: sut.fails(x) or sut.failsCheck(x)) #break elapsed = time.time() - startprog if running: if sut.newBranches() != set([]): print "ACTION:",a[i] for b in sut.newBranches(): print elapsed,len(sut.allBranches()),"New branch",b sawNew = True else: sawNew = False if sut.newStatements() != set([]): print "ACTION:",a[i] for s in sut.newStatements(): print elapsed,len(sut.allStatements()),"New statement",s sawNew = True else: sawNew = False if elapsed > timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test()) break if(actionelapsed > actiontime): if(naction < maxaction): naction+=1 if elapsed > timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test()) break if coverage: sut.internalReport() print time.time()-startprog, "TOTAL RUNTIME" print nops, "TOTAL TEST OPERATIONS" print opTime, "TIME SPENT EXECUTING TEST OPERATIONS" print bugs,"FAILED"
R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 quickCount = 0 repeatCount = 0 failures = [] cloudFailures = [] if config.gendepth != None: failCloud = {} allClouds = {} t = SUT.sut() if config.logging != None: t.setLog(config.logging) tacts = t.actions() a = None sawNew = False nops = 0 ntests = 0 reduceTime = 0.0 opTime = 0.0 checkTime = 0.0 guardTime = 0.0 restartTime = 0.0
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Triaging using config={}'.format(config))) if config.ignoreContaining is not None: ignoredStrings = config.ignoreContaining.split(";;") sut = SUT.sut() try: sut.stopCoverage() except BaseException: pass numTests = 0.0 numInvalid = 0 numPassing = 0 failingTests = {} if config.useFailures: sigs = {} for fn in glob.glob(config.testglob): try: t = sut.loadTest(fn) except BaseException: print("INVALID TEST CASE:", fn) numInvalid += 1 continue if config.ignoreContaining is not None: foundString = False for s in t: for istr in ignoredStrings: if istr in s[0]: foundString = True break if foundString: break if foundString: continue if not config.noCheck: fails = sut.failsAny(t) else: fails = sut.fails(t) if fails: ft = list(sut.test()) v = vector(ft, sut) if config.useFailures: e = repr(sut.failure()) e = e[:e.find("<traceback object at 0x")] + ")" sig = noDigits(e) if config.abstractStrings: sig = noStrings(sig) if sig in sigs: sigs[sig].append(fn) else: sigs[sig] = [fn] if not config.ignoreLast: vlast = [] for ac in sorted(sut.actionClasses()): if sut.actionClass(ft[-1]) == ac: vlast.append(1.0) else: vlast.append(0.0) v.extend(vlast) failingTests[fn] = (ft, v, sut.failure()) else: numPassing += 1 if config.useFailures: sk = sorted(sigs.keys()) for fn in failingTests: vsig = [] for sig in sk: if fn in sigs[sig]: vsig.append(1.0) else: vsig.append(0.0) _, v, _ = failingTests[fn] v.extend(vsig) distances = {} for fn in failingTests: distances[fn] = {} for fn1 in failingTests: for fn2 in failingTests: if fn1 == fn2: distances[fn1][fn2] = 0.0 continue if fn1 > fn2: continue distances[fn1][fn2] = euclidean(failingTests[fn1][1], failingTests[fn2][1]) distances[fn2][fn1] = distances[fn1][fn2] chosen = failingTests.keys()[0] print("=" * 80) print(1, chosen) if config.showTests: print() sut.prettyPrintTest(failingTests[chosen][0]) f = failingTests[chosen][2] print("EXCEPTION:", repr(f)) print("FAILURE:") traceback.print_tb(f[2], file=sys.stdout) ranked = [chosen] while len(ranked) != len(failingTests): chosen = None maxMinD = 0.0 for fn in failingTests: d = [] for r in ranked: d.append(distances[fn][r]) minD = min(d) if minD > maxMinD: chosen = fn maxMinD = minD if chosen is None: print("ALL REMAINING TESTS ARE ZERO DISTANCE FROM A RANKED TEST") break ranked.append(chosen) print("=" * 80) print(len(ranked), chosen, maxMinD) if config.showTests: print() sut.prettyPrintTest(failingTests[chosen][0]) print() f = failingTests[chosen][2] print("EXCEPTION:", repr(f)) print("FAILURE:") traceback.print_tb(f[2], file=sys.stdout) print() print("*" * 80) print(int(numTests), "TOTAL FAILING TESTS") if numInvalid > 0: print(numInvalid, "TOTAL INVALID TESTS") if numPassing > 0: print(numPassing, "TOTAL PASSING TESTS")
def main(): global start,config,sut,R,nonErrorSeq,ErrorSeq,file_name,num num = 0 file_name = 'failurefile' parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Random testing using config={}'.format(config)) nonErrorSeq = [] ErrorSeq = [] newSeq = None sut = SUT.sut() R = random.Random(config.seed) start = time.time() elapsed = time.time() - start states = [sut.state()] while(time.time() < start + config.timeout): for s in states: sut.restart() sut.backtrack(s) for w in xrange(0, config.width): for d in xrange(0, config.depth): action = sut.randomEnabled(R) ok = sut.safely(action) elapsed = time.time() - start if config.running: if len(sut.newBranches()) > 0: print "ACTION:", action[0] for b in sut.newBranches(): print elapsed, len(sut.allBranches()),"New branch",b if not ok: num += 1 print "Found Bug" , num print "REDUCING..." startReduce = time.time() test = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(test) print(sut.failure()) if config.fault: f = open((file_name + str(num) + ".test"),"w") f.writelines(str(sut.failure())) f.writelines('\nReduced test has ' + str(len(test)) + ' steps') f.close() ErrorSeq.append(sut.currStatements()) newSeq = sut.newStatements() if not ok: break if((len(newSeq)>0) and (not ((newSeq in nonErrorSeq) or (newSeq in ErrorSeq)))): states.insert(0,sut.state()) nonErrorSeq.append(sut.currStatements()) if config.coverage: sut.internalReport() print "Total Bugs: ",num
def main(): global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount, test_file, ntest parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start while elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): for j in xrange(0, config.width): if randomAct(): preTest = sut.state() else: sut.backtrack(preTest) elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break elapsed = time.time() - start if elapsed > config.timeout: break collectCoverage() elapsed = time.time() - start printCoverage() if config.coverage: sut.internalReport() print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
def main(): if "--help" in sys.argv: print( "Usage: tstl_graph <outfile> <depth> <width> [<seed>] [<traces> (default 1)] [<skip> (default none)]" ) sys.exit(0) outfile = sys.argv[1] depth = int(sys.argv[2]) k = int(sys.argv[3]) if len(sys.argv) > 5: seed = int(sys.argv[4]) random.seed(seed) if len(sys.argv) > 5: traces = int(sys.argv[5]) else: traces = 1 if len(sys.argv) > 6: skiplen = int(sys.argv[6]) else: skiplen = -1 print("Producing graph of", traces, "traces with depth", depth, "and width", k, "starting from", skiplen) dot = Digraph(comment="Depth " + str(depth)) for i in range(0, traces): d = 0 s = 0 state = str(i) + r"\<init\>" dot.node(state, r"\<init\>", penwidth="3.0", shape='box') t = SUT.sut() t.restart() s = 0 d = 1 last = state midFlip = True sd = 0 while sd <= skiplen: nexta = t.enabled() act = random.choice(nexta) t.safely(act) sd += 1 while d <= depth: nexta = t.enabled() act = random.choice(nexta) aname = t.prettyName(act[0]) nexts = [t.prettyName(a[0]) for a in nexta] eqnexts = nexts eqnexts = collapse(eqnexts) eqnexts = [x for x in eqnexts if x != aname] random.shuffle(eqnexts) eqnexts = eqnexts[-(k - 1):] mid = len(eqnexts) / 2 if (len(eqnexts) % 2) != 0: if midFlip: mid = mid + 1 midFlip = not midFlip eqnexts = eqnexts[:mid] + [aname] + eqnexts[mid:] for name in eqnexts: s += 1 state = str(i) + "s" + str(s) if name == aname: newLast = state dot.node(state, name, penwidth="3.0", shape='box') dot.edge(last, state, penwidth="3.0") else: dot.node(state, name, fontsize="10.0") dot.edge(last, state) last = newLast t.safely(act) d += 1 dot.render(outfile, view=True)
def main(): global config, R, sut, bugs, ntest, start global actCount, branchCount, statementCount global savedTest, savedFlag parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() if config.silence: sut.silenceCoverage() bugs = 0 actCount = 0 branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start print "Starting Phase 1" if elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() print "Finishing Phase 1" print "Starting Phase 2" elapsed = time.time() - start while elapsed < config.timeout: sut.restart() ntest += 1 savedTest = None if (savedTest != None) and (R.random() > 0.7): sut.backtrack(savedTest) savedFlag = False for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() elapsed = time.time() - start print "Finishing Phase 2" if time.time() - start > config.timeout: print "[TIMEOUT]" if config.coverdetail: printCoverage() if config.coverage: sut.internalReport() if config.resultdetail: print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
def __init__(self): self.errorSeqs = [] self.nonErrorSeqs = [] self.sut = sut.sut()
def main(): #global variables are defined here global sut, rgen, actCount, bugs, coveraged, visited, start, argv parsed_args, parser = parse_args() argv = make_config(parsed_args, parser) start = time.time() sut = sut.sut() sut.silenceCoverage() rgen = random.Random() rgen.seed(argv.SEED) actCount = 0 bugs = 0 coveraged = {} visited = [] print "STARTING PHASE 1" while time.time() - start < argv.TIMEOUT * 0.5: sut.restart() for s in xrange(0,argv.DEPTH): randomAction() collectCoverage() sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x]) for s in sortedCov: print s, coveraged[s] print "STARTING PHASE 2" while time.time() - start < argv.TIMEOUT : belowMedian = set([]) sort = sorted(coveraged.keys(),key = lambda x : coveraged[x]) a = len(sort)/2 for s in sort: if coveraged[s] < sort[a]: belowMedian.add(s) else: break activePool = [] for (t,c) in visited: for s in c: if s in belowMedian: activePool.append((t,c)) break sut.restart() if len(sut.newStatements()) != 0: print "new statement",sut.newStatements() visited.append((list(sut.test()), set(sut.currStatements()))) for s in xrange(0,argv.DEPTH): if not randomAction(): break collectCoverage() sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x]) for s in sortedCov: print s, coveraged[s] if argv.COVERAGE: sut.internalReport() print "FAILURES ",bugs, "TIMES" print "TOTAL ACTIONS", actCount print "TOTAL RUNTIME", time.time()-start
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('BFS exploration using config={}'.format(config)) R = random.Random(config.seed) start = time.time() elapsed = time.time()-start failCount = 0 maxDepth = 0 maxQueue = 0 t = SUT.sut() if config.logging != None: t.setLog(config.logging) queue = [] visited = [] test = [] t.restart() queue.append((t.state(), test)) while (queue != []): if len(queue) > maxQueue: maxQueue = len(queue) (s, test) = queue[0] queue = queue[1:] if len(test) > maxDepth: maxDepth = len(test) print "REACHED DEPTH",maxDepth,"QUEUE SIZE",len(queue)+1 if len(test) == config.depth: continue t.backtrack(s) shuffleActs = t.enabled() if not config.deterministic: R.shuffle(shuffleActs) for c in shuffleActs: stepOk = t.safely(c) test.append(c) thisBug = False if (not config.uncaught) and (not stepOk): handle_failure(test, "UNCAUGHT EXCEPTION", False) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" thisBug = True if (not config.ignoreprops) and (not t.check()): handle_failure(test, "PROPERLY VIOLATION", True) if not config.multiple: print "STOPPING TESTING DUE TO FAILED TEST" thisBug = True ns = t.state() if not thisBug: if config.novisited or (ns not in visited): if (random.random() < config.forget) and (not (queue == [])): break if len(queue) >= config.breadth: break if not config.novisited: visited.append(s) if config.verbose: print len(visited), "NEW STATE:" print s queue.append((ns, test)) elif not config.multiple: break elapsed = time.time() - start if config.running: if t.newBranches() != (set([])): print "ACTION:",action for b in t.newBranches(): print elapsed,len(t.allBranches()),"New branch",b if elapsed > config.timeout: print "STOPPING EXPLORATION DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test) break t.backtrack(s) test = test[:-1] if (not config.multiple) and (failCount > 0): break if elapsed > config.timeout: print "STOPPING TESTING DUE TO TIMEOUT" break if not config.nocover: print t.report(config.coverfile),"PERCENT COVERED" if config.html: t.htmlReport(config.html) print len(visited), "STATES VISITED" print maxDepth,"MAX SEARCH DEPTH" print maxQueue,"MAX QUEUE SIZE" if config.multiple: print failCount,"FAILED" if not config.nocover: print len(t.allBranches()),"BRANCHES COVERED" print len(t.allStatements()),"STATEMENTS COVERED"
key_list = pdict.keys() arg_list = [pdict[k] for k in key_list] Config = namedtuple('Config', key_list) nt_config = Config(*arg_list) return nt_config actCount = 0 bugs = 0 no_tests = 0 i = 0 covCount = {} leastCov = None savedTest = None sut = sut.sut() parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('My tester using config={}'.format(config)) rgen = random.Random(config.seed) start = time.time() while time.time()-start < config.timeout: for ts in xrange(0,config.width): sut.restart() no_tests += 1 if (savedTest != None) and (rgen.random() > 0.8): sut.backtrack(savedTest)
def main(): parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print(('Triaging using config={}'.format(config))) if config.ignoreContaining is not None: ignoredStrings = config.ignoreContaining.split(";;") sut = SUT.sut() try: sut.stopCoverage() except BaseException: pass signatures = {} numTests = 0.0 numInvalid = 0 numPassing = 0 for fn in glob.glob(config.testglob): try: t = sut.loadTest(fn) except BaseException: print("INVALID TEST CASE:", fn) numInvalid += 1 continue if config.ignoreContaining is not None: foundString = False for s in t: for istr in ignoredStrings: if istr in s[0]: foundString = True break if foundString: break if foundString: continue if not config.noCheck: fails = sut.failsAny(t) else: fails = sut.fails(t) if fails: numTests += 1 e = repr(sut.failure()) e = e[:e.find("<traceback object at 0x")] + ")" line = sut.test()[-1][0] tline = t[-1][0] if line != tline: print("TEST", fn, "FAILS BEFORE END OF TEST") sig = (noDigits(e), noDigits(line)) if config.abstractStrings: sig = (noStrings(sig[0]), sig[1]) if config.actionClasses: sig += (repr(set(map(sut.actionClass, t))), ) if (sig not in signatures): signatures[sig] = (fn, t, sut.failure(), sut.prettyName(t[-1][0]), 1) elif (len(t) < len(signatures[sig][1])): signatures[sig] = (fn, t, sut.failure(), sut.prettyName(t[-1][0]), signatures[sig][4] + 1) else: (sfn, st, sf, sp, count) = signatures[sig] signatures[sig] = (sfn, st, sf, sp, count + 1) else: numPassing += 1 for sig in sorted(signatures.keys(), key=lambda x: signatures[x][4], reverse=True): print("=" * 80) print("TEST:", signatures[sig][0], "LENGTH:", len(signatures[sig][1])) print("OPERATION:", signatures[sig][3]) if config.actionClasses: print("ACTION CLASSES:", set(map(lambda x: sut.prettyName(x[0]), signatures[sig][1]))) print("EXCEPTION:", repr(signatures[sig][2])) print("FAILURE:") traceback.print_tb(signatures[sig][2][2], file=sys.stdout) print("COUNT:", signatures[sig][4], "(" + str(round(signatures[sig][4] / numTests, 2) * 100) + "%)") if config.showTests: sut.prettyPrintTest(signatures[sig][1]) print() print("*" * 80) print(int(numTests), "TOTAL FAILING TESTS") if numInvalid > 0: print(numInvalid, "TOTAL INVALID TESTS") if numPassing > 0: print(numPassing, "TOTAL PASSING TESTS")