def main(argv): global sut, rgen, ntest, nbug, coverageCount, start, selectStatements, threshold parse_options(argv) # GLOBAL VARIABLES sut = sut.sut() rgen = random.Random(opts.seed) ntest = 0 nbug = 0 coverageCount = {} selectStatements = {} threshold = 0 start = time.time() # PHASES phaseLimit = 0 if opts.timeout < 30: phaseLimit = int(round(opts.timeout/3*2)) else: phaseLimit = int(round(opts.timeout/10)) while (time.time() - start) < (0.95 * opts.timeout): randomPool = phase1(phaseLimit) selectPool = filterPool(randomPool) phase2(randomPool, selectPool) print "FULL POOL:",len(randomPool),", SELECT POOL:",len(selectPool) print "EXECUTED:",ntest,"tests" print "BUGS:",nbug print "TOTAL ELAPSED:",round(time.time() - start,5),"seconds" if (opts.coverage): sut.internalReport()
def main(): global flag,timeout,sut,test,para,currStatements,coverageNum,sortedCov,weight,weightedCov,coverageWBM,bugs,actNum,start while time.time()-start < timeout: sut.restart() if (test != None): if (rgen.random() > exp): sut.backtrack(test) para = False for s in xrange(0,lenth): tester() if flag == 1: break for s in sut.currStatements(): if s not in coverageNum: coverageNum[s] = 0 coverageNum[s] += 1 sortedCov = sorted(coverageNum.keys(), key=lambda x: coverageNum[x]) for c in sortedCov: weightedCov = c * (lenth - coverageNum[c]) if weightedCov > 20: coverageWBM.append(weightedCov) print c sut.internalReport() for s in sortedCov: print s, coverageNum[s] print bugs,"Failures have been found" print "TOTAL ACTIONS",actNum print "TOTAL RUNTIME",time.time()-start
def main(): global ntests while time.time()-start < timeout: for ts in xrange(0,width): sut.restart() ntests += 1 loopBug() for s in sut.currBranches(): if s not in covCount: covCount[s] = 0 covCount[s] += 1 sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x]) LCov= sortedCov[0] if (time.time() - start > timeout): print "THE TEST IS STOPPED SINCE TIMEOUT" break sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x]) if coverage == 1: print "SHOW COVERAGE" sut.internalReport() print ntests, "TOTAL TESTS" print "TOTAL BUGS",bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNNINGTIME",time.time()-start
def main(): global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() R = random.Random(seed) states = [sut.state()] print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: break checkAlg() if coverage: sut.internalReport() print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global flag,timeout,sut,test,para,currStatements,coverageNum,sortedCov,weight,weightedCov,coverageWeightedBelowMean,bugs,actNum,start while time.time()-start < timeout: sut.restart() if (test != None): if (rgen.random() > 0.4): sut.backtrack(test) para = False print "Testing" for s in xrange(0,100): tester() if flag == 1: break for s in sut.currStatements(): if s not in coverageNum: coverageNum[s] = 0 coverageNum[s] += 1 sortedCov = sorted(coverageNum.keys(), key=lambda x: coverageNum[x]) print "Coverage" for c in sortedCov: weight = (100 - coverageNum[c]) weightedCov = c * weight if weightedCov > 25: coverageWeightedBelowMean.append(weightedCov) print c sut.internalReport() for s in sortedCov: print s, coverageNum[s] print bugs,"BUGS" print "TOTAL ACTIONS",actNum print "TOTAL RUNTIME",time.time()-start
def main(): global ntests,start while time.time()-start < timeout: for ts in xrange(0,width): sut.restart() ntests += 1 if (Stest != None ) and (r.random() > 0.6): sut.backtrack(Stest) randomAction() for b in sut.currBranches(): if b not in covCount: covCount[b] = 0 covCount[b] += 1 sortedCov = sorted(covCount.keys(), key = lambda x: covCount[x]) LCov = sortedCov[0] if time.time()-start > timeout: print "THE TEST IS STOPPED SINCE TIMEOUT" break sortedCov = sorted(covCount.keys(), key = lambda x: covCount[x]) if coverage: print"Show Coverage" sut.internalReport() print ntests,"TOTAL TESTS" print "TOTAL BUGS",bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global config, rgen, start parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) rgen.seed(config.seed) start = time.time() nseqs.append(getSeq()) while time.time() - start < config.timeout: seq = rgen.choice(nseqs)[:] sut.replay(seq) if rgen.randint(0, 9) == 0: n = rgen.randint(2, 100) ok, propok, timeover = appendAndExecuteSeq(seq, n, eseqs, nseqs) else: ok, propok, timeover = appendAndExecuteSeq(seq, 1, eseqs, nseqs) if timeover: break if ok and propok: nseqs.append(seq) if config.coverage: sut.internalReport()
def main(): global config, R, sut, bugs, actCount, branchCount, statementCount, ntest, start parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start print "Starting Test" while elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() elapsed = time.time() - start print "Finishing Test" if time.time() - start > config.timeout: print "[TIMEOUT]" printCoverage() if config.coverage: sut.internalReport() print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
def main(): global start,BUDGET,sortedCov,Cover_percent,Number while time.time()-start < BUDGET: newFunction() sut.internalReport() sortedCov = sorted(Cover_percent.keys(), key=lambda x: Cover_percent[x]) for s in sortedCov: print s, Cover_percent[s] print bugs_found,"FAILED" print "ACTIONS_total",Number print "RUNTIME_total",time.time()-start
def main(): global config,rgen,actCount,failCount,ntests,fullPool parsed_args, parser = parse_args() config = make_config(parsed_args, parser) rgen = random.Random(config.seed) actCount = 0 failCount = 0 fullPool = [] start = time.time() ntests = 0 while time.time() - start < config.timeout: ntests += 1 sut.restart() for d in xrange(0, config.depth): act = sut.randomEnabled(rgen) actCount += 1 ok = sut.safely(act) expandPool() if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print time.time()-start, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print time.time()-start, len(sut.allStatements()),"New statement",s if not ok: failCount += 1 if config.faults: failureHandle(failCount) print "FOUND A FAILURE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() sut.restart() break if time.time() - start > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests break if config.faults: print "TOTAL FAULTS", failCount if config.coverage: sut.internalReport() print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global running,start,TIMEBUDGET,sut,testSaving,gene,testStored,act,srun,flag,coverageCount,covrSort,calculateWei,allCoverage,cW,coverageWM,failCount,actionsCount,sd #global running,possible global config #global lastAddCoverage belowMeanCoverage = set([]) fullPoll = [] parsed_args, parser = parse_args() config = make_config(parsed_args, parser) while time.time()-start < TIMEBUDGET: sut.restart() if (testSaving != None) and (gene.random() > 0.3): sut.backtrack(testSaving) testStored = False #print "Firstly test the AVL tree" for s in xrange(0,config.depth): act = sut.randomEnabled(gene) srun = sut.safely(act) if config.running: if sut.newBranches() != set([]): for d in sut.newBranches(): print time.time()-start,len(sut.allBranches()),"New branch",d findFail() if flag == 1: break for sa in sut.currStatements(): if sa not in coverageCount: coverageCount[sa] = 0 coverageCount[sa] = coverageCount[sa] + 1 covrSort = sorted(coverageCount.keys(), key=lambda x: coverageCount[x]) #coverSum = sum(coverageCount.values()) #coverMean = coverSum / (1.0*len(coverageCount)) #print "the mean of coverage is:",coverMean #for st in covrSort: #print st, coverageCount[st] # if coverageCount[st]<coverMean: # belowMeanCoverage.add(st) # else: # break #print len(belowMeanCoverage), "statement below the mean coverage out of the total coverage:",len(coverageCount) for st in covrSort: if st*(allCoverage - coverageCount[st]) >25: coverageWM.append(st*(allCoverage - coverageCount[st])) print "Now I have the statement below coverage:", st for sd in covrSort: print sd,coverageCount[sd] sut.internalReport() print failCount,"FAILED" print len(sut.allBranches()),"BRANCH" print actionsCount,"TOTAL ACTIONS" print time.time()-start,"TOTAL RUNTIME"
def main(): global start,BUDGET,sut,COVERAGE_REPORT,savedcoverage,rgen,storedTest,act,ok,savedcoverage,running,savedcoverage,Num,faults,foundbug,savedTestState while time.time()-start < BUDGET: sut.restart() if (savedcoverage != None) and (rgen.random() > 0.3): print "Processing" sut.backtrack(savedcoverage) storedTest = False print "First: AVL tree" for s in xrange(0,100): act = sut.randomEnabled(rgen) ok = sut.safely(act) if running: if sut.newBranches() != set([]): ## print "ACTION:",a[0],tryStutter for d in sut.newBranches(): print time.time()-start,len(sut.allBranches()),"New branch",d if len(sut.newStatements()) > 0: savedcoverage = sut.state() storedTest = True if(running): print "New Statement",sut.newStatements() if (not storedTest) and (i != None) and (i in sut.currStatements()): savedcoverage = sut.state() storedTest = True Num += 1 if(faults): if not ok: foundbug += 1 print "Failed" print sut.failure() print "REDUCE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() filename = 'failure%d.test'%bugs_found sut.saveTest(R,filename) break savedTestState = sut.state() expandNewState() if coverage: sut.internalReport() print foundbug,"FAILED" print "ACTIVE",Num print "RUNTIME",time.time()-start
def main(): global sut, rgen, nbugs, start, CXPB, MUTPB, NGEN sut = sut.sut() rgen = random.Random(opts.seed) CXPB, MUTPB, NGEN = 0.5, 0.2, 1 best, timeclock = [], [] nbugs, g, prev_best = 0, 0, 1 init() start = time.time() while True: print("Building population") pop = toolbox.population(n=100) print(" n=%d" % len(pop)) print("Start of evolution") fitnesses = list(map(toolbox.evaluate, pop)) for ind, fit in zip(pop, fitnesses): ind.fitness.values = fit while True: g += 1 generation(g, pop) best_test = tools.selBest(pop, 1)[0] elapsed = time.time()-start if 0.02 > (best_test.fitness.values[0] - prev_best) / prev_best or elapsed > (0.95 * opts.timeout): break else: prev_best = max(best_test.fitness.values[0], prev_best) best_test = tools.selBest(pop, 1)[0] print("Best test has %s lines covered" % int(best_test.fitness.values[0])) best.append(best_test.fitness.values[0]) elapsed = time.time()-start timeclock.append(elapsed) if not timeclock else timeclock.append(elapsed - timeclock[-1]) average_time = sum(timeclock) / len(timeclock) print("elapsed: %f seconds out of %f seconds" % (elapsed, opts.timeout)) if (elapsed + average_time) > opts.timeout: break overall_best = max(best) print("Overall best test has %s lines covered" % int(overall_best)) if (opts.coverage): sut.internalReport()
def main(): global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() R = random.Random() R.seed(seed) states = [sut.state()] print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: error.append(sut.currStatements()) break checkAlg() #if not error ==[]: # print "SHOW ERROR SEQUENCE" # print error # f = open(("error" + str(actCount) + ".test"), 'w') # f.write(str(error)) # f.close() #else: # print "Data in noerror sequence" # print noerror if coverage: sut.internalReport() print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global start,sut,R,noerror,error,reduceTime,actCount, bugs,ntest,newseq,currseq,states, beforeReduceS, beforeReduceB,quickCount actCount = 0 bugs = 0 start = time.time() noerror = [] error = [] newseq = [] ntest=0 currseq=[] sut = sut.sut() reduceTime = 0.0 R = random.Random() R.seed(seed) beforeReduceS = set(sut.allStatements()) beforeReduceB = set(sut.allBranches()) states = [sut.state()] quickCount=0 print "STARTING PHASE 1" while(time.time() < start + timeout): for st in states: ntest+=1 if (time.time() > start + timeout): break sut.restart() sut.backtrack(st) for s in xrange(0, depth): if (time.time() > start + timeout): break ok = randomAction() if not ok: break checkAlg() if coverage: sut.internalReport() if quickTests==1: #if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])): handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True) print "TOTAL BUGS", bugs print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global start,BUDGET,sut,savedcoverage,rgen,storedTest,act,ok,savedcoverage,running,savedcoverage,Num,faults,foundbug,savedTestState while time.time()-start < BUDGET: sut.restart() if (savedcoverage != None) and (rgen.random() > 0.4): print "Processing" sut.backtrack(savedcoverage) storedTest = False print "First: AVL tree" for s in xrange(0,100): act = sut.randomEnabled(rgen) ok = sut.safely(act) if len(sut.newStatements()) > 0: savedcoverage = sut.state() storedTest = True if(running): print "New Statement",sut.newStatements() if (not storedTest) and (i != None) and (i in sut.currStatements()): savedcoverage = sut.state() storedTest = True Num += 1 if(faults): if not ok: foundbug += 1 print "Failed" print sut.failure() print "REDUCE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() break savedTestState = sut.state() expandNewState() print foundbug,"FAILED" print "ACTIVE",Num print "RUNTIME",time.time()-start for s in sortedCoverage: print s, coverageCount[s] if (COVERAGE_REPORT): sut.internalReport()
def main(): global running,start,TIMEBUDGET,sut,testSaving,gene,testStored,act,srun,flag,coverageCount,covrSort,calculateWei,allCoverage,cW,coverageWM,failCount,actionsCount,sd #global running,possible global config parsed_args, parser = parse_args() config = make_config(parsed_args, parser) while time.time()-start < TIMEBUDGET: sut.restart() if (testSaving != None) and (gene.random() > 0.3): sut.backtrack(testSaving) testStored = False print "Firstly test the AVL tree" for s in xrange(0,100): act = sut.randomEnabled(gene) srun = sut.safely(act) if config.running: if sut.newBranches() != set([]): for d in sut.newBranches(): print time.time()-start,len(sut.allBranches()),"New branch",d func1() if flag == 1: break for s in sut.currStatements(): if s not in coverageCount: coverageCount[s] = 0 coverageCount[s] = coverageCount[s] + 1 covrSort = sorted(coverageCount.keys(), key=lambda x: coverageCount[x]) for st in covrSort: if st*(allCoverage - coverageCount[st]) >25: coverageWM.append(st*(allCoverage - coverageCount[st])) print "Now I have the statement below coverage:", st for sd in covrSort: print sd,coverageCount[sd] sut.internalReport() print failCount,"FAILED" print len(sut.allBranches()),"BRANCH" print actionsCount,"TOTAL ACTIONS" print time.time()-start,"TOTAL RUNTIME"
i += 1 bugs += 1 saveFault = 'failure' + str(bugs) + '.test' sut.saveTest(sut.test(), saveFault) print "Number bugs found is" ,i sut.restart() #To see what is the least covered branch to do experiments on them for s in sut.currBranches(): if s not in covCount: covCount[s] = 0 covCount[s] += 1 sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x]) leastCov = sortedCov[0] if (time.time() - start > config.timeout): break #Take the name of all actions and sort them by their action count sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x]) # Produce coverage report if config.coverage: sut.internalReport() print "TOTAL NUMBER OF BUGS",bugs print "TOTAL NUMBER OF TESTS",no_tests print "TOTAL NUMBER OF ACTIONS",actCount print "TOTAL NUMBER OF RUNTIME",time.time()-start
def main(): #global variables are defined here global sut, rgen, actCount, bugs, coveraged, visited, start, argv parsed_args, parser = parse_args() argv = make_config(parsed_args, parser) start = time.time() sut = sut.sut() sut.silenceCoverage() rgen = random.Random() rgen.seed(argv.SEED) actCount = 0 bugs = 0 coveraged = {} visited = [] print "STARTING PHASE 1" while time.time() - start < argv.TIMEOUT * 0.5: sut.restart() for s in xrange(0,argv.DEPTH): randomAction() collectCoverage() sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x]) for s in sortedCov: print s, coveraged[s] print "STARTING PHASE 2" while time.time() - start < argv.TIMEOUT : belowMedian = set([]) sort = sorted(coveraged.keys(),key = lambda x : coveraged[x]) a = len(sort)/2 for s in sort: if coveraged[s] < sort[a]: belowMedian.add(s) else: break activePool = [] for (t,c) in visited: for s in c: if s in belowMedian: activePool.append((t,c)) break sut.restart() if len(sut.newStatements()) != 0: print "new statement",sut.newStatements() visited.append((list(sut.test()), set(sut.currStatements()))) for s in xrange(0,argv.DEPTH): if not randomAction(): break collectCoverage() sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x]) for s in sortedCov: print s, coveraged[s] if argv.COVERAGE: sut.internalReport() print "FAILURES ",bugs, "TIMES" print "TOTAL ACTIONS", actCount print "TOTAL RUNTIME", time.time()-start
def main(): global config, R, sut, bugs, ntest, start global actCount, branchCount, statementCount global savedTest, savedFlag parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() if config.silence: sut.silenceCoverage() bugs = 0 actCount = 0 branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start print "Starting Phase 1" if elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() print "Finishing Phase 1" print "Starting Phase 2" elapsed = time.time() - start while elapsed < config.timeout: sut.restart() ntest += 1 savedTest = None if (savedTest != None) and (R.random() > 0.7): sut.backtrack(savedTest) savedFlag = False for i in xrange(0, config.depth): if not randomAct(): break elapsed = time.time() - start if config.running: showRunning() elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() elapsed = time.time() - start print "Finishing Phase 2" if time.time() - start > config.timeout: print "[TIMEOUT]" if config.coverdetail: printCoverage() if config.coverage: sut.internalReport() if config.resultdetail: print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
def main(): global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount, test_file, ntest parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} ntest = 0 start = time.time() elapsed = time.time() - start while elapsed < config.timeout: sut.restart() ntest += 1 for i in xrange(0, config.depth): for j in xrange(0, config.width): if randomAct(): preTest = sut.state() else: sut.backtrack(preTest) elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break elapsed = time.time() - start if elapsed > config.timeout: break collectCoverage() elapsed = time.time() - start printCoverage() if config.coverage: sut.internalReport() print "Covered", len(sut.allBranches()), "branches" print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total tests", ntest print "Total actions", actCount print "Total runtime", time.time() - start
failureCount = failureCount + 1 #the failure count will go on increasing if (FAULT_CHECK): # this will check for the faults name = "failure" + str(failureCount) + ".test" f = sut.test() sut.saveTest(F,filename) for (s_reduces) in S: steps_reduce = "#no of steps taken " + str(j) print >> f, sut.prettyName(s_reduces).ljust(80 - len(steps_reduce), ' '), steps_reduce j =j + 1 else: print ""/n print "This is the same test" sut.prettyPrintTest(F) break; if (COVERAGE_REPORT): sut.internalReport() #printing the result
start = time.time() while time.time() - start < BUDGET: sut.restart() for s in xrange(0, depth): acts = sut.randomEnableds(rgen, k) sortActs = sorted(acts, key=lambda x: actionCount[abstract(x[0])]) act = sortActs[0] ok = sut.safely(act) actCount += 1 actionCount[abstract(act[0])] += 1 if not ok: bugs += 1 print "FOUND A FAILURE" #sut.prettyPrintTest(sut.test()) print sut.failure() print "REDUCING" R = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() sut.restart() sut.internalReport() orderedActs = sorted(actionCount.keys(), key=lambda x: actionCount[x]) for act in orderedActs: print act, actionCount[act] print bugs, "FAILED" print "TOTAL ACTIONS", actCount print "TOTAL RUNTIME", time.time() - start
def main(): if "--help" in sys.argv: print "Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover] [--verbose] [--running]" print "Options:" print " --noCheck: do not run property checks" print " --html: output an HTML coverage report to the chosen directory" print " --noCover: do not compute code coverage" print " --verbose: make actions verbose" print " --running: give a running report on code coverage" print " --keepGoing: don't stop on failed test" sys.exit(0) sut = sut.sut() nocover = False verbose = False running = False keepGoing = False ignoreProps = False lastWasHtml = False files = [] htmlOut = None for f in sys.argv[1:]: if lastWasHtml: htmlOut = f lastWasHtml = False elif "--" not in f: files.append(f) elif f == "--html": lastWasHtml = True if "--noCover" in sys.argv: nocover = True if "--verbose" in sys.argv: verbose = True if "--noCheck" in sys.argv: ignoreProps = True if "--running" in sys.argv: running = True if "--keepGoing" in sys.argv: keepGoing = True if verbose: sut.verbose(True) anyFailed = False stime = time.time() for f in files: print "RUNNING TEST", f t = sut.loadTest(f) ok = False try: ok = sut.replay(t, checkProp=(not ignoreProps)) except Exception as e: print "EXCEPTION RAISED:", e if running: if sut.newBranches() != set([]): for b in sut.newBranches(): print "New branch:", b if sut.newStatements() != set([]): for s in sut.newStatements(): print "New statement:", s if not ok: print "TEST", f, "FAILED:" print sut.failure() anyFailed = True if not keepGoing: sys.exit(255) print time.time() - stime, "ELAPSED" if not nocover: print "STATEMENTS:", len(sut.allStatements()), "BRANCHES:", len( sut.allBranches()) if not nocover: sut.internalReport() print sut.report("coverage.out"), "PERCENT COVERED" if htmlOut != None: sut.htmlReport(htmlOut) if not anyFailed: print "ALL TESTS SUCCESSFUL"
def randomOperation(): global bug_num, time_start operation = sut.randomEnabled(rand) good = sut.safely(operation) runtime = time.time() - time_start if (running): if (len(sut.newBranches())) > 0 print "Operation: ", operation[0] for i in sut.newBranches(): print runtime, len(sut.allBranches()), "New Branch", i if (not good): bug_num += 1 print "A bug is found! #", bug_num print sut.failure() print ("Reducing!") reduce = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(reduce) print(sut.failure()) if (faults): f = open((fail + str(bug_num) + ".test"), "w") print >> f, sut.failure() j = 0 for (s_reduce, _, _) in reduce: steps_reduce = "# Step " + str(j) print >> f, sut.prettyName(s_reduce).ljust(80 - len(steps_reduce), ' '), steps_reduce j += 1 f.close() print "Time: ", runtime return good time_start = time.time() current_depth = 1 state_queue = [sut.state()] Time_Phase1 = timeout / 4 print "PHASE 1 ..." while (time.time() < (time_start + Time_Phase1)): sut.restart() for d in xrange(0, depth): good = randomOperation() if (not good): break if (len(sut.newStatements()) > 0): new_state.append(sut.state()) new_statement.append(sut.newStatements()) for s in sut.currStatements(): if s not in coverageCount: coverageCount[s] = 0 coverageCount[s] += 1 sortedCoverage = sorted(coverageCount.keys(), key = lambda x: coverageCount[x]) sum_value = 0 for s in sortedCoverage: sum_value += coverageCount[s] mean_value = sum_value / len(coverageCount) sum_value = 0.0 for s in sortedCoverage: sum_value += math.pow(coverageCount[s] - mean_value, 2) sum_value = sum_value / (len(coverageCount) - 1) std = math.sqrt(sum_value) threshold = mean_value - (0.67 * std) print "Mean: ", mean_value print "Standard Deviation: ", std print "Threshold: ", threshold for s in sortedCoverage: if (coverageCount[s] > threshold): break for k in k_chosen: if s in new_statement[k]: continue for k in xrange(0, len(new_statement)): if s in new_statement[k]: k_chosen.append(k) for k in k_chosen: chosen_state.append(new_state[k]) print bug_num, " Bugs Found!" for s in sortedCoverage: print s, coverageCount[s] if (coverage): sut.internalReport() print "PHASE 2 ..." Time_Phase2 = timeout - Time_Phase1 all_state = [] new_statement = [] i = 0; time_start = time.time() while (time.time() < (time_start + Time_Phase2)): for state in chosen_state: i += 1 time_state = float(Time_Phase2) / (len(chosen_state) * (i + 1)) time_start2 = time.time() while (time.time() < time_start2 + time_state): sut.restart() sut.backtrack(state) for d in xrange(0, depth): good = randomOperation() if (not good): break if (len(sut.newStatements()) > 0): print "A New Statement is Found!" chosen_state.insert(i, sut.state()) for s in sut.currStatements(): if s not in coverageCount: coverageCount[s] = 0 coverageCount[s] += 1 sortedCoverage = sorted(coverageCount.keys(), key = lambda x: coverageCount[x]) print bug_num, " Bugs Found!" for s in sortedCoverage: print s, coverageCount[s] if (coverage): sut.internalReport()
def main(): global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount parsed_args, parser = parse_args() config = make_config(parsed_args, parser) print('Testing using config={}'.format(config)) R = random.Random(config.seed) sut = sut.sut() sut.silenceCoverage() bugs = 0 actCount = 0 coverageCount = {} branchCount = {} statementCount = {} if config.faults: test_file = open("failure1.test", "w") start = time.time() elapsed = time.time() - start for i in xrange(0, config.depth): sut.restart() for j in xrange(0, config.width): if not randomAct(): print "Function called randomAct() occurs error" break elapsed = time.time() - start if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s elapsed = time.time() - start if elapsed > config.timeout: print "Stopping test [TIMEOUT]" break collectCoverage() if config.faults: test_file.close() if config.coverage: sut.restart() sut.report("coverage.out") #printCoverage() sut.internalReport() #print "Covered", len(sut.allBranches()), "branches" #print "Covered", len(sut.allStatements()), "statements" print "Failed", bugs, "times" print "Total actions", actCount print "Total runtime", time.time() - start
def main(): parsed_args, parser = parse_args() Config = make_config(parsed_args, parser) rgen = random.Random(Config.seed) print ('testing using config={}'.format(Config)) sut.silenceCoverage() sut.restart() state_queue = [sut.state()] visited = [] tdepth = 30 d = 1 ntests = 0 bugs = 0 actCount = 0 faults_file = "failure" coverage_count = {} leastCovered = None statebuff = [] start = time.time() elapsed = time.time() - start while d <= Config.depth: print "Depth", d, "Size", len(state_queue), "Set", len(visited) w = 1 len_queue = len(state_queue) ntests += 1 frontier = [] depth_start = time.time() random.shuffle(state_queue) savedTest = False for s in state_queue: sut.backtrack(s) for a in sut.enabled(): depth_time = time.time() - depth_start act = sut.randomEnabled(rgen) actCount += 1 if depth_time >= tdepth: break isGood = sut.safely(a) if Config.running: if sut.newBranches() != set([]): print "Action:", a[0] for b in sut.newBranches(): print elapsed, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): print "Action:", a[0] for s in sut.newStatements(): print elapsed, len(sut.allStatements()), "New statement", s s_next = sut.state() if s_next not in visited: visited.append(s_next) frontier.append(s_next) if not isGood: failures(sut, bugs, Config, faults_file) if Config.faults == 1: sut.saveTest(R, faults_file + str(bugs) + ".test") sut.restart() if len(sut.newStatements()) > 0: frontier.insert(0, sut.state()) savedTest = True if (not savedTest) and (leastCovered != None) and (leastCovered in sut.currStatements()): frontier.insert(0, sut.state()) savedTest = True elapsed = time.time() - start if elapsed >= Config.timeout: break if depth_time >= tdepth: break if elapsed >= Config.timeout: break if w <= Config.width: w += 1 else: break statebuff.append((list(sut.test()), set(sut.currBranches()))); state_queue = frontier sortPop = sorted(statebuff,key = lambda x: len(x[1]),reverse=True) (t,b) = rgen.choice(statebuff) m = mutate(t, rgen, bugs) statebuff.append((m, sut.currBranches())) if elapsed >= Config.timeout: print "Stopping Test Due To Timeout, Terminated at Length", len(sut.test()) print ntests, "EXECUTED" break for s in sut.currStatements(): if s not in coverage_count: coverage_count[s] = 0 coverage_count[s] += 1 sortedCov = sorted(coverage_count.keys(), key=lambda x: coverage_count[x]) if len(sortedCov) != 0: leastCovered = sortedCov[0] d += 1 if Config.coverage == 1: sut.internalReport() print bugs,"FAILED" print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main(): global BUDGET,start,test,sut,storedTest,act,correct,running,coverage3,count,flag,savedTestState,coverageCount,sortedCov,weight,weightedCov,coverage2,bug,noTest while time.time()-start < BUDGET: sut.restart() noTest += 1 if (random.random() > 0.7) and (test != None): #print "Exploiting.." sut.backtrack(test) storedTest = False #print "Testing.." for i in xrange(0,depth): act = sut.randomEnabled(random) correct = sut.safely(act) if running: if sut.newBranches() != set([]): for d in sut.newBranches(): print time.time() - start, len(sut.allBranches()),"New branch",d if len(sut.newStatements()) > 0: test = sut.state() storedTest = True if(running): print "Found a statement",sut.newStatements() if (not storedTest) and (coverage3 != None) and (coverage3 in sut.currStatements()): test = sut.state() storedTest = True count += 1 if not correct: if not correct: bug += 1 print "Found a failure!" print sut.failure() print "Reducing.." R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() filename ='failure%d.test'%bug sut.saveTest(R,filename) #flag = 1 #if flag == 1: break savedTestState = sut.state() for i in sut.currStatements(): if i not in coverageCount: coverageCount[i] = 0 coverageCount[i] += 1 sortedCov = sorted(coverageCount.keys(), key=lambda x: coverageCount[x]) for i in sortedCov: weight = i * (defCover - coverageCount[i]) if weight > coverage4: coverage2.append(weight) print "Coverage:", i #sut.backtrack(sut.state()) if coverage: sut.internalReport() sortedCov = sorted(coverageCount.keys(), key=lambda x: coverageCount[x]) for i in sortedCov: print i, coverageCount[i] print bug,"failed" print "Total tests:", noTest print "Total actions:",count print "Total running time:",time.time()-start
def main(): global config,rgen,actCount,failCount,ntests,firstPool,coverageCount parsed_args, parser = parse_args() config = make_config(parsed_args, parser) rgen = random.Random(config.seed) actCount = 0 failCount = 0 coverageCount = {} firstPool = [] start = time.time() ntests = 0 while time.time() - start < (config.timeout)/2: ntests += 1 sut.restart() for d in xrange(0, config.depth): act = sut.randomEnabled(rgen) actCount += 1 ok = sut.safely(act) expandPool() if config.running: if sut.newBranches() != set([]): print "ACTION:", act[0] for b in sut.newBranches(): print time.time()-start, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): print "ACTION:", act[0] for s in sut.newStatements(): print time.time()-start, len(sut.allStatements()),"New statement",s if not ok: failCount += 1 print "FOUND A FAILURE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) if config.faults: filename = 'failure' + `failCount` + '.test' sut.saveTest(R,filename) print sut.failure() sut.restart() break if time.time() - start > (config.timeout)/2: print "FIRST POOL HAS BEEN CREATED, TERMINATED AT LENGTH",ntests break collectCoverage() end = time.time() while time.time() - end < (config.timeout)/2: buildSecondPool() sut.restart() if rgen.random() > 0.5: sut.replay(rgen.choice(secondPool)[0]) ntests += 1 for d in xrange(0, config.depth): act = sut.randomEnabled(rgen) actCount += 1 ok = sut.safely(act) expandPool() if config.running: if sut.newBranches() != set([]): print "ACTION:", act[0] for b in sut.newBranches(): print time.time()-end, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): print "ACTION:", act[0] for s in sut.newStatements(): print time.time()-end, len(sut.allStatements()),"New statement",s if not ok: failCount += 1 print "FOUND A FAILURE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) if config.faults: filename = 'failure' + `failCount` + '.test' sut.saveTest(R,filename) print sut.failure() sut.restart() break if time.time() - end > (config.timeout)/2: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests break collectCoverage() if config.faults: print "TOTAL FAULTS", failCount if config.coverage: sut.internalReport() print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def print_results(): if COVERAGE: sut.internalReport() print "Number of bugs found: " + str(bugs)