def main(argv):
global sut, rgen, ntest, nbug, coverageCount, start, selectStatements, threshold
parse_options(argv)
# GLOBAL VARIABLES
sut = sut.sut()
rgen = random.Random(opts.seed)
ntest = 0
nbug = 0
coverageCount = {}
selectStatements = {}
threshold = 0
start = time.time()
# PHASES
phaseLimit = 0
if opts.timeout < 30:
phaseLimit = int(round(opts.timeout/3*2))
else:
phaseLimit = int(round(opts.timeout/10))
while (time.time() - start) < (0.95 * opts.timeout):
randomPool = phase1(phaseLimit)
selectPool = filterPool(randomPool)
phase2(randomPool, selectPool)
print "FULL POOL:",len(randomPool),", SELECT POOL:",len(selectPool)
print "EXECUTED:",ntest,"tests"
print "BUGS:",nbug
print "TOTAL ELAPSED:",round(time.time() - start,5),"seconds"
if (opts.coverage):
sut.internalReport()
def main():
global flag,timeout,sut,test,para,currStatements,coverageNum,sortedCov,weight,weightedCov,coverageWBM,bugs,actNum,start
while time.time()-start < timeout:
sut.restart()
if (test != None):
if (rgen.random() > exp):
sut.backtrack(test)
para = False
for s in xrange(0,lenth):
tester()
if flag == 1:
break
for s in sut.currStatements():
if s not in coverageNum:
coverageNum[s] = 0
coverageNum[s] += 1
sortedCov = sorted(coverageNum.keys(), key=lambda x: coverageNum[x])
for c in sortedCov:
weightedCov = c * (lenth - coverageNum[c])
if weightedCov > 20:
coverageWBM.append(weightedCov)
print c
sut.internalReport()
for s in sortedCov:
print s, coverageNum[s]
print bugs,"Failures have been found"
print "TOTAL ACTIONS",actNum
print "TOTAL RUNTIME",time.time()-start
def main():
global ntests
while time.time()-start < timeout:
for ts in xrange(0,width):
sut.restart()
ntests += 1
loopBug()
for s in sut.currBranches():
if s not in covCount:
covCount[s] = 0
covCount[s] += 1
sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x])
LCov= sortedCov[0]
if (time.time() - start > timeout):
print "THE TEST IS STOPPED SINCE TIMEOUT"
break
sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x])
if coverage == 1:
print "SHOW COVERAGE"
sut.internalReport()
print ntests, "TOTAL TESTS"
print "TOTAL BUGS",bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNNINGTIME",time.time()-start
def main():
global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
R = random.Random(seed)
states = [sut.state()]
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
break
checkAlg()
if coverage:
sut.internalReport()
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global flag,timeout,sut,test,para,currStatements,coverageNum,sortedCov,weight,weightedCov,coverageWeightedBelowMean,bugs,actNum,start
while time.time()-start < timeout:
sut.restart()
if (test != None):
if (rgen.random() > 0.4):
sut.backtrack(test)
para = False
print "Testing"
for s in xrange(0,100):
tester()
if flag == 1:
break
for s in sut.currStatements():
if s not in coverageNum:
coverageNum[s] = 0
coverageNum[s] += 1
sortedCov = sorted(coverageNum.keys(), key=lambda x: coverageNum[x])
print "Coverage"
for c in sortedCov:
weight = (100 - coverageNum[c])
weightedCov = c * weight
if weightedCov > 25:
coverageWeightedBelowMean.append(weightedCov)
print c
sut.internalReport()
for s in sortedCov:
print s, coverageNum[s]
print bugs,"BUGS"
print "TOTAL ACTIONS",actNum
print "TOTAL RUNTIME",time.time()-start
def main():
global ntests,start
while time.time()-start < timeout:
for ts in xrange(0,width):
sut.restart()
ntests += 1
if (Stest != None ) and (r.random() > 0.6):
sut.backtrack(Stest)
randomAction()
for b in sut.currBranches():
if b not in covCount:
covCount[b] = 0
covCount[b] += 1
sortedCov = sorted(covCount.keys(), key = lambda x: covCount[x])
LCov = sortedCov[0]
if time.time()-start > timeout:
print "THE TEST IS STOPPED SINCE TIMEOUT"
break
sortedCov = sorted(covCount.keys(), key = lambda x: covCount[x])
if coverage:
print"Show Coverage"
sut.internalReport()
print ntests,"TOTAL TESTS"
print "TOTAL BUGS",bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global config, rgen, start
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
rgen.seed(config.seed)
start = time.time()
nseqs.append(getSeq())
while time.time() - start < config.timeout:
seq = rgen.choice(nseqs)[:]
sut.replay(seq)
if rgen.randint(0, 9) == 0:
n = rgen.randint(2, 100)
ok, propok, timeover = appendAndExecuteSeq(seq, n, eseqs, nseqs)
else:
ok, propok, timeover = appendAndExecuteSeq(seq, 1, eseqs, nseqs)
if timeover:
break
if ok and propok:
nseqs.append(seq)
if config.coverage:
sut.internalReport()
def main():
global config, R, sut, bugs, actCount, branchCount, statementCount, ntest, start
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
print "Starting Test"
while elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
elapsed = time.time() - start
print "Finishing Test"
if time.time() - start > config.timeout:
print "[TIMEOUT]"
printCoverage()
if config.coverage:
sut.internalReport()
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
def main(): global start,BUDGET,sortedCov,Cover_percent,Number while time.time()-start < BUDGET: newFunction() sut.internalReport() sortedCov = sorted(Cover_percent.keys(), key=lambda x: Cover_percent[x]) for s in sortedCov: print s, Cover_percent[s] print bugs_found,"FAILED" print "ACTIONS_total",Number print "RUNTIME_total",time.time()-start
def main(): global config,rgen,actCount,failCount,ntests,fullPool parsed_args, parser = parse_args() config = make_config(parsed_args, parser) rgen = random.Random(config.seed) actCount = 0 failCount = 0 fullPool = [] start = time.time() ntests = 0 while time.time() - start < config.timeout: ntests += 1 sut.restart() for d in xrange(0, config.depth): act = sut.randomEnabled(rgen) actCount += 1 ok = sut.safely(act) expandPool() if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print time.time()-start, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print time.time()-start, len(sut.allStatements()),"New statement",s if not ok: failCount += 1 if config.faults: failureHandle(failCount) print "FOUND A FAILURE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() sut.restart() break if time.time() - start > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests break if config.faults: print "TOTAL FAULTS", failCount if config.coverage: sut.internalReport() print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def main():
global running,start,TIMEBUDGET,sut,testSaving,gene,testStored,act,srun,flag,coverageCount,covrSort,calculateWei,allCoverage,cW,coverageWM,failCount,actionsCount,sd
#global running,possible
global config
#global lastAddCoverage
belowMeanCoverage = set([])
fullPoll = []
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
while time.time()-start < TIMEBUDGET:
sut.restart()
if (testSaving != None) and (gene.random() > 0.3):
sut.backtrack(testSaving)
testStored = False
#print "Firstly test the AVL tree"
for s in xrange(0,config.depth):
act = sut.randomEnabled(gene)
srun = sut.safely(act)
if config.running:
if sut.newBranches() != set([]):
for d in sut.newBranches():
print time.time()-start,len(sut.allBranches()),"New branch",d
findFail()
if flag == 1:
break
for sa in sut.currStatements():
if sa not in coverageCount:
coverageCount[sa] = 0
coverageCount[sa] = coverageCount[sa] + 1
covrSort = sorted(coverageCount.keys(), key=lambda x: coverageCount[x])
#coverSum = sum(coverageCount.values())
#coverMean = coverSum / (1.0*len(coverageCount))
#print "the mean of coverage is:",coverMean
#for st in covrSort:
#print st, coverageCount[st]
# if coverageCount[st]<coverMean:
# belowMeanCoverage.add(st)
# else:
# break
#print len(belowMeanCoverage), "statement below the mean coverage out of the total coverage:",len(coverageCount)
for st in covrSort:
if st*(allCoverage - coverageCount[st]) >25:
coverageWM.append(st*(allCoverage - coverageCount[st]))
print "Now I have the statement below coverage:", st
for sd in covrSort:
print sd,coverageCount[sd]
sut.internalReport()
print failCount,"FAILED"
print len(sut.allBranches()),"BRANCH"
print actionsCount,"TOTAL ACTIONS"
print time.time()-start,"TOTAL RUNTIME"
def main():
global start,BUDGET,sut,COVERAGE_REPORT,savedcoverage,rgen,storedTest,act,ok,savedcoverage,running,savedcoverage,Num,faults,foundbug,savedTestState
while time.time()-start < BUDGET:
sut.restart()
if (savedcoverage != None) and (rgen.random() > 0.3):
print "Processing"
sut.backtrack(savedcoverage)
storedTest = False
print "First: AVL tree"
for s in xrange(0,100):
act = sut.randomEnabled(rgen)
ok = sut.safely(act)
if running:
if sut.newBranches() != set([]):
## print "ACTION:",a[0],tryStutter
for d in sut.newBranches():
print time.time()-start,len(sut.allBranches()),"New branch",d
if len(sut.newStatements()) > 0:
savedcoverage = sut.state()
storedTest = True
if(running):
print "New Statement",sut.newStatements()
if (not storedTest) and (i != None) and (i in sut.currStatements()):
savedcoverage = sut.state()
storedTest = True
Num += 1
if(faults):
if not ok:
foundbug += 1
print "Failed"
print sut.failure()
print "REDUCE"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
filename = 'failure%d.test'%bugs_found
sut.saveTest(R,filename)
break
savedTestState = sut.state()
expandNewState()
if coverage:
sut.internalReport()
print foundbug,"FAILED"
print "ACTIVE",Num
print "RUNTIME",time.time()-start
def main():
global sut, rgen, nbugs, start, CXPB, MUTPB, NGEN
sut = sut.sut()
rgen = random.Random(opts.seed)
CXPB, MUTPB, NGEN = 0.5, 0.2, 1
best, timeclock = [], []
nbugs, g, prev_best = 0, 0, 1
init()
start = time.time()
while True:
print("Building population")
pop = toolbox.population(n=100)
print(" n=%d" % len(pop))
print("Start of evolution")
fitnesses = list(map(toolbox.evaluate, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
while True:
g += 1
generation(g, pop)
best_test = tools.selBest(pop, 1)[0]
elapsed = time.time()-start
if 0.02 > (best_test.fitness.values[0] - prev_best) / prev_best or elapsed > (0.95 * opts.timeout):
break
else:
prev_best = max(best_test.fitness.values[0], prev_best)
best_test = tools.selBest(pop, 1)[0]
print("Best test has %s lines covered" % int(best_test.fitness.values[0]))
best.append(best_test.fitness.values[0])
elapsed = time.time()-start
timeclock.append(elapsed) if not timeclock else timeclock.append(elapsed - timeclock[-1])
average_time = sum(timeclock) / len(timeclock)
print("elapsed: %f seconds out of %f seconds" % (elapsed, opts.timeout))
if (elapsed + average_time) > opts.timeout:
break
overall_best = max(best)
print("Overall best test has %s lines covered" % int(overall_best))
if (opts.coverage):
sut.internalReport()
def main():
global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
R = random.Random()
R.seed(seed)
states = [sut.state()]
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
error.append(sut.currStatements())
break
checkAlg()
#if not error ==[]:
# print "SHOW ERROR SEQUENCE"
# print error
# f = open(("error" + str(actCount) + ".test"), 'w')
# f.write(str(error))
# f.close()
#else:
# print "Data in noerror sequence"
# print noerror
if coverage:
sut.internalReport()
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global start,sut,R,noerror,error,reduceTime,actCount, bugs,ntest,newseq,currseq,states, beforeReduceS, beforeReduceB,quickCount
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
reduceTime = 0.0
R = random.Random()
R.seed(seed)
beforeReduceS = set(sut.allStatements())
beforeReduceB = set(sut.allBranches())
states = [sut.state()]
quickCount=0
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
break
checkAlg()
if coverage:
sut.internalReport()
if quickTests==1:
#if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])):
handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True)
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global start,BUDGET,sut,savedcoverage,rgen,storedTest,act,ok,savedcoverage,running,savedcoverage,Num,faults,foundbug,savedTestState
while time.time()-start < BUDGET:
sut.restart()
if (savedcoverage != None) and (rgen.random() > 0.4):
print "Processing"
sut.backtrack(savedcoverage)
storedTest = False
print "First: AVL tree"
for s in xrange(0,100):
act = sut.randomEnabled(rgen)
ok = sut.safely(act)
if len(sut.newStatements()) > 0:
savedcoverage = sut.state()
storedTest = True
if(running):
print "New Statement",sut.newStatements()
if (not storedTest) and (i != None) and (i in sut.currStatements()):
savedcoverage = sut.state()
storedTest = True
Num += 1
if(faults):
if not ok:
foundbug += 1
print "Failed"
print sut.failure()
print "REDUCE"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
break
savedTestState = sut.state()
expandNewState()
print foundbug,"FAILED"
print "ACTIVE",Num
print "RUNTIME",time.time()-start
for s in sortedCoverage:
print s, coverageCount[s]
if (COVERAGE_REPORT):
sut.internalReport()
def main():
global running,start,TIMEBUDGET,sut,testSaving,gene,testStored,act,srun,flag,coverageCount,covrSort,calculateWei,allCoverage,cW,coverageWM,failCount,actionsCount,sd
#global running,possible
global config
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
while time.time()-start < TIMEBUDGET:
sut.restart()
if (testSaving != None) and (gene.random() > 0.3):
sut.backtrack(testSaving)
testStored = False
print "Firstly test the AVL tree"
for s in xrange(0,100):
act = sut.randomEnabled(gene)
srun = sut.safely(act)
if config.running:
if sut.newBranches() != set([]):
for d in sut.newBranches():
print time.time()-start,len(sut.allBranches()),"New branch",d
func1()
if flag == 1:
break
for s in sut.currStatements():
if s not in coverageCount:
coverageCount[s] = 0
coverageCount[s] = coverageCount[s] + 1
covrSort = sorted(coverageCount.keys(), key=lambda x: coverageCount[x])
for st in covrSort:
if st*(allCoverage - coverageCount[st]) >25:
coverageWM.append(st*(allCoverage - coverageCount[st]))
print "Now I have the statement below coverage:", st
for sd in covrSort:
print sd,coverageCount[sd]
sut.internalReport()
print failCount,"FAILED"
print len(sut.allBranches()),"BRANCH"
print actionsCount,"TOTAL ACTIONS"
print time.time()-start,"TOTAL RUNTIME"
i += 1
bugs += 1
saveFault = 'failure' + str(bugs) + '.test'
sut.saveTest(sut.test(), saveFault)
print "Number bugs found is" ,i
sut.restart()
#To see what is the least covered branch to do experiments on them
for s in sut.currBranches():
if s not in covCount:
covCount[s] = 0
covCount[s] += 1
sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x])
leastCov = sortedCov[0]
if (time.time() - start > config.timeout):
break
#Take the name of all actions and sort them by their action count
sortedCov = sorted(covCount.keys(), key=lambda x: covCount[x])
# Produce coverage report
if config.coverage:
sut.internalReport()
print "TOTAL NUMBER OF BUGS",bugs
print "TOTAL NUMBER OF TESTS",no_tests
print "TOTAL NUMBER OF ACTIONS",actCount
print "TOTAL NUMBER OF RUNTIME",time.time()-start
def main():
#global variables are defined here
global sut, rgen, actCount, bugs, coveraged, visited, start, argv
parsed_args, parser = parse_args()
argv = make_config(parsed_args, parser)
start = time.time()
sut = sut.sut()
sut.silenceCoverage()
rgen = random.Random()
rgen.seed(argv.SEED)
actCount = 0
bugs = 0
coveraged = {}
visited = []
print "STARTING PHASE 1"
while time.time() - start < argv.TIMEOUT * 0.5:
sut.restart()
for s in xrange(0,argv.DEPTH):
randomAction()
collectCoverage()
sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x])
for s in sortedCov:
print s, coveraged[s]
print "STARTING PHASE 2"
while time.time() - start < argv.TIMEOUT :
belowMedian = set([])
sort = sorted(coveraged.keys(),key = lambda x : coveraged[x])
a = len(sort)/2
for s in sort:
if coveraged[s] < sort[a]:
belowMedian.add(s)
else:
break
activePool = []
for (t,c) in visited:
for s in c:
if s in belowMedian:
activePool.append((t,c))
break
sut.restart()
if len(sut.newStatements()) != 0:
print "new statement",sut.newStatements()
visited.append((list(sut.test()), set(sut.currStatements())))
for s in xrange(0,argv.DEPTH):
if not randomAction():
break
collectCoverage()
sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x])
for s in sortedCov:
print s, coveraged[s]
if argv.COVERAGE:
sut.internalReport()
print "FAILURES ",bugs, "TIMES"
print "TOTAL ACTIONS", actCount
print "TOTAL RUNTIME", time.time()-start
def main():
global config, R, sut, bugs, ntest, start
global actCount, branchCount, statementCount
global savedTest, savedFlag
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
if config.silence:
sut.silenceCoverage()
bugs = 0
actCount = 0
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
print "Starting Phase 1"
if elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
print "Finishing Phase 1"
print "Starting Phase 2"
elapsed = time.time() - start
while elapsed < config.timeout:
sut.restart()
ntest += 1
savedTest = None
if (savedTest != None) and (R.random() > 0.7):
sut.backtrack(savedTest)
savedFlag = False
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
elapsed = time.time() - start
print "Finishing Phase 2"
if time.time() - start > config.timeout:
print "[TIMEOUT]"
if config.coverdetail:
printCoverage()
if config.coverage:
sut.internalReport()
if config.resultdetail:
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount, test_file, ntest
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
while elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
for j in xrange(0, config.width):
if randomAct():
preTest = sut.state()
else:
sut.backtrack(preTest)
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
elapsed = time.time() - start
if elapsed > config.timeout:
break
collectCoverage()
elapsed = time.time() - start
printCoverage()
if config.coverage:
sut.internalReport()
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
failureCount = failureCount + 1 #the failure count will go on increasing
if (FAULT_CHECK): # this will check for the faults
name = "failure" + str(failureCount) + ".test"
f = sut.test()
sut.saveTest(F,filename)
for (s_reduces) in S:
steps_reduce = "#no of steps taken " + str(j)
print >> f, sut.prettyName(s_reduces).ljust(80 - len(steps_reduce), ' '), steps_reduce
j =j + 1
else:
print ""/n
print "This is the same test"
sut.prettyPrintTest(F)
break;
if (COVERAGE_REPORT):
sut.internalReport() #printing the result
start = time.time()
while time.time() - start < BUDGET:
sut.restart()
for s in xrange(0, depth):
acts = sut.randomEnableds(rgen, k)
sortActs = sorted(acts, key=lambda x: actionCount[abstract(x[0])])
act = sortActs[0]
ok = sut.safely(act)
actCount += 1
actionCount[abstract(act[0])] += 1
if not ok:
bugs += 1
print "FOUND A FAILURE"
#sut.prettyPrintTest(sut.test())
print sut.failure()
print "REDUCING"
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
sut.restart()
sut.internalReport()
orderedActs = sorted(actionCount.keys(), key=lambda x: actionCount[x])
for act in orderedActs:
print act, actionCount[act]
print bugs, "FAILED"
print "TOTAL ACTIONS", actCount
print "TOTAL RUNTIME", time.time() - start
def main():
if "--help" in sys.argv:
print "Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover] [--verbose] [--running]"
print "Options:"
print " --noCheck: do not run property checks"
print " --html: output an HTML coverage report to the chosen directory"
print " --noCover: do not compute code coverage"
print " --verbose: make actions verbose"
print " --running: give a running report on code coverage"
print " --keepGoing: don't stop on failed test"
sys.exit(0)
sut = sut.sut()
nocover = False
verbose = False
running = False
keepGoing = False
ignoreProps = False
lastWasHtml = False
files = []
htmlOut = None
for f in sys.argv[1:]:
if lastWasHtml:
htmlOut = f
lastWasHtml = False
elif "--" not in f:
files.append(f)
elif f == "--html":
lastWasHtml = True
if "--noCover" in sys.argv:
nocover = True
if "--verbose" in sys.argv:
verbose = True
if "--noCheck" in sys.argv:
ignoreProps = True
if "--running" in sys.argv:
running = True
if "--keepGoing" in sys.argv:
keepGoing = True
if verbose:
sut.verbose(True)
anyFailed = False
stime = time.time()
for f in files:
print "RUNNING TEST", f
t = sut.loadTest(f)
ok = False
try:
ok = sut.replay(t, checkProp=(not ignoreProps))
except Exception as e:
print "EXCEPTION RAISED:", e
if running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print "New branch:", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print "New statement:", s
if not ok:
print "TEST", f, "FAILED:"
print sut.failure()
anyFailed = True
if not keepGoing:
sys.exit(255)
print time.time() - stime, "ELAPSED"
if not nocover:
print "STATEMENTS:", len(sut.allStatements()), "BRANCHES:", len(
sut.allBranches())
if not nocover:
sut.internalReport()
print sut.report("coverage.out"), "PERCENT COVERED"
if htmlOut != None:
sut.htmlReport(htmlOut)
if not anyFailed:
print "ALL TESTS SUCCESSFUL"
def randomOperation():
global bug_num, time_start
operation = sut.randomEnabled(rand)
good = sut.safely(operation)
runtime = time.time() - time_start
if (running):
if (len(sut.newBranches())) > 0
print "Operation: ", operation[0]
for i in sut.newBranches():
print runtime, len(sut.allBranches()), "New Branch", i
if (not good):
bug_num += 1
print "A bug is found! #", bug_num
print sut.failure()
print ("Reducing!")
reduce = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(reduce)
print(sut.failure())
if (faults):
f = open((fail + str(bug_num) + ".test"), "w")
print >> f, sut.failure()
j = 0
for (s_reduce, _, _) in reduce:
steps_reduce = "# Step " + str(j)
print >> f, sut.prettyName(s_reduce).ljust(80 - len(steps_reduce), ' '), steps_reduce
j += 1
f.close()
print "Time: ", runtime
return good
time_start = time.time()
current_depth = 1
state_queue = [sut.state()]
Time_Phase1 = timeout / 4
print "PHASE 1 ..."
while (time.time() < (time_start + Time_Phase1)):
sut.restart()
for d in xrange(0, depth):
good = randomOperation()
if (not good):
break
if (len(sut.newStatements()) > 0):
new_state.append(sut.state())
new_statement.append(sut.newStatements())
for s in sut.currStatements():
if s not in coverageCount:
coverageCount[s] = 0
coverageCount[s] += 1
sortedCoverage = sorted(coverageCount.keys(), key = lambda x: coverageCount[x])
sum_value = 0
for s in sortedCoverage:
sum_value += coverageCount[s]
mean_value = sum_value / len(coverageCount)
sum_value = 0.0
for s in sortedCoverage:
sum_value += math.pow(coverageCount[s] - mean_value, 2)
sum_value = sum_value / (len(coverageCount) - 1)
std = math.sqrt(sum_value)
threshold = mean_value - (0.67 * std)
print "Mean: ", mean_value
print "Standard Deviation: ", std
print "Threshold: ", threshold
for s in sortedCoverage:
if (coverageCount[s] > threshold):
break
for k in k_chosen:
if s in new_statement[k]:
continue
for k in xrange(0, len(new_statement)):
if s in new_statement[k]:
k_chosen.append(k)
for k in k_chosen:
chosen_state.append(new_state[k])
print bug_num, " Bugs Found!"
for s in sortedCoverage:
print s, coverageCount[s]
if (coverage):
sut.internalReport()
print "PHASE 2 ..."
Time_Phase2 = timeout - Time_Phase1
all_state = []
new_statement = []
i = 0;
time_start = time.time()
while (time.time() < (time_start + Time_Phase2)):
for state in chosen_state:
i += 1
time_state = float(Time_Phase2) / (len(chosen_state) * (i + 1))
time_start2 = time.time()
while (time.time() < time_start2 + time_state):
sut.restart()
sut.backtrack(state)
for d in xrange(0, depth):
good = randomOperation()
if (not good):
break
if (len(sut.newStatements()) > 0):
print "A New Statement is Found!"
chosen_state.insert(i, sut.state())
for s in sut.currStatements():
if s not in coverageCount:
coverageCount[s] = 0
coverageCount[s] += 1
sortedCoverage = sorted(coverageCount.keys(), key = lambda x: coverageCount[x])
print bug_num, " Bugs Found!"
for s in sortedCoverage:
print s, coverageCount[s]
if (coverage):
sut.internalReport()
def main():
global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
if config.faults:
test_file = open("failure1.test", "w")
start = time.time()
elapsed = time.time() - start
for i in xrange(0, config.depth):
sut.restart()
for j in xrange(0, config.width):
if not randomAct():
print "Function called randomAct() occurs error"
break
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
if config.faults:
test_file.close()
if config.coverage:
sut.restart()
sut.report("coverage.out")
#printCoverage()
sut.internalReport()
#print "Covered", len(sut.allBranches()), "branches"
#print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
parsed_args, parser = parse_args()
Config = make_config(parsed_args, parser)
rgen = random.Random(Config.seed)
print ('testing using config={}'.format(Config))
sut.silenceCoverage()
sut.restart()
state_queue = [sut.state()]
visited = []
tdepth = 30
d = 1
ntests = 0
bugs = 0
actCount = 0
faults_file = "failure"
coverage_count = {}
leastCovered = None
statebuff = []
start = time.time()
elapsed = time.time() - start
while d <= Config.depth:
print "Depth", d, "Size", len(state_queue), "Set", len(visited)
w = 1
len_queue = len(state_queue)
ntests += 1
frontier = []
depth_start = time.time()
random.shuffle(state_queue)
savedTest = False
for s in state_queue:
sut.backtrack(s)
for a in sut.enabled():
depth_time = time.time() - depth_start
act = sut.randomEnabled(rgen)
actCount += 1
if depth_time >= tdepth:
break
isGood = sut.safely(a)
if Config.running:
if sut.newBranches() != set([]):
print "Action:", a[0]
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
print "Action:", a[0]
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
s_next = sut.state()
if s_next not in visited:
visited.append(s_next)
frontier.append(s_next)
if not isGood:
failures(sut, bugs, Config, faults_file)
if Config.faults == 1:
sut.saveTest(R, faults_file + str(bugs) + ".test")
sut.restart()
if len(sut.newStatements()) > 0:
frontier.insert(0, sut.state())
savedTest = True
if (not savedTest) and (leastCovered != None) and (leastCovered in sut.currStatements()):
frontier.insert(0, sut.state())
savedTest = True
elapsed = time.time() - start
if elapsed >= Config.timeout:
break
if depth_time >= tdepth:
break
if elapsed >= Config.timeout:
break
if w <= Config.width:
w += 1
else:
break
statebuff.append((list(sut.test()), set(sut.currBranches())));
state_queue = frontier
sortPop = sorted(statebuff,key = lambda x: len(x[1]),reverse=True)
(t,b) = rgen.choice(statebuff)
m = mutate(t, rgen, bugs)
statebuff.append((m, sut.currBranches()))
if elapsed >= Config.timeout:
print "Stopping Test Due To Timeout, Terminated at Length", len(sut.test())
print ntests, "EXECUTED"
break
for s in sut.currStatements():
if s not in coverage_count:
coverage_count[s] = 0
coverage_count[s] += 1
sortedCov = sorted(coverage_count.keys(), key=lambda x: coverage_count[x])
if len(sortedCov) != 0:
leastCovered = sortedCov[0]
d += 1
if Config.coverage == 1:
sut.internalReport()
print bugs,"FAILED"
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global BUDGET,start,test,sut,storedTest,act,correct,running,coverage3,count,flag,savedTestState,coverageCount,sortedCov,weight,weightedCov,coverage2,bug,noTest
while time.time()-start < BUDGET:
sut.restart()
noTest += 1
if (random.random() > 0.7) and (test != None):
#print "Exploiting.."
sut.backtrack(test)
storedTest = False
#print "Testing.."
for i in xrange(0,depth):
act = sut.randomEnabled(random)
correct = sut.safely(act)
if running:
if sut.newBranches() != set([]):
for d in sut.newBranches():
print time.time() - start, len(sut.allBranches()),"New branch",d
if len(sut.newStatements()) > 0:
test = sut.state()
storedTest = True
if(running):
print "Found a statement",sut.newStatements()
if (not storedTest) and (coverage3 != None) and (coverage3 in sut.currStatements()):
test = sut.state()
storedTest = True
count += 1
if not correct:
if not correct:
bug += 1
print "Found a failure!"
print sut.failure()
print "Reducing.."
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
filename ='failure%d.test'%bug
sut.saveTest(R,filename)
#flag = 1
#if flag == 1:
break
savedTestState = sut.state()
for i in sut.currStatements():
if i not in coverageCount:
coverageCount[i] = 0
coverageCount[i] += 1
sortedCov = sorted(coverageCount.keys(), key=lambda x: coverageCount[x])
for i in sortedCov:
weight = i * (defCover - coverageCount[i])
if weight > coverage4:
coverage2.append(weight)
print "Coverage:", i
#sut.backtrack(sut.state())
if coverage:
sut.internalReport()
sortedCov = sorted(coverageCount.keys(), key=lambda x: coverageCount[x])
for i in sortedCov:
print i, coverageCount[i]
print bug,"failed"
print "Total tests:", noTest
print "Total actions:",count
print "Total running time:",time.time()-start
def main():
global config,rgen,actCount,failCount,ntests,firstPool,coverageCount
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
rgen = random.Random(config.seed)
actCount = 0
failCount = 0
coverageCount = {}
firstPool = []
start = time.time()
ntests = 0
while time.time() - start < (config.timeout)/2:
ntests += 1
sut.restart()
for d in xrange(0, config.depth):
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
expandPool()
if config.running:
if sut.newBranches() != set([]):
print "ACTION:", act[0]
for b in sut.newBranches():
print time.time()-start, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
print "ACTION:", act[0]
for s in sut.newStatements():
print time.time()-start, len(sut.allStatements()),"New statement",s
if not ok:
failCount += 1
print "FOUND A FAILURE"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
if config.faults:
filename = 'failure' + `failCount` + '.test'
sut.saveTest(R,filename)
print sut.failure()
sut.restart()
break
if time.time() - start > (config.timeout)/2:
print "FIRST POOL HAS BEEN CREATED, TERMINATED AT LENGTH",ntests
break
collectCoverage()
end = time.time()
while time.time() - end < (config.timeout)/2:
buildSecondPool()
sut.restart()
if rgen.random() > 0.5:
sut.replay(rgen.choice(secondPool)[0])
ntests += 1
for d in xrange(0, config.depth):
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
expandPool()
if config.running:
if sut.newBranches() != set([]):
print "ACTION:", act[0]
for b in sut.newBranches():
print time.time()-end, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
print "ACTION:", act[0]
for s in sut.newStatements():
print time.time()-end, len(sut.allStatements()),"New statement",s
if not ok:
failCount += 1
print "FOUND A FAILURE"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
if config.faults:
filename = 'failure' + `failCount` + '.test'
sut.saveTest(R,filename)
print sut.failure()
sut.restart()
break
if time.time() - end > (config.timeout)/2:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests
break
collectCoverage()
if config.faults:
print "TOTAL FAULTS", failCount
if config.coverage:
sut.internalReport()
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def print_results(): if COVERAGE: sut.internalReport() print "Number of bugs found: " + str(bugs)