def main():
global start,config,sut,R,nonerror,error,file_name,num
num = 0
file_name = 'failurefile'
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
sut = SUT.sut()
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
states = [sut.state()]
nonerror = []
error = []
news = None
while(time.time() < start + config.timeout):
for s in states:
sut.restart()
sut.backtrack(s)
for w in xrange(0, config.width):
for d in xrange(0, config.depth):
ok = check_action()
news = sut.newStatements()
if not ok:
break
if((len(news)>0) and (not ((news in error) or (news in nonerror)))):
states.insert(0,sut.state())
nonerror.append(sut.currStatements())
if config.coverage:
sut.internalReport()
print "Bugs ",num
def main(argv):
global sut, rgen, ntest, nbug, coverageCount, start, selectStatements, threshold
parse_options(argv)
# GLOBAL VARIABLES
sut = sut.sut()
rgen = random.Random(opts.seed)
ntest = 0
nbug = 0
coverageCount = {}
selectStatements = {}
threshold = 0
start = time.time()
# PHASES
phaseLimit = 0
if opts.timeout < 30:
phaseLimit = int(round(opts.timeout/3*2))
else:
phaseLimit = int(round(opts.timeout/10))
while (time.time() - start) < (0.95 * opts.timeout):
randomPool = phase1(phaseLimit)
selectPool = filterPool(randomPool)
phase2(randomPool, selectPool)
print "FULL POOL:",len(randomPool),", SELECT POOL:",len(selectPool)
print "EXECUTED:",ntest,"tests"
print "BUGS:",nbug
print "TOTAL ELAPSED:",round(time.time() - start,5),"seconds"
if (opts.coverage):
sut.internalReport()
def main():
global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
R = random.Random(seed)
states = [sut.state()]
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
break
checkAlg()
if coverage:
sut.internalReport()
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def test_tstl_regressions():
sut = SUT.sut()
failCount = 0
passCount = 0
for f in glob.glob("tstl_tests/*.test"):
print()
print("*" * 50)
print()
t = sut.loadTest(f)
failed = sut.failsCheck(t)
if failed:
print("TEST", f, "FAILS:")
print()
sut.verbose(True)
sut.replay(t, checkProp=True, catchUncaught=True)
failure = sut.failure()
print(failure)
traceback.print_tb(failure[2], file=sys.stdout)
sut.verbose(False)
failCount += 1
else:
print("TEST", f, "PASSES")
passCount += 1
print()
print("*" * 50)
print()
print("TSTL REGRESSIONS RESULT:")
print(passCount, "TESTS PASSED,", failCount, "TESTS FAILED")
assert failCount == 0
def main():
global timeout
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Generalizing using config={}'.format(config)))
sut = SUT.sut()
timeout = config.timeout
if not config.coverage:
try:
sut.stopCoverage()
except BaseException:
pass
t = sut.loadTest(config.infile)
f = None
if config.matchException:
print("RUNNING TO OBTAIN FAILURE FOR EXCEPTION MATCHING...")
assert sut.fails(t)
f = sut.failure()
print("ERROR:", f)
print("TRACEBACK:")
traceback.print_tb(f[2], file=sys.stdout)
if not config.sandbox:
pred = (lambda x: sut.failsCheck(x, failure=f))
if not config.noCheck:
pred = (lambda x: sut.fails(x, failure=f))
else:
pred = sandboxReplay
if config.coverage:
print("EXECUTING TEST TO OBTAIN COVERAGE...")
sut.replay(t,
checkPropcheckProp=not config.noCheck,
catchUncaught=config.uncaught)
b = set(sut.currBranches())
s = set(sut.currStatements())
print("PRESERVING", len(b), "BRANCHES AND", len(s), "STATEMENTS")
pred = sut.coversAll(s,
b,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
print("GENERALIZING...")
start = time.time()
sut.generalize(t,
pred,
verbose=config.verbose,
fresh=not config.noFresh,
keepLast=config.keepLast)
print("GENERALIZED IN", time.time() - start, "SECONDS")
def __init__(self, pid): self.pid = pid # unique id for this pool self.sut = SUT.sut() # SUT object self.nseqs = [[]] # list of non-error sequences self.eseqs = [] # list of error sequences self.covered = dict() # dictionary, key: coverage by this pool, value: frequency of coverage by this pool self.time = 0.0 # how many seconds this pool is used in feedback() self.count = 0 # how many times this pool is selected in select_pool() self.score = 0.0 # score of this pool self.uniqueness = 0.0 # uniqueness of this pool
def main():
global config, R, sut, bugs, actCount, branchCount, statementCount, ntest, start
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
print "Starting Test"
while elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
elapsed = time.time() - start
print "Finishing Test"
if time.time() - start > config.timeout:
print "[TIMEOUT]"
printCoverage()
if config.coverage:
sut.internalReport()
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
global sut, rgen, nbugs, start, CXPB, MUTPB, NGEN
sut = sut.sut()
rgen = random.Random(opts.seed)
CXPB, MUTPB, NGEN = 0.5, 0.2, 1
best, timeclock = [], []
nbugs, g, prev_best = 0, 0, 1
init()
start = time.time()
while True:
print("Building population")
pop = toolbox.population(n=100)
print(" n=%d" % len(pop))
print("Start of evolution")
fitnesses = list(map(toolbox.evaluate, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
while True:
g += 1
generation(g, pop)
best_test = tools.selBest(pop, 1)[0]
elapsed = time.time()-start
if 0.02 > (best_test.fitness.values[0] - prev_best) / prev_best or elapsed > (0.95 * opts.timeout):
break
else:
prev_best = max(best_test.fitness.values[0], prev_best)
best_test = tools.selBest(pop, 1)[0]
print("Best test has %s lines covered" % int(best_test.fitness.values[0]))
best.append(best_test.fitness.values[0])
elapsed = time.time()-start
timeclock.append(elapsed) if not timeclock else timeclock.append(elapsed - timeclock[-1])
average_time = sum(timeclock) / len(timeclock)
print("elapsed: %f seconds out of %f seconds" % (elapsed, opts.timeout))
if (elapsed + average_time) > opts.timeout:
break
overall_best = max(best)
print("Overall best test has %s lines covered" % int(overall_best))
if (opts.coverage):
sut.internalReport()
def main():
global start,sut,R,noerror,error,actCount, bugs,ntest,newseq,currseq,states
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
R = random.Random()
R.seed(seed)
states = [sut.state()]
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
error.append(sut.currStatements())
break
checkAlg()
#if not error ==[]:
# print "SHOW ERROR SEQUENCE"
# print error
# f = open(("error" + str(actCount) + ".test"), 'w')
# f.write(str(error))
# f.close()
#else:
# print "Data in noerror sequence"
# print noerror
if coverage:
sut.internalReport()
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
global swarm, showActions, noCheck, alwaysSave, noSave, sut
if "--help" in sys.argv:
print(
"Usage: tstl_afl [--noCheck] [--swarm] [--verbose] [--showActions] [--noSave] [--alwaysSave]"
)
print("Options:")
print(" --noCheck: do not run property checks")
print(
" --swarm use first four bytes to determine a swarm configuration"
)
print(" --verbose: make actions verbose")
print(" --showActions: show actions in test")
print(
" --noSave: don't save failing tests as standard TSTL tests")
print(" --alwaysSave: save even non-failing tests")
sys.exit(0)
sut = SUT.sut()
try:
sut.stopCoverage()
except BaseException:
pass
sut.restart()
swarm = "--swarm" in sys.argv
showActions = "--showActions" in sys.argv
if "--verbose" in sys.argv:
sut.verbose(True)
noSave = "--noSave" in sys.argv
alwaysSave = "--alwaysSave" in sys.argv
noCheck = "--noCheck" in sys.argv
persist = "--persist" in sys.argv
if not persist:
afl.init()
runTest()
else:
while afl.loop():
runTest()
sut.restart()
os._exit(0)
def main():
if "--help" in sys.argv:
print("Usage: tstl_toafl <outputdir> <files>")
sys.exit(0)
sut = SUT.sut()
outputDir = sys.argv[1]
files = sys.argv[2:]
acts = sut.actions()
i = 0
for f in files:
t = sut.loadTest(f)
sut.saveTest(t,outputDir+"/"+os.path.basename(f)+".afl",afl=True)
i += 1
def main():
global start,sut,R,noerror,error,reduceTime,actCount, bugs,ntest,newseq,currseq,states, beforeReduceS, beforeReduceB,quickCount
actCount = 0
bugs = 0
start = time.time()
noerror = []
error = []
newseq = []
ntest=0
currseq=[]
sut = sut.sut()
reduceTime = 0.0
R = random.Random()
R.seed(seed)
beforeReduceS = set(sut.allStatements())
beforeReduceB = set(sut.allBranches())
states = [sut.state()]
quickCount=0
print "STARTING PHASE 1"
while(time.time() < start + timeout):
for st in states:
ntest+=1
if (time.time() > start + timeout):
break
sut.restart()
sut.backtrack(st)
for s in xrange(0, depth):
if (time.time() > start + timeout):
break
ok = randomAction()
if not ok:
break
checkAlg()
if coverage:
sut.internalReport()
if quickTests==1:
#if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])):
handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True)
print "TOTAL BUGS", bugs
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def main():
if "--help" in sys.argv:
print("Usage: tstl_fromafl <outputdir> <files> [--aflswarm]")
sys.exit(0)
sut = SUT.sut()
aflswarm = "--aflswarm" in sys.argv
if aflswarm:
sys.argv.remove("--aflswarm")
outputDir = sys.argv[1]
files = sys.argv[2:]
i = 0
for f in files:
t = sut.loadTest(f, afl=True, swarm=aflswarm)
sut.saveTest(t, outputDir + "/" + os.path.basename(f) + ".test")
i += 1
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Triaging using config={}'.format(config)))
sut = SUT.sut()
try:
sut.stopCoverage()
except:
pass
signatures = {}
for fn in glob.glob(config.testglob):
t = sut.loadTest(fn)
if not config.noCheck:
fails = sut.failsAny(t)
else:
fails = sut.fails(t)
if fails:
e = repr(sut.failure())
e = e[:e.find("<traceback object at 0x")] + ")"
l = t[-1][0]
sig = (noDigits(e),noDigits(l))
if (sig not in signatures):
signatures[sig] = (fn,t,sut.failure(),sut.prettyName(t[-1][0]),1)
elif (len(t) < signatures[sig][1]):
signatures[sig] = (fn,t,sut.failure(),sut.prettyName(t[-1][0]),signatures[sig][4]+1)
for sig in sorted(signatures.keys(),key=lambda x:signatures[x][4],reverse=True):
print ("="*80)
print ("TEST:",signatures[sig][0],"LENGTH:",len(signatures[sig][1]))
print ("OPERATION:",signatures[sig][3])
print ("FAILURE:")
traceback.print_tb(signatures[sig][2][2],file=sys.stdout)
print ("COUNT:",signatures[sig][4])
if config.showTests:
sut.prettyPrintTest(signatures[sig][1])
def main():
R = random.Random()
repeats = 0
pid = str(os.getpid())
depth = int(sys.argv[1])
sut = SUT.sut()
taken = defaultdict(lambda: 0)
takenClass = defaultdict(lambda: 0)
allTaken = defaultdict(lambda: 0)
allTakenClass = defaultdict(lambda: 0)
# takenPath = defaultdict(lambda: 0)
takenFull = {}
start = time.time()
lastEpoch = 0
count = 0
sut.standardSwarm(R)
print(sut.swarmConfig())
totalPaths = int(math.pow(len(sut.actions()), depth))
print(totalPaths, "UPPER BOUND ON POSSIBLE TESTS")
while count < totalPaths:
sut.restart()
path = []
i = 0
count += 1
while len(path) < depth:
possible = sut.actions()
possible = sorted(possible,
key=lambda act: (allTakenClass[sut.actionClass(
act)], allTaken[act[0]], takenClass[
(sut.actionClass(act), i)], taken[
(act[0], i)]))
a = None
for act in possible:
if act[1]():
a = act
break
a = sut.randomEnabled(R)
if a is None:
print("DEADLOCK!")
break
path.append(a)
# takenPath[sut.captureReplay(path)] += 1
taken[(a[0], i)] += 1
takenClass[(sut.actionClass(a), i)] += 1
allTaken[(a[0])] += 1
allTakenClass[sut.actionClass(a)] += 1
ok = sut.safely(a)
if not ok:
print("FALIURE IN TEST", count)
sut.saveTest(path, "failure.exhaust." + pid + ".test")
sut.prettyPrintTest(path)
print(sut.failure())
sys.exit(255)
if not "--noCheck" not in sys.argv:
okCheck = sut.check()
if not okCheck:
print("PROPERTY VIOLATION IN TEST", count)
sut.saveTest(path, "failure.exhaust." + pid + ".test")
sut.prettyPrintTest(path)
print(sut.failure())
sys.exit(255)
i += 1
if "--ultraVerbose" in sys.argv:
print("=" * 25)
sut.prettyPrintTest(path)
p = sut.captureReplay(path)
if p in takenFull:
repeats += 1
takenFull[p] = True
epoch = int((time.time() - start) / 2)
if epoch > lastEpoch:
lastEpoch = epoch
print(time.time() - start,
"ELAPSED",
count,
"TESTS",
repeats,
"REPEATS",
end=' ')
if "--noCover" not in sys.argv:
print("[",
len(sut.allStatements()),
"stmts",
len(sut.allBranches()),
"branches ]",
end=' ')
print()
if "--verbose" in sys.argv:
print("*" * 50)
print("PATH #", count)
sut.prettyPrintTest(path)
print()
print("COUNTS:")
print("=" * 20)
for c in sorted(list(allTakenClass.keys()),
key=lambda ac: allTakenClass[ac]):
print(c, allTakenClass[c])
print("=" * 20)
for a in sorted(list(allTaken.keys()),
key=lambda act: allTaken[act]):
print(a, allTaken[a])
print(repeats, "TOTAL REPEATED TESTS", len(takenFull), "DISTINCT TESTS")
def main():
global failCount, sut, config, reduceTime, quickCount, repeatCount, failures, cloudFailures, R, opTime, checkTime, guardTime, restartTime, nops, ntests
global fullPool, activePool, branchCoverageCount, statementCoverageCount, localizeSFail, localizeBFail
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
if config.gendepth != None:
failCloud = {}
allClouds = {}
if config.exploit != None:
fullPool = []
activePool = []
if config.quickAnalysis or (config.exploit != None):
branchCoverageCount = {}
statementCoverageCount = {}
if config.uniqueValuesAnalysis:
handledValues = {}
uniquef = open("unique.corpus", 'w')
allUniquePaths = []
sut = SUT.sut()
if config.relax:
sut.relax()
if config.verboseActions:
sut.verbose(True)
if config.readQuick:
print "REPLAYING QUICK TESTS"
sqrtime = time.time()
for f in glob.glob("quick*.test"):
fn = int(f.split("quick")[1].split(".")[0])
if fn >= quickCount:
quickCount = fn + 1
t = sut.loadTest(f)
sut.replay(t,
catchUncaught=True,
checkProp=(not config.ignoreProps))
print "EXECUTION TIME:", time.time() - sqrtime
print "BRANCH COVERAGE:", len(sut.allBranches())
print "STATEMENT COVERAGE:", len(sut.allStatements())
if config.logging != None:
sut.setLog(config.logging)
if config.profile:
profileTime = {}
profileCount = {}
for a in set(map(sut.actionClass, sut.actions())):
profileTime[a] = 0.0
profileCount[a] = 0
if config.markov != None:
nactions = len(sut.actions())
mprobs = {}
prefix = []
probs = []
inProbs = False
readSize = False
for l in open(config.markov):
if not readSize:
markovN = int(l)
readSize = True
elif "START CLASS" in l:
if (prefix != []):
mprobs[tuple(prefix)] = probs
prefix = []
probs = []
inProbs = False
elif inProbs:
ls = l.split("%%%%")
prob = float(ls[0])
ac = ls[1][1:-1]
probs.append((prob, ac))
elif "END CLASS" in l:
inProbs = True
else:
prefix.append(l[:-1])
tacts = sut.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
checkResult = True
if config.total:
fulltest = open("fulltest.test", 'w')
if config.localize:
localizeSFail = {}
localizeSPass = {}
localizeBFail = {}
localizeBPass = {}
testsPassed = 0
if config.computeFeatureStats:
featureStatsB = {}
featureStatsS = {}
featureStatsA = {}
if config.quickAnalysis:
quickcf = open("quick.corpus", 'w')
quickCorpus = []
quickDoneB = {}
quickDoneS = {}
quickAnalysisTotal = 0
quickAnalysisBCounts = {}
quickAnalysisSCounts = {}
quickAnalysisCounts = {}
quickClassCounts = {}
quickAnalysisRawCounts = {}
for c in set(map(sut.actionClass, sut.actions())):
quickAnalysisCounts[c] = 0
quickClassCounts[c] = 0
quickAnalysisRawCounts[c] = 0
quickAnalysisReducedB = {}
quickAnalysisReducedS = {}
if config.timedProgress:
lastInterval = 0
if config.verbose:
print "ABOUT TO START TESTING"
sys.stdout.flush()
while (config.maxtests == -1) or (ntests < config.maxtests):
if config.progress:
printStatus(elapsed)
if config.verbose:
print "STARTING TEST", ntests
sys.stdout.flush()
ntests += 1
if config.swarm:
sut.standardSwarm(R, file=config.swarmProbs, P=config.swarmP)
if config.progress:
print "CONFIG:", (sut.swarmConfig())
if config.highLowSwarm != None:
classP = sut.highLowSwarm(R,
file=config.swarmProbs,
highProb=config.highLowSwarm)
if config.swarmSwitch != None:
lastSwitch = 0
switches = []
for i in xrange(0, config.swarmSwitch):
switch = R.randrange(lastSwitch + 1,
config.depth - ((config.swarmSwitch - i)))
switches.append(switch)
lastSwitch = switch
startRestart = time.time()
sut.restart()
restartTime += time.time() - startRestart
if config.total:
fulltest.write("<<RESTART>>\n")
if config.replayable:
currtest = open("currtest.test", 'w')
if (config.exploit != None) and (
(time.time() - start) > config.startExploit):
tryExploit()
testFailed = False
for s in xrange(0, config.depth):
if config.verbose:
print "GENERATING STEP", s,
sys.stdout.flush()
if (config.swarmSwitch != None) and (s in switches):
if config.highLowSwarm == None:
sut.standardSwarm(R,
file=config.swarmProbs,
P=config.swarmP)
else:
classP = sut.highLowSwarm(R,
file=config.swarmProbs,
highProb=config.highLowSwarm)
if (config.swarmLength != None) and (((s + 1) % config.swarmLength)
== 0):
if config.highLowSwarm == None:
sut.standardSwarm(R,
file=config.swarmProbs,
P=config.swarmP)
else:
classP = sut.highLowSwarm(R,
file=config.swarmProbs,
highProb=config.highLowSwarm)
startGuard = time.time()
tryStutter = (a != None) and (a[1]()) and ((config.stutter != None)
or config.greedyStutter)
if tryStutter:
if (config.stutter == None) or (R.random() > config.stutter):
tryStutter = False
if (config.greedyStutter) and sawNew:
print "TRYING TO STUTTER DUE TO COVERAGE GAIN"
tryStutter = True
else:
if config.markov == None:
if config.highLowSwarm == None:
a = sut.randomEnabled(R)
else:
a = sut.randomEnabledClassProbs(R, classP)
else:
prefix = tuple(map(sut.actionClass, sut.test()[-markovN:]))
if prefix not in mprobs:
a = sut.randomEnabled(R)
else:
a = sut.randomEnabledClassProbs(R, mprobs[prefix])
if a == None:
a = sut.randomEnabled(R)
if a == None:
#sut.prettyPrintTest(sut.test())
print "WARNING: DEADLOCK (NO ENABLED ACTIONS)"
guardTime += time.time() - startGuard
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
sut.test())
break
if config.timedProgress != -1:
thisInterval = int(elapsed / config.timedProgress)
if thisInterval > lastInterval:
printStatus(elapsed)
lastInterval = thisInterval
sys.stdout.flush()
if a == None:
print "TERMINATING TEST DUE TO NO ENABLED ACTIONS"
break
if tryStutter:
print "STUTTERING WITH", a[0]
if config.replayable:
currtest.write(a[0] + "\n")
currtest.flush()
if config.total:
fulltest.write(a[0] + "\n")
fulltest.flush()
if config.verbose:
print "ACTION:", sut.prettyName(a[0])
sys.stdout.flush()
startOp = time.time()
if config.quickAnalysis:
quickClassCounts[sut.actionClass(a)] += 1
if config.showActions:
print "STEP #" + str(s), sut.prettyName(a[0])
stepOk = sut.safely(a)
thisOpTime = time.time() - startOp
nops += 1
if config.profile:
profileTime[sut.actionClass(a)] += thisOpTime
profileCount[sut.actionClass(a)] += 1
opTime += thisOpTime
if sut.warning() != None:
print "SUT WARNING:", sut.warning()
if tryStutter:
print "DONE STUTTERING"
if (stepOk or config.uncaught) and config.ignoreProps and (
config.exploit != None):
collectExploitable()
if (not config.uncaught) and (not stepOk):
testFailed = True
handle_failure(sut.test(), "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
startCheck = time.time()
if not config.ignoreProps:
checkResult = sut.check()
checkTime += time.time() - startCheck
if checkResult and (stepOk or config.uncaught) and (
config.exploit != None):
collectExploitable()
if not checkResult:
testFailed = True
handle_failure(sut.test(), "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
print "ACTION:", sut.prettyName(a[0])
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
sys.stdout.flush()
sawNew = True
else:
sawNew = False
if sut.newStatements() != set([]):
print "ACTION:", sut.prettyName(a[0])
for s in sut.newStatements():
print elapsed, len(
sut.allStatements()), "New statement", s
sys.stdout.flush()
sawNew = True
else:
sawNew = False
if config.uniqueValuesAnalysis:
uvals = sut.uniqueVals()
olds = sut.state()
currTest = list(sut.test())
for u in uvals:
if u not in handledValues:
print "ANALYZING NEW UNIQUE VALUE:", u
else:
continue
r = sut.reduce(currTest,
sut.coversUnique(u),
keepLast=False)
rc = map(sut.actionClass, r)
sut.replay(r)
for ru in sut.uniqueVals():
handledValues[ru] = True
if rc not in allUniquePaths:
print "NEW PATH DISCOVERED"
allUniquePaths.append(rc)
for s in rc:
uniquef.write(s + "\n")
uniquef.write(("=" * 50) + "\n")
uniquef.flush()
sut.backtrack(olds)
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
sut.test())
break
if (config.computeFeatureStats):
for act in sut.swarmConfig():
if act in featureStatsA:
featureStatsA[act] += 1
else:
featureStatsA[act] = 1
for b in sut.currBranches():
if b not in featureStatsB:
featureStatsB[b] = [1, {}]
else:
featureStatsB[b][0] += 1
for act in sut.swarmConfig():
if act not in featureStatsB[b][1]:
featureStatsB[b][1][act] = 1
else:
featureStatsB[b][1][act] += 1
for s in sut.currStatements():
if s not in featureStatsS:
featureStatsS[s] = [1, {}]
else:
featureStatsS[s][0] += 1
for act in sut.swarmConfig():
if act not in featureStatsS[s][1]:
featureStatsS[s][1][act] = 1
else:
featureStatsS[s][1][act] += 1
if (config.exploit != None) and (not config.quickAnalysis):
for b in sut.currBranches():
if b not in branchCoverageCount:
branchCoverageCount[b] = 1
else:
branchCoverageCount[b] += 1
for s in sut.currStatements():
if s not in statementCoverageCount:
statementCoverageCount[s] = 1
else:
statementCoverageCount[s] += 1
if config.localize and not testFailed:
testsPassed += 1
for s in sut.currStatements():
if s not in localizeSPass:
localizeSPass[s] = 0
localizeSPass[s] += 1
for b in sut.currBranches():
if b not in localizeBPass:
localizeBPass[b] = 0
localizeBPass[b] += 1
if config.quickAnalysis:
currTest = list(sut.test())
sut.replay(currTest)
currB = sut.currBranches()
currS = sut.currStatements()
clen = len(currTest)
#print "GATHERING QUICK ANALYSIS DATA FOR",len(currB),"BRANCHES"
for b in currB:
if config.fastQuickAnalysis and (b in quickDoneB):
continue
print "ANALYZING BRANCH", b
if b not in branchCoverageCount:
branchCoverageCount[b] = 0
quickAnalysisReducedB[b] = 0
branchCoverageCount[b] += 1
r = sut.reduce(currTest,
sut.coversBranches([b]),
keepLast=False)
print "REDUCED FROM", clen, "TO", len(r)
sys.stdout.flush()
sut.replay(r)
for br in sut.currBranches():
quickDoneB[br] = True
for sr in sut.currStatements():
quickDoneS[sr] = True
rc = map(sut.actionClass, r)
if rc not in quickCorpus:
quickCorpus.append(rc)
for s in rc:
quickcf.write(s + "\n")
quickcf.write(("=" * 50) + "\n")
quickcf.flush()
sut.replay(r)
for b2 in sut.currBranches():
if b2 not in quickAnalysisReducedB:
quickAnalysisReducedB[b2] = 0
quickAnalysisReducedB[b2] += 1
for s2 in sut.currStatements():
if s2 not in quickAnalysisReducedS:
quickAnalysisReducedS[s2] = 0
quickAnalysisReducedS[s2] += 1
if b not in quickAnalysisBCounts:
quickAnalysisBCounts[b] = {}
quickAnalysisTotal += 1
for c in map(sut.actionClass, r):
quickAnalysisRawCounts[c] += 1
for c in set(map(sut.actionClass, r)):
quickAnalysisCounts[c] += 1
if c not in quickAnalysisBCounts[b]:
quickAnalysisBCounts[b][c] = 0
quickAnalysisBCounts[b][c] += 1
#print "GATHERING QUICK ANALYSIS DATA FOR",len(currS),"STATEMENTS"
for s in currS:
if config.fastQuickAnalysis and (s in quickDoneS):
continue
if s not in statementCoverageCount:
statementCoverageCount[s] = 0
quickAnalysisReducedS[s] = 0
statementCoverageCount[s] += 1
print "ANALYZING STATEMENT", s
r = sut.reduce(currTest,
sut.coversStatements([s]),
keepLast=False)
print "REDUCED FROM", clen, "TO", len(r)
sys.stdout.flush()
sut.replay(r)
for br in sut.currBranches():
quickDoneB[br] = True
for sr in sut.currStatements():
quickDoneS[sr] = True
rc = map(sut.actionClass, r)
if rc not in quickCorpus:
quickCorpus.append(rc)
sut.replay(r)
for b2 in sut.currBranches():
if b2 not in quickAnalysisReducedB:
quickAnalysisReducedB[b2] = 0
quickAnalysisReducedB[b2] += 1
for s2 in sut.currStatements():
quickAnalysisReducedS[s2] += 1
if s not in quickAnalysisSCounts:
quickAnalysisSCounts[s] = {}
quickAnalysisTotal += 1
for c in map(sut.actionClass, r):
quickAnalysisRawCounts[c] += 1
for c in set(map(sut.actionClass, r)):
quickAnalysisCounts[c] += 1
if c not in quickAnalysisSCounts[s]:
quickAnalysisSCounts[s][c] = 0
quickAnalysisSCounts[s][c] += 1
if config.throughput:
print "ACTION THROUGHPUT:", nops / (time.time() - start)
if config.replayable:
currtest.close()
if config.quickTests:
if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements()
!= set([])):
handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True)
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if config.total:
fulltest.close()
if not config.nocover:
sut.restart()
print sut.report(config.coverfile), "PERCENT COVERED"
if config.internal:
sut.internalReport()
if config.html:
sut.htmlReport(config.html)
if config.uniqueValuesAnalysis:
uniquef.close()
if config.computeFeatureStats:
fstatsf = open(
"feature.stats." + str(os.getpid()) +
str(R.randrange(1000, 10000)), 'w')
fstatsf.write("TESTS:" + str(ntests) + "\n")
for act in featureStatsA:
fstatsf.write(act + " %%ACTCOUNT%% " + str(featureStatsA[act]) +
"\n")
for b in sut.allBranches():
fstatsf.write("BRANCH:" + str(b) + "\n")
count = featureStatsB[b][0]
fstatsf.write("COUNT:" + str(count) + "\n")
for act in featureStatsB[b][1]:
fstatsf.write(act + " %%%% " + str(featureStatsB[b][1][act]) +
"\n")
for s in sut.allStatements():
fstatsf.write("STATEMENT:" + str(s) + "\n")
count = featureStatsS[s][0]
fstatsf.write("COUNT:" + str(count) + "\n")
for act in featureStatsS[s][1]:
fstatsf.write(act + " %%%% " + str(featureStatsS[s][1][act]) +
"\n")
fstatsf.close()
if config.quickAnalysis:
quickcf.close()
print "*" * 70
print "QUICK ANALYSIS RESULTS:"
print "*" * 70
print "TEST PATTERNS:"
for rc in quickCorpus:
print "=" * 50
for s in rc:
print s
print "*" * 70
print "OVERALL ACTION ANALYSIS:"
totalTaken = sum(quickClassCounts.values())
actSort = sorted(quickAnalysisRawCounts.keys(),
key=lambda x: quickAnalysisCounts.get(x, 0),
reverse=True)
for a in actSort:
print "=" * 50
print "ACTION CLASS:"
print a
print "APPEARS", quickClassCounts[a], "TIMES IN TESTS"
print "APPEARS", quickAnalysisRawCounts[
a], "TIMES IN REDUCED TESTS"
print "APPEARS IN", quickAnalysisCounts[
a], "REDUCED TESTS (" + str(
round((quickAnalysisCounts[a] /
(quickAnalysisTotal * 1.0)) * 100, 2)) + "%)"
#baselineRate = quickClassCounts[a]/(totalTaken*1.0)
#reducedRate = quickAnalysisRawCounts[a]/(quickAnalysisTotal*1.0)
#if reducedRate > 0.0:
# print "RATIO:",(baselineRate/reducedRate)
#else:
# print "RATIO: INFINITE"
print "*" * 70
print "DETAILED BRANCH ANALYSIS"
branchCoverageCountSort = sorted(branchCoverageCount.keys(),
key=lambda x: branchCoverageCount[x])
for b in branchCoverageCountSort:
print "=" * 50
print "BRANCH:", b
baselineRate = branchCoverageCount[b] / (ntests * 1.0)
print "IN", str(round(baselineRate * 100,
2)) + "% OF TESTS (" + str(
branchCoverageCount[b]) + " TESTS)"
reducedRate = quickAnalysisReducedB[b] / (quickAnalysisTotal * 1.0)
print "IN", str(round(reducedRate * 100, 2)) + "% OF REDUCED TESTS"
if reducedRate > 0.0:
print "RATIO:", (baselineRate / reducedRate)
print "REDUCED TEST ACTION ANALYSIS:"
sortAs = sorted(quickAnalysisBCounts[b].keys(),
key=lambda x: quickAnalysisBCounts[b][x],
reverse=True)
print branchCoverageCount[b], "TESTS"
for a in sortAs:
print a, str(
round(
quickAnalysisBCounts[b][a] /
(branchCoverageCount[b] * 1.0) * 100, 2)) + "%"
print "*" * 70
print "DETAILED STATEMENT ANALYSIS"
statementCoverageCountSort = sorted(
statementCoverageCount.keys(),
key=lambda x: statementCoverageCount[x])
for s in statementCoverageCountSort:
print "=" * 50
print "STATEMENT:", s
baselineRate = statementCoverageCount[s] / (ntests * 1.0)
print "IN", str(round(baselineRate * 100, 2)) + "% OF TESTS"
reducedRate = quickAnalysisReducedS[s] / (quickAnalysisTotal * 1.0)
print "IN", str(round(reducedRate * 100, 2)) + "% OF REDUCED TESTS"
if reducedRate > 0.0:
print "RATIO:", (baselineRate / reducedRate)
print "REDUCED TEST ACTION ANALYSIS:"
print statementCoverageCount[s], "TESTS"
sortAs = sorted(quickAnalysisSCounts[s].keys(),
key=lambda x: quickAnalysisSCounts[s][x],
reverse=True)
for a in sortAs:
print a, str(
round(
quickAnalysisSCounts[s][a] /
(statementCoverageCount[s] * 1.0) * 100, 2)) + "%"
print time.time() - start, "TOTAL RUNTIME"
print ntests, "EXECUTED"
print nops, "TOTAL TEST OPERATIONS"
print opTime, "TIME SPENT EXECUTING TEST OPERATIONS"
print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS"
if not config.ignoreProps:
print checkTime, "TIME SPENT CHECKING PROPERTIES"
print(opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT"
print restartTime, "TIME SPENT RESTARTING"
print reduceTime, "TIME SPENT REDUCING TEST CASES"
if config.multiple:
print failCount, "FAILED"
print repeatCount, "REPEATS OF FAILURES"
print len(failures), "ACTUAL DISTINCT FAILURES"
print
n = 0
for (test, err) in failures:
print "FAILURE", n
sut.prettyPrintTest(test)
n += 1
if err != None:
print "ERROR:", err
print "TRACEBACK:"
traceback.print_tb(err[2])
i = -1
if config.compareFails: # Comparison feature normally not useful, just for researching normalization
for test1 in failures:
i += 1
j = -1
for test2 in failures:
j += 1
if (j > i):
print "COMPARING FAILURE", i, "AND FAILURE", j
for k in xrange(0, max(len(test1), len(test2))):
if k >= len(test1):
print "STEP", k, "-->", test2[k][0]
elif k >= len(test2):
print "STEP", k, test1[k][0], "-->"
elif test1[k] != test2[k]:
print "STEP", k, test1[k][0], "-->", test2[k][
0]
if config.profile:
print "ACTION PROFILE:"
averages = []
for a in profileTime:
if profileCount[a] != 0:
averages.append((a, profileTime[a] / profileCount[a]))
averages = sorted(averages, key=lambda x: x[1])
maxAvg = averages[-1][1]
minAvg = averages[0][1]
sumAvg = sum(map(lambda x: x[1], averages))
for (a, t) in averages:
print a, profileCount[a], t, round(t / maxAvg, 2), round(
t / minAvg, 2), round(t / sumAvg, 2)
if config.localize and failCount > 0:
scoresS = {}
scoresB = {}
for s in sut.allStatements():
if s not in localizeSPass:
localizeSPass[s] = 0
if s not in localizeSFail:
localizeSFail[s] = 0
denom = math.sqrt(failCount *
(localizeSFail[s] + localizeSPass[s]))
if denom == 0.0:
continue
scoresS[s] = localizeSFail[s] / denom
for b in sut.allBranches():
if b not in localizeBPass:
localizeBPass[b] = 0
if b not in localizeBFail:
localizeBFail[b] = 0
denom = math.sqrt(failCount *
(localizeBFail[b] + localizeBPass[b]))
if denom == 0.0:
continue
scoresB[b] = localizeBFail[b] / denom
sortedS = sorted(scoresS.keys(), key=lambda x: scoresS[x])
sortedB = sorted(scoresB.keys(), key=lambda x: scoresB[x])
print "FAULT LOCALIZATION RESULTS:"
for s in sortedS:
if scoresS[s] > 0.0:
print s, scoresS[s]
for b in sortedB:
if scoresB[b] > 0.0:
print b, scoresB[b]
if not config.nocover:
print len(sut.allBranches()), "BRANCHES COVERED"
print len(sut.allStatements()), "STATEMENTS COVERED"
def main():
global config, R, sut, fails, actCount
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
fails = 0
actCount = 0
if config.faults:
test_file = open("failure1.test", "w")
start = time.time()
elapsed = time.time() - start
for i in xrange(0, config.depth):
sut.restart()
for j in xrange(0, config.width):
act = None
act = sut.randomEnabled(R)
actCount += 1
if act == None:
print "No enabled actions"
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
ok = sut.safely(act)
propok = sut.check()
if ((not ok) or (not propok)):
fails += 1
if config.faults == 1:
test_file.write(str(sut.failure()) + "\n")
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
sut.restart()
if config.faults:
test_file.close()
if config.coverage:
sut.restart()
sut.report("coverage.out")
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", fails, "times"
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
global failCount,t,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
if config.gendepth != None:
failCloud = {}
allClouds = {}
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
tacts = t.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
checkResult = True
if config.total:
fulltest = open("fulltest.txt",'w')
if config.verbose:
print "ABOUT TO START TESTING"
sys.stdout.flush()
while (config.maxtests == -1) or (ntests < config.maxtests):
if config.verbose:
print "STARTING TEST",ntests
sys.stdout.flush()
ntests += 1
startRestart = time.time()
t.restart()
restartTime += time.time() - startRestart
test = []
if config.total:
fulltest.write("<<RESTART>>\n")
if config.replayable:
currtest = open("currtest.txt",'w')
for s in xrange(0,config.depth):
if config.verbose:
print "GENERATING STEP",s
startGuard = time.time()
acts = tacts
while True:
elapsed = time.time() - start
if elapsed > config.timeout:
break
tryStutter = (a != None)
if tryStutter:
if (config.stutter == None) and (not config.greedyStutter):
tryStutter = False
if tryStutter:
if (config.stutter == None) or (R.random() > config.stutter):
tryStutter = False
if (config.greedyStutter) and sawNew:
print "TRYING TO STUTTER DUE TO COVERAGE GAIN"
tryStutter = True
if not tryStutter:
if len(acts) == 1:
p = 0
else:
p = R.randint(0,len(acts)-1)
a = acts[p]
if a[1]():
break
else:
a = None
acts = acts[:p] + acts[p+1:]
guardTime += time.time()-startGuard
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
if tryStutter:
print "STUTTERING WITH",a[0]
test.append(a)
nops += 1
if config.replayable:
currtest.write(a[0] + "\n")
currtest.flush()
if config.total:
fulltest.write(a[0] + "\n")
fulltest.flush()
if config.verbose:
print "ACTION:",t.prettyName(a[0])
startOp = time.time()
stepOk = t.safely(a)
opTime += (time.time()-startOp)
if tryStutter:
print "DONE STUTTERING"
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
startCheck = time.time()
if not config.ignoreprops:
checkResult = t.check()
checkTime += time.time()-startCheck
if not checkResult:
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if config.running:
if t.newBranches() != set([]):
print "ACTION:",a[0],tryStutter
for b in t.newBranches():
print elapsed,len(t.allBranches()),"New branch",b
sawNew = True
else:
sawNew = False
if t.newStatements() != set([]):
print "ACTION:",a[0],tryStutter
for s in t.newStatements():
print elapsed,len(t.allStatements()),"New statement",s
sawNew = True
else:
sawNew = False
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
if config.replayable:
currtest.close()
if config.quickTests:
if (t.newCurrBranches() != set([])) or (t.newCurrStatements() != set([])):
handle_failure(test, "NEW COVERAGE", False, newCov=True)
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if config.total:
fulltest.close()
if not config.nocover:
t.restart()
print t.report(config.coverfile),"PERCENT COVERED"
if config.internal:
t.internalReport()
if config.html:
t.htmlReport(config.html)
print time.time()-start, "TOTAL RUNTIME"
print ntests, "EXECUTED"
print nops, "TOTAL TEST OPERATIONS"
print opTime, "TIME SPENT EXECUTING TEST OPERATIONS"
print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS"
if not config.ignoreprops:
print checkTime, "TIME SPENT CHECKING PROPERTIES"
print (opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT"
print restartTime, "TIME SPENT RESTARTING"
print reduceTime, "TIME SPENT REDUCING TEST CASES"
if config.multiple:
print failCount,"FAILED"
print repeatCount,"REPEATS OF FAILURES"
print len(failures),"ACTUAL DISTINCT FAILURES"
print
n = 0
for (test, err) in failures:
print "FAILURE",n
t.prettyPrintTest(test)
n += 1
if err != None:
print "ERROR:", err
print "TRACEBACK:"
traceback.print_tb(err[2])
i = -1
if False: # Comparison feature normally not useful, just for researching normalization
for test1 in failures:
i += 1
j = -1
for test2 in failures:
j += 1
if (j > i):
print "COMPARING FAILURE",i,"AND FAILURE",j
for k in xrange(0,max(len(test1),len(test2))):
if k >= len(test1):
print "STEP",k,"-->",test2[k][0]
elif k >= len(test2):
print "STEP",k,test1[k][0],"-->"
elif test1[k] != test2[k]:
print "STEP",k,test1[k][0],"-->",test2[k][0]
if not config.nocover:
print len(t.allBranches()),"BRANCHES COVERED"
print len(t.allStatements()),"STATEMENTS COVERED"
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random variation of beam search using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
failCount = 0
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
tacts = t.actions()
ntests = 0
while (config.maxtests == -1) or (ntests < config.maxtests):
ntests += 1
t.restart()
test = []
for s in xrange(0, config.depth):
count = 0
newCover = False
acts = tacts
old = t.state()
while (count <= config.width):
count += 1
while True:
p = R.randint(0, len(acts) - 1)
a = acts[p]
if a[1]():
break
acts = acts[:p] + acts[p + 1:]
test.append(a)
stepOk = t.safely(a)
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
if (not config.ignoreprops) and (not t.check()):
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if t.newBranches() != set([]):
if config.running:
print "ACTION:", a[0]
for b in t.newBranches():
print elapsed, len(
t.allBranches()), "New branch", b
break # Continue with this choice if new branch exposed
if elapsed > config.timeout:
break
test = test[:-1]
t.backtrack(old)
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
test)
break
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if not config.nocover:
print t.report(config.coverfile), "PERCENT COVERED"
if config.html:
t.htmlReport(config.html)
print ntests, "EXECUTED"
if config.multiple:
print failCount, "FAILED"
if not config.nocover:
print len(t.allBranches()), "BRANCHES COVERED"
print len(t.allStatements()), "STATEMENTS COVERED"
savedTest = None
failureCount = 0
actioncount = 0
testsCovered = []
option = parser.parse_args() # a object named pointer is made for this
SEED = option.SEED
TIMEOUT = option.TIMEOUT
DEPTH = option.DEPTH
WIDTH = option.WIDTH
FAULT_CHECK = option.FAULT_CHECK
COVERAGE_REPORT = option.COVERAGE_REPORT
DETAIL_OF_RUNNING = option.DETAIL_OF_RUNNING
#INITIAL_POP = option.INITIAL_POP
rgen = random.Random()
rgen.seed(SEED)
sut = sut.sut() #this will reset the system state
start = time.time() #this will start the system time
if len(sut.newStatements()) > 0:
savedTest = sut.state()
storedTest = True
print "FOUND NEW STATEMENTS",sut.newStatements()
while time.time()-start < TIMEOUT:
for f in xrange(0,DEPTH):
action = sut.randomEnabled(rgen)
for f in xrange(0,WIDTH):
action = sut.randomEnabled(rgen)
def main():
if "--help" in sys.argv:
print("Usage: tstl_replay <test file> [--noCheck] [--logging loglevel] [--verbose] [--showActions] [--coverage] [--internal] [--html directory] [--delay secs] [--trace] [--afl] [--aflswarm]")
print("Options:")
print("--noCheck: do not run property checks")
print("--logging: set the logging level for the test")
print("--verbose: run with verbose action output")
print("--hideOpaque: hide opaque values in verbose actions")
print("--showActions: show all actions")
print("--coverage: report code coverage")
print("--internal: report detailed code coverage information")
print("--html: produce HTML report on coverage")
print("--delay: delay to inject between steps")
print("--trace: trace lines executed (does not work with SUTs compiled with coverage)")
print("--afl: test is in afl format")
print("--aflswarm: test is in afl swarm format")
sys.exit(0)
sut = SUT.sut()
if not (("--coverage" in sys.argv) or ("--internal" in sys.argv)):
try:
sut.stopCoverage()
except:
pass
if ("--trace" in sys.argv):
goodToTrace = False
try:
sut.stopCoverage()
except:
goodToTrace = True
if not goodToTrace:
print("CANNOT TRACE WHEN SUT IS COMPILED WITH COVERAGE. REBUILD WITH --noCover")
sys.exit(1)
rout = open("replay.out",'w')
file = sys.argv[1]
nocheck = "--noCheck" in sys.argv
verbose = "--verbose" in sys.argv
logLevel = None
if "--logging" in sys.argv:
lastWasLogging = False
for l in sys.argv:
if lastWasLogging:
logLevel = int(l)
if l == "--logging":
lastWasLogging = True
else:
lastWasLogging = False
delay = None
if "--delay" in sys.argv:
lastWasDelay = False
for l in sys.argv:
if lastWasDelay:
delay = float(l)
if l == "--delay":
lastWasDelay = True
else:
lastWasDelay = False
htmlOut = None
lastWasHtml = False
for f in sys.argv[1:]:
if lastWasHtml:
htmlOut = f
lastWasHtml = False
elif f == "--html":
lastWasHtml = True
else:
lastWasHtml = False
sut.restart()
if logLevel != None:
sut.setLog(logLevel)
i = 0
if verbose:
sut.verbose(True)
if "--hideOpaque" in sys.argv:
sut.verboseOpaque(False)
if not "--afl" in sys.argv:
with open(file,'r') as f:
theTest = f.readlines()
else:
readTest = sut.loadTest(file,afl=True,swarm=("--aflswarm" in sys.argv))
theTest = map(lambda x:x[0]+"\n",readTest)
for l in theTest:
name = l[:-1]
if name == "<<RESTART>>":
if "--showActions" in sys.argv:
print("<<RESTART>>")
#print "RESTART"
rout.write("<<RESTART>>\n")
rout.flush()
sut.restart()
else:
if verbose:
print("STEP #"+str(i)+":", end=' ')
rout.write(l)
rout.flush()
action = sut.playable(name)
if "--showActions" in sys.argv:
print(sut.prettyName(action[0]))
if action[1](): # check the guard
if "--trace" in sys.argv:
sys.settrace(trace_calls)
stepOk = sut.safely(action)
if "--trace" in sys.argv:
sys.settrace(None)
if not stepOk:
print("FAILED STEP")
print(sut.failure())
traceback.print_tb(sut.failure()[2],file=sys.stdout)
if "--internal" in sys.argv:
sut.internalReport()
if "--coverage" in sys.argv:
print(sut.report("coverage.out"),"PERCENT COVERED")
if htmlOut != None:
sut.htmlReport(htmlOut)
sys.exit(255)
if not nocheck:
checkResult = sut.check()
if not checkResult:
print("FAILED PROPERTY")
print(sut.failure())
traceback.print_tb(sut.failure()[2],file=sys.stdout)
if "--internal" in sys.argv:
sut.internalReport()
if "--coverage" in sys.argv:
print(sut.report("coverage.out"),"PERCENT COVERED")
if htmlOut != None:
sut.htmlReport(htmlOut)
sys.exit(255)
if delay != None:
time.sleep(delay)
i += 1
rout.write("TEST REPLAYED SUCCESSFULLY\n")
rout.close()
if "--internal" in sys.argv:
sut.internalReport()
if "--coverage" in sys.argv:
print(sut.report("coverage.out"),"PERCENT COVERED")
if htmlOut != None:
sut.htmlReport(htmlOut)
sys.exit(0)
def main():
global failCount,t,config,quickCount,repeatCount,failures,R,opTime,checkTime,guardTime,restartTime,nops,lastact,parts , pt , ptlast
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
failures = []
t = SUT.sut()
tacts = t.actions()
a = None
sawNew = False
nops = 0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
test = []
while time.time() - start < config.timeout:
t.restart()
lastact = 0
pt = 0
ptlast = 0
for s in xrange(0,config.depth):
acts = tacts
while True:
if len(acts) > 10 :
partsize = len(acts)/3
pt = R.randint(0,2)
while pt == ptlast:
pt = R.randint(0,2)
else:
pt = 0
partsize = len(acts)
p = R.randint(pt*partsize,(pt+1)*partsize-1)
a = acts[p]
lastact = p
ptlast = pt
if a[1]():
break
else:
a = None
acts = acts[:p] + acts[p+1:]
test.append(a)
nops += 1
stepOk = t.safely(a)
if not stepOk:
print "STOPPING TESTING DUE TO FAILED TEST"
failCount+=1
if config.faults == 1:
f = "failure" + str(failCount) + ".test"
t.saveTest(t.test(),f)
break
if config.running:
if t.newBranches() != set([]):
print "ACTION:",a[0]
for b in t.newBranches():
print elapsed,len(t.allBranches()),"New branch",b
sawNew = True
else:
sawNew = False
if t.newStatements() != set([]):
print "ACTION:",a[0]
for s in t.newStatements():
print elapsed,len(t.allStatements()),"New statement",s
sawNew = True
else:
sawNew = False
print time.time()-start, "TOTAL RUNTIME"
print nops, "TOTAL TEST OPERATIONS"
if config.coverage:
t.restart()
print t.report("coverage.out"),"PERCENT COVERED"
t.internalReport()
def main():
if "--help" in sys.argv:
print "Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover] [--verbose] [--running]"
print "Options:"
print " --noCheck: do not run property checks"
print " --html: output an HTML coverage report to the chosen directory"
print " --noCover: do not compute code coverage"
print " --verbose: make actions verbose"
print " --running: give a running report on code coverage"
print " --keepGoing: don't stop on failed test"
sys.exit(0)
sut = sut.sut()
nocover = False
verbose = False
running = False
keepGoing = False
ignoreProps = False
lastWasHtml = False
files = []
htmlOut = None
for f in sys.argv[1:]:
if lastWasHtml:
htmlOut = f
lastWasHtml = False
elif "--" not in f:
files.append(f)
elif f == "--html":
lastWasHtml = True
if "--noCover" in sys.argv:
nocover = True
if "--verbose" in sys.argv:
verbose = True
if "--noCheck" in sys.argv:
ignoreProps = True
if "--running" in sys.argv:
running = True
if "--keepGoing" in sys.argv:
keepGoing = True
if verbose:
sut.verbose(True)
anyFailed = False
stime = time.time()
for f in files:
print "RUNNING TEST", f
t = sut.loadTest(f)
ok = False
try:
ok = sut.replay(t, checkProp=(not ignoreProps))
except Exception as e:
print "EXCEPTION RAISED:", e
if running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print "New branch:", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print "New statement:", s
if not ok:
print "TEST", f, "FAILED:"
print sut.failure()
anyFailed = True
if not keepGoing:
sys.exit(255)
print time.time() - stime, "ELAPSED"
if not nocover:
print "STATEMENTS:", len(sut.allStatements()), "BRANCHES:", len(
sut.allBranches())
if not nocover:
sut.internalReport()
print sut.report("coverage.out"), "PERCENT COVERED"
if htmlOut != None:
sut.htmlReport(htmlOut)
if not anyFailed:
print "ALL TESTS SUCCESSFUL"
def main():
if "--help" in sys.argv:
print(
"Usage: tstl_regress <test files> [--noCheck] [--html dir] [--noCover]",
"[--verbose] [--running] [--afl] [--aflswarm]")
print("Options:")
print(
" --glob: <test files> are glob expressions (used when too many files to use shell expansion)"
)
print(" --noCheck: do not run property checks")
print(
" --html: output an HTML coverage report to the chosen directory"
)
print(" --noCover: do not compute code coverage")
print(" --verbose: make actions verbose")
print(" --running: give a running report on code coverage")
print(" --keepGoing: don't stop on failed test")
print(" --afl: tests are in afl format")
print(" --aflswarm: tests are in afl swarm format")
sys.exit(0)
sut = SUT.sut()
nocover = False
verbose = False
running = False
keepGoing = False
afl = False
aflswarm = False
ignoreProps = False
lastWasHtml = False
files = []
htmlOut = None
for f in sys.argv[1:]:
if lastWasHtml:
htmlOut = f
lastWasHtml = False
elif "--" not in f:
files.append(f)
elif f == "--html":
lastWasHtml = True
else:
lastWasHtml = False
if "--noCover" in sys.argv:
nocover = True
try:
sut.stopCoverage()
except BaseException:
pass
if "--glob" in sys.argv:
newFiles = []
for f in files:
newFiles.extend(glob.glob(f))
files = newFiles
if "--verbose" in sys.argv:
verbose = True
if "--noCheck" in sys.argv:
ignoreProps = True
if "--running" in sys.argv:
running = True
if "--keepGoing" in sys.argv:
keepGoing = True
if "--afl" in sys.argv:
afl = True
if "--aflswarm" in sys.argv:
aflswarm = True
if verbose:
sut.verbose(True)
failedTests = []
anyFailed = False
noNewCover = []
newCover = []
totalTests = 0
invalidTests = []
stime = time.time()
for f in files:
totalTests += 1
print("RUNNING TEST", f)
try:
t = sut.loadTest(f, afl=afl, swarm=aflswarm)
except KeyError:
print("INVALID TEST, SKIPPING...")
invalidTests.append(f)
continue
ok = False
try:
ok = sut.replay(t, checkProp=(not ignoreProps))
except Exception as e:
print("EXCEPTION RAISED:", e)
if not nocover:
if (len(sut.newCurrBranches()) == 0) and (len(
sut.newCurrStatements()) == 0):
noNewCover.append(f)
else:
newCover.append(f)
if running:
if sut.newCurrBranches() != set([]):
for b in sut.newCurrBranches():
print("New branch:", b)
if sut.newCurrStatements() != set([]):
for s in sut.newCurrStatements():
print("New statement:", s)
if not ok:
print("TEST", f, "FAILED:")
print(sut.failure())
failedTests.append(f)
anyFailed = True
if not keepGoing:
sys.exit(255)
print(time.time() - stime, "ELAPSED")
if not nocover:
print("STATEMENTS:", len(sut.allStatements()), "BRANCHES:",
len(sut.allBranches()))
if f not in noNewCover:
print("NEW STATEMENTS:", len(sut.newCurrStatements()),
"BRANCHES:", len(sut.newCurrBranches()))
sys.stdout.flush()
if not nocover:
sut.internalReport()
print(sut.report("coverage.out"), "PERCENT COVERED")
if htmlOut is not None:
sut.htmlReport(htmlOut)
if (not nocover) and (len(noNewCover) > 0):
for f in noNewCover:
print("TEST", f, "REDUNDANT WITH RESPECT TO COVERAGE")
print()
print(len(newCover), "TESTS NEEDED FOR FULL COVERAGE:",
", ".join(newCover))
print()
print("EXECUTED", totalTests, "TESTS")
if len(invalidTests) > 0:
print(len(invalidTests), "INVALID TESTS:")
for f in invalidTests:
print(f, end=' ')
print()
if not anyFailed:
print("ALL TESTS SUCCESSFUL")
else:
print(len(failedTests), "FAILED TESTS:")
for f in failedTests:
print(f, end=' ')
print()
def main():
if "--help" in sys.argv:
print(
"Usage: tstl_afl [--noCheck] [--swarm] [--verbose] [--showActions] [--noSave] [--alwaysSave]"
)
print("Options:")
print(" --noCheck: do not run property checks")
print(
" --swarm use first four bytes to determine a swarm configuration"
)
print(" --verbose: make actions verbose")
print(" --showActions: show actions in test")
print(
" --noSave: don't save failing tests as standard TSTL tests")
print(" --alwaysSave: save even non-failing tests")
sys.exit(0)
sut = SUT.sut()
try:
sut.stopCoverage()
except:
pass
sut.restart()
if "--swarm" in sys.argv:
swarm = True
R = random.Random()
else:
swarm = False
showActions = "--showActions" in sys.argv
if "--verbose" in sys.argv:
sut.verbose(True)
noSave = "--noSave" in sys.argv
alwaysSave = "--alwaysSave" in sys.argv
noCheck = "--noCheck" in sys.argv
afl.init()
bytesin = sys.stdin.read()
if len(bytesin) < 4:
os._exit(0)
if swarm:
R.seed(struct.unpack("<L", bytesin[0:4])[0])
sut.standardSwarm(R)
bytesin = bytesin[4:]
alen = len(sut.actions())
test = sut.bytesToTest(bytesin)
for a in test:
if a[1]():
if showActions:
print(sut.prettyName(a[0]))
ok = sut.safely(a)
if (not noSave) and not ok:
i = 0
saveFile = "aflfail." + str(
os.getpid()) + "." + str(i) + ".test"
while os.path.exists(saveFile):
i += 1
saveFile = "aflfail." + str(
os.getpid()) + "." + str(i) + ".test"
sut.saveTest(sut.test(), saveFile)
assert (ok)
if not noCheck:
checkResult = sut.check()
if (not noSave) and not checkResult:
i = 0
saveFile = "aflfail." + str(
os.getpid()) + "." + str(i) + ".test"
while os.path.exists(saveFile):
i += 1
saveFile = "aflfail." + str(
os.getpid()) + "." + str(i) + ".test"
sut.saveTest(sut.test(), saveFile)
assert (checkResult)
if alwaysSave:
i = 0
saveFile = "afltest." + str(os.getpid()) + "." + str(i) + ".test"
while os.path.exists(saveFile):
i += 1
saveFile = "afltest." + str(os.getpid()) + "." + str(i) + ".test"
sut.saveTest(sut.test(), saveFile)
os._exit(0)
def __init__(self): #initiialize lists self.errorSeqs = [] self.nonErrorSeqs = [] self.sut = sut.sut()
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Generating swarm probabilties using config={}'.format(config)))
sut = SUT.sut()
verbose = False
try:
sut.stopCoverage()
except BaseException:
pass
numTests = 0
mode = "CONFIG"
entryClasses = []
entryBranches = []
entryStatements = []
count = {}
branches = set([])
statements = set([])
hits = {}
print("READING DATA...")
for line in open(config.swarmData):
if "::::" in line:
if mode == "CONFIG":
ac = line.split("::::")[1][:-1]
entryClasses.append(ac)
if ac in count:
count[ac] += 1
else:
count[ac] = 1
elif mode == "BRANCHES":
b = eval(line.split("::::")[1][:-1])
entryBranches.append(b)
if b in count:
count[b] += 1
else:
count[b] = 1
hits[b] = []
branches.add(b)
elif mode == "STATEMENTS":
s = eval(line.split("::::")[1][:-1])
entryStatements.append(s)
if s in count:
count[s] += 1
else:
count[s] = 1
hits[s] = []
statements.add(s)
elif "CONFIG:" in line:
if entryClasses != []:
numTests += 1
for b in entryBranches:
hits[b].append(entryClasses)
for s in entryStatements:
hits[s].append(entryClasses)
entryClasses = []
entryBranches = []
entryStatements = []
mode = "CONFIG"
elif "BRANCHES:" in line:
mode = "BRANCHES"
elif "STATEMENTS:" in line:
mode = "STATEMENTS"
numTests += 1
for b in entryBranches:
hits[b].append(entryClasses)
for s in entryStatements:
hits[s].append(entryClasses)
print("DONE")
eqClasses = {}
targets = branches.union(statements)
print("ANALYZING", len(targets), "TARGETS")
analyzed = 0
for target in targets:
if float(count[target]) / numTests >= config.cutoff:
analyzed += 1
continue
triggers = []
suppressors = []
hitT = hits[target]
if len(hitT) < config.minHits:
analyzed += 1
continue
for ac in sut.actionClasses():
rate = float(count[ac]) / numTests
if rate == 1.0:
# Neither a suppressor nor trigger if present in every test!
continue
chits = 0
for t in hitT:
if ac in t:
chits += 1
low, high = proportion_confint(chits,
len(hitT),
method='wilson',
alpha=1 - config.confidence)
if low > rate:
triggers.append(ac)
if high < rate:
suppressors.append(ac)
signature = (tuple(sorted(triggers)), tuple(sorted(suppressors)))
data = (target, count[target])
analyzed += 1
if signature not in eqClasses:
if verbose:
print("NEW EQUIVALENCE CLASS AFTER ANALYZING", analyzed,
"TARGETS")
print("NOW", len(eqClasses), "EQUIVALENCE CLASSES")
print("TRIGGERS:", triggers)
print("SUPPRESSORS:", suppressors)
print("TARGET:", target)
print("FREQUENCY:", count[target])
eqClasses[signature] = (triggers, suppressors, [data])
else:
eqClasses[signature][2].append(data)
pcount = 0
with open(config.prefix + ".classes", 'w') as cfile:
for c in sorted(
eqClasses.keys(),
key=lambda x: min(map(lambda y: y[1], eqClasses[x][2]))):
triggers, suppressors, targets = eqClasses[c]
if (triggers == []) and (suppressors == []):
continue # Ignore the no-data class
print("=" * 80)
print("# TARGETS:", len(targets))
print("MINIMUM FREQUENCY:", min(map(lambda x: x[1], targets)))
print("TRIGGERS:", triggers)
print("SUPPRESSORS:", suppressors)
cfile.write("TRIGGERS:\n")
cp = {}
for t in triggers:
cfile.write(t + "\n")
cp[t] = 1.0
cfile.write("SUPPRESSORS:\n")
for s in suppressors:
cfile.write(s + "\n")
cp[s] = 0.0
cfile.write("TARGETS:\n")
for t in targets:
cfile.write(repr(t[0]) + " :::: " + str(t[1]) + "\n")
cfile.write("FILE: " + config.prefix + "." + str(pcount) +
".prob\n")
sut.writeProbFile(config.prefix + "." + str(pcount) + ".prob", cp)
pcount += 1
def main():
global failCount, t, config, reduceTime, quickCount, repeatCount, failures, cloudFailures, R, opTime, checkTime, guardTime, restartTime, nops, ntests
# region define the config object by parsing the command-line arguments
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
# endregion
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
if config.gendepth != None:
failCloud = {}
allClouds = {}
# create an instance of object SUT
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
tacts = t.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
checkResult = True
if config.total:
fulltest = open("fulltest.txt", 'w')
if config.verbose:
print "ABOUT TO START TESTING"
sys.stdout.flush()
while (config.maxtests == -1) or (ntests < config.maxtests):
if config.verbose:
print "STARTING TEST", ntests
sys.stdout.flush()
ntests += 1
startRestart = time.time()
t.restart()
restartTime += time.time() - startRestart
test = []
if config.total:
fulltest.write("<<RESTART>>\n")
if config.replayable:
currtest = open("currtest.txt", 'w')
for s in xrange(0, config.depth):
if config.verbose:
print "GENERATING STEP", s
startGuard = time.time()
acts = tacts
while True:
elapsed = time.time() - start
if elapsed > config.timeout:
break
tryStutter = (a != None)
if tryStutter:
if (config.stutter == None) and (not config.greedyStutter):
tryStutter = False
if tryStutter:
if (config.stutter
== None) or (R.random() > config.stutter):
tryStutter = False
if (config.greedyStutter) and sawNew:
print "TRYING TO STUTTER DUE TO COVERAGE GAIN"
tryStutter = True
if not tryStutter:
if len(acts) == 1:
p = 0
else:
p = R.randint(0, len(acts) - 1)
a = acts[p]
if a[1]():
break
else:
a = None
acts = acts[:p] + acts[p + 1:]
guardTime += time.time() - startGuard
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
test)
break
if tryStutter:
print "STUTTERING WITH", a[0]
test.append(a)
nops += 1
if config.replayable:
currtest.write(a[0] + "\n")
currtest.flush()
if config.total:
fulltest.write(a[0] + "\n")
fulltest.flush()
if config.verbose:
print "ACTION:", t.prettyName(a[0])
startOp = time.time()
stepOk = t.safely(a)
if t.warning() != None:
print "SUT WARNING:", t.warning()
opTime += (time.time() - startOp)
if tryStutter:
print "DONE STUTTERING"
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
startCheck = time.time()
if not config.ignoreprops:
checkResult = t.check()
checkTime += time.time() - startCheck
if not checkResult:
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if config.running:
if t.newBranches() != set([]):
print "ACTION:", a[0], tryStutter
for b in t.newBranches():
print elapsed, len(t.allBranches()), "New branch", b
sawNew = True
else:
sawNew = False
if t.newStatements() != set([]):
print "ACTION:", a[0], tryStutter
for s in t.newStatements():
print elapsed, len(
t.allStatements()), "New statement", s
sawNew = True
else:
sawNew = False
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
test)
break
if config.replayable:
currtest.close()
if config.quickTests:
if (t.newCurrBranches() != set([])) or (t.newCurrStatements() !=
set([])):
handle_failure(test, "NEW COVERAGE", False, newCov=True)
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if config.total:
fulltest.close()
if not config.nocover:
t.restart()
print t.report(config.coverfile), "PERCENT COVERED"
if config.internal:
t.internalReport()
if config.html:
t.htmlReport(config.html)
print time.time() - start, "TOTAL RUNTIME"
print ntests, "EXECUTED"
print nops, "TOTAL TEST OPERATIONS"
print opTime, "TIME SPENT EXECUTING TEST OPERATIONS"
print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS"
if not config.ignoreprops:
print checkTime, "TIME SPENT CHECKING PROPERTIES"
print(opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT"
print restartTime, "TIME SPENT RESTARTING"
print reduceTime, "TIME SPENT REDUCING TEST CASES"
if config.multiple:
print failCount, "FAILED"
print repeatCount, "REPEATS OF FAILURES"
print len(failures), "ACTUAL DISTINCT FAILURES"
print
n = 0
for (test, err) in failures:
print "FAILURE", n
t.prettyPrintTest(test)
n += 1
if err != None:
print "ERROR:", err
print "TRACEBACK:"
traceback.print_tb(err[2])
i = -1
if False: # Comparison feature normally not useful, just for researching normalization
for test1 in failures:
i += 1
j = -1
for test2 in failures:
j += 1
if (j > i):
print "COMPARING FAILURE", i, "AND FAILURE", j
for k in xrange(0, max(len(test1), len(test2))):
if k >= len(test1):
print "STEP", k, "-->", test2[k][0]
elif k >= len(test2):
print "STEP", k, test1[k][0], "-->"
elif test1[k] != test2[k]:
print "STEP", k, test1[k][0], "-->", test2[k][
0]
if not config.nocover:
print len(t.allBranches()), "BRANCHES COVERED"
print len(t.allStatements()), "STATEMENTS COVERED"
def main():
global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
if config.faults:
test_file = open("failure1.test", "w")
start = time.time()
elapsed = time.time() - start
for i in xrange(0, config.depth):
sut.restart()
for j in xrange(0, config.width):
if not randomAct():
print "Function called randomAct() occurs error"
break
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
if config.faults:
test_file.close()
if config.coverage:
sut.restart()
sut.report("coverage.out")
#printCoverage()
sut.internalReport()
#print "Covered", len(sut.allBranches()), "branches"
#print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
mytester = simplerRandomTester(sys.argv[1:])
mysut = SUT.sut()
mysut.testWith(mytester)
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Generating swarm probabilties using config={}'.format(config)))
sut = SUT.sut()
try:
sut.stopCoverage()
except BaseException:
pass
target = eval(config.target)
tests = []
mode = "CONFIG"
entryClasses = []
entryBranches = []
entryStatements = []
for line in open(config.swarmData):
if "::::" in line:
if mode == "CONFIG":
ac = line.split("::::")[1][:-1]
entryClasses.append(ac)
elif mode == "BRANCHES":
b = eval(line.split("::::")[1][:-1])
entryBranches.append(b)
elif mode == "STATEMENTS":
s = eval(line.split("::::")[1][:-1])
entryStatements.append(s)
elif "CONFIG:" in line:
if entryClasses != []:
tests.append((entryClasses, entryBranches, entryStatements))
entryClasses = []
entryBranches = []
entryStatements = []
mode = "CONFIG"
elif "BRANCHES:" in line:
mode = "BRANCHES"
elif "STATEMENTS:" in line:
mode = "STATEMENTS"
count = {}
for t in tests:
_, branches, statements = t
for branch in branches:
if branch not in count:
count[branch] = 1
else:
count[branch] += 1
for stmt in statements:
if stmt not in count:
count[stmt] = 1
else:
count[stmt] += 1
rates = {}
for ac in sut.actionClasses():
acCount = 0
for t in tests:
if ac in t[0]:
acCount += 1
rates[ac] = acCount / float(len(tests))
triggers = []
suppressors = []
hitT = []
for t in tests:
if (target in t[1]) or (target in t[2]):
hitT.append(t)
for ac in sut.actionClasses():
if rates[ac] == 1.0:
# Neither a suppressor nor trigger if present in every test!
continue
rate = rates[ac]
hits = 0
for t in hitT:
if ac in t[0]:
hits += 1
low, high = proportion_confint(hits,
len(hitT),
method='wilson',
alpha=1 - config.confidence)
if low > rate:
triggers.append(ac)
if high < rate:
suppressors.append(ac)
print("=" * 80)
sratio = float(count[target]) / len(tests)
print(target, count[target], "HITS", sratio, "RATIO")
print("TRIGGERS:")
for tr in triggers:
print(tr, rates[tr])
print()
print("SUPPRESSORS:")
for sp in suppressors:
print(sp, rates[sp])
print()
cp = {}
for ac in sut.actionClasses():
if ac in triggers:
cp[ac] = 1.0
elif ac in suppressors:
cp[ac] = 0.0
else:
if config.mode == "halfswarm":
cp[ac] = 0.5
elif config.mode == "triggers-only":
cp[ac] = 0.0
else:
cp[ac] = 1.0
sut.writeProbFile(config.probFile, cp)
import os
import sys
current_working_dir = os.getcwd()
sys.path.append(current_working_dir)
from collections import namedtuple
import sut as SUT
import random
import time
import traceback
import argparse
sut = SUT.sut()
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('timeout', type=int, default=60, help='Timeout in seconds. (60 default)')
parser.add_argument('seed', type=int, default=None, help='Random seed. (default = None)')
parser.add_argument('depth', type=int, default=100, help='Maximum search depth. (100 default)')
parser.add_argument('width', type=int, default=10000, help='Maximum memory. (10000 default)')
parser.add_argument('faults', type=int, default=0, choices=[0, 1], help='Check for faults or not. 1 for check, 0 for do not check (0 default)')
parser.add_argument('coverage', type=int, default=0, choices=[0, 1] ,help='report coverage or not. 1 for report, 0 for do not report(0 default)')
parser.add_argument('running', type=int, default=0, choices=[0, 1], help='Produce running branch coverage report.')
parsed_args = parser.parse_args(sys.argv[1:])
return (parsed_args, parser)
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Calibrating using config={}'.format(config)))
calibFile = open(".tstl_calibration", 'w')
sut = SUT.sut()
R = random.Random()
if config.seed:
R.seed(config.seed)
print("=" * 80)
print("CALIBRATING COST OF COVERAGE...\n")
oldOut = sys.stdout
fnull = open(os.devnull, 'w')
sys.stdout = fnull
sut.stopCoverage()
oldOut.write(
"GENERATING 30 TESTS (OR 30+s of TESTS) WITHOUT COVERAGE...\n")
oldOut.flush()
start = time.time()
noCovCount = 0
for i in range(0, 30):
oldOut.write(str(i) + "...")
oldOut.flush()
(t, ok) = sut.makeTest(100, R, stopFail=False)
noCovCount += 1
if (time.time() - start) > 30:
oldOut.write("TIMEOUT AT " + str(time.time() - start))
break
noCovTime = time.time() - start
sut.startCoverage()
oldOut.write(
"\n\nGENERATING 30 TESTS (OR 30+s of TESTS) WITH COVERAGE...\n")
start = time.time()
covCount = 0
for i in range(0, 30):
oldOut.write(str(i) + "...")
oldOut.flush()
(t, ok) = sut.makeTest(100, R, stopFail=False)
covCount += 1
if (time.time() - start) > 30:
oldOut.write("TIMEOUT AT " + str(time.time() - start))
break
covTime = time.time() - start
sys.stdout = oldOut
print()
print()
sys.stdout.flush()
covR = (covCount * 100) / covTime
noCovR = (noCovCount * 100) / noCovTime
if covR < noCovR:
print("WITH COVERAGE:", covR, "ACTIONS/s")
print("WITHOUT COVERAGE:", noCovR, "ACTIONS/s")
overhead = (noCovR - covR) / noCovR
print("COVERAGE OVERHEAD:", str(round(overhead * 100, 2)) + "%")
else:
print("NO DETECTABLE COVERAGE OVERHEAD")
calibFile.write("COVERAGE OVERHEAD: " + str(overhead))
print("=" * 80)
print("ESTIMATING LINES OF CODE IN ACTIONS...\n")
subprocess.call([
"tstl_rt --timeout 180 --generateLOC .tstl_calibration_loc --noCover"
],
shell=True,
stdout=fnull,
stderr=fnull)
classLOCVals = {}
for c in sut.actionClasses():
classLOCVals[c] = 0.0
totalLOCs = 0.0
num0 = 0.0
with open(".tstl_calibration_loc", 'r') as cf:
for l in cf:
ls = l.split(" %%%% ")
c = ls[0]
loc = float(ls[1])
totalLOCs += loc
classLOCVals[c] = loc
classP = {}
for c in sut.actionClasses():
if classLOCVals[c] == 0.0:
num0 += 1
for c in sut.actionClasses():
if classLOCVals[c] == 0.0:
classP[c] = (0.20 / num0)
else:
classP[c] = (classLOCVals[c] / totalLOCs)
sortLOC = sorted(classLOCVals.keys(),
key=lambda x: classP[x],
reverse=True)
print("HIGHEST LOC-BASED PROBABILITY ACTIONS:")
for c in sortLOC[:3]:
print(" ", c, round(classP[c], 5))
print("\nLOWEST LOC-BASED PROBABILITY ACTIONS:")
for c in sortLOC[-3:]:
print(" ", c, round(classP[c], 5))
calibFile.close()
def main():
if "--help" in sys.argv:
print(
"Usage: tstl_aflcorpus <outputdir> <length> <time> [--noCheck] [--noReduce] [--noCover] [--swarm] [--skipFails]"
)
print("Options:")
print(" --noCheck: do not check properties")
print(" --noReduce: do not reduce inputs by coverage")
print(" --noCover: do not check for new coverage")
print(" --swarm: use swarm format, generate tests using swarm")
print(" --skipFails: just pass over failures, don't try to fix")
sys.exit(0)
sut = SUT.sut()
pid = str(os.getpid())
outputDir = sys.argv[1]
length = int(sys.argv[2])
timeout = int(sys.argv[3])
checkProp = not "--noCheck" in sys.argv
noReduce = "--noReduce" in sys.argv
noCover = "--noCover" in sys.argv
swarm = "--swarm" in sys.argv
skipFails = "--skipFails" in sys.argv
if noCover:
sut.stopCoverage()
R = random.Random()
Rswarm = random.Random()
acts = sut.actions()
i = 0
stime = time.time()
while (time.time() - stime) < timeout:
i += 1
if swarm:
seed = R.randint(0, 2**32)
Rswarm.seed(seed)
sut.standardSwarm(Rswarm)
(t, ok) = sut.makeTest(length, R, stopFail=True, checkProp=checkProp)
if (not noCover) and (len(sut.newCurrBranches()) == 0):
continue
else:
print("INPUT", i, "GENERATED", end=" ")
if not noCover:
print("NEW BRANCHES:", len(sut.newCurrBranches()), end=" ")
type = "branch" + str(len(sut.newCurrBranches()))
if ok: # failing tests won't work with afl
if (not noCover) and (not noReduce):
b = set(sut.currBranches())
s = set(sut.currStatements())
pred = sut.coversAll(s, b, checkProp=True, catchUncaught=False)
r = sut.reduce(t, pred, verbose=False)
else:
r = t
elif skipFails:
print("SKIPPING FAILED TEST...")
continue
else:
type = "nearfail"
print("SAVING ALL BUT LAST STEP OF FAILED TEST")
r = t[:-1]
# always alpha convert to make actions clearer to afl, easier to splice
r = sut.alphaConvert(r)
if ok and (not noReduce):
print("REDUCED LENGTH:", len(r))
sut.prettyPrintTest(r)
print("=" * 80)
if not swarm:
sut.saveTest(r,
outputDir + "/tstl." + type + "." + pid + "." +
str(i) + ".afl",
afl=True)
else:
bytes = sut.testToBytes(r)
with open(
outputDir + "/tstl." + type + "." + pid + "." + str(i) +
".afl", 'wb') as f:
f.write(struct.pack("<L", seed))
f.write(bytes)
def main():
global timeout
if "--help" in sys.argv:
print "Usage: tstl_reduce <test file> <output test file> [--noCheck] [--noReduce] [--noAlpha] [--noNormalized] [--verbose verbosity] [--sandbox] [--quietSandbox] [--timeout secs]"
print "Options:"
print " --noCheck: do not run property checks"
print " --noReduce do not reduce test (useful for normalizing an already reduced test)"
print " --noAlpha do not alpha convert test"
print " --noNormalize after reducing, do not also normalize"
print " --verbose: set verbosity level for reduction/normalization (defaults to silent reduction/normalization)"
print " --sandbox: run tests in a subprocess sandbox, for tests that crash Python interpreter;"
print " due to common difficulties, sandbox is by default very verbose!"
print " WARNING: if not needed, sandbox mode is VERY SLOW"
print " --quietSandbox: run sandbox in a quiet mode"
print " --timeout: if tests are run in a sandbox, consider tests running longer than this to have failed"
sys.exit(0)
sut = SUT.sut()
vLevel = False
if "--verbose" in sys.argv:
lastWasVerbose = False
for l in sys.argv:
if lastWasVerbose:
vLevel = l
if l == "--verbose":
lastWasVerbose = True
if vLevel == "True":
vLevel = True
if vLevel == "False":
vLevel = False
timeout = None
if "--timeout" in sys.argv:
lastWasTimeout = False
for l in sys.argv:
if lastWasTimeout:
timeout = l
if l == "--timeout":
lastWasTimeout = True
if not "--sandbox" in sys.argv:
pred = sut.failsCheck
if "--noCheck" is sys.argv:
pred = sut.fails
else:
pred = sandboxReplay
r = sut.loadTest(sys.argv[1])
print "STARTING WITH TEST OF LENGTH", len(r)
if not ("--noReduce" in sys.argv):
start = time.time()
print "REDUCING..."
r = sut.reduce(r, pred, verbose=vLevel)
print "REDUCED IN", time.time() - start, "SECONDS"
print "NEW LENGTH", len(r)
if not ("--noAlpha" in sys.argv):
print "ALPHA CONVERTING..."
r = sut.alphaConvert(r)
if not ("--noNormalize" in sys.argv):
start = time.time()
print "NORMALIZING..."
r = sut.normalize(r, pred, verbose=vLevel)
print "NORMALIZED IN", time.time() - start, "SECONDS"
print "NEW LENGTH", len(r)
sut.saveTest(r, sys.argv[2])
print "TEST WRITTEN TO", sys.argv[2]
def main():
if "--help" in sys.argv:
print(
"Usage: tstl_standalone <test file> <output Python file> [<sut file>] [--noCheck] [--noRefs] [--regression] [--verbose] [--afl] [--aflswarm]"
)
print(" default for <sut file> is sut.py")
print("Options:")
print(" --noCheck: do not include property checks")
print(" --noRefs: do not include reference actions")
print(
" --regression: produce a regression test that captures values")
print(
" --verbose: produce a verbose test that shows actions taken")
print(" --afl: test is in afl format")
print(" --aflswarm: test is in afl swarm format")
sys.exit(0)
testFile = sys.argv[1]
outFile = sys.argv[2]
if (len(sys.argv) > 3) and (".py" in sys.argv[3]):
sutFile = sys.argv[3]
else:
sutFile = "sut.py"
checkProps = not "--noCheck" in sys.argv
checkRefs = not "--noRefs" in sys.argv
makeRegression = "--regression" in sys.argv
verbose = "--verbose" in sys.argv
afl = "--afl" in sys.argv
aflswarm = "--aflswarm" in sys.argv
t = SUT.sut()
def globalCheck(str):
if " # CHECK POOL INIT" not in str:
return str
newStr = str.replace("(", '("')
newStr = newStr.replace(" != None", '" in globals()')
return newStr.replace("# CHECK POOL INIT", "")
print("STR=", str)
condBegin = str.find("(")
theCheck = str[condBegin + 1:str.find(" != None):")]
print(theCheck)
strNew = str[:condBegin +
1] + '"' + theCheck + '"' + str[str.find(" != None):"):]
print(strNew)
strNew = strNew.replace(" != None", " in globals()")
print(strNew)
strNew = strNew.replace("# CHECK POOL INIT", "")
print(strNew)
return strNew
outf = open(outFile, 'w')
startPrelim = False
startCheck = False
startReload = False
startInit = False
reloadCode = []
initCode = []
outf.write("from __future__ import print_function\n\n")
for l in open(sutFile):
if "END STANDALONE CODE" in l:
startPrelim = False
if "END CHECK CODE" in l:
startCheck = False
if "END RELOAD CODE" in l:
startReload = False
if "END INITIALIZATION CODE" in l:
startInit = False
if startPrelim:
outf.write(l)
if startCheck:
outf.write(
globalCheck(t.prettyName(l.replace("# GLOBAL ", "global "))))
if startReload:
reloadCode.append(l.lstrip())
if startInit:
initCode.append(l.lstrip())
if "BEGIN STANDALONE CODE" in l:
startPrelim = True
if checkProps and ("BEGIN CHECK CODE" in l):
startCheck = True
outf.write("\n\ndef check():\n")
if "BEGIN RELOAD CODE" in l:
startReload = True
if "BEGIN INITIALIZATION CODE" in l:
startInit = True
outf.write("\n\n")
for i in initCode:
outf.write(t.prettyName(i))
outf.write("\n\n")
for l in open(testFile):
if l == "<<RESTART>>":
if makeRegression:
t.restart()
outf.write("\n# RESTART\n\n")
for r in reloadCode:
outf.write(r)
for i in initCode:
outf.write(i)
name = l[:-1]
if t.getPreCode(name) != None:
for p in t.getPreCode(name):
outf.write(t.prettyName(p) + "\n")
if makeRegression:
t.safely(t.playable(name))
if verbose:
outf.write("print ('''" + t.prettyName(name) + "''')\n")
if t.getOkExceptions(name) == "":
outf.write(t.prettyName(name) + "\n")
else:
outf.write("try:\n")
outf.write(" " + t.prettyName(name) + "\n")
outf.write("except (" + t.getOkExceptions(name) + "):\n")
outf.write(" pass\n")
if t.getPropCode(name) != None:
outf.write("assert " + t.prettyName(t.getPropCode(name)) + "\n")
if checkRefs:
if t.getRefCode(name) != None:
for r in t.getRefCode(name):
outf.write("try:\n")
outf.write(" " + t.prettyName(r) + "\n")
outf.write("except:\n")
outf.write(" pass\n")
if checkProps:
outf.write("check()\n")
if makeRegression:
v = t.shallowState()
for (p, vals) in v:
if t.prettyName(p) not in name:
continue
if p in t.opaque():
continue
if t.abstraction(p) != None:
absFun = t.abstraction(p)
else:
absFun = ""
for i in vals:
if vals[i] != None:
pname = t.prettyName(p + "[" + str(i) + "]")
#outf.write("print (repr("+pname+"))\n")
outf.write("assert (repr(" + absFun + "(" + pname +
')) == (' + repr(repr(vals[i])) + '))\n')
outf.write('\n\nprint ("TEST COMPLETED SUCCESSFULLY")\n')
outf.close()
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('BFS exploration using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
failCount = 0
maxDepth = 0
maxQueue = 0
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
queue = []
visited = []
test = []
t.restart()
queue.append((t.state(), test))
while (queue != []):
if len(queue) > maxQueue:
maxQueue = len(queue)
(s, test) = queue[0]
queue = queue[1:]
if len(test) > maxDepth:
maxDepth = len(test)
print "REACHED DEPTH", maxDepth, "QUEUE SIZE", len(queue) + 1
if len(test) == config.depth:
continue
t.backtrack(s)
shuffleActs = t.enabled()
if not config.deterministic:
R.shuffle(shuffleActs)
for c in shuffleActs:
stepOk = t.safely(c)
test.append(c)
thisBug = False
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
thisBug = True
if (not config.ignoreprops) and (not t.check()):
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
thisBug = True
ns = t.state()
if not thisBug:
if config.novisited or (ns not in visited):
if (random.random() < config.forget) and (not (queue
== [])):
break
if len(queue) >= config.breadth:
break
if not config.novisited:
visited.append(s)
if config.verbose:
print len(visited), "NEW STATE:"
print s
queue.append((ns, test))
elif not config.multiple:
break
elapsed = time.time() - start
if config.running:
if t.newBranches() != (set([])):
print "ACTION:", action
for b in t.newBranches():
print elapsed, len(t.allBranches()), "New branch", b
if elapsed > config.timeout:
print "STOPPING EXPLORATION DUE TO TIMEOUT, TERMINATED AT LENGTH", len(
test)
break
t.backtrack(s)
test = test[:-1]
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if not config.nocover:
print t.report(config.coverfile), "PERCENT COVERED"
if config.html:
t.htmlReport(config.html)
print len(visited), "STATES VISITED"
print maxDepth, "MAX SEARCH DEPTH"
print maxQueue, "MAX QUEUE SIZE"
if config.multiple:
print failCount, "FAILED"
if not config.nocover:
print len(t.allBranches()), "BRANCHES COVERED"
print len(t.allStatements()), "STATEMENTS COVERED"
def main():
global failCount,t,config,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,lastact,parts , pt , ptlast
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
t = SUT.sut()
tacts = t.actions()
a = None
sawNew = False
nops = 0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
lastact = 0
parts = R.randint(2,10)
pt = 0
ptlast = 0
startRestart = time.time()
t.restart()
restartTime += time.time() - startRestart
test = []
for s in xrange(0,config.depth):
acts = tacts
while True:
elapsed = time.time() - start
partsize = len(acts)/parts
pt = R.randint(0,parts-1)
while pt == ptlast:
pt = R.randint(0,parts-1)
p = R.randint(pt*partsize,(pt+1)*partsize-1)
a = acts[p]
lastact = p
ptlast = pt
if a[1]():
break
else:
a = None
acts = acts[:p] + acts[p+1:]
test.append(a)
nops += 1
stepOk = t.safely(a)
if not stepOk:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
stepOk = t.safely(a)
checkResult = t.check()
if not checkResult:
print "STOPPING TESTING DUE TO FAILED TEST"
break
if config.running:
if t.newBranches() != set([]):
print "ACTION:",a[0]
for b in t.newBranches():
print elapsed,len(t.allBranches()),"New branch",b
sawNew = True
else:
sawNew = False
if t.newStatements() != set([]):
print "ACTION:",a[0]
for s in t.newStatements():
print elapsed,len(t.allStatements()),"New statement",s
sawNew = True
else:
sawNew = False
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
print time.time()-start, "TOTAL RUNTIME"
print nops, "TOTAL TEST OPERATIONS"
if config.coverage:
t.restart()
print t.report("coverage.out"),"PERCENT COVERED"
t.internalReport()
def main():
global start,config,sut,R,nonerror,error,file_name,num,actioncount,Z
num = 0
actioncount = 0
file_name = 'failurefile'
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing={}'.format(config))
sut = SUT.sut()
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
states = [sut.state()]
nonerror = []
error = []
news = None
i = 0
while(time.time() < start + config.timeout):
for s in states:
i += 1
if (time.time() > start + config.timeout):
break
sut.restart() #the sut is restarted
sut.backtrack(s)
for w in xrange(0, config.width):
for d in xrange(0, config.depth):
if (time.time() > start + config.timeout):
break
ok = check_action()
news = sut.newStatements()
if not ok:
error.append(sut.currStatements())
break
if((len(news)>0) and (not ((news in error) or (news in nonerror)))):
states.insert(i-1,sut.state())
nonerror.append(sut.currStatements())
#if (config.fault): #if fault is found
# for (s_reduces, _, _) in Z:
# steps_reduce = "# STEP " + str(j)
# print >> f, sut.prettyName(s_reduces).ljust(80 - len(steps_reduce), ' '), steps_reduce
# j += 1
# f.close()
if config.coverage:
sut.internalReport() #this will print the internal report
print "####### mytester.py ######"
print "Running time: ", time.time() - start
print "Bugs Found: ",num
print "No. of actions: ", actioncount
print "Random count is",(randint(0,actioncount))
print "Actual count", actioncount/(randint(0,actioncount))
print "Complex count", num/actioncount/(randint(0,actioncount))
#print n,co.co_filename,"ALREADY ADDED"
return traceLOC
lastFuncs[(n, co.co_filename)] = True
if co.co_filename == sut.__file__.replace(".pyc", ".py"):
#print n,"FROM SUT"
return traceLOC
loc = len(inspect.getsourcelines(co)[0])
print "==>", n, co.co_filename, loc
lastLOCs += loc
if not onlyFirst:
return traceLOC
else:
return None
SUT = sut.sut()
R = random.Random()
try:
SUT.stopCoverage()
except:
pass
actLOCs = {}
N = len(SUT.actions())
numClasses = len(SUT.actionClasses())
print "CONSTRUCTING ESTIMATE FOR", numClasses, "CLASSES"
start = time.time()
TIMEOUT = int(sys.argv[1])
onlyFirst = "--onlyFirst" in sys.argv
def main():
global failCount,sut,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests
global fullPool,activePool,branchCoverageCount,statementCoverageCount,localizeSFail,localizeBFail
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
if config.gendepth != None:
failCloud = {}
allClouds = {}
if config.exploit != None:
fullPool = []
activePool = []
if config.quickAnalysis or (config.exploit != None):
branchCoverageCount = {}
statementCoverageCount = {}
if config.uniqueValuesAnalysis:
handledValues = {}
uniquef = open("unique.corpus",'w')
allUniquePaths = []
sut = SUT.sut()
if config.relax:
sut.relax()
if config.readQuick:
print "REPLAYING QUICK TESTS"
sqrtime = time.time()
for f in glob.glob("quicktest.*"):
fn = int(f.split("quicktest.")[1])
if fn >= quickCount:
quickCount = fn + 1
t = sut.loadTest(f)
sut.replay(t,catchUncaught=True,checkProp=(not config.ignoreprops))
print "EXECUTION TIME:",time.time()-sqrtime
print "BRANCH COVERAGE:",len(sut.allBranches())
print "STATEMENT COVERAGE:",len(sut.allStatements())
if config.logging != None:
sut.setLog(config.logging)
if config.profile:
profileTime = {}
profileCount = {}
for a in set(map(sut.actionClass, sut.actions())):
profileTime[a] = 0.0
profileCount[a] = 0
if config.markov != None:
nactions = len(sut.actions())
mprobs = {}
prefix = []
probs = []
inProbs = False
readSize = False
for l in open(config.markov):
if not readSize:
markovN = int(l)
readSize = True
elif "START CLASS" in l:
if (prefix != []):
mprobs[tuple(prefix)] = probs
prefix = []
probs = []
inProbs = False
elif inProbs:
ls = l.split("%%%%")
prob = float(ls[0])
ac = ls[1][1:-1]
probs.append((prob,ac))
elif "END CLASS" in l:
inProbs = True
else:
prefix.append(l[:-1])
tacts = sut.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
checkResult = True
if config.total:
fulltest = open("fulltest.txt",'w')
if config.localize:
localizeSFail = {}
localizeSPass = {}
localizeBFail = {}
localizeBPass = {}
testsPassed = 0
if config.quickAnalysis:
quickcf = open("quick.corpus",'w')
quickCorpus = []
quickDoneB = {}
quickDoneS = {}
quickAnalysisTotal = 0
quickAnalysisBCounts = {}
quickAnalysisSCounts = {}
quickAnalysisCounts = {}
quickClassCounts = {}
quickAnalysisRawCounts = {}
for c in set(map(sut.actionClass,sut.actions())):
quickAnalysisCounts[c] = 0
quickClassCounts[c] = 0
quickAnalysisRawCounts[c] = 0
quickAnalysisReducedB = {}
quickAnalysisReducedS = {}
if config.verbose:
print "ABOUT TO START TESTING"
sys.stdout.flush()
while (config.maxtests == -1) or (ntests < config.maxtests):
if config.verbose:
print "STARTING TEST",ntests
sys.stdout.flush()
ntests += 1
if config.swarm:
sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP)
#print "CONFIG:",(sut.swarmConfig())
if config.highLowSwarm != None:
classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm)
if config.swarmSwitch != None:
lastSwitch = 0
switches = []
for i in xrange(0,config.swarmSwitch):
switch = R.randrange(lastSwitch+1,config.depth-((config.swarmSwitch-i)))
switches.append(switch)
lastSwitch = switch
startRestart = time.time()
sut.restart()
restartTime += time.time() - startRestart
if config.total:
fulltest.write("<<RESTART>>\n")
if config.replayable:
currtest = open("currtest.txt",'w')
if (config.exploit != None) and ((time.time() - start) > config.startExploit):
tryExploit()
testFailed = False
for s in xrange(0,config.depth):
if config.verbose:
print "GENERATING STEP",s
if (config.swarmSwitch != None) and (s in switches):
if config.highLowSwarm == None:
sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP)
else:
classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm)
if (config.swarmLength != None) and (((s + 1) % config.swarmLength) == 0):
if config.highLowSwarm == None:
sut.standardSwarm(R,file=config.swarmProbs,P=config.swarmP)
else:
classP = sut.highLowSwarm(R,file=config.swarmProbs,highProb=config.highLowSwarm)
startGuard = time.time()
tryStutter = (a != None) and (a[1]()) and ((config.stutter != None) or config.greedyStutter)
if tryStutter:
if (config.stutter == None) or (R.random() > config.stutter):
tryStutter = False
if (config.greedyStutter) and sawNew:
print "TRYING TO STUTTER DUE TO COVERAGE GAIN"
tryStutter = True
else:
if config.markov == None:
if config.highLowSwarm == None:
a = sut.randomEnabled(R)
else:
a = sut.randomEnabledClassProbs(R,classP)
else:
prefix = tuple(map(sut.actionClass,sut.test()[-markovN:]))
if prefix not in mprobs:
a = sut.randomEnabled(R)
else:
a = sut.randomEnabledClassProbs(R,mprobs[prefix])
if a == None:
print "WARNING: DEADLOCK (NO ENABLED ACTIONS)"
guardTime += time.time()-startGuard
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test())
break
if a == None:
print "TERMINATING TEST DUE TO NO ENABLED ACTIONS"
break
if tryStutter:
print "STUTTERING WITH",a[0]
if config.replayable:
currtest.write(a[0] + "\n")
currtest.flush()
if config.total:
fulltest.write(a[0] + "\n")
fulltest.flush()
if config.verbose:
print "ACTION:",sut.prettyName(a[0])
startOp = time.time()
if config.quickAnalysis:
quickClassCounts[sut.actionClass(a)] += 1
stepOk = sut.safely(a)
thisOpTime = time.time()-startOp
nops += 1
if config.profile:
profileTime[sut.actionClass(a)] += thisOpTime
profileCount[sut.actionClass(a)] += 1
opTime += thisOpTime
if sut.warning() != None:
print "SUT WARNING:",sut.warning()
if tryStutter:
print "DONE STUTTERING"
if (stepOk or config.uncaught) and config.ignoreprops and (config.exploit != None):
collectExploitable()
if (not config.uncaught) and (not stepOk):
testFailed = True
handle_failure(sut.test(), "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
startCheck = time.time()
if not config.ignoreprops:
checkResult = sut.check()
checkTime += time.time()-startCheck
if checkResult and (stepOk or config.uncaught) and (config.exploit != None):
collectExploitable()
if not checkResult:
testFailed = True
handle_failure(sut.test(), "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
print "ACTION:",sut.prettyName(a[0])
for b in sut.newBranches():
print elapsed,len(sut.allBranches()),"New branch",b
sys.stdout.flush()
sawNew = True
else:
sawNew = False
if sut.newStatements() != set([]):
print "ACTION:",sut.prettyName(a[0])
for s in sut.newStatements():
print elapsed,len(sut.allStatements()),"New statement",s
sys.stdout.flush()
sawNew = True
else:
sawNew = False
if config.uniqueValuesAnalysis:
uvals = sut.uniqueVals()
olds = sut.state()
currTest = list(sut.test())
for u in uvals:
if u not in handledValues:
print "ANALYZING NEW UNIQUE VALUE:",u
else:
continue
r = sut.reduce(currTest, sut.coversUnique(u), keepLast=False)
rc = map(sut.actionClass, r)
sut.replay(r)
for ru in sut.uniqueVals():
handledValues[ru] = True
if rc not in allUniquePaths:
print "NEW PATH DISCOVERED"
allUniquePaths.append(rc)
for s in rc:
uniquef.write(s+"\n")
uniquef.write(("="*50)+"\n")
uniquef.flush()
sut.backtrack(olds)
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test())
break
if (config.exploit != None) and (not config.quickAnalysis):
for b in sut.currBranches():
if b not in branchCoverageCount:
branchCoverageCount[b] = 1
else:
branchCoverageCount[b] += 1
for s in sut.currStatements():
if s not in statementCoverageCount:
statementCoverageCount[s] = 1
else:
statementCoverageCount[s] += 1
if config.localize and not testFailed:
testsPassed += 1
for s in sut.currStatements():
if s not in localizeSPass:
localizeSPass[s] = 0
localizeSPass[s] += 1
for b in sut.currBranches():
if b not in localizeBPass:
localizeBPass[b] = 0
localizeBPass[b] += 1
if config.quickAnalysis:
currTest = list(sut.test())
sut.replay(currTest)
currB = sut.currBranches()
currS = sut.currStatements()
clen = len(currTest)
#print "GATHERING QUICK ANALYSIS DATA FOR",len(currB),"BRANCHES"
for b in currB:
if config.fastQuickAnalysis and (b in quickDoneB):
continue
print "ANALYZING BRANCH",b
if b not in branchCoverageCount:
branchCoverageCount[b] = 0
quickAnalysisReducedB[b] = 0
branchCoverageCount[b] += 1
r = sut.reduce(currTest,sut.coversBranches([b]),keepLast=False)
print "REDUCED FROM",clen,"TO",len(r)
sys.stdout.flush()
sut.replay(r)
for br in sut.currBranches():
quickDoneB[br] = True
for sr in sut.currStatements():
quickDoneS[sr] = True
rc = map(sut.actionClass,r)
if rc not in quickCorpus:
quickCorpus.append(rc)
for s in rc:
quickcf.write(s+"\n")
quickcf.write(("="*50)+"\n")
quickcf.flush()
sut.replay(r)
for b2 in sut.currBranches():
if b2 not in quickAnalysisReducedB:
quickAnalysisReducedB[b2] = 0
quickAnalysisReducedB[b2] += 1
for s2 in sut.currStatements():
if s2 not in quickAnalysisReducedS:
quickAnalysisReducedS[s2] = 0
quickAnalysisReducedS[s2] += 1
if b not in quickAnalysisBCounts:
quickAnalysisBCounts[b] = {}
quickAnalysisTotal += 1
for c in map(sut.actionClass,r):
quickAnalysisRawCounts[c] += 1
for c in set(map(sut.actionClass,r)):
quickAnalysisCounts[c] += 1
if c not in quickAnalysisBCounts[b]:
quickAnalysisBCounts[b][c] = 0
quickAnalysisBCounts[b][c] += 1
#print "GATHERING QUICK ANALYSIS DATA FOR",len(currS),"STATEMENTS"
for s in currS:
if config.fastQuickAnalysis and (s in quickDoneS):
continue
if s not in statementCoverageCount:
statementCoverageCount[s] = 0
quickAnalysisReducedS[s] = 0
statementCoverageCount[s] += 1
print "ANALYZING STATEMENT",s
r = sut.reduce(currTest,sut.coversStatements([s]),keepLast=False)
print "REDUCED FROM",clen,"TO",len(r)
sys.stdout.flush()
sut.replay(r)
for br in sut.currBranches():
quickDoneB[br] = True
for sr in sut.currStatements():
quickDoneS[sr] = True
rc = map(sut.actionClass,r)
if rc not in quickCorpus:
quickCorpus.append(rc)
sut.replay(r)
for b2 in sut.currBranches():
if b2 not in quickAnalysisReducedB:
quickAnalysisReducedB[b2] = 0
quickAnalysisReducedB[b2] += 1
for s2 in sut.currStatements():
quickAnalysisReducedS[s2] += 1
if s not in quickAnalysisSCounts:
quickAnalysisSCounts[s] = {}
quickAnalysisTotal += 1
for c in map(sut.actionClass,r):
quickAnalysisRawCounts[c] += 1
for c in set(map(sut.actionClass,r)):
quickAnalysisCounts[c] += 1
if c not in quickAnalysisSCounts[s]:
quickAnalysisSCounts[s][c] = 0
quickAnalysisSCounts[s][c] += 1
if config.throughput:
print "ACTION THROUGHPUT:",nops/(time.time()-start)
if config.replayable:
currtest.close()
if config.quickTests:
if (sut.newCurrBranches() != set([])) or (sut.newCurrStatements() != set([])):
handle_failure(sut.test(), "NEW COVERAGE", False, newCov=True)
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if config.total:
fulltest.close()
if not config.nocover:
sut.restart()
print sut.report(config.coverfile),"PERCENT COVERED"
if config.internal:
sut.internalReport()
if config.html:
sut.htmlReport(config.html)
if config.uniqueValuesAnalysis:
uniquef.close()
if config.quickAnalysis:
quickcf.close()
print "*" * 70
print "QUICK ANALYSIS RESULTS:"
print "*" * 70
print "TEST PATTERNS:"
for rc in quickCorpus:
print "="*50
for s in rc:
print s
print "*" * 70
print "OVERALL ACTION ANALYSIS:"
totalTaken = sum(quickClassCounts.values())
actSort = sorted(quickAnalysisRawCounts.keys(),key=lambda x: quickAnalysisCounts.get(x,0), reverse=True)
for a in actSort:
print "="*50
print "ACTION CLASS:"
print a
print "APPEARS",quickClassCounts[a],"TIMES IN TESTS"
print "APPEARS",quickAnalysisRawCounts[a],"TIMES IN REDUCED TESTS"
print "APPEARS IN",quickAnalysisCounts[a],"REDUCED TESTS ("+str(round((quickAnalysisCounts[a]/(quickAnalysisTotal*1.0))*100,2)) + "%)"
#baselineRate = quickClassCounts[a]/(totalTaken*1.0)
#reducedRate = quickAnalysisRawCounts[a]/(quickAnalysisTotal*1.0)
#if reducedRate > 0.0:
# print "RATIO:",(baselineRate/reducedRate)
#else:
# print "RATIO: INFINITE"
print "*" * 70
print "DETAILED BRANCH ANALYSIS"
branchCoverageCountSort = sorted(branchCoverageCount.keys(), key = lambda x: branchCoverageCount[x])
for b in branchCoverageCountSort:
print "="*50
print "BRANCH:",b
baselineRate = branchCoverageCount[b]/(ntests*1.0)
print "IN",str(round(baselineRate*100,2))+"% OF TESTS ("+str(branchCoverageCount[b])+" TESTS)"
reducedRate = quickAnalysisReducedB[b]/(quickAnalysisTotal*1.0)
print "IN",str(round(reducedRate*100,2))+"% OF REDUCED TESTS"
if reducedRate > 0.0:
print "RATIO:",(baselineRate/reducedRate)
print "REDUCED TEST ACTION ANALYSIS:"
sortAs = sorted(quickAnalysisBCounts[b].keys(),key=lambda x: quickAnalysisBCounts[b][x],reverse=True)
print branchCoverageCount[b],"TESTS"
for a in sortAs:
print a,str(round(quickAnalysisBCounts[b][a]/(branchCoverageCount[b]*1.0)*100,2))+"%"
print "*" * 70
print "DETAILED STATEMENT ANALYSIS"
statementCoverageCountSort = sorted(statementCoverageCount.keys(), key = lambda x: statementCoverageCount[x])
for s in statementCoverageCountSort:
print "="*50
print "STATEMENT:",s
baselineRate = statementCoverageCount[s]/(ntests*1.0)
print "IN",str(round(baselineRate*100,2))+"% OF TESTS"
reducedRate = quickAnalysisReducedS[s]/(quickAnalysisTotal*1.0)
print "IN",str(round(reducedRate*100,2))+"% OF REDUCED TESTS"
if reducedRate > 0.0:
print "RATIO:",(baselineRate/reducedRate)
print "REDUCED TEST ACTION ANALYSIS:"
print statementCoverageCount[s],"TESTS"
sortAs = sorted(quickAnalysisSCounts[s].keys(),key=lambda x: quickAnalysisSCounts[s][x],reverse=True)
for a in sortAs:
print a,str(round(quickAnalysisSCounts[s][a]/(statementCoverageCount[s]*1.0)*100,2))+"%"
print time.time()-start, "TOTAL RUNTIME"
print ntests, "EXECUTED"
print nops, "TOTAL TEST OPERATIONS"
print opTime, "TIME SPENT EXECUTING TEST OPERATIONS"
print guardTime, "TIME SPENT EVALUATING GUARDS AND CHOOSING ACTIONS"
if not config.ignoreprops:
print checkTime, "TIME SPENT CHECKING PROPERTIES"
print (opTime + checkTime), "TOTAL TIME SPENT RUNNING SUT"
print restartTime, "TIME SPENT RESTARTING"
print reduceTime, "TIME SPENT REDUCING TEST CASES"
if config.multiple:
print failCount,"FAILED"
print repeatCount,"REPEATS OF FAILURES"
print len(failures),"ACTUAL DISTINCT FAILURES"
print
n = 0
for (test, err) in failures:
print "FAILURE",n
sut.prettyPrintTest(test)
n += 1
if err != None:
print "ERROR:", err
print "TRACEBACK:"
traceback.print_tb(err[2])
i = -1
if config.compareFails: # Comparison feature normally not useful, just for researching normalization
for test1 in failures:
i += 1
j = -1
for test2 in failures:
j += 1
if (j > i):
print "COMPARING FAILURE",i,"AND FAILURE",j
for k in xrange(0,max(len(test1),len(test2))):
if k >= len(test1):
print "STEP",k,"-->",test2[k][0]
elif k >= len(test2):
print "STEP",k,test1[k][0],"-->"
elif test1[k] != test2[k]:
print "STEP",k,test1[k][0],"-->",test2[k][0]
if config.profile:
print "ACTION PROFILE:"
averages = []
for a in profileTime:
if profileCount[a] != 0:
averages.append((a,profileTime[a]/profileCount[a]))
averages = sorted(averages, key = lambda x: x[1])
maxAvg = averages[-1][1]
minAvg = averages[0][1]
sumAvg = sum(map(lambda x: x[1], averages))
for (a,t) in averages:
print a,profileCount[a],t,round(t/maxAvg,2),round(t/minAvg,2),round(t/sumAvg,2)
if config.localize and failCount > 0:
scoresS = {}
scoresB = {}
for s in sut.allStatements():
if s not in localizeSPass:
localizeSPass[s] = 0
if s not in localizeSFail:
localizeSFail[s] = 0
denom = math.sqrt(failCount*(localizeSFail[s]+localizeSPass[s]))
if denom == 0.0:
continue
scoresS[s] = localizeSFail[s]/denom
for b in sut.allBranches():
if b not in localizeBPass:
localizeBPass[b] = 0
if b not in localizeBFail:
localizeBFail[b] = 0
denom = math.sqrt(failCount*(localizeBFail[b]+localizeBPass[b]))
if denom == 0.0:
continue
scoresB[b] = localizeBFail[b]/denom
sortedS = sorted(scoresS.keys(),key = lambda x:scoresS[x])
sortedB = sorted(scoresB.keys(),key = lambda x:scoresB[x])
print "FAULT LOCALIZATION RESULTS:"
for s in sortedS:
if scoresS[s] > 0.0:
print s, scoresS[s]
for b in sortedB:
if scoresB[b] > 0.0:
print b, scoresB[b]
if not config.nocover:
print len(sut.allBranches()),"BRANCHES COVERED"
print len(sut.allStatements()),"STATEMENTS COVERED"
def main():
global timeout
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Reducing using config={}'.format(config)))
sut = SUT.sut()
timeout = config.timeout
if not ((config.coverage) or (config.coverMore) or (config.decompose)):
try:
sut.stopCoverage()
except BaseException:
pass
R = None
if config.random:
R = random.Random()
if config.seed is not None:
R.seed(config.seed)
r = sut.loadTest(config.infile, afl=config.afl, swarm=config.aflswarm)
f = None
if config.matchException:
print("EXECUTING TEST TO OBTAIN FAILURE FOR EXCEPTION MATCHING...")
assert sut.fails(r)
f = sut.failure()
print("ERROR:", f)
print("TRACEBACK:")
traceback.print_tb(f[2], file=sys.stdout)
if not config.sandbox:
pred = (lambda x: sut.failsCheck(x, failure=f))
if config.noCheck:
pred = (lambda x: sut.fails(x, failure=f))
else:
pred = sandboxReplay
if config.coverage or config.coverMore or (config.decompose
and not config.noNormalize):
print("EXECUTING TEST TO OBTAIN COVERAGE FOR CAUSE REDUCTION...")
sut.replay(r,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
# Replay it twice, due to odd coverage on startup sometimes
sut.replay(r,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
b = set(sut.currBranches())
s = set(sut.currStatements())
print("PRESERVING", len(b), "BRANCHES AND", len(s), "STATEMENTS")
if config.coverMore:
pred = sut.coversMore(s,
b,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
else:
pred = sut.coversAll(s,
b,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
if config.checkDeterminism:
pred = (
lambda t: sut.nondeterministic(t,
delay=config.determinismDelay,
tries=config.determinismTries,
delay0=config.determinismDelay0))
if config.checkStepDeterminism:
pred = (lambda t: sut.stepNondeterministic(
t,
delay=config.determinismDelay,
tries=config.determinismTries,
delay0=config.determinismDelay0))
if config.checkProcessDeterminism:
pred = (lambda t: sut.processNondeterministic(
t,
delay=config.determinismDelay,
tries=config.determinismTries,
iterate=config.iterate))
if config.probability is not None:
Ppred = pred
pred = (lambda t: sut.forceP(t,
Ppred,
P=config.probability,
samples=config.samples,
replications=config.replications))
print("STARTING WITH TEST OF LENGTH", len(r))
if not config.noReduce:
start = time.time()
print("REDUCING...")
if (not config.multiple) and (not config.decompose):
r = sut.reduce(r,
pred,
verbose=config.verbose,
tryFast=not config.ddmin,
keepLast=config.keepLast,
pruneGuards=not config.noPruneGuards,
rgen=R)
elif config.multiple:
rs = sut.reductions(r,
pred,
verbose=config.verbose,
recursive=config.recursive,
limit=config.limit,
keepLast=config.keepLast,
pruneGuards=not config.noPruneGuards,
tryFast=not config.ddmin)
elif config.decompose:
print("DECOMPOSING...")
rs = sut.coverDecompose(r,
verbose=config.verbose,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
print("REDUCED IN", time.time() - start, "SECONDS")
if (not config.multiple) and (not config.decompose):
print("NEW LENGTH", len(r))
else:
print("NEW LENGTHS", list(map(len, rs)))
if not config.noAlpha:
print("ALPHA CONVERTING...")
if (not config.multiple) and (not config.decompose):
r = sut.alphaConvert(r)
else:
rs = list(map(sut.alphaConvert, rs))
if not config.noNormalize:
start = time.time()
print("NORMALIZING...")
if (not config.multiple) and (not config.decompose):
r = sut.normalize(r,
pred,
verbose=config.verbose,
keepLast=config.keepLast,
checkEnabled=not config.noCheckEnabled,
pruneGuards=not config.noPruneGuards,
tryFast=not config.ddmin,
speed=config.speed)
else:
newrs = []
for r in rs:
f = None
if config.matchException:
print(
"EXECUTING TEST TO OBTAIN FAILURE FOR EXCEPTION MATCHING..."
)
assert sut.fails(r)
f = sut.failure()
print("ERROR:", f)
print("TRACEBACK:")
traceback.print_tb(f[2], file=sys.stdout)
if not config.sandbox:
pred = (lambda x: sut.failsCheck(x, failure=f))
if not config.noCheck:
pred = (lambda x: sut.fails(x, failure=f))
else:
pred = sandboxReplay
if config.coverage or config.coverMore or (
config.decompose and not config.noNormalize):
print(
"EXECUTING TEST TO OBTAIN COVERAGE FOR CAUSE REDUCTION..."
)
sut.replay(r,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
b = set(sut.currBranches())
s = set(sut.currStatements())
print("PRESERVING", len(b), "BRANCHES AND", len(s),
"STATEMENTS")
if config.coverMore:
pred = sut.coversMore(s,
b,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
else:
pred = sut.coversAll(s,
b,
checkProp=not config.noCheck,
catchUncaught=config.uncaught)
newrs.append(
sut.normalize(r,
pred,
verbose=config.verbose,
keepLast=config.keepLast,
pruneGuards=not config.noPruneGuards,
tryFast=not config.ddmin))
rs = newrs
print("NORMALIZED IN", time.time() - start, "SECONDS")
if (not config.multiple) and (not config.decompose):
print("NEW LENGTH", len(r))
else:
print("NEW LENGTHS", list(map(len, rs)))
if (not config.multiple) and (not config.decompose):
sut.saveTest(r, config.outfile, afl=config.writeafl)
sut.prettyPrintTest(r)
print()
print("TEST WRITTEN TO", config.outfile)
else:
i = 0
for r in rs:
print("TEST #" + str(i) + ":")
sut.saveTest(r,
config.outfile + "." + str(i) + ".test",
afl=config.writeafl)
sut.prettyPrintTest(r)
print()
print("TEST WRITTEN TO", config.outfile + "." + str(i))
print()
i += 1
def create_new_pool():
return [SUT.sut(), [[]], [], 0.0, dict(), 0.0]
def main():
if "--help" in sys.argv:
print("Usage: tstl_markov <outfile> <prefix size> <test files> [--notRaw]")
print("Options:")
print(" --notRaw: corpus files are not raw TSTL tests, but action classes")
sys.exit(0)
sut = SUT.sut()
classes = []
nacts = len(sut.actions())
for a in sut.actions():
if sut.actionClass(a) not in classes:
classes.append(sut.actionClass(a))
#print len(classes)
n = int(sys.argv[2])
outfile = sys.argv[1]
corpfiles = sys.argv[3:]
corpfiles = [x for x in corpfiles if x != "--notRaw"]
tests = []
test = []
for f in corpfiles:
for l in open(f):
if "--notRaw" in sys.argv:
test.append(l[:-1])
else:
test.append(sut.actionClass(sut.playable(l[:-1])))
if test != []:
tests.append(test)
test = []
chains = {}
for t in tests:
for pos in range(n+1,len(t)):
prefix = tuple(t[pos-n:pos])
#print prefix,"==>",t[pos]
if prefix not in chains:
chains[prefix] = []
chains[prefix].append(t[pos])
mout = open(outfile,'w')
mout.write(str(n)+"\n")
for c in chains:
print("PREFIX:",c)
mout.write("START CLASS\n")
for ac in c:
mout.write(ac+"\n")
mout.write("END CLASS\n")
counts = {}
total = 0.0
for suffix in chains[c]:
total += 1
if suffix not in counts:
counts[suffix] = 0
counts[suffix] += 1
for suffix in counts:
print(suffix,counts[suffix]/total)
mout.write(str(counts[suffix]/total) + " %%%% "+suffix+"\n")
mout.close()
def main():
global start,config,sut,R,nonErrorSeq,ErrorSeq,bug,act
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
nonErrorSeq = []
ErrorSeq = []
newSeq = None
sut = SUT.sut()
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
states = [sut.state()]
n = -1
bug = 0
act = 0
while(time.time() < start + config.timeout):
for s in states:
if (time.time() > start + config.timeout):
break
n+=1
sut.restart()
sut.backtrack(s)
for i in xrange(0, config.width):
for j in xrange(0, config.depth):
if (time.time() > start + config.timeout):
break
action = sut.randomEnabled(R)
act+=1
ok = sut.safely(action)
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()),"New branch",b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
if elapsed > config.timeout:
print "Timeout: Stop testing."
break
if not ok:
bug += 1
print "Bug" , bug
startReduce = time.time()
r = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(r)
print(sut.failure())
if config.fault:
filename = 'failure%d.test' % bug
sut.saveTest(r, filename)
ErrorSeq.append(sut.currStatements())
newSeq = sut.newStatements()
if not ok:
break
if not (newSeq in nonErrorSeq):
if (len(newSeq)>0) :
states.insert(n,sut.state())
nonErrorSeq.append(sut.currStatements())
else:
continue
else:
states.insert(n,sut.state())
nonErrorSeq.append(sut.currStatements())
if config.coverage:
sut.internalReport()
print "Total Bugs: ",bug
print "Total actions: ", act
print "Total Running time: ", time.time() - start
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random variation of beam search using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
tacts = t.actions()
ntests = 0
while (config.maxtests == -1) or (ntests < config.maxtests):
ntests += 1
t.restart()
test = []
for s in xrange(0,config.depth):
count = 0
newCover = False
acts = tacts
old = t.state()
while (count <= config.width):
count += 1
while True:
p = R.randint(0,len(acts)-1)
a = acts[p]
if a[1]():
break
acts = acts[:p] + acts[p+1:]
test.append(a)
stepOk = t.safely(a)
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
if (not config.ignoreprops) and (not t.check()):
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
break
elapsed = time.time() - start
if t.newBranches() != set([]):
if config.running:
print "ACTION:",a[0]
for b in t.newBranches():
print elapsed,len(t.allBranches()),"New branch",b
break # Continue with this choice if new branch exposed
if elapsed > config.timeout:
break
test = test[:-1]
t.backtrack(old)
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if not config.nocover:
print t.report(config.coverfile),"PERCENT COVERED"
if config.html:
t.htmlReport(config.html)
print ntests, "EXECUTED"
if config.multiple:
print failCount,"FAILED"
if not config.nocover:
print len(t.allBranches()),"BRANCHES COVERED"
print len(t.allStatements()),"STATEMENTS COVERED"
from __future__ import print_function
import sys
import sut
import glob
sut = sut.sut()
def abstraction(s):
if len(s) <= 3:
return (sut.actionClass(s), )
else:
return (sut.actionClass(s), s[3])
seedFiles = sys.argv[1]
test = sys.argv[2]
newTestName = sys.argv[3]
abstract = "--abstract" in sys.argv
seeds = []
for f in glob.glob(seedFiles):
t = sut.loadTest(f)
if abstract:
t = list(map(abstraction, t))
seeds.append((t, f))
original = sut.loadTest(test)
t = list(original)
def main():
global failCount,sut,config,reduceTime,quickCount,repeatCount,failures,cloudFailures,R,opTime,checkTime,guardTime,restartTime,nops,ntests
rand = random.Random(seed)
start = time.time()
#elapsed = time.time()-start
runningtime = time.time()-startprog
sut = SUT.sut()
tacts = sut.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
bugs = 0
checkResult = True
naction = 10
maxaction = 107
actiontime = 1
while runningtime < timeout:
sut.restart()
elapsed = time.time()-startprog
for d in xrange(0,depth):
elapsed = time.time()-startprog
if elapsed > timeout:
break
startaction = time.time()
for w in xrange(0,width):
elapsed = time.time()-startprog
#a = sut.randomEnabled(R)
a = sut.randomEnableds(rand, naction)
actionelapsed = time.time() - startaction
for i in xrange(0,naction):
if a[0] == None:
print "WARNING: DEADLOCK (NO ENABLED ACTIONS)"
if elapsed > timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test())
break
if a[0] == None:
print "TERMINATING TEST DUE TO NO ENABLED ACTIONS"
break
nops += 1
startOp = time.time()
stepOk = sut.safely(a[0])
propok = sut.check()
if sut.warning() != None:
print "SUT WARNING:",sut.warning()
opTime += (time.time()-startOp)
if (not propok) or (not stepOk):
bugs += 1
print "TEST FAILED"
print "REDUCING..."
R = sut.reduce(sut.test(), lambda x: sut.fails(x) or sut.failsCheck(x))
sut.prettyPrintTest(R)
print sut.failure()
if fault:
filename='failure'+str(bugs)+'.test'
for i in range(len(R)):
with open(filename,'w') as f:
f.write('\n'+'Bug no.'+str(bugs)+'\n')
f.write(str(sut.failure())+'\n')
f.write(str(R)+'\n')
#print "NORMALIZING..."
#N = sut.normalize(R, lambda x: sut.fails(x) or sut.failsCheck(x))
#sut.prettyPrintTest(N)
#print "GENERALIZING..."
#sut.generalize(N, lambda x: sut.fails(x) or sut.failsCheck(x))
#break
elapsed = time.time() - startprog
if running:
if sut.newBranches() != set([]):
print "ACTION:",a[i]
for b in sut.newBranches():
print elapsed,len(sut.allBranches()),"New branch",b
sawNew = True
else:
sawNew = False
if sut.newStatements() != set([]):
print "ACTION:",a[i]
for s in sut.newStatements():
print elapsed,len(sut.allStatements()),"New statement",s
sawNew = True
else:
sawNew = False
if elapsed > timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test())
break
if(actionelapsed > actiontime):
if(naction < maxaction):
naction+=1
if elapsed > timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",len(sut.test())
break
if coverage:
sut.internalReport()
print time.time()-startprog, "TOTAL RUNTIME"
print nops, "TOTAL TEST OPERATIONS"
print opTime, "TIME SPENT EXECUTING TEST OPERATIONS"
print bugs,"FAILED"
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
quickCount = 0
repeatCount = 0
failures = []
cloudFailures = []
if config.gendepth != None:
failCloud = {}
allClouds = {}
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
tacts = t.actions()
a = None
sawNew = False
nops = 0
ntests = 0
reduceTime = 0.0
opTime = 0.0
checkTime = 0.0
guardTime = 0.0
restartTime = 0.0
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Triaging using config={}'.format(config)))
if config.ignoreContaining is not None:
ignoredStrings = config.ignoreContaining.split(";;")
sut = SUT.sut()
try:
sut.stopCoverage()
except BaseException:
pass
numTests = 0.0
numInvalid = 0
numPassing = 0
failingTests = {}
if config.useFailures:
sigs = {}
for fn in glob.glob(config.testglob):
try:
t = sut.loadTest(fn)
except BaseException:
print("INVALID TEST CASE:", fn)
numInvalid += 1
continue
if config.ignoreContaining is not None:
foundString = False
for s in t:
for istr in ignoredStrings:
if istr in s[0]:
foundString = True
break
if foundString:
break
if foundString:
continue
if not config.noCheck:
fails = sut.failsAny(t)
else:
fails = sut.fails(t)
if fails:
ft = list(sut.test())
v = vector(ft, sut)
if config.useFailures:
e = repr(sut.failure())
e = e[:e.find("<traceback object at 0x")] + ")"
sig = noDigits(e)
if config.abstractStrings:
sig = noStrings(sig)
if sig in sigs:
sigs[sig].append(fn)
else:
sigs[sig] = [fn]
if not config.ignoreLast:
vlast = []
for ac in sorted(sut.actionClasses()):
if sut.actionClass(ft[-1]) == ac:
vlast.append(1.0)
else:
vlast.append(0.0)
v.extend(vlast)
failingTests[fn] = (ft, v, sut.failure())
else:
numPassing += 1
if config.useFailures:
sk = sorted(sigs.keys())
for fn in failingTests:
vsig = []
for sig in sk:
if fn in sigs[sig]:
vsig.append(1.0)
else:
vsig.append(0.0)
_, v, _ = failingTests[fn]
v.extend(vsig)
distances = {}
for fn in failingTests:
distances[fn] = {}
for fn1 in failingTests:
for fn2 in failingTests:
if fn1 == fn2:
distances[fn1][fn2] = 0.0
continue
if fn1 > fn2:
continue
distances[fn1][fn2] = euclidean(failingTests[fn1][1],
failingTests[fn2][1])
distances[fn2][fn1] = distances[fn1][fn2]
chosen = failingTests.keys()[0]
print("=" * 80)
print(1, chosen)
if config.showTests:
print()
sut.prettyPrintTest(failingTests[chosen][0])
f = failingTests[chosen][2]
print("EXCEPTION:", repr(f))
print("FAILURE:")
traceback.print_tb(f[2], file=sys.stdout)
ranked = [chosen]
while len(ranked) != len(failingTests):
chosen = None
maxMinD = 0.0
for fn in failingTests:
d = []
for r in ranked:
d.append(distances[fn][r])
minD = min(d)
if minD > maxMinD:
chosen = fn
maxMinD = minD
if chosen is None:
print("ALL REMAINING TESTS ARE ZERO DISTANCE FROM A RANKED TEST")
break
ranked.append(chosen)
print("=" * 80)
print(len(ranked), chosen, maxMinD)
if config.showTests:
print()
sut.prettyPrintTest(failingTests[chosen][0])
print()
f = failingTests[chosen][2]
print("EXCEPTION:", repr(f))
print("FAILURE:")
traceback.print_tb(f[2], file=sys.stdout)
print()
print("*" * 80)
print(int(numTests), "TOTAL FAILING TESTS")
if numInvalid > 0:
print(numInvalid, "TOTAL INVALID TESTS")
if numPassing > 0:
print(numPassing, "TOTAL PASSING TESTS")
def main():
global start,config,sut,R,nonErrorSeq,ErrorSeq,file_name,num
num = 0
file_name = 'failurefile'
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Random testing using config={}'.format(config))
nonErrorSeq = []
ErrorSeq = []
newSeq = None
sut = SUT.sut()
R = random.Random(config.seed)
start = time.time()
elapsed = time.time() - start
states = [sut.state()]
while(time.time() < start + config.timeout):
for s in states:
sut.restart()
sut.backtrack(s)
for w in xrange(0, config.width):
for d in xrange(0, config.depth):
action = sut.randomEnabled(R)
ok = sut.safely(action)
elapsed = time.time() - start
if config.running:
if len(sut.newBranches()) > 0:
print "ACTION:", action[0]
for b in sut.newBranches():
print elapsed, len(sut.allBranches()),"New branch",b
if not ok:
num += 1
print "Found Bug" , num
print "REDUCING..."
startReduce = time.time()
test = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(test)
print(sut.failure())
if config.fault:
f = open((file_name + str(num) + ".test"),"w")
f.writelines(str(sut.failure()))
f.writelines('\nReduced test has ' + str(len(test)) + ' steps')
f.close()
ErrorSeq.append(sut.currStatements())
newSeq = sut.newStatements()
if not ok:
break
if((len(newSeq)>0) and (not ((newSeq in nonErrorSeq) or (newSeq in ErrorSeq)))):
states.insert(0,sut.state())
nonErrorSeq.append(sut.currStatements())
if config.coverage:
sut.internalReport()
print "Total Bugs: ",num
def main():
global config, R, sut, bugs, actCount, coverageCount, branchCount, statementCount, test_file, ntest
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
sut.silenceCoverage()
bugs = 0
actCount = 0
coverageCount = {}
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
while elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
for j in xrange(0, config.width):
if randomAct():
preTest = sut.state()
else:
sut.backtrack(preTest)
elapsed = time.time() - start
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print elapsed, len(sut.allStatements()), "New statement", s
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
elapsed = time.time() - start
if elapsed > config.timeout:
break
collectCoverage()
elapsed = time.time() - start
printCoverage()
if config.coverage:
sut.internalReport()
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
def main():
if "--help" in sys.argv:
print(
"Usage: tstl_graph <outfile> <depth> <width> [<seed>] [<traces> (default 1)] [<skip> (default none)]"
)
sys.exit(0)
outfile = sys.argv[1]
depth = int(sys.argv[2])
k = int(sys.argv[3])
if len(sys.argv) > 5:
seed = int(sys.argv[4])
random.seed(seed)
if len(sys.argv) > 5:
traces = int(sys.argv[5])
else:
traces = 1
if len(sys.argv) > 6:
skiplen = int(sys.argv[6])
else:
skiplen = -1
print("Producing graph of", traces, "traces with depth", depth,
"and width", k, "starting from", skiplen)
dot = Digraph(comment="Depth " + str(depth))
for i in range(0, traces):
d = 0
s = 0
state = str(i) + r"\<init\>"
dot.node(state, r"\<init\>", penwidth="3.0", shape='box')
t = SUT.sut()
t.restart()
s = 0
d = 1
last = state
midFlip = True
sd = 0
while sd <= skiplen:
nexta = t.enabled()
act = random.choice(nexta)
t.safely(act)
sd += 1
while d <= depth:
nexta = t.enabled()
act = random.choice(nexta)
aname = t.prettyName(act[0])
nexts = [t.prettyName(a[0]) for a in nexta]
eqnexts = nexts
eqnexts = collapse(eqnexts)
eqnexts = [x for x in eqnexts if x != aname]
random.shuffle(eqnexts)
eqnexts = eqnexts[-(k - 1):]
mid = len(eqnexts) / 2
if (len(eqnexts) % 2) != 0:
if midFlip:
mid = mid + 1
midFlip = not midFlip
eqnexts = eqnexts[:mid] + [aname] + eqnexts[mid:]
for name in eqnexts:
s += 1
state = str(i) + "s" + str(s)
if name == aname:
newLast = state
dot.node(state, name, penwidth="3.0", shape='box')
dot.edge(last, state, penwidth="3.0")
else:
dot.node(state, name, fontsize="10.0")
dot.edge(last, state)
last = newLast
t.safely(act)
d += 1
dot.render(outfile, view=True)
def main():
global config, R, sut, bugs, ntest, start
global actCount, branchCount, statementCount
global savedTest, savedFlag
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('Testing using config={}'.format(config))
R = random.Random(config.seed)
sut = sut.sut()
if config.silence:
sut.silenceCoverage()
bugs = 0
actCount = 0
branchCount = {}
statementCount = {}
ntest = 0
start = time.time()
elapsed = time.time() - start
print "Starting Phase 1"
if elapsed < config.timeout:
sut.restart()
ntest += 1
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
print "Finishing Phase 1"
print "Starting Phase 2"
elapsed = time.time() - start
while elapsed < config.timeout:
sut.restart()
ntest += 1
savedTest = None
if (savedTest != None) and (R.random() > 0.7):
sut.backtrack(savedTest)
savedFlag = False
for i in xrange(0, config.depth):
if not randomAct():
break
elapsed = time.time() - start
if config.running:
showRunning()
elapsed = time.time() - start
if elapsed > config.timeout:
print "Stopping test [TIMEOUT]"
break
collectCoverage()
elapsed = time.time() - start
print "Finishing Phase 2"
if time.time() - start > config.timeout:
print "[TIMEOUT]"
if config.coverdetail:
printCoverage()
if config.coverage:
sut.internalReport()
if config.resultdetail:
print "Covered", len(sut.allBranches()), "branches"
print "Covered", len(sut.allStatements()), "statements"
print "Failed", bugs, "times"
print "Total tests", ntest
print "Total actions", actCount
print "Total runtime", time.time() - start
def __init__(self): self.errorSeqs = [] self.nonErrorSeqs = [] self.sut = sut.sut()
def main():
#global variables are defined here
global sut, rgen, actCount, bugs, coveraged, visited, start, argv
parsed_args, parser = parse_args()
argv = make_config(parsed_args, parser)
start = time.time()
sut = sut.sut()
sut.silenceCoverage()
rgen = random.Random()
rgen.seed(argv.SEED)
actCount = 0
bugs = 0
coveraged = {}
visited = []
print "STARTING PHASE 1"
while time.time() - start < argv.TIMEOUT * 0.5:
sut.restart()
for s in xrange(0,argv.DEPTH):
randomAction()
collectCoverage()
sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x])
for s in sortedCov:
print s, coveraged[s]
print "STARTING PHASE 2"
while time.time() - start < argv.TIMEOUT :
belowMedian = set([])
sort = sorted(coveraged.keys(),key = lambda x : coveraged[x])
a = len(sort)/2
for s in sort:
if coveraged[s] < sort[a]:
belowMedian.add(s)
else:
break
activePool = []
for (t,c) in visited:
for s in c:
if s in belowMedian:
activePool.append((t,c))
break
sut.restart()
if len(sut.newStatements()) != 0:
print "new statement",sut.newStatements()
visited.append((list(sut.test()), set(sut.currStatements())))
for s in xrange(0,argv.DEPTH):
if not randomAction():
break
collectCoverage()
sortedCov = sorted(coveraged.keys(), key=lambda x: coveraged[x])
for s in sortedCov:
print s, coveraged[s]
if argv.COVERAGE:
sut.internalReport()
print "FAILURES ",bugs, "TIMES"
print "TOTAL ACTIONS", actCount
print "TOTAL RUNTIME", time.time()-start
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('BFS exploration using config={}'.format(config))
R = random.Random(config.seed)
start = time.time()
elapsed = time.time()-start
failCount = 0
maxDepth = 0
maxQueue = 0
t = SUT.sut()
if config.logging != None:
t.setLog(config.logging)
queue = []
visited = []
test = []
t.restart()
queue.append((t.state(), test))
while (queue != []):
if len(queue) > maxQueue:
maxQueue = len(queue)
(s, test) = queue[0]
queue = queue[1:]
if len(test) > maxDepth:
maxDepth = len(test)
print "REACHED DEPTH",maxDepth,"QUEUE SIZE",len(queue)+1
if len(test) == config.depth:
continue
t.backtrack(s)
shuffleActs = t.enabled()
if not config.deterministic:
R.shuffle(shuffleActs)
for c in shuffleActs:
stepOk = t.safely(c)
test.append(c)
thisBug = False
if (not config.uncaught) and (not stepOk):
handle_failure(test, "UNCAUGHT EXCEPTION", False)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
thisBug = True
if (not config.ignoreprops) and (not t.check()):
handle_failure(test, "PROPERLY VIOLATION", True)
if not config.multiple:
print "STOPPING TESTING DUE TO FAILED TEST"
thisBug = True
ns = t.state()
if not thisBug:
if config.novisited or (ns not in visited):
if (random.random() < config.forget) and (not (queue == [])):
break
if len(queue) >= config.breadth:
break
if not config.novisited:
visited.append(s)
if config.verbose:
print len(visited), "NEW STATE:"
print s
queue.append((ns, test))
elif not config.multiple:
break
elapsed = time.time() - start
if config.running:
if t.newBranches() != (set([])):
print "ACTION:",action
for b in t.newBranches():
print elapsed,len(t.allBranches()),"New branch",b
if elapsed > config.timeout:
print "STOPPING EXPLORATION DUE TO TIMEOUT, TERMINATED AT LENGTH",len(test)
break
t.backtrack(s)
test = test[:-1]
if (not config.multiple) and (failCount > 0):
break
if elapsed > config.timeout:
print "STOPPING TESTING DUE TO TIMEOUT"
break
if not config.nocover:
print t.report(config.coverfile),"PERCENT COVERED"
if config.html:
t.htmlReport(config.html)
print len(visited), "STATES VISITED"
print maxDepth,"MAX SEARCH DEPTH"
print maxQueue,"MAX QUEUE SIZE"
if config.multiple:
print failCount,"FAILED"
if not config.nocover:
print len(t.allBranches()),"BRANCHES COVERED"
print len(t.allStatements()),"STATEMENTS COVERED"
key_list = pdict.keys()
arg_list = [pdict[k] for k in key_list]
Config = namedtuple('Config', key_list)
nt_config = Config(*arg_list)
return nt_config
actCount = 0
bugs = 0
no_tests = 0
i = 0
covCount = {}
leastCov = None
savedTest = None
sut = sut.sut()
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print('My tester using config={}'.format(config))
rgen = random.Random(config.seed)
start = time.time()
while time.time()-start < config.timeout:
for ts in xrange(0,config.width):
sut.restart()
no_tests += 1
if (savedTest != None) and (rgen.random() > 0.8):
sut.backtrack(savedTest)
def main():
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
print(('Triaging using config={}'.format(config)))
if config.ignoreContaining is not None:
ignoredStrings = config.ignoreContaining.split(";;")
sut = SUT.sut()
try:
sut.stopCoverage()
except BaseException:
pass
signatures = {}
numTests = 0.0
numInvalid = 0
numPassing = 0
for fn in glob.glob(config.testglob):
try:
t = sut.loadTest(fn)
except BaseException:
print("INVALID TEST CASE:", fn)
numInvalid += 1
continue
if config.ignoreContaining is not None:
foundString = False
for s in t:
for istr in ignoredStrings:
if istr in s[0]:
foundString = True
break
if foundString:
break
if foundString:
continue
if not config.noCheck:
fails = sut.failsAny(t)
else:
fails = sut.fails(t)
if fails:
numTests += 1
e = repr(sut.failure())
e = e[:e.find("<traceback object at 0x")] + ")"
line = sut.test()[-1][0]
tline = t[-1][0]
if line != tline:
print("TEST", fn, "FAILS BEFORE END OF TEST")
sig = (noDigits(e), noDigits(line))
if config.abstractStrings:
sig = (noStrings(sig[0]), sig[1])
if config.actionClasses:
sig += (repr(set(map(sut.actionClass, t))), )
if (sig not in signatures):
signatures[sig] = (fn, t, sut.failure(),
sut.prettyName(t[-1][0]), 1)
elif (len(t) < len(signatures[sig][1])):
signatures[sig] = (fn, t, sut.failure(),
sut.prettyName(t[-1][0]),
signatures[sig][4] + 1)
else:
(sfn, st, sf, sp, count) = signatures[sig]
signatures[sig] = (sfn, st, sf, sp, count + 1)
else:
numPassing += 1
for sig in sorted(signatures.keys(),
key=lambda x: signatures[x][4],
reverse=True):
print("=" * 80)
print("TEST:", signatures[sig][0], "LENGTH:", len(signatures[sig][1]))
print("OPERATION:", signatures[sig][3])
if config.actionClasses:
print("ACTION CLASSES:",
set(map(lambda x: sut.prettyName(x[0]), signatures[sig][1])))
print("EXCEPTION:", repr(signatures[sig][2]))
print("FAILURE:")
traceback.print_tb(signatures[sig][2][2], file=sys.stdout)
print("COUNT:", signatures[sig][4],
"(" + str(round(signatures[sig][4] / numTests, 2) * 100) + "%)")
if config.showTests:
sut.prettyPrintTest(signatures[sig][1])
print()
print("*" * 80)
print(int(numTests), "TOTAL FAILING TESTS")
if numInvalid > 0:
print(numInvalid, "TOTAL INVALID TESTS")
if numPassing > 0:
print(numPassing, "TOTAL PASSING TESTS")
