def mutate(test): tcopy = list(test) i = rgen.randint(0,len(tcopy)) sut.replay(tcopy[i:]) e = sut.randomEnabled(rgen) ok = sut.safely(e) elapsed = time.time() - start if RUNNING: #should branch coverage be produced if sut.newBranches() != set([]): for b in sut.newBranches(): print elapsed,len(sut.allBranches()),"New branch",b if not ok: #found a bug, report it bugs+=1 print "Found a FAILURE" #sut.prettyPrintTest(sut.test()) print sut.failure() print "REDUCING" R = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() if FAULT: #save test to a file if FAULT is enabled filename = "failure" + str(bugs) + ".test" sut.saveTest(R,filename) trest = [e] for s in tcopy[i+1:]: if s[1](): trest.append(s) ok = sut.safely(s) if not ok: #found a bug, report it bugs+=1 print "Found a FAILURE" #sut.prettyPrintTest(sut.test()) print sut.failure() print "REDUCING" R = sut.reduce(sut.test(), sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() if FAULT: #save test to a file if FAULT is enabled filename = "failure" + str(bugs) + ".test" sut.saveTest(R,filename) tcopy = test[i:] +trest return tcopy
def action():
global actCount, bugs, failPool
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
if running == 1:
if len(sut.newBranches()) > 0:
print "ACTION:", act[0]
for b in sut.newBranches():
print time.time() - start, len(sut.allBranches()), "New branch", b
for s in sut.newStatements():
print time.time() - start, len(sut.allStatements()),"New statement",s
if not ok:
if faults:
bugs += 1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
failPool.append(sut.test())
collectCoverage()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
fname="failure" + str(bugs) + ".test"
sut.saveTest(sut.test(),fname)
errorSeqs.append(sut.currStatements())
sut.restart()
else:
expand()
return ok
def Find_bugs(list_actions): # print "Find_bugs" global coverage_counter global bugs global Record global string safe = sut.safely(list_actions) running_oh() if not safe: bugs = bugs + 1 print " *** find a bug: ",bugs # bugs = bugs + 1 for t in sut.currStatements(): if t not in coverage_counter: coverage_counter[t]=0 coverage_counter[t] = coverage_counter[t] + 1 # Collect_coverage() print sut.failure() # string = str(sut.failure) Record = True print "reducing..." R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) # record_failures(R) if FAULTS: print"create file" # for x in xrange(1,bugs): # print "save" sut.saveTest(R, failure + str(bugs)+ ".test") # filename = "failure" + str(bugs) + ".test" # sut.saveTest(sut.test(), filename) # sut.restart() return safe
def randomAction():
global actCount, bugs, coverageCount, belowMean,lastAddCoverage
act = sut.randomEnabled(random.Random())
actCount += 1
ok = sut.safely(act)
if running:
if len(sut.newBranches()) > 0:
print "ACTION:",sut.randomEnabled(random.Random(seed))[0]
for b in sut.newBranches():
print time.time() - start, len(sut.allBranches()),"New branch",b
for s1 in sut.newStatements():
print time.time() - start, len(sut.allStatements()),"New statement",b
if not ok or not sut.check():
bugs += 1
print "FOUND A FAILURE"
if faults:
print("SHOW FAULT")
print sut.failure()
error.append(sut.test())
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
sut.restart()
return ok
def randomAction():
global actCount, bugs, fails
sawNew = False
act = sut.randomEnabled(seeds)
actCount += 1
ok = sut.safely(act)
propok = sut.check()
if running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print time.time()-start,len(sut.allBranches()),"New branch",b
sawNew = True
else:
sawNew = False
if not ok or not propok:
if faults:
bugs += 1
fail.append(sut.test())
saveCover()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.restart()
print "FAILURE FOUND.....FAILURES ARE STORING IN FILES"
fault = sut.failure()
fname = 'failure' + str(bugs) + '.test'
wfile = open(fname, 'w+')
wfile.write(str(fault))
wfile.close()
sut.restart()
else:
expandPool()
return ok
def randomAction():
global actCount, bugs, failPool, faults, start, failureFiles
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
elapsed = time.time() - start
if running:
if len(sut.newBranches()) > 0:
print "ACTION:", act[0]
for branch in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", branch
if not ok:
bugs += 1
print "FOUND A FAILURE, REDUCING..."
failPool.append(sut.test())
collectCoverage()
R = sut.reduce(sut.test(),sut.fails, True, True)
if faults:
print "SAVING INTO FILE NAMED failurefile"+str(bugs)
failureFile = "failure" + str(bugs) + ".test"
sut.saveTest(sut.test(), failureFile)
failureFiles.append(failureFile)
sut.restart()
else:
expandPool()
return ok
def newState():
global rgen,sut,collected_test,storedTest,action_cnt,bugfound
for i in xrange(0,DEPTH):
action = sut.randomEnabled(rgen)
no_bug_found = sut.safely(action)
if RUNNING:
if sut.newBranches() != set([]):
for d in sut.newBranches():
print time.time() - start,len(sut.allBranches()),"New Branches",d
if len(sut.newStatements()) > 0:
collected_test = sut.state()
storedTest = True
print "new statement:",sut.newStatements()
if (not storedTest) and (lst_coverage != None) and (lst_coverage in sut.currStatements()):
collected_test = sut.state()
storedTest = True
action_cnt += 1
if not no_bug_found:
bugfound += 1
print "A failure happened here."
rds = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(rds)
print sut.failure()
break
def randomAction():
global bug_no, time_start, allAction, ii
action = sut.randomEnabled(rand)
ok = sut.safely(action)
elapsed = time.time() - time_start
if RUNNING_DETAIL:
if len(sut.newBranches()) > 0:
print "ACTION:", action[0]
for b in sut.newBranches():
print elapsed, len(sut.allBranches()), "New branch", b
if not ok: # found a bug
bug_no += 1
print "FOUND A BUG! #", bug_no
print sut.failure()
print("REDUCING")
reduce = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(reduce)
print(sut.failure())
# write the bug to file if FAULT_ARGUMENT is enabled
if (FAULT_CHECK):
f = open((failure_file + str(bug_no) + ".test"), "w")
print >> f, sut.failure()
# print sequence that causes failure
j = 0
for (s_reduces, _, _) in reduce:
steps_reduce = "# STEP " + str(j)
print >> f, sut.prettyName(s_reduces).ljust(80 - len(steps_reduce), ' '), steps_reduce
j += 1
f.close()
print "time: ", time.time() - time_start
return ok
def randomAction():
global actCount, bugs,newseq,currseq
act = sut.randomEnabled(R)
actCount += 1
ok = sut.safely(act)
if running:
if len(sut.newBranches()) > 0:
print "ACTION:",sut.randomEnabled(random.Random(seed))[0]
for b in sut.newBranches():
print time.time() - start, len(sut.allBranches()),"New branch",b
for s1 in sut.newStatements():
print time.time() - start, len(sut.allStatements()),"New statement",s1
if not ok:
bugs += 1
print "FOUND A FAILURE"
if faults:
f = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(f)
currseq.append((f,set(sut.currStatements())))
print("SHOW FAULT")
file = 'failure' + str(actCount) + '.test'
sut.saveTest(sut.test(), file)
sut.restart()
print sut.failure()
return ok
def tester():
global sut,act,n,test,para,lCovered,actNum,bugs,M,flag, filename
act = sut.randomEnabled(rgen)
flag = 0
n = sut.safely(act)
if running:
if sut.newBranches() != set([]):
for d in sut.newBranches():
print time.time() - start, len(sut.allBranches()),"New branch",d
if len(sut.newStatements()) > 0:
test = sut.state()
para = True
print sut.newStatements()
if (para == False):
if (lCovered != None):
if (lCovered in sut.currStatements()):
test = sut.state()
para = True
actNum += 1
if (n == 0):
bugs += 1
print "A failure has been found!", bugs
print sut.failure()
M = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(M)
print sut.failure()
filename ='failure%d.test'%bugs
sut.saveTest(M,filename)
flag = 1
def action():
global actCount, bugs, failPool
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
if running == 1:
if len(sut.newBranches()) > 0:
print "ACTION:", act[0]
for b in sut.newBranches():
print time.time() - start, len(sut.allBranches()), "New branch", b
if faults:
if not ok:
bugs += 1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
failPool.append(sut.test())
collectCoverage()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
f = open(("failure" + str(bugs) + ".test"),"w")
f.writelines(str(sut.failure()))
f.close()
sut.restart()
else:
expand()
return ok
def running_oh(): global actinos, elapsed, lastAddCoverage, median_even, median_odd if len(sut.newBranches()) != set ([]): print "ACTION:", list_actions[0] for s in sut.newBranches(): print elapsed,len(sut.allBranches()),"New branch",s if len(sut.newStatements())> 0: print "ACTION:", list_actions[0] for s in sut.newStatements(): print elapsed,len(sut.allStatements()),"New statement",s old_test = list(sut.test()) store_new_test = sut.reduce(old_test,sut.coversStatements(sut.newStatements())) sut.replay(old_test) testBuffer.append(store_new_test) currentStatementBuffer.append(set(sut.currStatements())) lastAddCoverage = set(sut.currStatements()) for s in median_odd or median_even: if s in sut.currStatements() and s not in lastAddCoverage: print "find new path to lower coverstatements: ", s testBuffer.append(list(sut.test()) ) currentStatementBuffer.append(set(sut.currStatements())) lastAddCoverage = set(sut.currStatements() )
def notOK():
global bugs, sut
bugs += 1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
def newStatement():
global lastAddCoverage
print "NEW STATEMENTS DISCOVERED",sut.newStatements()
oldTest = list(sut.test())
storeTest = sut.reduce(oldTest,sut.coversStatements(sut.newStatements()))
print "OLD LENGTH = ",len(oldTest),"NEW LENGTH = ",len(storeTest)
sut.replay(oldTest)
fullPool.append((storeTest, set(sut.currStatements())))
lastAddCoverage = set(sut.currStatements())
return
def failure(): global bugs bugs += 1 collectCoverage() R = sut.reduce(sut.test(), sut.fails, True, True) if config.faults : filename = 'failure%d.test'%bugs sut.saveTest(sut.test(), filename) sut.prettyPrintTest(R) sut.restart() return
def main():
global config,rgen,actCount,failCount,ntests,coverageCount
parsed_args, parser = parse_args()
config = make_config(parsed_args, parser)
rgen = random.Random(config.seed)
actCount = 0
failCount = 0
coverageCount = {}
start = time.time()
ntests = 0
while True:
elapsed = time.time() - start
if elapsed > config.timeout:
print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests
break
for i in xrange(0, config.depth):
sut.restart()
ntests += 1
for j in xrange(0, config.width):
act = sut.randomEnabled(rgen)
actCount += 1
ok = sut.safely(act)
if not ok:
failCount += 1
print "FOUND A FAILURE"
collectCoverage()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
break
if config.running:
if sut.newBranches() != set([]):
for b in sut.newBranches():
print time.time()-start, len(sut.allBranches()), "New branch", b
if sut.newStatements() != set([]):
for s in sut.newStatements():
print time.time()-start,len(sut.allStatements()),"New statement",s
collectCoverage()
if config.faults:
print "TOTAL FAULTS", failCount
if config.coverage:
print len(sut.allBranches()),"BRANCHES COVERED"
print len(sut.allStatements()),"STATEMENTS COVERED"
print "TOTAL ACTIONS",actCount
print "TOTAL RUNTIME",time.time()-start
def failures(sut,bugs, Config, faults_file):
bugs +=1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
failPool.append(sut.test())
collectCoverage()
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
def main(): global config,rgen,actCount,failCount,ntests,fullPool parsed_args, parser = parse_args() config = make_config(parsed_args, parser) rgen = random.Random(config.seed) actCount = 0 failCount = 0 fullPool = [] start = time.time() ntests = 0 while time.time() - start < config.timeout: ntests += 1 sut.restart() for d in xrange(0, config.depth): act = sut.randomEnabled(rgen) actCount += 1 ok = sut.safely(act) expandPool() if config.running: if sut.newBranches() != set([]): for b in sut.newBranches(): print time.time()-start, len(sut.allBranches()), "New branch", b if sut.newStatements() != set([]): for s in sut.newStatements(): print time.time()-start, len(sut.allStatements()),"New statement",s if not ok: failCount += 1 if config.faults: failureHandle(failCount) print "FOUND A FAILURE" R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R) print sut.failure() sut.restart() break if time.time() - start > config.timeout: print "STOPPING TEST DUE TO TIMEOUT, TERMINATED AT LENGTH",ntests break if config.faults: print "TOTAL FAULTS", failCount if config.coverage: sut.internalReport() print "TOTAL ACTIONS",actCount print "TOTAL RUNTIME",time.time()-start
def failure():
global bugs, failPool
bugs += 1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
failPool.append(sut.test())
collectCoverage()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
sut.restart()
return
def captureFault():
'Print failure state and reduction'
print "FAILURE LOCATED:"
print sut.failure()
print "REDUCING FAILURE:"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
# output to file for each fault
fname = 'failure%d.test' % nbugs
fo = open(fname, "wb")
fo.write(sut.failure())
fo.close()
def captureFault():
'Print failure state and reduction, save to file'
global nbugs
nbugs += 1
print "FAILURE LOCATED:"
print sut.failure()
print "REDUCING FAILURE:"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
# output to file for each fault
filename = 'failure%d.test' % nbugs
sut.saveTest(R, filename)
def takeTest(actt): global fullPool,bugs,sut ok = sut.safely(actt) collectCoverage() if len(sut.newStatements()) != 0: fullPool.append((list(sut.test()), set(sut.currStatements()))) if not ok : bugs += 1 if Faults : R = sut.reduce(sut.test(),sut.fails, True, True) filename = "failure" + str(bugs) + ".test" sut.saveTest(R,filename)
def Statement(): global belowMid,lastAddCoverage if len(sut.newStatements()) != 0: oldTest = list(sut.test()) storeTest = sut.reduce(oldTest,sut.coversStatements(sut.newStatements())) sut.replay(oldTest) visited.append((storeTest, set(sut.currStatements()))) lastAddCoverage = set(sut.currStatements()) return for s in belowMid: if s in sut.currStatements() and s not in lastAddCoverage: visited.append((list(sut.test()), set(sut.currStatements()))) lastAddCoverage = set(sut.currStatements()) return
def main():
global start,BUDGET,sut,COVERAGE_REPORT,savedcoverage,rgen,storedTest,act,ok,savedcoverage,running,savedcoverage,Num,faults,foundbug,savedTestState
while time.time()-start < BUDGET:
sut.restart()
if (savedcoverage != None) and (rgen.random() > 0.3):
print "Processing"
sut.backtrack(savedcoverage)
storedTest = False
print "First: AVL tree"
for s in xrange(0,100):
act = sut.randomEnabled(rgen)
ok = sut.safely(act)
if running:
if sut.newBranches() != set([]):
## print "ACTION:",a[0],tryStutter
for d in sut.newBranches():
print time.time()-start,len(sut.allBranches()),"New branch",d
if len(sut.newStatements()) > 0:
savedcoverage = sut.state()
storedTest = True
if(running):
print "New Statement",sut.newStatements()
if (not storedTest) and (i != None) and (i in sut.currStatements()):
savedcoverage = sut.state()
storedTest = True
Num += 1
if(faults):
if not ok:
foundbug += 1
print "Failed"
print sut.failure()
print "REDUCE"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
filename = 'failure%d.test'%bugs_found
sut.saveTest(R,filename)
break
savedTestState = sut.state()
expandNewState()
if coverage:
sut.internalReport()
print foundbug,"FAILED"
print "ACTIVE",Num
print "RUNTIME",time.time()-start
def notOK():
global bugs, sut
bugs += 1
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
# save test
name = "failure" + str(bugs) + ".test"
f = sut.test()
sut.saveTest(f, name)
def fault_collection():
if(FAULT_CHECK):
print "use"+str(time.time()-startAll)
print "FOUND A FAILURE"
print sut.failure()
print "REDUCING"
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print "this is fault:"+ str(sut.failure())
fname = 'failure'+str(bugs)+'.test'
sut.saveTest(R,fname)
def Find_bugs(a): #print "aaaaa" global coverage_counter global bugs safe = sut.safely(a) if not safe: print " find a bug: " bugs = bugs + 1 for s in sut.currStatements(): if s not in coverage_counter: coverage_counter[s]=0 coverage_counter[s] = coverage_counter[s] + 1 print sut.failure() R = sut.reduce(sut.test(),sut.fails, True, True) sut.prettyPrintTest(R)
def expand():
global lastAddCoverage
nonErrorSeqs.append(sut.currStatements())
if len(sut.newStatements()) != 0:
print "NEW STATEMENTS DISCOVERED",sut.newStatements()
newSeq = sut.newStatements()
if (newSeq in nonErrorSeqs or newSeq in errorSeqs):
action()
#sut.restart()
else:
oldTest = list(sut.test())
storeTest = sut.reduce(oldTest,sut.coversStatements(newSeq))
print "OLD LENGTH = ",len(oldTest),"NEW LENGTH = ",len(storeTest)
sut.replay(oldTest)
fullPool.append((storeTest, set(sut.currStatements())))
lastAddCoverage = set(sut.currStatements())
return
def expandPool():
global belowMean,lastAddCoverage
if len(sut.newStatements()) != 0:
print "NEW STATEMENTS DISCOVERED",sut.newStatements()
oldTest = list(sut.test())
storeTest = sut.reduce(oldTest,sut.coversStatements(sut.newStatements()))
print "OLD LENGTH = ",len(oldTest),"NEW LENGTH = ",len(storeTest)
sut.replay(oldTest)
fullPool.append((storeTest, set(sut.currStatements())))
lastAddCoverage = set(sut.currStatements())
return
for s in belowMean:
if s in sut.currStatements() and s not in lastAddCoverage:
print "NEW PATH TO LOW COVERAGE STATEMENT",s
fullPool.append((list(sut.test()), set(sut.currStatements())))
lastAddCoverage = set(sut.currStatements())
return
def newFunction():
global savecoverage_test,rgen,depth,explore,savecoverage_test,Number,sut,BUDGET,seed,width,faults,coverage,running,bugs_found,Cover_percent,Coverage_W,Least,weight,Coverage_all,start
sut.restart()
if (savecoverage_test != None) and (rgen.random() > explore):
print "processing"
sut.backtrack(savecoverage_test)
storedTest = False
print "part1: AVL"
for s in xrange(0,100):
act = sut.randomEnabled(rgen)
ok = sut.safely(act)
if len(sut.newStatements()) > 0:
savecoverage_test = sut.state()
storedTest = True
print "New state",sut.newStatements()
if (not storedTest) and (Least != None) and (Least in sut.currStatements()):
#print "SAW LEAST COVERED STATEMENT, STORING TEST"
savecoverage_test = sut.state()
storedTest = True
Number += 1
if not ok:
bugs_found += 1
print "FAILURE"
#sut.prettyPrintTest(sut.test())
print sut.failure()
R = sut.reduce(sut.test(),sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
break
for s in sut.currStatements():
if s not in Cover_percent:
Cover_percent[s] = 0
Cover_percent[s] += 1
sortedCov = sorted(Cover_percent.keys(), key=lambda x: Cover_percent[x])
# go through the sortedCov and assign weights on them
print "part2: coverage"
# weight is calculated by: coverage * (mean - Cover_percent),
# the greater the difference between the mean and the coverage count,
# the larger your weight will be
for t in sortedCov:
weight = (Coverage_all - Cover_percent[t])
weightedCov = t*weight
if weightedCov > 20:
Coverage_W.append(weightedCov)
print "Coverage:", t
import sys
rgen = random.Random()
DEPTH = 100
NUM_TESTS = 100
sut = sut.sut()
#sut.silenceCoverage()
for t in xrange(0,NUM_TESTS):
sut.restart()
for s in xrange(0,DEPTH):
action = sut.randomEnabled(rgen)
ok = sut.safely(action)
propok = sut.check()
if ((not ok) or (not propok)):
#sut.prettyPrintTest(sut.test())
print "TEST FAILED"
print "REDUCING..."
R = sut.reduce(sut.test(), lambda x: sut.fails(x) or sut.failsCheck(x))
sut.prettyPrintTest(R)
print "NORMALIZING..."
N = sut.normalize(R, lambda x: sut.fails(x) or sut.failsCheck(x))
#sut.prettyPrintTest(N)
sut.generalize(N, lambda x: sut.fails(x) or sut.failsCheck(x))
sys.exit(1)
sut.internalReport()
start = time.time()
while time.time() - start < BUDGET:
sut.restart()
for s in xrange(0, depth):
acts = sut.randomEnableds(rgen, k)
sortActs = sorted(acts, key=lambda x: actionCount[abstract(x[0])])
act = sortActs[0]
ok = sut.safely(act)
actCount += 1
actionCount[abstract(act[0])] += 1
if not ok:
bugs += 1
print "FOUND A FAILURE"
#sut.prettyPrintTest(sut.test())
print sut.failure()
print "REDUCING"
R = sut.reduce(sut.test(), sut.fails, True, True)
sut.prettyPrintTest(R)
print sut.failure()
sut.restart()
sut.internalReport()
orderedActs = sorted(actionCount.keys(), key=lambda x: actionCount[x])
for act in orderedActs:
print act, actionCount[act]
print bugs, "FAILED"
print "TOTAL ACTIONS", actCount
print "TOTAL RUNTIME", time.time() - start
random.shuffle(forder)
for f in forder:
if os.path.exists(f.replace("full","normalized")):
continue
print "PROCESSING",f
print "EXECUTING"
t = sut.loadTest(f)
assert(sut.failsAny(t))
tfail = sut.failure()
print tfail
sys.stdout.flush()
with open(f.replace("full","failure"),'w') as fout:
fout.write(repr(sut.failure()[1]) + "\n")
sameFail = lambda x: sut.failsAny(x,failure=tfail)
tstart = time.time()
r = sut.reduce(t, sameFail, keepLast=False)
rtime = time.time()-tstart
sys.stdout.flush()
with open(f.replace("full","reducetime"),'w') as fout:
fout.write(str(rtime) + "\n")
sut.saveTest(r, f.replace("full","reduced"))
print "NORMALIZING"
sut.clearNormalizationCache()
tstart = time.time()
n = sut.normalize(r, sameFail, keepLast=False, verbose=True)
ntime = time.time()-tstart
sys.stdout.flush()
with open(f.replace("full","normtime"),'w') as fout:
fout.write(str(ntime) + "\n")
sut.saveTest(n, f.replace("full","normalized"))
break