def retestAll(options, buildInfo): ''' Retest all testcases in options.retestRoot, starting with the newest, without modifying that subtree (because it might be rsync'ed). ''' assert options.testType == "dom" testcases = [] retestSkips = readSkips(options.retestSkips) # Find testcases to retest for jobTypeDir in (os.path.join(options.retestRoot, x) for x in os.listdir(options.retestRoot) if x.startswith(options.testType + "-")): for j in os.listdir(jobTypeDir): if "-asan" in buildInfo.buildDir and "-asan" not in jobTypeDir: pass elif j.split("_")[0] in retestSkips: print "Skipping " + j + " for " + j.split("_")[0] elif "_0_lines" in j: print "Skipping a 0-line testcase" elif "_reduced" in j: job = os.path.join(jobTypeDir, j) testcase_leafs = filter(lambda s: s.find("reduced") != -1, os.listdir(job)) if len(testcase_leafs) == 1: testcase = os.path.join(job, testcase_leafs[0]) mtime = os.stat(testcase).st_mtime testcases.append({'testcase': testcase, 'mtime': mtime}) # Sort so the newest testcases are first print "Retesting " + str(len(testcases)) + " testcases..." testcases.sort(key=lambda t: t['mtime'], reverse=True) i = 0 levelAndLines, domInterestingOptions = domInteresting.rdfInit([buildInfo.buildDir]) tempDir = tempfile.mkdtemp("retesting") # Retest all the things! for t in testcases: testcase = t['testcase'] print testcase i += 1 logPrefix = os.path.join(tempDir, str(i)) extraPrefs = randomPrefs.grabExtraPrefs(testcase) testcaseURL = loopdomfuzz.asFileURL(testcase) level, lines = levelAndLines(testcaseURL, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=True) #if level > domInteresting.DOM_FINE: # print "Reproduced: " + testcase # with open(logPrefix + "-summary.txt") as f: # for line in f: # print line, # Would it be easier to do it this way? #with open(os.devnull, "w") as devnull: # p = subprocess.Popen([loopdomfuzz.domInterestingpy, "mozilla-central/obj-firefox-asan-debug/", testcase], stdout=devnull, stderr=subprocess.STDOUT) # if p.wait() > 0: # print "Still reproduces: " + testcase # Ideally we'd use something like "lithium-command.txt" to get the right --valgrind args, etc... # (but we don't want the --min-level option) # Or this way? #lithArgs = ["--strategy=check-only", loopdomfuzz.domInterestingpy, buildInfo.buildDir, testcase] # #(lithResult, lithDetails) = lithOps.runLithium(lithArgs, logPrefix, options.targetTime) #if lithResult == lithOps.LITH_RETESTED_STILL_INTERESTING: # print "Reproduced: " + testcase shutil.rmtree(tempDir)
def many_timed_runs(targetTime, tempDir, args, quiet=True): startTime = time.time() iteration = 0 fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js")) linkFuzzer(fuzzerJS) os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS levelAndLines, options = domInteresting.rdfInit(args) browserDir = options.browserDir reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir reftestURLs = getURLs(os.path.abspath(reftestFilesDir)) while True: if targetTime and time.time() > startTime + targetTime: print "Out of time!" os.remove(fuzzerJS) if len(os.listdir(tempDir)) == 0: os.rmdir(tempDir) return (lithOps.HAPPY, None) iteration += 1 url = options.argURL or (random.choice(reftestURLs) + randomHash()) extraPrefs = randomPrefs.randomPrefs() logPrefix = os.path.join(tempDir, "q" + str(iteration)) now = datetime.datetime.isoformat(datetime.datetime.now(), " ") print print "%%% " + now + " starting q" + str(iteration) + ": " + url level, lines = levelAndLines(url, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=quiet) if level > domInteresting.DOM_FINE: print "loopdomfuzz.py: will try reducing from " + url rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix) if platform.system() == "Windows": rFN = rFN.replace("/", "\\") # Ensure both Lithium and Firefox understand the filename extraRDFArgs = ["--valgrind"] if options.valgrind else [] lithArgs = [domInterestingpy] + extraRDFArgs + ["-m%d" % level, browserDir, rFN] (lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2) if lithresult == lithOps.LITH_NO_REPRO: os.remove(rFN) print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all." level2, _ = levelAndLines(url, logPrefix=logPrefix + "-retry", extraPrefs=extraPrefs) if level2 > domInteresting.DOM_FINE: print "%%% Lithium can't reproduce, but I can!" with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile: reproOnlyFile.write( "I was able to reproduce an issue at the same URL, but Lithium was not.\n\n" ) reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL else: print "%%% Lithium can't reproduce, and neither can I." with open(logPrefix + "-sorry.txt", "w") as sorryFile: sorryFile.write("I wasn't even able to reproduce with the same URL.\n\n") sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_AT_ALL print "" if targetTime: return (lithresult, lithdetails) if options.argURL: break
def retestAll(options, buildInfo): ''' Retest all testcases in options.retestRoot, starting with the newest, without modifying that subtree (because it might be rsync'ed). ''' assert options.testType == "dom" testcases = [] retestSkips = readSkips(options.retestSkips) # Find testcases to retest for jobTypeDir in (os.path.join(options.retestRoot, x) for x in os.listdir(options.retestRoot) if x.startswith(options.testType + "-")): for j in os.listdir(jobTypeDir): if "-asan" in buildInfo.buildDir and "-asan" not in jobTypeDir: pass elif j.split("_")[0] in retestSkips: print "Skipping " + j + " for " + j.split("_")[0] elif "_0_lines" in j: print "Skipping a 0-line testcase" elif "_reduced" in j: job = os.path.join(jobTypeDir, j) testcase_leafs = filter(lambda s: s.find("reduced") != -1, os.listdir(job)) if len(testcase_leafs) == 1: testcase = os.path.join(job, testcase_leafs[0]) mtime = os.stat(testcase).st_mtime testcases.append({'testcase': testcase, 'mtime': mtime}) # Sort so the newest testcases are first print "Retesting " + str(len(testcases)) + " testcases..." testcases.sort(key=lambda t: t['mtime'], reverse=True) i = 0 levelAndLines, domInterestingOptions = domInteresting.rdfInit( [buildInfo.buildDir]) tempDir = tempfile.mkdtemp("retesting") # Retest all the things! for t in testcases: testcase = t['testcase'] print testcase i += 1 logPrefix = os.path.join(tempDir, str(i)) extraPrefs = randomPrefs.grabExtraPrefs(testcase) testcaseURL = loopdomfuzz.asFileURL(testcase) level, lines = levelAndLines(testcaseURL, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=True) #if level > domInteresting.DOM_FINE: # print "Reproduced: " + testcase # with open(logPrefix + "-summary.txt") as f: # for line in f: # print line, # Would it be easier to do it this way? #with open(os.devnull, "w") as devnull: # p = subprocess.Popen([loopdomfuzz.domInterestingpy, "mozilla-central/obj-firefox-asan-debug/", testcase], stdout=devnull, stderr=subprocess.STDOUT) # if p.wait() > 0: # print "Still reproduces: " + testcase # Ideally we'd use something like "lithium-command.txt" to get the right --valgrind args, etc... # (but we don't want the --min-level option) # Or this way? #lithArgs = ["--strategy=check-only", loopdomfuzz.domInterestingpy, buildInfo.buildDir, testcase] # #(lithResult, lithDetails) = lithOps.runLithium(lithArgs, logPrefix, options.targetTime) #if lithResult == lithOps.LITH_RETESTED_STILL_INTERESTING: # print "Reproduced: " + testcase shutil.rmtree(tempDir)
def many_timed_runs(targetTime, tempDir, args, quiet=True): startTime = time.time() iteration = 0 fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js")) linkFuzzer(fuzzerJS) os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS levelAndLines, options = domInteresting.rdfInit(args) browserDir = options.browserDir reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir reftestURLs = getURLs(os.path.abspath(reftestFilesDir)) while True: if targetTime and time.time() > startTime + targetTime: print "Out of time!" os.remove(fuzzerJS) if len(os.listdir(tempDir)) == 0: os.rmdir(tempDir) return (lithOps.HAPPY, None) iteration += 1 url = options.argURL or (random.choice(reftestURLs) + randomHash()) extraPrefs = randomPrefs.randomPrefs() logPrefix = os.path.join(tempDir, "q" + str(iteration)) now = datetime.datetime.isoformat(datetime.datetime.now(), " ") print print "%%% " + now + " starting q" + str(iteration) + ": " + url level, lines = levelAndLines(url, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=quiet) if level > domInteresting.DOM_FINE: print "loopdomfuzz.py: will try reducing from " + url rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix) if platform.system() == "Windows": rFN = rFN.replace( "/", "\\" ) # Ensure both Lithium and Firefox understand the filename extraRDFArgs = ["--valgrind"] if options.valgrind else [] lithArgs = [domInterestingpy ] + extraRDFArgs + ["-m%d" % level, browserDir, rFN] (lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2) if lithresult == lithOps.LITH_NO_REPRO: os.remove(rFN) print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all." level2, _ = levelAndLines(url, logPrefix=logPrefix + "-retry", extraPrefs=extraPrefs) if level2 > domInteresting.DOM_FINE: print "%%% Lithium can't reproduce, but I can!" with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile: reproOnlyFile.write( "I was able to reproduce an issue at the same URL, but Lithium was not.\n\n" ) reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL else: print "%%% Lithium can't reproduce, and neither can I." with open(logPrefix + "-sorry.txt", "w") as sorryFile: sorryFile.write( "I wasn't even able to reproduce with the same URL.\n\n" ) sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_AT_ALL print "" if targetTime: return (lithresult, lithdetails) if options.argURL: break