def lithReduceCmd(strategy): """Lithium reduction commands accepting various strategies.""" reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print sps.shellify([lithiumpy] + fullLithArgs) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium(fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc, targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def lithReduceCmd(strategy): '''Lithium reduction commands accepting various strategies.''' reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print sps.shellify([lithiumpy] + fullLithArgs) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium( fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc, targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def lithReduceCmd(strategy): """Lithium reduction commands accepting various strategies.""" reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print( sps.shellify([sys.executable, "-u", "-m", "lithium"] + fullLithArgs)) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium( fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def many_timed_runs(targetTime, tempDir, args, collector, quiet=True): startTime = time.time() iteration = 0 results = {} fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js")) linkFuzzer(fuzzerJS) os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS bc = domInteresting.BrowserConfig(args, collector) browserDir = bc.options.browserDir reftestFilesDir = bc.dirs.reftestFilesDir reftestURLs = getURLs(os.path.abspath(reftestFilesDir)) writeStats(tempDir, iteration, results) while True: if targetTime and time.time() > startTime + targetTime: print "Out of time!" os.remove(fuzzerJS) if len(os.listdir(tempDir)) == 0: os.rmdir(tempDir) break iteration += 1 url = bc.options.argURL or (random.choice(reftestURLs) + randomHash()) extraPrefs = randomPrefs.randomPrefs() logPrefix = os.path.join(tempDir, "q" + str(iteration)) now = datetime.datetime.isoformat(datetime.datetime.now(), " ") print print "%%% " + now + " starting q" + str(iteration) + ": " + url result = domInteresting.BrowserResult(bc, url, logPrefix, extraPrefs=extraPrefs, quiet=quiet) while result.level > domInteresting.DOM_FINE: if collector: _, cacheMetadata = collector.search(result.crashInfo) if cacheMetadata is not None: if cacheMetadata['frequent']: print "Frequent crash matched existing signature: %s" % cacheMetadata[ "shortDescription"] break print "loopdomfuzz.py: will try reducing from " + url rFN = createReproFile(fuzzerJS, extraPrefs, result.lines, logPrefix) if platform.system() == "Windows": rFN = rFN.replace( "/", "\\" ) # Ensure both Lithium and Firefox understand the filename extraRDFArgs = ["--valgrind"] if bc.options.valgrind else [] lithArgs = [domInterestingpy] + extraRDFArgs + [ "-m%d" % result.level, browserDir, rFN ] (lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2) if lithresult == lithOps.LITH_NO_REPRO: print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all." retestResult = domInteresting.BrowserResult( bc, url, logPrefix + "-retry", extraPrefs=extraPrefs) if retestResult.level > domInteresting.DOM_FINE: print "%%% Lithium can't reproduce, but I can!" quality = 9 with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile: reproOnlyFile.write( "I was able to reproduce an issue at the same URL, but Lithium was not.\n\n" ) reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL else: print "%%% Lithium can't reproduce, and neither can I." quality = 10 with open(logPrefix + "-sorry.txt", "w") as sorryFile: sorryFile.write( "I wasn't even able to reproduce with the same URL.\n\n" ) sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_AT_ALL elif lithresult == lithOps.LITH_FINISHED: # Upload the output and crash information for the reduced testcase, not the original reducedResult = domInteresting.BrowserResult( bc, rFN, logPrefix + "-final", extraPrefs=extraPrefs) if reducedResult.level > domInteresting.DOM_FINE: result = reducedResult quality = 0 else: quality = 6 else: quality = 12 results.setdefault(quality, 0) results[quality] += 1 if collector: # ddsize = lithOps.ddsize(rFN) collector.submit(result.crashInfo, rFN, quality) break writeStats(tempDir, iteration, results) if bc.options.argURL: break
def botmain(options): options.tempDir = tempfile.mkdtemp("fuzzbot") print options.tempDir if options.remote_host: printMachineInfo() #sendEmail("justInWhileLoop", "Platform details , " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong") buildInfo = ensureBuild(options) assert os.path.isdir(buildInfo.buildDir) if options.retestRoot: print "Retesting time!" retestAll(options, buildInfo) else: options.relevantJobsDirName = options.testType + "-" + buildInfo.buildType options.relevantJobsDir = options.baseDir + options.relevantJobsDirName + options.remoteSep runCommand(options.remote_host, "mkdir -p " + options.baseDir) # don't want this created recursively, because "mkdir -p" is weird with modes runCommand(options.remote_host, "chmod og+rx " + options.baseDir) runCommand(options.remote_host, "mkdir -p " + options.relevantJobsDir) runCommand(options.remote_host, "chmod og+rx " + options.relevantJobsDir) (job, oldjobname, takenNameOnServer) = grabJob(options, "_needsreduction") if job: print "Reduction time!" lithArgs = readTinyFile(job + "lithium-command.txt").strip().split(" ") lithArgs[-1] = job + splitSlash(lithArgs[-1])[-1] # options.tempDir may be different if platform.system() == "Windows": # Ensure both Lithium and Firefox understand the filename lithArgs[-1] = lithArgs[-1].replace("/", "\\") logPrefix = job + "reduce" + timestamp() (lithResult, lithDetails) = lithOps.runLithium(lithArgs, logPrefix, options.targetTime) uploadJob(options, lithResult, lithDetails, job, oldjobname) runCommand(options.remote_host, "rm -rf " + takenNameOnServer) else: print "Fuzz time!" #if options.testType == 'js': # if sps.isLinux: # Test to see whether releng AWS Linux instances can send email # print "Sending email..." # sendEmail("justFuzzTime", "Platform details (" + str(multiprocessing.cpu_count()) + " cores), " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong") # print "Email sent!" numProcesses = multiprocessing.cpu_count() if "-asan" in buildInfo.buildDir: # This should really be based on the amount of RAM available, but I don't know how to compute that in Python. # I could guess 1 GB RAM per core, but that wanders into sketchyville. numProcesses = max(numProcesses // 2, 1) if sps.isARMv7l: # Even though ARM boards generally now have many cores, each core is not as powerful # as x86/64 ones, so restrict fuzzing to only 1 core for now. numProcesses = 1 forkJoin.forkJoin(options.tempDir, numProcesses, fuzzUntilBug, options, buildInfo) # Remove build directory if we created it if options.testType == 'dom' and not \ (options.retestRoot or options.existingBuildDir or options.buildOptions is not None): shutil.rmtree(buildInfo.buildDir) shutil.rmtree(options.tempDir)
def botmain(options): options.tempDir = tempfile.mkdtemp("fuzzbot") print options.tempDir if options.remote_host: printMachineInfo() #sendEmail("justInWhileLoop", "Platform details , " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong") buildInfo = ensureBuild(options) assert os.path.isdir(buildInfo.buildDir) if options.retestRoot: print "Retesting time!" retestAll(options, buildInfo) else: options.relevantJobsDirName = options.testType + "-" + buildInfo.buildType options.relevantJobsDir = options.baseDir + options.relevantJobsDirName + options.remoteSep runCommand( options.remote_host, "mkdir -p " + options.baseDir ) # don't want this created recursively, because "mkdir -p" is weird with modes runCommand(options.remote_host, "chmod og+rx " + options.baseDir) runCommand(options.remote_host, "mkdir -p " + options.relevantJobsDir) runCommand(options.remote_host, "chmod og+rx " + options.relevantJobsDir) (job, oldjobname, takenNameOnServer) = grabJob(options, "_needsreduction") if job: print "Reduction time!" lithArgs = readTinyFile(job + "lithium-command.txt").strip().split(" ") lithArgs[-1] = job + splitSlash( lithArgs[-1])[-1] # options.tempDir may be different if platform.system() == "Windows": # Ensure both Lithium and Firefox understand the filename lithArgs[-1] = lithArgs[-1].replace("/", "\\") logPrefix = job + "reduce" + timestamp() (lithResult, lithDetails) = lithOps.runLithium(lithArgs, logPrefix, options.targetTime) uploadJob(options, lithResult, lithDetails, job, oldjobname) runCommand(options.remote_host, "rm -rf " + takenNameOnServer) else: print "Fuzz time!" #if options.testType == 'js': # if sps.isLinux: # Test to see whether releng AWS Linux instances can send email # print "Sending email..." # sendEmail("justFuzzTime", "Platform details (" + str(multiprocessing.cpu_count()) + " cores), " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong") # print "Email sent!" numProcesses = multiprocessing.cpu_count() if "-asan" in buildInfo.buildDir: # This should really be based on the amount of RAM available, but I don't know how to compute that in Python. # I could guess 1 GB RAM per core, but that wanders into sketchyville. numProcesses = max(numProcesses // 2, 1) if sps.isARMv7l: # Even though ARM boards generally now have many cores, each core is not as powerful # as x86/64 ones, so restrict fuzzing to only 1 core for now. numProcesses = 1 forkJoin.forkJoin(options.tempDir, numProcesses, fuzzUntilBug, options, buildInfo) # Remove build directory if we created it if options.testType == 'dom' and not \ (options.retestRoot or options.existingBuildDir or options.buildOptions is not None): shutil.rmtree(buildInfo.buildDir) shutil.rmtree(options.tempDir)
def many_timed_runs(targetTime, tempDir, args, quiet=True): startTime = time.time() iteration = 0 fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js")) linkFuzzer(fuzzerJS) os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS levelAndLines, options = domInteresting.rdfInit(args) browserDir = options.browserDir reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir reftestURLs = getURLs(os.path.abspath(reftestFilesDir)) while True: if targetTime and time.time() > startTime + targetTime: print "Out of time!" os.remove(fuzzerJS) if len(os.listdir(tempDir)) == 0: os.rmdir(tempDir) return (lithOps.HAPPY, None) iteration += 1 url = options.argURL or (random.choice(reftestURLs) + randomHash()) extraPrefs = randomPrefs.randomPrefs() logPrefix = os.path.join(tempDir, "q" + str(iteration)) now = datetime.datetime.isoformat(datetime.datetime.now(), " ") print print "%%% " + now + " starting q" + str(iteration) + ": " + url level, lines = levelAndLines(url, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=quiet) if level > domInteresting.DOM_FINE: print "loopdomfuzz.py: will try reducing from " + url rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix) if platform.system() == "Windows": rFN = rFN.replace("/", "\\") # Ensure both Lithium and Firefox understand the filename extraRDFArgs = ["--valgrind"] if options.valgrind else [] lithArgs = [domInterestingpy] + extraRDFArgs + ["-m%d" % level, browserDir, rFN] (lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2) if lithresult == lithOps.LITH_NO_REPRO: os.remove(rFN) print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all." level2, _ = levelAndLines(url, logPrefix=logPrefix + "-retry", extraPrefs=extraPrefs) if level2 > domInteresting.DOM_FINE: print "%%% Lithium can't reproduce, but I can!" with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile: reproOnlyFile.write( "I was able to reproduce an issue at the same URL, but Lithium was not.\n\n" ) reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL else: print "%%% Lithium can't reproduce, and neither can I." with open(logPrefix + "-sorry.txt", "w") as sorryFile: sorryFile.write("I wasn't even able to reproduce with the same URL.\n\n") sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_AT_ALL print "" if targetTime: return (lithresult, lithdetails) if options.argURL: break
def many_timed_runs(targetTime, tempDir, args, quiet=True): startTime = time.time() iteration = 0 fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js")) linkFuzzer(fuzzerJS) os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS levelAndLines, options = domInteresting.rdfInit(args) browserDir = options.browserDir reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir reftestURLs = getURLs(os.path.abspath(reftestFilesDir)) while True: if targetTime and time.time() > startTime + targetTime: print "Out of time!" os.remove(fuzzerJS) if len(os.listdir(tempDir)) == 0: os.rmdir(tempDir) return (lithOps.HAPPY, None) iteration += 1 url = options.argURL or (random.choice(reftestURLs) + randomHash()) extraPrefs = randomPrefs.randomPrefs() logPrefix = os.path.join(tempDir, "q" + str(iteration)) now = datetime.datetime.isoformat(datetime.datetime.now(), " ") print print "%%% " + now + " starting q" + str(iteration) + ": " + url level, lines = levelAndLines(url, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=quiet) if level > domInteresting.DOM_FINE: print "loopdomfuzz.py: will try reducing from " + url rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix) if platform.system() == "Windows": rFN = rFN.replace( "/", "\\" ) # Ensure both Lithium and Firefox understand the filename extraRDFArgs = ["--valgrind"] if options.valgrind else [] lithArgs = [domInterestingpy ] + extraRDFArgs + ["-m%d" % level, browserDir, rFN] (lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2) if lithresult == lithOps.LITH_NO_REPRO: os.remove(rFN) print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all." level2, _ = levelAndLines(url, logPrefix=logPrefix + "-retry", extraPrefs=extraPrefs) if level2 > domInteresting.DOM_FINE: print "%%% Lithium can't reproduce, but I can!" with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile: reproOnlyFile.write( "I was able to reproduce an issue at the same URL, but Lithium was not.\n\n" ) reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL else: print "%%% Lithium can't reproduce, and neither can I." with open(logPrefix + "-sorry.txt", "w") as sorryFile: sorryFile.write( "I wasn't even able to reproduce with the same URL.\n\n" ) sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n") lithresult = lithOps.NO_REPRO_AT_ALL print "" if targetTime: return (lithresult, lithdetails) if options.argURL: break