def lithReduceCmd(strategy):
"""Lithium reduction commands accepting various strategies."""
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements
print sps.shellify([lithiumpy] + fullLithArgs)
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(fullLithArgs, logPrefix + "-" +
str(reductionCount[0]) + desc, targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def lithReduceCmd(strategy):
'''Lithium reduction commands accepting various strategies.'''
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs)
if x] # Remove empty elements
print sps.shellify([lithiumpy] + fullLithArgs)
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(
fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc,
targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def lithReduceCmd(strategy):
"""Lithium reduction commands accepting various strategies."""
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs)
if x] # Remove empty elements
print(
sps.shellify([sys.executable, "-u", "-m", "lithium"] +
fullLithArgs))
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(
fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc),
targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def many_timed_runs(targetTime, tempDir, args, collector, quiet=True):
startTime = time.time()
iteration = 0
results = {}
fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js"))
linkFuzzer(fuzzerJS)
os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS
bc = domInteresting.BrowserConfig(args, collector)
browserDir = bc.options.browserDir
reftestFilesDir = bc.dirs.reftestFilesDir
reftestURLs = getURLs(os.path.abspath(reftestFilesDir))
writeStats(tempDir, iteration, results)
while True:
if targetTime and time.time() > startTime + targetTime:
print "Out of time!"
os.remove(fuzzerJS)
if len(os.listdir(tempDir)) == 0:
os.rmdir(tempDir)
break
iteration += 1
url = bc.options.argURL or (random.choice(reftestURLs) + randomHash())
extraPrefs = randomPrefs.randomPrefs()
logPrefix = os.path.join(tempDir, "q" + str(iteration))
now = datetime.datetime.isoformat(datetime.datetime.now(), " ")
print
print "%%% " + now + " starting q" + str(iteration) + ": " + url
result = domInteresting.BrowserResult(bc,
url,
logPrefix,
extraPrefs=extraPrefs,
quiet=quiet)
while result.level > domInteresting.DOM_FINE:
if collector:
_, cacheMetadata = collector.search(result.crashInfo)
if cacheMetadata is not None:
if cacheMetadata['frequent']:
print "Frequent crash matched existing signature: %s" % cacheMetadata[
"shortDescription"]
break
print "loopdomfuzz.py: will try reducing from " + url
rFN = createReproFile(fuzzerJS, extraPrefs, result.lines,
logPrefix)
if platform.system() == "Windows":
rFN = rFN.replace(
"/", "\\"
) # Ensure both Lithium and Firefox understand the filename
extraRDFArgs = ["--valgrind"] if bc.options.valgrind else []
lithArgs = [domInterestingpy] + extraRDFArgs + [
"-m%d" % result.level, browserDir, rFN
]
(lithresult,
lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime
and targetTime // 2)
if lithresult == lithOps.LITH_NO_REPRO:
print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all."
retestResult = domInteresting.BrowserResult(
bc, url, logPrefix + "-retry", extraPrefs=extraPrefs)
if retestResult.level > domInteresting.DOM_FINE:
print "%%% Lithium can't reproduce, but I can!"
quality = 9
with open(logPrefix + "-repro-only.txt",
"w") as reproOnlyFile:
reproOnlyFile.write(
"I was able to reproduce an issue at the same URL, but Lithium was not.\n\n"
)
reproOnlyFile.write(domInterestingpy + " " +
browserDir + " " + url + "\n")
lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL
else:
print "%%% Lithium can't reproduce, and neither can I."
quality = 10
with open(logPrefix + "-sorry.txt", "w") as sorryFile:
sorryFile.write(
"I wasn't even able to reproduce with the same URL.\n\n"
)
sorryFile.write(domInterestingpy + " " + browserDir +
" " + url + "\n")
lithresult = lithOps.NO_REPRO_AT_ALL
elif lithresult == lithOps.LITH_FINISHED:
# Upload the output and crash information for the reduced testcase, not the original
reducedResult = domInteresting.BrowserResult(
bc, rFN, logPrefix + "-final", extraPrefs=extraPrefs)
if reducedResult.level > domInteresting.DOM_FINE:
result = reducedResult
quality = 0
else:
quality = 6
else:
quality = 12
results.setdefault(quality, 0)
results[quality] += 1
if collector:
# ddsize = lithOps.ddsize(rFN)
collector.submit(result.crashInfo, rFN, quality)
break
writeStats(tempDir, iteration, results)
if bc.options.argURL:
break
def botmain(options):
options.tempDir = tempfile.mkdtemp("fuzzbot")
print options.tempDir
if options.remote_host:
printMachineInfo()
#sendEmail("justInWhileLoop", "Platform details , " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong")
buildInfo = ensureBuild(options)
assert os.path.isdir(buildInfo.buildDir)
if options.retestRoot:
print "Retesting time!"
retestAll(options, buildInfo)
else:
options.relevantJobsDirName = options.testType + "-" + buildInfo.buildType
options.relevantJobsDir = options.baseDir + options.relevantJobsDirName + options.remoteSep
runCommand(options.remote_host, "mkdir -p " + options.baseDir) # don't want this created recursively, because "mkdir -p" is weird with modes
runCommand(options.remote_host, "chmod og+rx " + options.baseDir)
runCommand(options.remote_host, "mkdir -p " + options.relevantJobsDir)
runCommand(options.remote_host, "chmod og+rx " + options.relevantJobsDir)
(job, oldjobname, takenNameOnServer) = grabJob(options, "_needsreduction")
if job:
print "Reduction time!"
lithArgs = readTinyFile(job + "lithium-command.txt").strip().split(" ")
lithArgs[-1] = job + splitSlash(lithArgs[-1])[-1] # options.tempDir may be different
if platform.system() == "Windows":
# Ensure both Lithium and Firefox understand the filename
lithArgs[-1] = lithArgs[-1].replace("/", "\\")
logPrefix = job + "reduce" + timestamp()
(lithResult, lithDetails) = lithOps.runLithium(lithArgs, logPrefix, options.targetTime)
uploadJob(options, lithResult, lithDetails, job, oldjobname)
runCommand(options.remote_host, "rm -rf " + takenNameOnServer)
else:
print "Fuzz time!"
#if options.testType == 'js':
# if sps.isLinux: # Test to see whether releng AWS Linux instances can send email
# print "Sending email..."
# sendEmail("justFuzzTime", "Platform details (" + str(multiprocessing.cpu_count()) + " cores), " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong")
# print "Email sent!"
numProcesses = multiprocessing.cpu_count()
if "-asan" in buildInfo.buildDir:
# This should really be based on the amount of RAM available, but I don't know how to compute that in Python.
# I could guess 1 GB RAM per core, but that wanders into sketchyville.
numProcesses = max(numProcesses // 2, 1)
if sps.isARMv7l:
# Even though ARM boards generally now have many cores, each core is not as powerful
# as x86/64 ones, so restrict fuzzing to only 1 core for now.
numProcesses = 1
forkJoin.forkJoin(options.tempDir, numProcesses, fuzzUntilBug, options, buildInfo)
# Remove build directory if we created it
if options.testType == 'dom' and not \
(options.retestRoot or options.existingBuildDir or options.buildOptions is not None):
shutil.rmtree(buildInfo.buildDir)
shutil.rmtree(options.tempDir)
def botmain(options):
options.tempDir = tempfile.mkdtemp("fuzzbot")
print options.tempDir
if options.remote_host:
printMachineInfo()
#sendEmail("justInWhileLoop", "Platform details , " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong")
buildInfo = ensureBuild(options)
assert os.path.isdir(buildInfo.buildDir)
if options.retestRoot:
print "Retesting time!"
retestAll(options, buildInfo)
else:
options.relevantJobsDirName = options.testType + "-" + buildInfo.buildType
options.relevantJobsDir = options.baseDir + options.relevantJobsDirName + options.remoteSep
runCommand(
options.remote_host, "mkdir -p " + options.baseDir
) # don't want this created recursively, because "mkdir -p" is weird with modes
runCommand(options.remote_host, "chmod og+rx " + options.baseDir)
runCommand(options.remote_host, "mkdir -p " + options.relevantJobsDir)
runCommand(options.remote_host,
"chmod og+rx " + options.relevantJobsDir)
(job, oldjobname, takenNameOnServer) = grabJob(options,
"_needsreduction")
if job:
print "Reduction time!"
lithArgs = readTinyFile(job +
"lithium-command.txt").strip().split(" ")
lithArgs[-1] = job + splitSlash(
lithArgs[-1])[-1] # options.tempDir may be different
if platform.system() == "Windows":
# Ensure both Lithium and Firefox understand the filename
lithArgs[-1] = lithArgs[-1].replace("/", "\\")
logPrefix = job + "reduce" + timestamp()
(lithResult,
lithDetails) = lithOps.runLithium(lithArgs, logPrefix,
options.targetTime)
uploadJob(options, lithResult, lithDetails, job, oldjobname)
runCommand(options.remote_host, "rm -rf " + takenNameOnServer)
else:
print "Fuzz time!"
#if options.testType == 'js':
# if sps.isLinux: # Test to see whether releng AWS Linux instances can send email
# print "Sending email..."
# sendEmail("justFuzzTime", "Platform details (" + str(multiprocessing.cpu_count()) + " cores), " + platform.node() + " , Python " + sys.version[:5] + " , " + " ".join(platform.uname()), "gkwong")
# print "Email sent!"
numProcesses = multiprocessing.cpu_count()
if "-asan" in buildInfo.buildDir:
# This should really be based on the amount of RAM available, but I don't know how to compute that in Python.
# I could guess 1 GB RAM per core, but that wanders into sketchyville.
numProcesses = max(numProcesses // 2, 1)
if sps.isARMv7l:
# Even though ARM boards generally now have many cores, each core is not as powerful
# as x86/64 ones, so restrict fuzzing to only 1 core for now.
numProcesses = 1
forkJoin.forkJoin(options.tempDir, numProcesses, fuzzUntilBug,
options, buildInfo)
# Remove build directory if we created it
if options.testType == 'dom' and not \
(options.retestRoot or options.existingBuildDir or options.buildOptions is not None):
shutil.rmtree(buildInfo.buildDir)
shutil.rmtree(options.tempDir)
def many_timed_runs(targetTime, tempDir, args, quiet=True):
startTime = time.time()
iteration = 0
fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js"))
linkFuzzer(fuzzerJS)
os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS
levelAndLines, options = domInteresting.rdfInit(args)
browserDir = options.browserDir
reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir
reftestURLs = getURLs(os.path.abspath(reftestFilesDir))
while True:
if targetTime and time.time() > startTime + targetTime:
print "Out of time!"
os.remove(fuzzerJS)
if len(os.listdir(tempDir)) == 0:
os.rmdir(tempDir)
return (lithOps.HAPPY, None)
iteration += 1
url = options.argURL or (random.choice(reftestURLs) + randomHash())
extraPrefs = randomPrefs.randomPrefs()
logPrefix = os.path.join(tempDir, "q" + str(iteration))
now = datetime.datetime.isoformat(datetime.datetime.now(), " ")
print
print "%%% " + now + " starting q" + str(iteration) + ": " + url
level, lines = levelAndLines(url, logPrefix=logPrefix, extraPrefs=extraPrefs, quiet=quiet)
if level > domInteresting.DOM_FINE:
print "loopdomfuzz.py: will try reducing from " + url
rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix)
if platform.system() == "Windows":
rFN = rFN.replace("/", "\\") # Ensure both Lithium and Firefox understand the filename
extraRDFArgs = ["--valgrind"] if options.valgrind else []
lithArgs = [domInterestingpy] + extraRDFArgs + ["-m%d" % level, browserDir, rFN]
(lithresult, lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime and targetTime // 2)
if lithresult == lithOps.LITH_NO_REPRO:
os.remove(rFN)
print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all."
level2, _ = levelAndLines(url, logPrefix=logPrefix + "-retry", extraPrefs=extraPrefs)
if level2 > domInteresting.DOM_FINE:
print "%%% Lithium can't reproduce, but I can!"
with open(logPrefix + "-repro-only.txt", "w") as reproOnlyFile:
reproOnlyFile.write(
"I was able to reproduce an issue at the same URL, but Lithium was not.\n\n"
)
reproOnlyFile.write(domInterestingpy + " " + browserDir + " " + url + "\n")
lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL
else:
print "%%% Lithium can't reproduce, and neither can I."
with open(logPrefix + "-sorry.txt", "w") as sorryFile:
sorryFile.write("I wasn't even able to reproduce with the same URL.\n\n")
sorryFile.write(domInterestingpy + " " + browserDir + " " + url + "\n")
lithresult = lithOps.NO_REPRO_AT_ALL
print ""
if targetTime:
return (lithresult, lithdetails)
if options.argURL:
break
def many_timed_runs(targetTime, tempDir, args, quiet=True):
startTime = time.time()
iteration = 0
fuzzerJS = os.path.abspath(os.path.join(tempDir, "fuzzer-combined.js"))
linkFuzzer(fuzzerJS)
os.environ["DOM_FUZZER_SCRIPT"] = fuzzerJS
levelAndLines, options = domInteresting.rdfInit(args)
browserDir = options.browserDir
reftestFilesDir = domInteresting.FigureOutDirs(browserDir).reftestFilesDir
reftestURLs = getURLs(os.path.abspath(reftestFilesDir))
while True:
if targetTime and time.time() > startTime + targetTime:
print "Out of time!"
os.remove(fuzzerJS)
if len(os.listdir(tempDir)) == 0:
os.rmdir(tempDir)
return (lithOps.HAPPY, None)
iteration += 1
url = options.argURL or (random.choice(reftestURLs) + randomHash())
extraPrefs = randomPrefs.randomPrefs()
logPrefix = os.path.join(tempDir, "q" + str(iteration))
now = datetime.datetime.isoformat(datetime.datetime.now(), " ")
print
print "%%% " + now + " starting q" + str(iteration) + ": " + url
level, lines = levelAndLines(url,
logPrefix=logPrefix,
extraPrefs=extraPrefs,
quiet=quiet)
if level > domInteresting.DOM_FINE:
print "loopdomfuzz.py: will try reducing from " + url
rFN = createReproFile(fuzzerJS, extraPrefs, lines, logPrefix)
if platform.system() == "Windows":
rFN = rFN.replace(
"/", "\\"
) # Ensure both Lithium and Firefox understand the filename
extraRDFArgs = ["--valgrind"] if options.valgrind else []
lithArgs = [domInterestingpy
] + extraRDFArgs + ["-m%d" % level, browserDir, rFN]
(lithresult,
lithdetails) = lithOps.runLithium(lithArgs, logPrefix, targetTime
and targetTime // 2)
if lithresult == lithOps.LITH_NO_REPRO:
os.remove(rFN)
print "%%% Lithium can't reproduce. One more shot to see if it's reproducible at all."
level2, _ = levelAndLines(url,
logPrefix=logPrefix + "-retry",
extraPrefs=extraPrefs)
if level2 > domInteresting.DOM_FINE:
print "%%% Lithium can't reproduce, but I can!"
with open(logPrefix + "-repro-only.txt",
"w") as reproOnlyFile:
reproOnlyFile.write(
"I was able to reproduce an issue at the same URL, but Lithium was not.\n\n"
)
reproOnlyFile.write(domInterestingpy + " " +
browserDir + " " + url + "\n")
lithresult = lithOps.NO_REPRO_EXCEPT_BY_URL
else:
print "%%% Lithium can't reproduce, and neither can I."
with open(logPrefix + "-sorry.txt", "w") as sorryFile:
sorryFile.write(
"I wasn't even able to reproduce with the same URL.\n\n"
)
sorryFile.write(domInterestingpy + " " + browserDir +
" " + url + "\n")
lithresult = lithOps.NO_REPRO_AT_ALL
print ""
if targetTime:
return (lithresult, lithdetails)
if options.argURL:
break
