def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo,
buildOptionsStr, targetTime, suspiciousLevel):
"""
Run Lithium and autobisect.
itest must be an array of the form [module, ...] where module is an interestingness module.
The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename
(If it's not prepared to accept engineFlags, engineFlags must be empty.)
"""
lithArgs = itest + [jsEngine] + engineFlags + [infilename]
(lithResult, lithDetails) = strategicReduction(logPrefix, infilename,
lithArgs, targetTime,
suspiciousLevel)
print "\nDone running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:"
print sps.shellify([lithiumpy, "--strategy=check-only"] + lithArgs) + '\n'
if bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and buildOptionsStr is not None:
if platform.uname()[2] == 'XP':
print 'Not pinpointing to exact changeset since autoBisect does not work well in WinXP.'
elif testJsShellOrXpcshell(jsEngine) != "xpcshell":
autobisectCmd = ([sys.executable, autobisectpy] +
["-b", buildOptionsStr] +
["-p", ' '.join(engineFlags + [infilename])] +
["-i"] + itest)
print sps.shellify(autobisectCmd)
subprocess.call(autobisectCmd,
stdout=open(logPrefix + "-autobisect.txt", "w"),
stderr=subprocess.STDOUT)
print "Done running autobisect. Log: " + logPrefix + "-autobisect.txt"
return (lithResult, lithDetails)
def runLithium(lithArgs, logPrefix, targetTime):
"""
Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level.
Returns a tuple of (lithlogfn, LITH_*, details).
"""
deletableLithTemp = None
if targetTime:
# FIXME: this could be based on whether bot.py has a remoteHost
# loopdomfuzz.py or loopjsfuzz.py is being used by bot.py
deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium")
lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs
else:
# loopdomfuzz.py or loopjsfuzz.py is being run standalone
lithtmp = logPrefix + "-lith-tmp"
os.mkdir(lithtmp)
lithArgs = ["--tempdir=" + lithtmp] + lithArgs
lithlogfn = logPrefix + "-lith-out.txt"
print "Preparing to run Lithium, log file " + lithlogfn
print sps.shellify(runlithiumpy + lithArgs)
subprocess.call(runlithiumpy + lithArgs, stdout=open(lithlogfn, "w"), stderr=subprocess.STDOUT)
print "Done running Lithium"
if deletableLithTemp:
shutil.rmtree(deletableLithTemp)
r = readLithiumResult(lithlogfn)
subprocess.call(["gzip", "-f", lithlogfn])
return r
def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo, buildOptionsStr, targetTime, suspiciousLevel):
"""
Run Lithium and autobisect.
itest must be an array of the form [module, ...] where module is an interestingness module.
The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename
(If it's not prepared to accept engineFlags, engineFlags must be empty.)
"""
lithArgs = itest + [jsEngine] + engineFlags + [infilename]
(lithResult, lithDetails) = strategicReduction(logPrefix, infilename, lithArgs, bisectRepo,
buildOptionsStr, targetTime, suspiciousLevel)
print "\nDone running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:"
print sps.shellify([lithiumpy, "--strategy=check-only"] + lithArgs) + '\n'
if bisectRepo is not "none" and targetTime >= 3*60*60 and buildOptionsStr is not None:
if platform.uname()[2] == 'XP':
print 'Not pinpointing to exact changeset since autoBisect does not work well in WinXP.'
elif testJsShellOrXpcshell(jsEngine) != "xpcshell":
autobisectCmd = (
[sys.executable, autobisectpy] +
["-b", buildOptionsStr] +
["-p", ' '.join(engineFlags + [infilename])] +
["-i"] + itest
)
print sps.shellify(autobisectCmd)
subprocess.call(autobisectCmd, stdout=open(logPrefix + "-autobisect", "w"), stderr=subprocess.STDOUT)
print "Done running autobisect. Log: " + logPrefix + "-autobisect"
return (lithResult, lithDetails)
def runLithium(lithArgs, logPrefix, targetTime):
"""
Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level.
Returns a tuple of (lithlogfn, LITH_*, details).
"""
deletableLithTemp = None
if targetTime:
# FIXME: this could be based on whether bot.py has a remoteHost
# loopdomfuzz.py or loopjsfuzz.py is being used by bot.py
deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium")
lithArgs = [
"--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp
] + lithArgs
else:
# loopdomfuzz.py or loopjsfuzz.py is being run standalone
lithtmp = logPrefix + "-lith-tmp"
os.mkdir(lithtmp)
lithArgs = ["--tempdir=" + lithtmp] + lithArgs
lithlogfn = logPrefix + "-lith-out.txt"
print "Preparing to run Lithium, log file " + lithlogfn
print sps.shellify(runlithiumpy + lithArgs)
subprocess.call(runlithiumpy + lithArgs,
stdout=open(lithlogfn, "w"),
stderr=subprocess.STDOUT)
print "Done running Lithium"
if deletableLithTemp:
shutil.rmtree(deletableLithTemp)
r = readLithiumResult(lithlogfn)
subprocess.call(["gzip", "-f", lithlogfn])
return r
def envDump(shell, log):
'''Dumps environment to a .fuzzmanagerconf file.'''
# Platform and OS detection for the spec, part of which is in:
# https://wiki.mozilla.org/Security/CrashSignatures
if sps.isARMv7l:
fmconfPlatform = 'ARM'
elif sps.isARMv7l and not shell.buildOptions.enable32:
print 'ARM64 is not supported in .fuzzmanagerconf yet.'
fmconfPlatform = 'ARM64'
elif shell.buildOptions.enable32:
fmconfPlatform = 'x86'
else:
fmconfPlatform = 'x86-64'
if sps.isLinux:
fmconfOS = 'linux'
elif sps.isMac:
fmconfOS = 'macosx'
elif sps.isWin:
fmconfOS = 'windows'
with open(log, 'ab') as f:
f.write('# Information about shell:\n# \n')
f.write('# Create another shell in shell-cache like this one:\n')
f.write('# python -u %s -b "%s" -r %s\n# \n' %
('~/funfuzz/js/compileShell.py',
shell.buildOptions.buildOptionsStr, shell.getHgHash()))
f.write('# Full environment is:\n')
f.write('# %s\n# \n' % str(shell.getEnvFull()))
f.write(
'# Full configuration command with needed environment variables is:\n'
)
f.write('# %s %s\n# \n' % (sps.shellify(
shell.getEnvAdded()), sps.shellify(shell.getCfgCmdExclEnv())))
# .fuzzmanagerconf details
f.write('\n')
f.write('[Main]\n')
f.write('platform = %s\n' % fmconfPlatform)
f.write('product = %s\n' % shell.getRepoName())
f.write('product_version = %s\n' % shell.getHgHash())
f.write('os = %s\n' % fmconfOS)
f.write('\n')
f.write('[Metadata]\n')
f.write('buildFlags = %s\n' % shell.buildOptions.buildOptionsStr)
f.write('majorVersion = %s\n' % shell.getMajorVersion())
f.write('pathPrefix = %s%s\n' %
(shell.getRepoDir(),
'/' if not shell.getRepoDir().endswith('/') else ''))
f.write('version = %s\n' % shell.getVersion())
def envDump(shell, log):
'''Dumps environment to a .fuzzmanagerconf file.'''
# Platform and OS detection for the spec, part of which is in:
# https://wiki.mozilla.org/Security/CrashSignatures
if sps.isARMv7l:
fmconfPlatform = 'ARM'
elif sps.isARMv7l and not shell.buildOptions.enable32:
print 'ARM64 is not supported in .fuzzmanagerconf yet.'
fmconfPlatform = 'ARM64'
elif shell.buildOptions.enable32:
fmconfPlatform = 'x86'
else:
fmconfPlatform = 'x86-64'
if sps.isLinux:
fmconfOS = 'linux'
elif sps.isMac:
fmconfOS = 'macosx'
elif sps.isWin:
fmconfOS = 'windows'
with open(log, 'ab') as f:
f.write('# Information about shell:\n# \n')
f.write('# Create another shell in shell-cache like this one:\n')
f.write('# python -u %s -b "%s" -r %s\n# \n' %
('~/funfuzz/js/compileShell.py',
shell.buildOptions.buildOptionsStr, shell.getHgHash()))
f.write('# Full environment is:\n')
f.write('# %s\n# \n' % str(shell.getEnvFull()))
f.write(
'# Full configuration command with needed environment variables is:\n'
)
f.write('# %s %s\n# \n' % (sps.shellify(shell.getEnvAdded()),
sps.shellify(shell.getCfgCmdExclEnv())))
# .fuzzmanagerconf details
f.write('\n')
f.write('[Main]\n')
f.write('platform = %s\n' % fmconfPlatform)
f.write('product = %s\n' % shell.getRepoName())
f.write('product_version = %s\n' % shell.getHgHash())
f.write('os = %s\n' % fmconfOS)
f.write('\n')
f.write('[Metadata]\n')
f.write('pathPrefix = %s%s\n' %
(shell.getRepoDir(),
'/' if not shell.getRepoDir().endswith('/') else ''))
f.write('buildFlags = %s\n' % shell.buildOptions.buildOptionsStr)
def lithReduceCmd(strategy):
"""Lithium reduction commands accepting various strategies."""
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements
print sps.shellify([lithiumpy] + fullLithArgs)
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(fullLithArgs, logPrefix + "-" +
str(reductionCount[0]) + desc, targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def envDump(shell, log):
"""Dumps environment to a .fuzzmanagerconf file."""
# Platform and OS detection for the spec, part of which is in:
# https://wiki.mozilla.org/Security/CrashSignatures
if sps.isARMv7l:
fmconfPlatform = "ARM"
elif sps.isARMv7l and not shell.buildOptions.enable32:
print "ARM64 is not supported in .fuzzmanagerconf yet."
fmconfPlatform = "ARM64"
elif shell.buildOptions.enable32:
fmconfPlatform = "x86"
else:
fmconfPlatform = "x86-64"
if sps.isLinux:
fmconfOS = "linux"
elif sps.isMac:
fmconfOS = "macosx"
elif sps.isWin:
fmconfOS = "windows"
with open(log, "ab") as f:
f.write("# Information about shell:\n# \n")
f.write("# Create another shell in shell-cache like this one:\n")
f.write(
'# python -u %s -b "%s" -r %s\n# \n'
% ("~/funfuzz/js/compileShell.py", shell.buildOptions.buildOptionsStr, shell.getHgHash())
)
f.write("# Full environment is:\n")
f.write("# %s\n# \n" % str(shell.getEnvFull()))
f.write("# Full configuration command with needed environment variables is:\n")
f.write("# %s %s\n# \n" % (sps.shellify(shell.getEnvAdded()), sps.shellify(shell.getCfgCmdExclEnv())))
# .fuzzmanagerconf details
f.write("\n")
f.write("[Main]\n")
f.write("platform = %s\n" % fmconfPlatform)
f.write("product = %s\n" % shell.getRepoName())
f.write("product_version = %s\n" % shell.getHgHash())
f.write("os = %s\n" % fmconfOS)
f.write("\n")
f.write("[Metadata]\n")
f.write("pathPrefix = %s%s\n" % (shell.getRepoDir(), "/" if not shell.getRepoDir().endswith("/") else ""))
f.write("buildFlags = %s\n" % shell.buildOptions.buildOptionsStr)
def lithReduceCmd(strategy):
'''Lithium reduction commands accepting various strategies.'''
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs)
if x] # Remove empty elements
print sps.shellify([lithiumpy] + fullLithArgs)
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(
fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc,
targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def readFromURL(url):
'''
Reads in a URL and returns its contents as a list.
'''
inpCmdList = ['curl', '--silent', url
] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', '-', url]
p = subprocess.Popen(inpCmdList,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
if not useCurl and p.returncode == 5:
print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
'"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
'in ~/.wgetrc'
raise Exception(
'Unable to read from URL. Please check your ~/.wgetrc file.')
elif p.returncode != 0:
print 'inpCmdList is: ' + sps.shellify(inpCmdList)
print 'stdout: ' + repr(out)
print 'stderr: ' + repr(err)
raise Exception('The following exit code was returned: ' +
str(p.returncode))
else:
# Ignore whatever verbose output wget spewed to stderr.
return out
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def readFromURL(url):
"""Read in a URL and returns its contents as a list."""
inpCmdList = ["curl", "--silent", url] if useCurl else ["wget"] + wgetMaybeNCC + ["-O", "-", url]
p = subprocess.Popen(inpCmdList, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if not useCurl and p.returncode == 5:
print "Unable to read from URL. If you installed wget using MacPorts, you should put " + '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + "in ~/.wgetrc"
raise Exception("Unable to read from URL. Please check your ~/.wgetrc file.")
elif p.returncode != 0:
print "inpCmdList is: " + sps.shellify(inpCmdList)
print "stdout: " + repr(out)
print "stderr: " + repr(err)
raise Exception("The following exit code was returned: " + str(p.returncode))
else:
# Ignore whatever verbose output wget spewed to stderr.
return out
def lithReduceCmd(strategy):
"""Lithium reduction commands accepting various strategies."""
reductionCount[0] += 1
fullLithArgs = [x for x in (strategy + lithArgs)
if x] # Remove empty elements
print(
sps.shellify([sys.executable, "-u", "-m", "lithium"] +
fullLithArgs))
desc = '-chars' if strategy == '--char' else '-lines'
(lithResult, lithDetails) = runLithium(
fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc),
targetTime)
if lithResult == LITH_FINISHED:
shutil.copy2(infilename, backupFilename)
return lithResult, lithDetails
def readFromURL(url):
'''
Reads in a URL and returns its contents as a list.
'''
inpCmdList = ['curl', '--silent', url] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', '-', url]
p = subprocess.Popen(inpCmdList, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if not useCurl and p.returncode == 5:
print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
'"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
'in ~/.wgetrc'
raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.')
elif p.returncode != 0:
print 'inpCmdList is: ' + sps.shellify(inpCmdList)
print 'stdout: ' + repr(out)
print 'stderr: ' + repr(err)
raise Exception('The following exit code was returned: ' + str(p.returncode))
else:
# Ignore whatever verbose output wget spewed to stderr.
return out
def jsfunfuzzLevel(options, logPrefix, quiet=False):
(lev, issues, runinfo) = baseLevel(options.jsengineWithArgs,
options.timeout,
options.knownPath,
logPrefix,
valgrind=options.valgrind)
if lev == JS_FINE:
# Check for unexplained exits and for jsfunfuzz saying "Found a bug".
understoodExit = False
# Read in binary mode, because otherwise Python on Windows will
# throw a fit when it encounters certain unicode. Note that this
# makes line endings platform-specific.
if '-dm-' in options.jsengineWithArgs[0]:
# Since this is an --enable-more-deterministic build, we should get messages on stderr
# if the shell quit() or terminate() functions are called.
# (We use a sketchy filename-matching check because it's faster than inspecting the binary.)
with open(logPrefix + "-err.txt", "rb") as f:
for line in f:
if "terminate called" in line or "quit called" in line:
understoodExit = True
if "can't allocate region" in line:
understoodExit = True
else:
understoodExit = True
with open(logPrefix + "-out.txt", "rb") as f:
for line in f:
if line.startswith("It's looking good!") or line.startswith(
"jsfunfuzz broke its own scripting environment: "):
understoodExit = True
if line.startswith("Found a bug: "):
understoodExit = True
if not ("NestTest" in line and oomed(logPrefix)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
# FIXME: if not quiet:
# FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug."
if not understoodExit:
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# FIXME: if not quiet:
# FIXME: output the last tryItOut line
if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored.
sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev))
lev = JS_FINE
issues = []
if lev != JS_FINE:
# FIXME: compareJIT failures do not generate this -summary file.
statusIssueList = []
for i in issues:
statusIssueList.append('Status: ' + i)
assert len(statusIssueList) != 0
fileManipulation.writeLinesToFile([
'Number: ' + logPrefix + '\n',
'Command: ' + sps.shellify(options.jsengineWithArgs) + '\n'
] + [i + '\n' for i in statusIssueList], logPrefix + '-summary.txt')
if not quiet:
print logPrefix + " | " + summaryString(issues, lev,
runinfo.elapsedtime)
return lev
def cfgBin(shell):
"""This function configures a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt["AR"] = "ar"
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang"))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang++"))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, "arm7vl boards are only 32-bit, armv8 boards will be 64-bit."
if not shell.buildOptions.enableHardFp:
cfgEnvDt["CC"] = "gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgEnvDt["CXX"] = "g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append("--target=arm-linux-gnueabi")
elif shell.buildOptions.enable32 and os.name == "posix":
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = "clang" + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = "clang++" + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt["CC"] = cfgEnvDt["CC"] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt["CXX"] = cfgEnvDt["CXX"] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt["RANLIB"] = "ranlib"
cfgEnvDt["AS"] = "$CC"
cfgEnvDt["LD"] = "ld"
cfgEnvDt["STRIP"] = "strip -x -S"
cfgEnvDt["CROSS_COMPILE"] = "1"
if sps.isProgramInstalled("brew"):
cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=i386-apple-darwin9.2.0") # Leopard 10.5.2
cfgCmdList.append("--enable-macos-target=10.5")
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig"
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = (
CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
)
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = (
CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
)
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt["CC"] = "gcc -m32" + SSE2_FLAGS
cfgEnvDt["CXX"] = "g++ -m32" + SSE2_FLAGS
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=i686-pc-linux")
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
else:
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt["CC"] = "clang" + CLANG_PARAMS
cfgEnvDt["CXX"] = "clang++" + CLANG_PARAMS
if sps.isProgramInstalled("brew"):
cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=x86_64-apple-darwin12.5.0") # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append("--enable-simulator=arm64")
elif sps.isWin:
cfgEnvDt["MAKE"] = "mozmake" # Workaround for bug 948534
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
else:
cfgCmdList.append("--host=x86_64-pc-mingw32")
cfgCmdList.append("--target=x86_64-pc-mingw32")
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append("--enable-simulator=arm64")
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.buildWithAsan:
assert "clang" in cfgEnvDt["CC"]
assert "clang++" in cfgEnvDt["CXX"]
cfgCmdList.append("--disable-jemalloc") # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append("--enable-debug")
elif shell.buildOptions.disableDbg:
cfgCmdList.append("--disable-debug")
if shell.buildOptions.enableOpt:
cfgCmdList.append("--enable-optimize" + ("=-O1" if shell.buildOptions.buildWithVg else ""))
elif shell.buildOptions.disableOpt:
cfgCmdList.append("--disable-optimize")
if shell.buildOptions.enableProfiling:
cfgCmdList.append("--enable-profiling")
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append("--enable-more-deterministic")
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append("--enable-oom-breakpoint")
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append("--without-intl-api")
if shell.buildOptions.buildWithVg:
cfgCmdList.append("--enable-valgrind")
cfgCmdList.append("--disable-jemalloc")
# We add the following flags by default.
if os.name == "posix":
cfgCmdList.append("--with-ccache")
cfgCmdList.append("--enable-gczeal")
cfgCmdList.append("--enable-debug-symbols") # gets debug symbols on opt shells
cfgCmdList.append("--disable-tests")
if os.name == "nt":
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, "//")
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = (
envVar + '="' + cfgEnvDt[envVar] + '"' if " " in cfgEnvDt[envVar] else envVar + "=" + cfgEnvDt[envVar]
)
envVarList.append(strToBeAppended)
sps.vdump("Command to be run is: " + sps.shellify(envVarList) + " " + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if "\\" in entry:
entry = entry.replace("\\", "/")
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[
'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt[
'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [
10, 7
] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt[
'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt[
'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7
] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' +
('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append(
'--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag(
) and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append(
'--with-nspr-prefix=' + sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(
), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append(
'--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' +
sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir(
) if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
# cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert 'clang-cl' in cfgEnvDt['CC']
assert 'clang-cl' in cfgEnvDt['CXX']
else:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def __init__(self, options, runthis, logPrefix, inCompareJIT):
pathToBinary = runthis[0]
# This relies on the shell being a local one from compileShell.py:
# Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like './js'
pc = ProgramConfiguration.fromBinary(
os.path.abspath(pathToBinary).split('.')[0])
pc.addProgramArguments(runthis[1:-1])
if options.valgrind:
runthis = (inspectShell.constructVgCmdList(
errorCode=VALGRIND_ERROR_EXIT_CODE) +
valgrindSuppressions(options.knownPath) + runthis)
preexec_fn = ulimitSet if os.name == 'posix' else None
runinfo = timed_run.timed_run(runthis,
options.timeout,
logPrefix,
preexec_fn=preexec_fn)
lev = JS_FINE
issues = []
auxCrashData = []
# FuzzManager expects a list of strings rather than an iterable, so bite the
# bullet and 'readlines' everything into memory.
with open(logPrefix + "-out.txt") as f:
out = f.readlines()
with open(logPrefix + "-err.txt") as f:
err = f.readlines()
if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE:
issues.append("valgrind reported an error")
lev = max(lev, JS_VG_AMISS)
valgrindErrorPrefix = "==" + str(runinfo.pid) + "=="
for line in err:
if valgrindErrorPrefix and line.startswith(
valgrindErrorPrefix):
issues.append(line.rstrip())
elif runinfo.sta == timed_run.CRASHED:
if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True):
with open(logPrefix + "-crash.txt") as f:
auxCrashData = [line.strip() for line in f.readlines()]
elif detect_malloc_errors.amiss(logPrefix):
issues.append("malloc error")
lev = max(lev, JS_NEW_ASSERT_OR_CRASH)
elif runinfo.return_code == 0 and not inCompareJIT:
# We might have(??) run jsfunfuzz directly, so check for special kinds of bugs
for line in out:
if line.startswith("Found a bug: ") and not ("NestTest" in line
and oomed(err)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
if options.shellIsDeterministic and not understoodJsfunfuzzExit(
out, err) and not oomed(err):
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# Copy non-crash issues to where FuzzManager's "AssertionHelper.py" can see it.
if lev != JS_FINE:
for issue in issues:
err.append("[Non-crash bug] " + issue)
# Finally, make a CrashInfo object and parse stack traces for asan/crash/assertion bugs
crashInfo = CrashInfo.CrashInfo.fromRawCrashData(
out, err, pc, auxCrashData=auxCrashData)
createCollector.printCrashInfo(crashInfo)
# We only care about crashes and assertion failures on shells with no symbols
# Note that looking out for the Assertion failure message is highly SpiderMonkey-specific
if not isinstance(crashInfo, CrashInfo.NoCrashInfo) or \
'Assertion failure: ' in str(crashInfo.rawStderr) or \
'Segmentation fault' in str(crashInfo.rawStderr) or \
'Bus error' in str(crashInfo.rawStderr):
lev = max(lev, JS_NEW_ASSERT_OR_CRASH)
match = options.collector.search(crashInfo)
if match[0] is not None:
createCollector.printMatchingSignature(match)
lev = JS_FINE
print("%s | %s" %
(logPrefix, summaryString(issues, lev, runinfo.elapsedtime)))
if lev != JS_FINE:
fileManipulation.writeLinesToFile([
'Number: ' + logPrefix + '\n',
'Command: ' + sps.shellify(runthis) + '\n'
] + ['Status: ' + i + "\n" for i in issues],
logPrefix + '-summary.txt')
self.lev = lev
self.out = out
self.err = err
self.issues = issues
self.crashInfo = crashInfo
self.match = match
self.runinfo = runinfo
self.return_code = runinfo.return_code
def compareLevel(jsEngine, flags, infilename, logPrefix, options,
showDetailedDiffs, quickMode):
# options dict must be one we can pass to jsInteresting.ShellResult
# we also use it directly for knownPath, timeout, and collector
# Return: (lev, crashInfo) or (jsInteresting.JS_FINE, None)
combos = shellFlags.basicFlagSets(jsEngine)
if quickMode:
# Only used during initial fuzzing. Allowed to have false negatives.
combos = [combos[0]]
if len(flags):
combos.append(flags)
commands = [[jsEngine] + combo + [infilename] for combo in combos]
for i in range(0, len(commands)):
prefix = logPrefix + "-r" + str(i)
command = commands[i]
r = jsInteresting.ShellResult(options, command, prefix, True)
oom = jsInteresting.oomed(r.err)
r.err = ignoreSomeOfStderr(r.err)
if (r.rc == 1 or r.rc == 2) and (
anyLineContains(r.out, '[[script] scriptArgs*]')
or anyLineContains(r.err, '[scriptfile] [scriptarg...]')):
print "Got usage error from:"
print " " + sps.shellify(command)
assert i > 0
jsInteresting.deleteLogs(prefix)
elif r.lev > jsInteresting.JS_OVERALL_MISMATCH:
# would be more efficient to run lithium on one or the other, but meh
print infilename + " | " + jsInteresting.summaryString(
r.issues + ["compareJIT found a more serious bug"], r.lev,
r.runinfo.elapsedtime)
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write('\n'.join(r.issues + [
sps.shellify(command),
"compareJIT found a more serious bug"
]) + '\n')
print " " + sps.shellify(command)
return (r.lev, r.crashInfo)
elif r.lev != jsInteresting.JS_FINE or r.rc != 0:
print infilename + " | " + jsInteresting.summaryString(
r.issues + [
"compareJIT is not comparing output, because the shell exited strangely"
], r.lev, r.runinfo.elapsedtime)
print " " + sps.shellify(command)
jsInteresting.deleteLogs(prefix)
if i == 0:
return (jsInteresting.JS_FINE, None)
elif oom:
# If the shell or python hit a memory limit, we consider the rest of the computation
# "tainted" for the purpose of correctness comparison.
message = "compareJIT is not comparing output: OOM"
print infilename + " | " + jsInteresting.summaryString(
r.issues + [message], r.lev, r.runinfo.elapsedtime)
jsInteresting.deleteLogs(prefix)
if i == 0:
return (jsInteresting.JS_FINE, None)
elif i == 0:
# Stash output from this run (the first one), so for subsequent runs, we can compare against it.
(r0, prefix0) = (r, prefix)
else:
# Compare the output of this run (r.out) to the output of the first run (r0.out), etc.
def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg):
fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err
fpuOptionDiffers = (("--no-fpu" in commands[0]) !=
("--no-fpu" in command))
return fpuOptionDisabledAsm and fpuOptionDiffers
def optionDisabledAsmOnOneSide():
asmMsg = "asm.js type error: Disabled by javascript.options.asmjs"
optionDisabledAsm = anyLineContains(
r0.err, asmMsg) or anyLineContains(r.err, asmMsg)
optionDiffers = (("--no-asmjs" in commands[0]) !=
("--no-asmjs" in command))
return optionDisabledAsm and optionDiffers
mismatchErr = (
r.err != r0.err and
# --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things.
# This should only affect asm.js diagnostics on stderr.
not fpuOptionDisabledAsmOnOneSide(
"asm.js type error: Disabled by lack of floating point support"
) and
# And also wasm stuff. See bug 1243031.
not fpuOptionDisabledAsmOnOneSide(
"WebAssembly is not supported on the current device")
and not optionDisabledAsmOnOneSide())
mismatchOut = (r.out != r0.out)
if mismatchErr or mismatchOut:
# Generate a short summary for stdout and a long summary for a "*-summary.txt" file.
rerunCommand = sps.shellify([
'~/funfuzz/js/compareJIT.py', "--flags=" + ' '.join(flags),
"--timeout=" + str(options.timeout), options.knownPath,
jsEngine,
os.path.basename(infilename)
])
(summary, issues) = summarizeMismatch(mismatchErr, mismatchOut,
prefix0, prefix)
summary = " " + sps.shellify(
commands[0]) + "\n " + sps.shellify(
command) + "\n\n" + summary
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write(rerunCommand + "\n\n" + summary)
print infilename + " | " + jsInteresting.summaryString(
issues, jsInteresting.JS_OVERALL_MISMATCH,
r.runinfo.elapsedtime)
if quickMode:
print rerunCommand
if showDetailedDiffs:
print summary
print ""
# Create a crashInfo object with empty stdout, and stderr showing diffs
pc = ProgramConfiguration.fromBinary(jsEngine)
pc.addProgramArguments(flags)
crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary,
pc)
return (jsInteresting.JS_OVERALL_MISMATCH, crashInfo)
else:
# print "compareJIT: match"
jsInteresting.deleteLogs(prefix)
# All matched :)
jsInteresting.deleteLogs(prefix0)
return (jsInteresting.JS_FINE, None)
def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDiffs, quickMode):
# options dict must be one we can pass to jsInteresting.ShellResult
# we also use it directly for knownPath, timeout, and collector
# Return: (lev, crashInfo) or (jsInteresting.JS_FINE, None)
combos = shellFlags.basicFlagSets(jsEngine)
if quickMode:
# Only used during initial fuzzing. Allowed to have false negatives.
combos = [combos[0]]
if len(flags):
combos.append(flags)
commands = [[jsEngine] + combo + [infilename] for combo in combos]
for i in range(0, len(commands)):
prefix = logPrefix + "-r" + str(i)
command = commands[i]
r = jsInteresting.ShellResult(options, command, prefix, True)
oom = jsInteresting.oomed(r.err)
r.err = ignoreSomeOfStderr(r.err)
if (r.rc == 1 or r.rc == 2) and (anyLineContains(r.out, '[[script] scriptArgs*]') or anyLineContains(r.err, '[scriptfile] [scriptarg...]')):
print "Got usage error from:"
print " " + sps.shellify(command)
assert i > 0
jsInteresting.deleteLogs(prefix)
elif r.lev > jsInteresting.JS_OVERALL_MISMATCH:
# would be more efficient to run lithium on one or the other, but meh
print infilename + " | " + jsInteresting.summaryString(r.issues + ["compareJIT found a more serious bug"], r.lev, r.runinfo.elapsedtime)
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write('\n'.join(r.issues + [sps.shellify(command), "compareJIT found a more serious bug"]) + '\n')
print " " + sps.shellify(command)
return (r.lev, r.crashInfo)
elif r.lev != jsInteresting.JS_FINE or r.rc != 0:
print infilename + " | " + jsInteresting.summaryString(r.issues + ["compareJIT is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime)
print " " + sps.shellify(command)
jsInteresting.deleteLogs(prefix)
if i == 0:
return (jsInteresting.JS_FINE, None)
elif oom:
# If the shell or python hit a memory limit, we consider the rest of the computation
# "tainted" for the purpose of correctness comparison.
message = "compareJIT is not comparing output: OOM"
print infilename + " | " + jsInteresting.summaryString(r.issues + [message], r.lev, r.runinfo.elapsedtime)
jsInteresting.deleteLogs(prefix)
if i == 0:
return (jsInteresting.JS_FINE, None)
elif i == 0:
# Stash output from this run (the first one), so for subsequent runs, we can compare against it.
(r0, prefix0) = (r, prefix)
else:
# Compare the output of this run (r.out) to the output of the first run (r0.out), etc.
def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg):
fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err
fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command))
return fpuOptionDisabledAsm and fpuOptionDiffers
def optionDisabledAsmOnOneSide():
asmMsg = "asm.js type error: Disabled by javascript.options.asmjs"
optionDisabledAsm = anyLineContains(r0.err, asmMsg) or anyLineContains(r.err, asmMsg)
optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command))
return optionDisabledAsm and optionDiffers
mismatchErr = (r.err != r0.err and
# --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things.
# This should only affect asm.js diagnostics on stderr.
not fpuOptionDisabledAsmOnOneSide("asm.js type error: Disabled by lack of floating point support") and
# And also wasm stuff. See bug 1243031.
not fpuOptionDisabledAsmOnOneSide("WebAssembly is not supported on the current device") and
not optionDisabledAsmOnOneSide())
mismatchOut = (r.out != r0.out)
if mismatchErr or mismatchOut:
# Generate a short summary for stdout and a long summary for a "*-summary.txt" file.
rerunCommand = sps.shellify(['~/funfuzz/js/compareJIT.py', "--flags="+' '.join(flags),
"--timeout="+str(options.timeout), options.knownPath, jsEngine,
os.path.basename(infilename)])
(summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix)
summary = " " + sps.shellify(commands[0]) + "\n " + sps.shellify(command) + "\n\n" + summary
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write(rerunCommand + "\n\n" + summary)
print infilename + " | " + jsInteresting.summaryString(issues, jsInteresting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime)
if quickMode:
print rerunCommand
if showDetailedDiffs:
print summary
print ""
# Create a crashInfo object with empty stdout, and stderr showing diffs
pc = ProgramConfiguration.fromBinary(jsEngine)
pc.addProgramArguments(flags)
crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary, pc)
return (jsInteresting.JS_OVERALL_MISMATCH, crashInfo)
else:
# print "compareJIT: match"
jsInteresting.deleteLogs(prefix)
# All matched :)
jsInteresting.deleteLogs(prefix0)
return (jsInteresting.JS_FINE, None)
def __init__(self, options, runthis, logPrefix, inCompareJIT):
pathToBinary = runthis[0]
# This relies on the shell being a local one from compileShell.py:
pc = ProgramConfiguration.fromBinary(pathToBinary.split('.')[0]) # Ignore trailing ".exe" in Win
pc.addProgramArguments(runthis[1:-1])
if options.valgrind:
runthis = (
inspectShell.constructVgCmdList(errorCode=VALGRIND_ERROR_EXIT_CODE) +
valgrindSuppressions(options.knownPath) +
runthis)
preexec_fn = ulimitSet if os.name == 'posix' else None
runinfo = timedRun.timed_run(runthis, options.timeout, logPrefix, preexec_fn=preexec_fn)
lev = JS_FINE
issues = []
auxCrashData = []
# FuzzManager expects a list of strings rather than an iterable, so bite the
# bullet and 'readlines' everything into memory.
with open(logPrefix + "-out.txt") as f:
out = f.readlines()
with open(logPrefix + "-err.txt") as f:
err = f.readlines()
if options.valgrind and runinfo.rc == VALGRIND_ERROR_EXIT_CODE:
issues.append("valgrind reported an error")
lev = max(lev, JS_VG_AMISS)
valgrindErrorPrefix = "==" + str(runinfo.pid) + "=="
for line in err:
if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix):
issues.append(line.rstrip())
elif runinfo.sta == timedRun.CRASHED:
if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True):
with open(logPrefix + "-crash.txt") as f:
auxCrashData = [line.strip() for line in f.readlines()]
elif detect_malloc_errors.amiss(logPrefix):
issues.append("malloc error")
lev = max(lev, JS_NEW_ASSERT_OR_CRASH)
elif runinfo.rc == 0 and not inCompareJIT:
# We might have(??) run jsfunfuzz directly, so check for special kinds of bugs
for line in out:
if line.startswith("Found a bug: ") and not ("NestTest" in line and oomed(err)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
if options.shellIsDeterministic and not understoodJsfunfuzzExit(out, err) and not oomed(err):
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# Copy non-crash issues to where FuzzManager's "AssertionHelper.py" can see it.
if lev != JS_FINE:
for issue in issues:
err.append("[Non-crash bug] " + issue)
# Finally, make a CrashInfo object and parse stack traces for asan/crash/assertion bugs
crashInfo = CrashInfo.CrashInfo.fromRawCrashData(out, err, pc, auxCrashData=auxCrashData)
createCollector.printCrashInfo(crashInfo)
if not isinstance(crashInfo, CrashInfo.NoCrashInfo):
lev = max(lev, JS_NEW_ASSERT_OR_CRASH)
match = options.collector.search(crashInfo)
if match[0] is not None:
createCollector.printMatchingSignature(match)
lev = JS_FINE
print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime)
if lev != JS_FINE:
fileManipulation.writeLinesToFile(
['Number: ' + logPrefix + '\n',
'Command: ' + sps.shellify(runthis) + '\n'] +
['Status: ' + i + "\n" for i in issues],
logPrefix + '-summary.txt')
self.lev = lev
self.out = out
self.err = err
self.issues = issues
self.crashInfo = crashInfo
self.match = match
self.runinfo = runinfo
self.rc = runinfo.rc
def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout, showDetailedDiffs, quickMode):
combos = shellFlags.basicFlagSets(jsEngine)
if quickMode:
# Only used during initial fuzzing. Allowed to have false negatives.
combos = [combos[0]]
if len(flags):
combos.append(flags)
commands = [[jsEngine] + combo + [infilename] for combo in combos]
for i in range(0, len(commands)):
prefix = logPrefix + "-r" + str(i)
command = commands[i]
(lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath, prefix)
with open(prefix + "-out.txt") as f:
r.out = f.read(lengthLimit)
with open(prefix + "-err.txt") as f:
r.err = f.read(lengthLimit)
oom = jsInteresting.hitMemoryLimit(r.err)
if (not oom) and (len(r.err) + 5 > lengthLimit):
# The output was too long for Python to read it in all at once. Assume the worst.
oom = "stderr too long"
r.err = ignoreSomeOfStderr(r.err)
if (r.rc == 1 or r.rc == 2) and (r.out.find('[[script] scriptArgs*]') != -1 or r.err.find('[scriptfile] [scriptarg...]') != -1):
print "Got usage error from:"
print " " + sps.shellify(command)
assert i > 0
jsInteresting.deleteLogs(prefix)
elif lev > jsInteresting.JS_OVERALL_MISMATCH:
# would be more efficient to run lithium on one or the other, but meh
print infilename + " | " + jsInteresting.summaryString(issues + ["compareJIT found a more serious bug"], lev, r.elapsedtime)
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write('\n'.join(issues + [sps.shellify(command), "compareJIT found a more serious bug"]) + '\n')
print " " + sps.shellify(command)
return lev
elif lev != jsInteresting.JS_FINE:
print infilename + " | " + jsInteresting.summaryString(issues + ["compareJIT is not comparing output, because the shell exited strangely"], lev, r.elapsedtime)
print " " + sps.shellify(command)
jsInteresting.deleteLogs(prefix)
if i == 0:
return jsInteresting.JS_FINE
elif oom:
# If the shell or python hit a memory limit, we consider the rest of the computation
# "tainted" for the purpose of correctness comparison.
message = "compareJIT is not comparing output: OOM (" + oom + ")"
print infilename + " | " + jsInteresting.summaryString(issues + [message], lev, r.elapsedtime)
jsInteresting.deleteLogs(prefix)
if i == 0:
return jsInteresting.JS_FINE
elif i == 0:
# Stash output from this run (the first one), so for subsequent runs, we can compare against it.
(r0, prefix0) = (r, prefix)
else:
# Compare the output of this run (r.out) to the output of the first run (r0.out), etc.
def fpuOptionDisabledAsmOnOneSide():
# --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things.
# This should only affect asm.js diagnostics on stderr.
fpuAsmMsg = "asm.js type error: Disabled by lack of floating point support"
fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err
fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command))
return (fpuOptionDisabledAsm and fpuOptionDiffers)
def optionDisabledAsmOnOneSide():
asmMsg = "asm.js type error: Disabled by javascript.options.asmjs"
optionDisabledAsm = asmMsg in r0.err or asmMsg in r.err
optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command))
return (optionDisabledAsm and optionDiffers)
mismatchErr = (r.err != r0.err and not fpuOptionDisabledAsmOnOneSide() and not optionDisabledAsmOnOneSide())
mismatchOut = (r.out != r0.out)
if mismatchErr or mismatchOut:
# Generate a short summary for stdout and a long summary for a "*-summary.txt" file.
rerunCommand = sps.shellify(['~/funfuzz/js/compareJIT.py', "--flags="+' '.join(flags),
"--timeout="+str(timeout), knownPath, jsEngine,
os.path.basename(infilename)])
(summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix)
summary = " " + sps.shellify(commands[0]) + "\n " + sps.shellify(command) + "\n\n" + summary
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write(rerunCommand + "\n\n" + summary)
print infilename + " | " + jsInteresting.summaryString(issues, jsInteresting.JS_OVERALL_MISMATCH, r.elapsedtime)
if quickMode:
print rerunCommand
if showDetailedDiffs:
print summary
print ""
return jsInteresting.JS_OVERALL_MISMATCH
else:
#print "compareJIT: match"
jsInteresting.deleteLogs(prefix)
# All matched :)
jsInteresting.deleteLogs(prefix0)
return jsInteresting.JS_FINE
def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout,
showDetailedDiffs, quickMode):
combos = shellFlags.basicFlagSets(jsEngine)
if quickMode:
# Only used during initial fuzzing. Allowed to have false negatives.
combos = [combos[0]]
if len(flags):
combos.append(flags)
commands = [[jsEngine] + combo + [infilename] for combo in combos]
for i in range(0, len(commands)):
prefix = logPrefix + "-r" + str(i)
command = commands[i]
(lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath,
prefix)
with open(prefix + "-out.txt") as f:
r.out = f.read(lengthLimit)
with open(prefix + "-err.txt") as f:
r.err = f.read(lengthLimit)
oom = jsInteresting.hitMemoryLimit(r.err)
if (not oom) and (len(r.err) + 5 > lengthLimit):
# The output was too long for Python to read it in all at once. Assume the worst.
oom = "stderr too long"
r.err = ignoreSomeOfStderr(r.err)
if (r.rc == 1 or r.rc
== 2) and (r.out.find('[[script] scriptArgs*]') != -1
or r.err.find('[scriptfile] [scriptarg...]') != -1):
print "Got usage error from:"
print " " + sps.shellify(command)
assert i > 0
jsInteresting.deleteLogs(prefix)
elif lev > jsInteresting.JS_OVERALL_MISMATCH:
# would be more efficient to run lithium on one or the other, but meh
print infilename + " | " + jsInteresting.summaryString(
issues + ["compareJIT found a more serious bug"], lev,
r.elapsedtime)
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write('\n'.join(issues + [
sps.shellify(command),
"compareJIT found a more serious bug"
]) + '\n')
print " " + sps.shellify(command)
return lev
elif lev != jsInteresting.JS_FINE:
print infilename + " | " + jsInteresting.summaryString(
issues + [
"compareJIT is not comparing output, because the shell exited strangely"
], lev, r.elapsedtime)
print " " + sps.shellify(command)
jsInteresting.deleteLogs(prefix)
if i == 0:
return jsInteresting.JS_FINE
elif oom:
# If the shell or python hit a memory limit, we consider the rest of the computation
# "tainted" for the purpose of correctness comparison.
message = "compareJIT is not comparing output: OOM (" + oom + ")"
print infilename + " | " + jsInteresting.summaryString(
issues + [message], lev, r.elapsedtime)
jsInteresting.deleteLogs(prefix)
if i == 0:
return jsInteresting.JS_FINE
elif i == 0:
# Stash output from this run (the first one), so for subsequent runs, we can compare against it.
(r0, prefix0) = (r, prefix)
else:
# Compare the output of this run (r.out) to the output of the first run (r0.out), etc.
def fpuOptionDisabledAsmOnOneSide():
# --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things.
# This should only affect asm.js diagnostics on stderr.
fpuAsmMsg = "asm.js type error: Disabled by lack of floating point support"
fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err
fpuOptionDiffers = (("--no-fpu" in commands[0]) !=
("--no-fpu" in command))
return (fpuOptionDisabledAsm and fpuOptionDiffers)
def optionDisabledAsmOnOneSide():
asmMsg = "asm.js type error: Disabled by javascript.options.asmjs"
optionDisabledAsm = asmMsg in r0.err or asmMsg in r.err
optionDiffers = (("--no-asmjs" in commands[0]) !=
("--no-asmjs" in command))
return (optionDisabledAsm and optionDiffers)
mismatchErr = (r.err != r0.err
and not fpuOptionDisabledAsmOnOneSide()
and not optionDisabledAsmOnOneSide())
mismatchOut = (r.out != r0.out)
if mismatchErr or mismatchOut:
# Generate a short summary for stdout and a long summary for a "*-summary.txt" file.
rerunCommand = sps.shellify([
'~/funfuzz/js/compareJIT.py', "--flags=" + ' '.join(flags),
"--timeout=" + str(timeout), knownPath, jsEngine,
os.path.basename(infilename)
])
(summary, issues) = summarizeMismatch(mismatchErr, mismatchOut,
prefix0, prefix)
summary = " " + sps.shellify(
commands[0]) + "\n " + sps.shellify(
command) + "\n\n" + summary
with open(logPrefix + "-summary.txt", 'wb') as f:
f.write(rerunCommand + "\n\n" + summary)
print infilename + " | " + jsInteresting.summaryString(
issues, jsInteresting.JS_OVERALL_MISMATCH, r.elapsedtime)
if quickMode:
print rerunCommand
if showDetailedDiffs:
print summary
print ""
return jsInteresting.JS_OVERALL_MISMATCH
else:
#print "compareJIT: match"
jsInteresting.deleteLogs(prefix)
# All matched :)
jsInteresting.deleteLogs(prefix0)
return jsInteresting.JS_FINE
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt[b"AR"] = b"ar"
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt[b"RANLIB"] = b"ranlib"
cfgEnvDt[b"AS"] = b"$CC"
cfgEnvDt[b"LD"] = b"ld"
cfgEnvDt[b"STRIP"] = b"strip -x -S"
cfgEnvDt[b"CROSS_COMPILE"] = b"1"
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
# Hacked up for new and old Homebrew configs, we can probably just call autoconf213
if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])):
cfgEnvDt[b"AUTOCONF"] = b"autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig"
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str(
"clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str(
"clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS
cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib "
b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib "
b"clang_rt.asan_dynamic-x86_64.dll")
cfgEnvDt[b"HOST_CFLAGS"] = b" "
cfgEnvDt[b"HOST_CXXFLAGS"] = b" "
cfgEnvDt[b"HOST_LDFLAGS"] = b" "
cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert b"clang-cl" in cfgEnvDt[b"CC"]
assert b"clang-cl" in cfgEnvDt[b"CXX"]
else:
assert b"clang" in cfgEnvDt[b"CC"]
assert b"clang++" in cfgEnvDt[b"CXX"]
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.disableProfiling:
cfgCmdList.append('--disable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] +
'"' if " " in cfgEnvDt[str(envVar)] else envVar +
"=" + cfgEnvDt[str(envVar)])
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def jsfunfuzzLevel(options, logPrefix, quiet=False):
(lev, issues, runinfo) = baseLevel(
options.jsengineWithArgs, options.timeout, options.knownPath, logPrefix, valgrind=options.valgrind
)
if lev == JS_FINE:
# Check for unexplained exits and for jsfunfuzz saying "Found a bug".
understoodExit = False
# Read in binary mode, because otherwise Python on Windows will
# throw a fit when it encounters certain unicode. Note that this
# makes line endings platform-specific.
if "-dm-" in options.jsengineWithArgs[0]:
# Since this is an --enable-more-deterministic build, we should get messages on stderr
# if the shell quit() or terminate() functions are called.
# (We use a sketchy filename-matching check because it's faster than inspecting the binary.)
with open(logPrefix + "-err.txt", "rb") as f:
for line in f:
if "terminate called" in line or "quit called" in line:
understoodExit = True
if "can't allocate region" in line:
understoodExit = True
else:
understoodExit = True
with open(logPrefix + "-out.txt", "rb") as f:
for line in f:
if line.startswith("It's looking good!") or line.startswith(
"jsfunfuzz broke its own scripting environment: "
):
understoodExit = True
if line.startswith("Found a bug: "):
understoodExit = True
if not ("NestTest" in line and oomed(logPrefix)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
# FIXME: if not quiet:
# FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug."
if not understoodExit:
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# FIXME: if not quiet:
# FIXME: output the last tryItOut line
if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored.
sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev))
lev = JS_FINE
issues = []
if lev != JS_FINE:
# FIXME: compareJIT failures do not generate this -summary file.
statusIssueList = []
for i in issues:
statusIssueList.append("Status: " + i)
assert len(statusIssueList) != 0
fileManipulation.writeLinesToFile(
["Number: " + logPrefix + "\n", "Command: " + sps.shellify(options.jsengineWithArgs) + "\n"]
+ [i + "\n" for i in statusIssueList],
logPrefix + "-summary.txt",
)
if not quiet:
print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime)
return lev
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append('--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append('--with-nspr-prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[
'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt[
'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [
10, 10
] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgEnvDt['CC'] = cfgEnvDt[
'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt[
'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10
] and not shell.buildOptions.enable32:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
# cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert 'clang-cl' in cfgEnvDt['CC']
assert 'clang-cl' in cfgEnvDt['CXX']
else:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' +
('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append(
'--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' +
sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
