def getRepoHashAndId(repoDir, repoRev='parents() and default'):
'''
This function returns the repository hash and id, and whether it is on default.
It also asks what the user would like to do, should the repository not be on default.
'''
# This returns null if the repository is not on default.
hgLogTmplList = [
'hg', '-R', repoDir, 'log', '-r', repoRev, '--template',
'{node|short} {rev}'
]
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input('Not on default tip! ' + \
'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
if updateDefault == 'a':
print 'Aborting...'
sys.exit(0)
elif updateDefault == 'd':
subprocess.check_call(['hg', '-R', repoDir, 'update', 'default'])
onDefault = True
elif updateDefault == 'u':
hgLogTmplList = [
'hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
'{node|short} {rev}'
]
else:
raise Exception('Invalid choice.')
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ''
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
sps.vdump(
'Finished getting the hash and local id number of the repository.')
return hgIdChangesetHash, hgIdLocalNum, onDefault
def getRepoHashAndId(repoDir, repoRev="parents() and default"):
"""Return the repository hash and id, and whether it is on default.
It will also ask what the user would like to do, should the repository not be on default.
"""
# This returns null if the repository is not on default.
hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", repoRev, "--template", "{node|short} {rev}"]
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input(
"Not on default tip! " + "Would you like to (a)bort, update to (d)efault, or (u)se this rev: "
)
if updateDefault == "a":
print "Aborting..."
sys.exit(0)
elif updateDefault == "d":
subprocess.check_call(["hg", "-R", repoDir, "update", "default"])
onDefault = True
elif updateDefault == "u":
hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", "parents()", "--template", "{node|short} {rev}"]
else:
raise Exception("Invalid choice.")
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ""
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(" ")
sps.vdump("Finished getting the hash and local id number of the repository.")
return hgIdChangesetHash, hgIdLocalNum, onDefault
def makeShell(shellCacheDir,
sourceDir,
archNum,
compileType,
valgrindSupport,
currRev,
verbose=False):
tempDir = tempfile.mkdtemp(prefix="abc-" + currRev + "-")
compileJsSrcPath = normExpUserPath(
os.path.join(tempDir, 'compilePath', 'js', 'src'))
vdump("Compiling in " + tempDir)
# Copy the js tree.
jsSrcDir = normExpUserPath(os.path.join(sourceDir, 'js', 'src'))
if sys.version_info >= (2, 6):
shutil.copytree(
jsSrcDir,
compileJsSrcPath,
ignore=shutil.ignore_patterns(
# ignore_patterns does not work in Python 2.5.
'jit-test',
'tests',
'trace-test',
'xpconnect'))
else:
shutil.copytree(jsSrcDir, compileJsSrcPath)
jsPubSrcDir = normExpUserPath(os.path.join(sourceDir, 'js', 'public'))
if os.path.isdir(jsPubSrcDir):
shutil.copytree(jsPubSrcDir,
os.path.join(compileJsSrcPath, '..', 'public'))
mfbtSrcDir = normExpUserPath(os.path.join(sourceDir, 'mfbt'))
if os.path.isdir(mfbtSrcDir):
shutil.copytree(mfbtSrcDir,
os.path.join(compileJsSrcPath, '..', '..', 'mfbt'))
# Run autoconf.
autoconfRun(compileJsSrcPath)
# Create objdir within the compileJsSrcPath.
objdir = os.path.join(compileJsSrcPath, compileType + '-objdir')
os.mkdir(objdir)
# Run configure.
threadsafe = False # Let's disable support for threadsafety in the js shell
cfgPath = normExpUserPath(os.path.join(compileJsSrcPath, 'configure'))
cfgJsBin(archNum, compileType, threadsafe, cfgPath, objdir)
# Compile and copy the first binary.
# Only pymake was tested on Windows.
usePymake = True if platform.system() == 'Windows' else False
try:
shell = compileCopy(archNum, compileType, currRev, usePymake,
sourceDir, shellCacheDir, objdir, valgrindSupport,
verbose)
finally:
assert os.path.isdir(tempDir) is True
rmDirInclSubDirs(tempDir)
assert os.path.isdir(tempDir) is False
return shell
def getRepoHashAndId(repoDir, repoRev='parents() and default'):
'''
This function returns the repository hash and id, and whether it is on default.
It also asks what the user would like to do, should the repository not be on default.
'''
# This returns null if the repository is not on default.
hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev,
'--template', '{node|short} {rev}']
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input('Not on default tip! ' + \
'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
if updateDefault == 'a':
print 'Aborting...'
sys.exit(0)
elif updateDefault == 'd':
subprocess.check_call(['hg', '-R', repoDir, 'update', 'default'])
onDefault = True
elif updateDefault == 'u':
hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
'{node|short} {rev}']
else:
raise Exception('Invalid choice.')
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ''
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
sps.vdump('Finished getting the hash and local id number of the repository.')
return hgIdChangesetHash, hgIdLocalNum, onDefault
def parseOpts(args):
parser = OptionParser()
parser.disable_interspersed_args()
parser.add_option(
"--comparejit",
action="store_true",
dest="useCompareJIT",
default=False,
help="After running the fuzzer, run the FCM lines against the engine in two configurations and compare the output.",
)
parser.add_option(
"--random-flags",
action="store_true",
dest="randomFlags",
default=False,
help="Pass a random set of flags (e.g. --ion-eager) to the js engine",
)
parser.add_option(
"--repo",
action="store",
dest="repo",
default=os.path.expanduser("~/trees/mozilla-central/"),
help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection",
)
parser.add_option(
"--build", action="store", dest="buildOptionsStr", help="The build options, for bisection", default=None
) # if you run loopjsfunfuzz.py directly without a --build, pinpoint will try to guess
parser.add_option(
"--valgrind",
action="store_true",
dest="valgrind",
default=False,
help="use valgrind with a reasonable set of options",
)
options, args = parser.parse_args(args)
if options.valgrind and options.useCompareJIT:
print "Note: When running comparejit, the --valgrind option will be ignored"
# kill js shell if it runs this long.
# jsfunfuzz will quit after half this time if it's not ilooping.
# higher = more complex mixing, especially with regression tests.
# lower = less time wasted in timeouts and in compareJIT testcases that are thrown away due to OOMs.
options.timeout = int(args[0])
options.knownPath = os.path.expanduser(args[1])
# FIXME: findIgnoreLists.py should probably check this automatically.
reposWithKnownLists = ["mozilla-central", "mozilla-esr45", "ionmonkey", "jscore", "v8"]
if options.knownPath not in reposWithKnownLists:
sps.vdump(
"Known bugs for the "
+ options.knownPath
+ " repository does not exist. Using the list for mozilla-central instead."
)
options.knownPath = "mozilla-central"
options.jsEngine = args[2]
options.engineFlags = args[3:]
return options
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
"""Tell hg what we learned about the revision."""
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix +
['bisect', '-U', '--' +
hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
if options.buildOptions:
repoDir = options.buildOptions.repoDir
if re.compile(
"Due to skipped revisions, the first (good|bad) revision could be any of:"
).match(outputLines[0]):
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print()
print()
print(
"autoBisect shows this is probably related to the following changeset:"
)
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print("Resetting to default revision...")
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def httpDirList(directory):
"""Read an Apache-style directory listing and returns a list of its contents, as relative URLs."""
print "Looking in " + directory + " ..."
page = readFromURL(directory)
sps.vdump('Finished reading from: ' + directory)
parser = MyHTMLParser()
fileList = parser.getHrefLinks(page, directory)
return fileList
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def httpDirList(directory):
"""Read an Apache-style directory listing and returns a list of its contents, as relative URLs."""
print("Looking in %s ..." % directory)
page = readFromURL(directory)
sps.vdump('Finished reading from: ' + directory)
parser = MyHTMLParser()
fileList = parser.getHrefLinks(page, directory)
return fileList
def testBinary(shell, file, flagsRequired, valgSupport, verbose=False, timeout=None):
testBinaryCmd = [shell] + flagsRequired + [file]
if valgSupport:
valgPrefixCmd = []
valgPrefixCmd.append('valgrind')
if platform.system() == 'Darwin':
valgPrefixCmd.append('--dsymutil=yes')
valgPrefixCmd.append('--smc-check=all-non-file')
valgPrefixCmd.append('--leak-check=full')
testBinaryCmd = valgPrefixCmd + testBinaryCmd
vdump('The testing command is:' + ' '.join(testBinaryCmd))
# Capture stdout and stderr into the same string.
p = subprocess.Popen(testBinaryCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out,err) = ('', '')
retCode = 0
if (timeout != None):
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout) # 5 minutes
try:
out, err = p.communicate()
signal.alarm(0)
retCode = p.returncode
except Alarm:
p.terminate()
if verbose:
print 'The exit code is:', retCode
if len(out) > 0:
print 'stdout shows:', out
if len(err) > 0:
print 'stderr shows:', err
# Switch to interactive input mode similar to `cat testcase.js | ./js -j -i`.
# Doesn't work, stdout shows:
#can't open : No such file or directory
#The exit code is: 4
#The second output is: None
#if retCode == 0:
# # Append the quit() function to make the testcase quit.
# # Doesn't work if retCode is something other than 0, that watchExitCode specified.
# testcaseFile = open(file, 'a')
# testcaseFile.write('\nquit()\n')
# testcaseFile.close()
#
# # Test interactive input.
# print 'Switching to interactive input mode in case passing as a CLI ' + \
# 'argument does not reproduce the issue..'
# testBinaryCmd3 = subprocess.Popen([shell, methodJit, tracingJit, '-i'],
# stdin=(subprocess.Popen(['cat', file])).stdout)
# output2 = testBinaryCmd3.communicate()[0]
# retCode = testBinaryCmd3.returncode
# print 'The exit code is:', retCode
# print 'The second output is:', output2
return out + "\n" + err, retCode
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def httpDirList(directory):
'''
Reads in an input directory and returns the href links in the directory as a list.
'''
print "Looking in " + directory + " ..."
page = readFromURL(directory)
sps.vdump('Finished reading from: ' + directory)
parser = MyHTMLParser()
fileList = parser.getHrefLinks(page)
return fileList
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
'''Tell hg what we learned about the revision.'''
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir
if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]):
print '\n' + sanitizeCsetMsg(outputResult, repoDir) + '\n'
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print '\n\nautoBisect shows this is probably related to the following changeset:\n'
print sanitizeCsetMsg(outputResult, repoDir) + '\n'
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print 'Resetting to default revision...'
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def parseOpts(args):
parser = OptionParser()
parser.disable_interspersed_args()
parser.add_option("--comparejit",
action="store_true", dest="useCompareJIT",
default=False,
help="After running the fuzzer, run the FCM lines against the engine in two configurations and compare the output.")
parser.add_option("--random-flags",
action="store_true", dest="randomFlags",
default=False,
help="Pass a random set of flags (e.g. --ion-eager) to the js engine")
parser.add_option("--repo",
action="store", dest="repo",
default=os.path.expanduser("~/trees/mozilla-central/"),
help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection")
parser.add_option("--build",
action="store", dest="buildOptionsStr",
help="The build options, for bisection",
default=None) # if you run loopjsfunfuzz.py directly without a --build, pinpoint will try to guess
parser.add_option("--valgrind",
action="store_true", dest="valgrind",
default=False,
help="use valgrind with a reasonable set of options")
options, args = parser.parse_args(args)
if options.valgrind and options.useCompareJIT:
print "Note: When running comparejit, the --valgrind option will be ignored"
# kill js shell if it runs this long.
# jsfunfuzz will quit after half this time if it's not ilooping.
# higher = more complex mixing, especially with regression tests.
# lower = less time wasted in timeouts and in compareJIT testcases that are thrown away due to OOMs.
options.timeout = int(args[0])
options.knownPath = os.path.expanduser(args[1])
# FIXME: findIgnoreLists.py should probably check this automatically.
reposWithKnownLists = ['mozilla-central', 'ionmonkey', 'jscore', 'v8']
if options.knownPath not in reposWithKnownLists:
sps.vdump('Known bugs for the ' + options.knownPath + ' repository does not exist. Using the list for mozilla-central instead.')
options.knownPath = 'mozilla-central'
options.jsEngine = args[2]
options.engineFlags = args[3:]
return options
def hgHashAddToFuzzPath(fuzzPath, repoDir):
'''
This function finds the mercurial revision and appends it to the directory name.
It also prompts if the user wants to continue, should the repository not be on tip.
'''
hgIdCmdList = ['hg', 'identify', '-i', '-n', '-b', repoDir]
vdump('About to start running `' + ' '.join(hgIdCmdList) + '` ...')
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdFull = captureStdout(hgIdCmdList,
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0]
hgIdChangesetHash = hgIdFull.split(' ')[0]
hgIdLocalNum = hgIdFull.split(' ')[1]
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir)[0]
onDefaultTip = True
if 'tip' not in hgIdBranch:
print 'The repository is at this changeset -', hgIdLocalNum + ':' + hgIdChangesetHash
notOnDefaultTipApproval = str(
raw_input(
'Not on default tip! Are you sure you want to continue? (y/n): '
))
if notOnDefaultTipApproval == ('y' or 'yes'):
onDefaultTip = False
else:
switchToDefaultTipApproval = str(
raw_input('Do you want to switch to the default tip? (y/n): '))
if switchToDefaultTipApproval == ('y' or 'yes'):
subprocess.check_call(['hg', 'up', 'default'], cwd=repoDir)
else:
raise Exception('Not on default tip.')
fuzzPath = '-'.join([fuzzPath, hgIdLocalNum, hgIdChangesetHash])
vdump('Finished running `' + ' '.join(hgIdCmdList) + '`.')
return normExpUserPath(fuzzPath), onDefaultTip
def patchHgRepoUsingMq(patchLoc, cwd=os.getcwdu()):
# We may have passed in the patch with or without the full directory.
p = os.path.abspath(normExpUserPath(patchLoc))
pname = os.path.basename(p)
assert (p, pname) != ('', '')
subprocess.check_call(['hg', 'qimport', p], cwd=cwd)
vdump("Patch qimport'ed.")
try:
subprocess.check_call(['hg', 'qpush', pname], cwd=cwd)
vdump("Patch qpush'ed.")
except subprocess.CalledProcessError:
subprocess.check_call(['hg', 'qpop'], cwd=cwd)
subprocess.check_call(['hg', 'qdelete', pname], cwd=cwd)
print 'You may have untracked .rej files in the repository.'
print '`hg st` output of the repository in ' + cwd + ' :'
subprocess.check_call(['hg', 'st'], cwd=cwd)
hgPurgeAns = str(raw_input('Do you want to run `hg purge`? (y/n): '))
assert hgPurgeAns.lower() in ('y', 'n')
if hgPurgeAns == 'y':
subprocess.check_call(['hg', 'purge'], cwd=cwd)
raise Exception(format_exc())
return pname
def cfgBin(shell):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[
'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt[
'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [
10, 10
] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgEnvDt['CC'] = cfgEnvDt[
'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt[
'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10
] and not shell.buildOptions.enable32:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
# cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert 'clang-cl' in cfgEnvDt['CC']
assert 'clang-cl' in cfgEnvDt['CXX']
else:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' +
('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append(
'--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' +
sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append('--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append('--with-nspr-prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt[b"AR"] = b"ar"
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt[b"RANLIB"] = b"ranlib"
cfgEnvDt[b"AS"] = b"$CC"
cfgEnvDt[b"LD"] = b"ld"
cfgEnvDt[b"STRIP"] = b"strip -x -S"
cfgEnvDt[b"CROSS_COMPILE"] = b"1"
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
# Hacked up for new and old Homebrew configs, we can probably just call autoconf213
if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])):
cfgEnvDt[b"AUTOCONF"] = b"autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig"
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str(
"clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str(
"clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS
cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib "
b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib "
b"clang_rt.asan_dynamic-x86_64.dll")
cfgEnvDt[b"HOST_CFLAGS"] = b" "
cfgEnvDt[b"HOST_CXXFLAGS"] = b" "
cfgEnvDt[b"HOST_LDFLAGS"] = b" "
cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert b"clang-cl" in cfgEnvDt[b"CC"]
assert b"clang-cl" in cfgEnvDt[b"CXX"]
else:
assert b"clang" in cfgEnvDt[b"CC"]
assert b"clang++" in cfgEnvDt[b"CXX"]
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.disableProfiling:
cfgCmdList.append('--disable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] +
'"' if " " in cfgEnvDt[str(envVar)] else envVar +
"=" + cfgEnvDt[str(envVar)])
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def bisectUsingTboxBins(options):
"""Download treeherder binaries and bisect them."""
testedIDs = {}
desiredArch = '32' if options.buildOptions.enable32 else '64'
buildType = downloadBuild.defaultBuildType(options.nameOfTreeherderBranch,
desiredArch,
options.buildOptions.enableDbg)
# Get list of treeherder IDs
urlsTbox = downloadBuild.getBuildList(buildType,
earliestBuild=options.startRepo,
latestBuild=options.endRepo)
# Download and test starting point.
print '\nExamining starting point...'
sID, startResult, _sReason, _sPosition, urlsTbox, testedIDs, _startSkippedNum = testBuildOrNeighbour(
options, 0, urlsTbox, buildType, testedIDs)
if sID is None:
raise Exception('No complete builds were found.')
print 'Numeric ID ' + sID + ' was tested.'
# Download and test ending point.
print '\nExamining ending point...'
eID, endResult, _eReason, _ePosition, urlsTbox, testedIDs, _endSkippedNum = testBuildOrNeighbour(
options,
len(urlsTbox) - 1, urlsTbox, buildType, testedIDs)
if eID is None:
raise Exception('No complete builds were found.')
print 'Numeric ID ' + eID + ' was tested.'
if startResult == endResult:
raise Exception(
'Starting and ending points should have opposite results')
count = 0
print '\nStarting bisection...\n'
while count < MAX_ITERATIONS:
sps.vdump('Unsorted dictionary of tested IDs is: ' + str(testedIDs))
count += 1
print 'Test number ' + str(count) + ':'
sortedUrlsTbox = sorted(urlsTbox)
if len(sortedUrlsTbox) >= 3:
mPosition = len(sortedUrlsTbox) // 2
else:
print '\nWARNING: ' + str(sortedUrlsTbox) + ' has size smaller than 3. ' + \
'Impossible to return "middle" element.\n'
mPosition = len(sortedUrlsTbox)
# Test the middle revision. If it is not a complete build, test ones around it.
mID, mResult, _mReason, mPosition, urlsTbox, testedIDs, middleRevSkippedNum = testBuildOrNeighbour(
options, mPosition, urlsTbox, buildType, testedIDs)
if mID is None:
print 'Middle ID is None.'
break
# Refresh the range of treeherder IDs depending on mResult.
if mResult == endResult:
urlsTbox = urlsTbox[0:(mPosition + 1)]
else:
urlsTbox = urlsTbox[(mPosition):len(urlsTbox)]
print 'Numeric ID ' + mID + ' was tested.',
# Exit infinite loop once we have tested the starting point, ending point and any points
# in the middle with results returning "incomplete".
if (len(urlsTbox) - middleRevSkippedNum) <= 2 and mID in testedIDs:
break
elif len(urlsTbox) < 2:
print 'urlsTbox is: ' + str(urlsTbox)
raise Exception('Length of urlsTbox should not be smaller than 2.')
elif (len(testedIDs) - 2) > 30:
raise Exception('Number of testedIDs has exceeded 30.')
print showRemainingNumOfTests(urlsTbox)
print
sps.vdump('Build URLs are: ' + str(urlsTbox))
assert getIdFromTboxUrl(
urlsTbox[0]) in testedIDs, 'Starting ID should have been tested.'
assert getIdFromTboxUrl(
urlsTbox[-1]) in testedIDs, 'Ending ID should have been tested.'
outputTboxBisectionResults(options, urlsTbox, testedIDs)
def findBlamedCset(options, repoDir, testRev):
print "%s | Bisecting on: %s" % (sps.dateStr(), repoDir)
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(
repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir,
repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(
fileManipulation.firstLine(
sps.captureStdout(hgPrefix +
['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev,
realStartRepo, realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'],
ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def testBinary(shell,
file,
flagsRequired,
valgSupport,
verbose=False,
timeout=None):
testBinaryCmd = [shell] + flagsRequired + [file]
if valgSupport:
valgPrefixCmd = []
valgPrefixCmd.append('valgrind')
if platform.system() == 'Darwin':
valgPrefixCmd.append('--dsymutil=yes')
valgPrefixCmd.append('--smc-check=all-non-file')
valgPrefixCmd.append('--leak-check=full')
testBinaryCmd = valgPrefixCmd + testBinaryCmd
vdump('The testing command is:' + ' '.join(testBinaryCmd))
# Capture stdout and stderr into the same string.
p = subprocess.Popen(testBinaryCmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = ('', '')
retCode = 0
if (timeout != None):
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout) # 5 minutes
try:
out, err = p.communicate()
signal.alarm(0)
retCode = p.returncode
except Alarm:
p.terminate()
if verbose:
print 'The exit code is:', retCode
if len(out) > 0:
print 'stdout shows:', out
if len(err) > 0:
print 'stderr shows:', err
# Switch to interactive input mode similar to `cat testcase.js | ./js -j -i`.
# Doesn't work, stdout shows:
#can't open : No such file or directory
#The exit code is: 4
#The second output is: None
#if retCode == 0:
# # Append the quit() function to make the testcase quit.
# # Doesn't work if retCode is something other than 0, that watchExitCode specified.
# testcaseFile = open(file, 'a')
# testcaseFile.write('\nquit()\n')
# testcaseFile.close()
#
# # Test interactive input.
# print 'Switching to interactive input mode in case passing as a CLI ' + \
# 'argument does not reproduce the issue..'
# testBinaryCmd3 = subprocess.Popen([shell, methodJit, tracingJit, '-i'],
# stdin=(subprocess.Popen(['cat', file])).stdout)
# output2 = testBinaryCmd3.communicate()[0]
# retCode = testBinaryCmd3.returncode
# print 'The exit code is:', retCode
# print 'The second output is:', output2
return out + "\n" + err, retCode
def bisectUsingTboxBins(options):
'''
Downloads treeherder binaries and bisects them.
'''
testedIDs = {}
desiredArch = '32' if options.buildOptions.enable32 else '64'
buildType = downloadBuild.defaultBuildType(options.nameOfTreeherderBranch, desiredArch, options.buildOptions.enableDbg)
# Get list of treeherder IDs
urlsTbox = downloadBuild.getBuildList(buildType, earliestBuild=options.startRepo, latestBuild=options.endRepo)
# Download and test starting point.
print '\nExamining starting point...'
sID, startResult, sReason, sPosition, urlsTbox, testedIDs, startSkippedNum = testBuildOrNeighbour(
options, 0, urlsTbox, buildType, testedIDs)
if sID is None:
raise Exception('No complete builds were found.')
print 'Numeric ID ' + sID + ' was tested.'
# Download and test ending point.
print '\nExamining ending point...'
eID, endResult, eReason, ePosition, urlsTbox, testedIDs, endSkippedNum = testBuildOrNeighbour(
options, len(urlsTbox) - 1, urlsTbox, buildType, testedIDs)
if eID is None:
raise Exception('No complete builds were found.')
print 'Numeric ID ' + eID + ' was tested.'
if startResult == endResult:
raise Exception('Starting and ending points should have opposite results')
count = 0
print '\nStarting bisection...\n'
while count < MAX_ITERATIONS:
sps.vdump('Unsorted dictionary of tested IDs is: ' + str(testedIDs))
count += 1
print 'Test number ' + str(count) + ':'
sortedUrlsTbox = sorted(urlsTbox)
if len(sortedUrlsTbox) >= 3:
mPosition = len(sortedUrlsTbox) // 2
else:
print '\nWARNING: ' + str(sortedUrlsTbox) + ' has size smaller than 3. ' + \
'Impossible to return "middle" element.\n'
mPosition = len(sortedUrlsTbox)
# Test the middle revision. If it is not a complete build, test ones around it.
mID, mResult, mReason, mPosition, urlsTbox, testedIDs, middleRevSkippedNum = testBuildOrNeighbour(
options, mPosition, urlsTbox, buildType, testedIDs)
if mID is None:
print 'Middle ID is None.'
break
# Refresh the range of treeherder IDs depending on mResult.
if mResult == endResult:
urlsTbox = urlsTbox[0:(mPosition + 1)]
else:
urlsTbox = urlsTbox[(mPosition):len(urlsTbox)]
print 'Numeric ID ' + mID + ' was tested.',
# Exit infinite loop once we have tested the starting point, ending point and any points
# in the middle with results returning "incomplete".
if (len(urlsTbox) - middleRevSkippedNum) <= 2 and mID in testedIDs:
break
elif len(urlsTbox) < 2:
print 'urlsTbox is: ' + str(urlsTbox)
raise Exception('Length of urlsTbox should not be smaller than 2.')
elif (len(testedIDs) - 2) > 30:
raise Exception('Number of testedIDs has exceeded 30.')
print showRemainingNumOfTests(urlsTbox)
print
sps.vdump('Build URLs are: ' + str(urlsTbox))
assert getIdFromTboxUrl(urlsTbox[0]) in testedIDs, 'Starting ID should have been tested.'
assert getIdFromTboxUrl(urlsTbox[-1]) in testedIDs, 'Ending ID should have been tested.'
outputTboxBisectionResults(options, urlsTbox, testedIDs)
def cfgBin(shell):
"""This function configures a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt["AR"] = "ar"
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang"))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang++"))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, "arm7vl boards are only 32-bit, armv8 boards will be 64-bit."
if not shell.buildOptions.enableHardFp:
cfgEnvDt["CC"] = "gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgEnvDt["CXX"] = "g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append("--target=arm-linux-gnueabi")
elif shell.buildOptions.enable32 and os.name == "posix":
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = "clang" + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = "clang++" + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt["CC"] = cfgEnvDt["CC"] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt["CXX"] = cfgEnvDt["CXX"] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt["RANLIB"] = "ranlib"
cfgEnvDt["AS"] = "$CC"
cfgEnvDt["LD"] = "ld"
cfgEnvDt["STRIP"] = "strip -x -S"
cfgEnvDt["CROSS_COMPILE"] = "1"
if sps.isProgramInstalled("brew"):
cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=i386-apple-darwin9.2.0") # Leopard 10.5.2
cfgCmdList.append("--enable-macos-target=10.5")
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig"
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = (
CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
)
cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = (
CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
)
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt["CC"] = "gcc -m32" + SSE2_FLAGS
cfgEnvDt["CXX"] = "g++ -m32" + SSE2_FLAGS
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=i686-pc-linux")
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
else:
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt["CC"] = "clang" + CLANG_PARAMS
cfgEnvDt["CXX"] = "clang++" + CLANG_PARAMS
if sps.isProgramInstalled("brew"):
cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append("--target=x86_64-apple-darwin12.5.0") # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append("--enable-simulator=arm64")
elif sps.isWin:
cfgEnvDt["MAKE"] = "mozmake" # Workaround for bug 948534
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append("--enable-arm-simulator")
cfgCmdList.append("--enable-simulator=arm")
else:
cfgCmdList.append("--host=x86_64-pc-mingw32")
cfgCmdList.append("--target=x86_64-pc-mingw32")
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append("--enable-simulator=arm64")
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append("sh")
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append("--enable-address-sanitizer")
if shell.buildOptions.buildWithAsan:
assert "clang" in cfgEnvDt["CC"]
assert "clang++" in cfgEnvDt["CXX"]
cfgCmdList.append("--disable-jemalloc") # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append("--enable-debug")
elif shell.buildOptions.disableDbg:
cfgCmdList.append("--disable-debug")
if shell.buildOptions.enableOpt:
cfgCmdList.append("--enable-optimize" + ("=-O1" if shell.buildOptions.buildWithVg else ""))
elif shell.buildOptions.disableOpt:
cfgCmdList.append("--disable-optimize")
if shell.buildOptions.enableProfiling:
cfgCmdList.append("--enable-profiling")
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append("--enable-more-deterministic")
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append("--enable-oom-breakpoint")
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append("--without-intl-api")
if shell.buildOptions.buildWithVg:
cfgCmdList.append("--enable-valgrind")
cfgCmdList.append("--disable-jemalloc")
# We add the following flags by default.
if os.name == "posix":
cfgCmdList.append("--with-ccache")
cfgCmdList.append("--enable-gczeal")
cfgCmdList.append("--enable-debug-symbols") # gets debug symbols on opt shells
cfgCmdList.append("--disable-tests")
if os.name == "nt":
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, "//")
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = (
envVar + '="' + cfgEnvDt[envVar] + '"' if " " in cfgEnvDt[envVar] else envVar + "=" + cfgEnvDt[envVar]
)
envVarList.append(strToBeAppended)
sps.vdump("Command to be run is: " + sps.shellify(envVarList) + " " + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if "\\" in entry:
entry = entry.replace("\\", "/")
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def findBlamedCset(options, repoDir, testRev):
print sps.dateStr()
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine(
sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo,
realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def jsfunfuzzLevel(options, logPrefix, quiet=False):
(lev, issues, runinfo) = baseLevel(
options.jsengineWithArgs, options.timeout, options.knownPath, logPrefix, valgrind=options.valgrind
)
if lev == JS_FINE:
# Check for unexplained exits and for jsfunfuzz saying "Found a bug".
understoodExit = False
# Read in binary mode, because otherwise Python on Windows will
# throw a fit when it encounters certain unicode. Note that this
# makes line endings platform-specific.
if "-dm-" in options.jsengineWithArgs[0]:
# Since this is an --enable-more-deterministic build, we should get messages on stderr
# if the shell quit() or terminate() functions are called.
# (We use a sketchy filename-matching check because it's faster than inspecting the binary.)
with open(logPrefix + "-err.txt", "rb") as f:
for line in f:
if "terminate called" in line or "quit called" in line:
understoodExit = True
if "can't allocate region" in line:
understoodExit = True
else:
understoodExit = True
with open(logPrefix + "-out.txt", "rb") as f:
for line in f:
if line.startswith("It's looking good!") or line.startswith(
"jsfunfuzz broke its own scripting environment: "
):
understoodExit = True
if line.startswith("Found a bug: "):
understoodExit = True
if not ("NestTest" in line and oomed(logPrefix)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
# FIXME: if not quiet:
# FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug."
if not understoodExit:
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# FIXME: if not quiet:
# FIXME: output the last tryItOut line
if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored.
sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev))
lev = JS_FINE
issues = []
if lev != JS_FINE:
# FIXME: compareJIT failures do not generate this -summary file.
statusIssueList = []
for i in issues:
statusIssueList.append("Status: " + i)
assert len(statusIssueList) != 0
fileManipulation.writeLinesToFile(
["Number: " + logPrefix + "\n", "Command: " + sps.shellify(options.jsengineWithArgs) + "\n"]
+ [i + "\n" for i in statusIssueList],
logPrefix + "-summary.txt",
)
if not quiet:
print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime)
return lev
def jsfunfuzzLevel(options, logPrefix, quiet=False):
(lev, issues, runinfo) = baseLevel(options.jsengineWithArgs,
options.timeout,
options.knownPath,
logPrefix,
valgrind=options.valgrind)
if lev == JS_FINE:
# Check for unexplained exits and for jsfunfuzz saying "Found a bug".
understoodExit = False
# Read in binary mode, because otherwise Python on Windows will
# throw a fit when it encounters certain unicode. Note that this
# makes line endings platform-specific.
if '-dm-' in options.jsengineWithArgs[0]:
# Since this is an --enable-more-deterministic build, we should get messages on stderr
# if the shell quit() or terminate() functions are called.
# (We use a sketchy filename-matching check because it's faster than inspecting the binary.)
with open(logPrefix + "-err.txt", "rb") as f:
for line in f:
if "terminate called" in line or "quit called" in line:
understoodExit = True
if "can't allocate region" in line:
understoodExit = True
else:
understoodExit = True
with open(logPrefix + "-out.txt", "rb") as f:
for line in f:
if line.startswith("It's looking good!") or line.startswith(
"jsfunfuzz broke its own scripting environment: "):
understoodExit = True
if line.startswith("Found a bug: "):
understoodExit = True
if not ("NestTest" in line and oomed(logPrefix)):
lev = JS_DECIDED_TO_EXIT
issues.append(line.rstrip())
# FIXME: if not quiet:
# FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug."
if not understoodExit:
issues.append("jsfunfuzz didn't finish")
lev = JS_DID_NOT_FINISH
# FIXME: if not quiet:
# FIXME: output the last tryItOut line
if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored.
sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev))
lev = JS_FINE
issues = []
if lev != JS_FINE:
# FIXME: compareJIT failures do not generate this -summary file.
statusIssueList = []
for i in issues:
statusIssueList.append('Status: ' + i)
assert len(statusIssueList) != 0
fileManipulation.writeLinesToFile([
'Number: ' + logPrefix + '\n',
'Command: ' + sps.shellify(options.jsengineWithArgs) + '\n'
] + [i + '\n' for i in statusIssueList], logPrefix + '-summary.txt')
if not quiet:
print logPrefix + " | " + summaryString(issues, lev,
runinfo.elapsedtime)
return lev
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
# cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert 'clang-cl' in cfgEnvDt['CC']
assert 'clang-cl' in cfgEnvDt['CXX']
else:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[
'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt[
'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [
10, 7
] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt[
'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt[
'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7
] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' +
('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append(
'--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag(
) and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append(
'--with-nspr-prefix=' + sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(
), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append(
'--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' +
sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir(
) if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)