def untarbz2(fn, dest):
'''
Extracts .tar.bz2 files to their destination.
'''
if not os.path.exists(dest):
os.mkdir(dest)
sps.captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])
def getRepoHashAndId(repoDir, repoRev="parents() and default"):
"""Return the repository hash and id, and whether it is on default.
It will also ask what the user would like to do, should the repository not be on default.
"""
# This returns null if the repository is not on default.
hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", repoRev, "--template", "{node|short} {rev}"]
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input(
"Not on default tip! " + "Would you like to (a)bort, update to (d)efault, or (u)se this rev: "
)
if updateDefault == "a":
print "Aborting..."
sys.exit(0)
elif updateDefault == "d":
subprocess.check_call(["hg", "-R", repoDir, "update", "default"])
onDefault = True
elif updateDefault == "u":
hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", "parents()", "--template", "{node|short} {rev}"]
else:
raise Exception("Invalid choice.")
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ""
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(" ")
sps.vdump("Finished getting the hash and local id number of the repository.")
return hgIdChangesetHash, hgIdLocalNum, onDefault
def printMachineInfo():
# Log information about the machine.
print "Platform details: " + " ".join(platform.uname())
print "hg version: " + sps.captureStdout(['hg', '-q', 'version'])[0]
# In here temporarily to see if mock Linux slaves on TBPL have gdb installed
try:
print "gdb version: " + sps.captureStdout(['gdb', '--version'], combineStderr=True,
ignoreStderr=True, ignoreExitCode=True)[0]
except (KeyboardInterrupt, Exception) as e:
print('Error involving gdb is: ' + repr(e))
# FIXME: Should have if os.path.exists(path to git) or something
#print "git version: " + sps.captureStdout(['git', 'version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0]
print "Python version: " + sys.version.split()[0]
print "Number of cores visible to OS: " + str(multiprocessing.cpu_count())
print 'Free space (GB): ' + str('%.2f') % sps.getFreeSpace('/', 3)
hgrcLocation = os.path.join(path0, '.hg', 'hgrc')
if os.path.isfile(hgrcLocation):
print 'The hgrc of this repository is:'
with open(hgrcLocation, 'rb') as f:
hgrcContentList = f.readlines()
for line in hgrcContentList:
print line.rstrip()
if os.name == 'posix':
# resource library is only applicable to Linux or Mac platforms.
import resource
print "Corefile size (soft limit, hard limit) is: " + \
repr(resource.getrlimit(resource.RLIMIT_CORE))
def printMachineInfo():
# Log information about the machine.
print "Platform details: " + " ".join(platform.uname())
print "hg version: " + sps.captureStdout(['hg', '-q', 'version'])[0]
# In here temporarily to see if mock Linux slaves on TBPL have gdb installed
try:
print "gdb version: " + sps.captureStdout(['gdb', '--version'],
combineStderr=True,
ignoreStderr=True,
ignoreExitCode=True)[0]
except (KeyboardInterrupt, Exception) as e:
print 'Error involving gdb is: ' + repr(e)
# FIXME: Should have if os.path.exists(path to git) or something
# print "git version: " + sps.captureStdout(['git', 'version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0]
print "Python version: " + sys.version.split()[0]
print "Number of cores visible to OS: " + str(multiprocessing.cpu_count())
print 'Free space (GB): ' + str('%.2f') % sps.getFreeSpace('/', 3)
hgrcLocation = os.path.join(path0, '.hg', 'hgrc')
if os.path.isfile(hgrcLocation):
print 'The hgrc of this repository is:'
with open(hgrcLocation, 'rb') as f:
hgrcContentList = f.readlines()
for line in hgrcContentList:
print line.rstrip()
if os.name == 'posix':
# resource library is only applicable to Linux or Mac platforms.
import resource
print "Corefile size (soft limit, hard limit) is: " + \
repr(resource.getrlimit(resource.RLIMIT_CORE))
def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()):
# We may have passed in the patch with or without the full directory.
patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile))
pname = os.path.basename(patchAbsPath)
assert pname != ''
qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qimportRetCode != 0:
if 'already exists' in qimportOutput:
print "A patch with the same name has already been qpush'ed. Please qremove it first."
raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode))
print("Patch qimport'ed..."),
qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname],
combineStderr=True, ignoreStderr=True)
assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty."
if qpushRetCode != 0:
hgQpopQrmAppliedPatch(patchFile, workingDir)
print 'You may have untracked .rej or .orig files in the repository.'
print '`hg status` output of the repository of interesting files in ' + workingDir + ' :'
subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added',
'--removed', '--deleted'])
raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode))
print("Patch qpush'ed. Continuing..."),
return pname
def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()):
# We may have passed in the patch with or without the full directory.
patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile))
pname = os.path.basename(patchAbsPath)
assert pname != ''
qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qimportRetCode != 0:
if 'already exists' in qimportOutput:
print "A patch with the same name has already been qpush'ed. Please qremove it first."
raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode))
print("Patch qimport'ed..."),
qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname],
combineStderr=True, ignoreStderr=True)
assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty."
if qpushRetCode != 0:
hgQpopQrmAppliedPatch(patchFile, workingDir)
print 'You may have untracked .rej or .orig files in the repository.'
print '`hg status` output of the repository of interesting files in ' + workingDir + ' :'
subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added',
'--removed', '--deleted'])
raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode))
print("Patch qpush'ed. Continuing..."),
return pname
def untarbz2(fn, dest):
'''
Extracts .tar.bz2 files to their destination.
'''
if not os.path.exists(dest):
os.mkdir(dest)
sps.captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])
def compileJs(shell):
'''This function compiles and copies a binary.'''
try:
cmdList = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s']
out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and \
('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
# FIXME: Absolute hack to retry after hitting OOM.
print 'Trying once more due to the compiler running out of memory...'
out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
else:
print out
raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def getRepoHashAndId(repoDir, repoRev='parents() and default'):
'''
This function returns the repository hash and id, and whether it is on default.
It also asks what the user would like to do, should the repository not be on default.
'''
# This returns null if the repository is not on default.
hgLogTmplList = [
'hg', '-R', repoDir, 'log', '-r', repoRev, '--template',
'{node|short} {rev}'
]
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input('Not on default tip! ' + \
'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
if updateDefault == 'a':
print 'Aborting...'
sys.exit(0)
elif updateDefault == 'd':
subprocess.check_call(['hg', '-R', repoDir, 'update', 'default'])
onDefault = True
elif updateDefault == 'u':
hgLogTmplList = [
'hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
'{node|short} {rev}'
]
else:
raise Exception('Invalid choice.')
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ''
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
sps.vdump(
'Finished getting the hash and local id number of the repository.')
return hgIdChangesetHash, hgIdLocalNum, onDefault
def getRepoHashAndId(repoDir, repoRev='parents() and default'):
'''
This function returns the repository hash and id, and whether it is on default.
It also asks what the user would like to do, should the repository not be on default.
'''
# This returns null if the repository is not on default.
hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev,
'--template', '{node|short} {rev}']
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
onDefault = bool(hgIdFull)
if not onDefault:
updateDefault = raw_input('Not on default tip! ' + \
'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
if updateDefault == 'a':
print 'Aborting...'
sys.exit(0)
elif updateDefault == 'd':
subprocess.check_call(['hg', '-R', repoDir, 'update', 'default'])
onDefault = True
elif updateDefault == 'u':
hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
'{node|short} {rev}']
else:
raise Exception('Invalid choice.')
hgIdFull = sps.captureStdout(hgLogTmplList)[0]
assert hgIdFull != ''
(hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
sps.vdump('Finished getting the hash and local id number of the repository.')
return hgIdChangesetHash, hgIdLocalNum, onDefault
def compileJs(shell):
"""Compile and copy a binary."""
try:
cmdList = [
MAKE_BINARY, '-C',
shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'
]
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and \
('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
# FIXME: Absolute hack to retry after hitting OOM.
print 'Trying once more due to the compiler running out of memory...'
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(),
shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
majorVersion, version = extractVersions(shell.getJsObjdir())
shell.setMajorVersion(majorVersion)
shell.setVersion(version)
if sps.isLinux:
# Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a)
# files in the objdir or else the stacks from failing testcases will lack symbols.
shutil.rmtree(
sps.normExpUserPath(
os.path.join(shell.getShellCacheDir(), 'objdir-js')))
else:
print out
raise Exception(MAKE_BINARY +
" did not result in a js shell, no exception thrown.")
def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo):
"""If bisect blamed a merge, try to figure out why."""
bisectLied = False
missedCommonAncestor = False
parents = sps.captureStdout(["hg", "-R", repoDir] + ["parent", '--template={node|short},',
"-r", blamedRev])[0].split(",")[:-1]
if len(parents) == 1:
return
for p in parents:
# Ensure we actually tested the parent.
if labels.get(p) is None:
print ""
print ("Oops! We didn't test rev %s, a parent of the blamed revision! " +
"Let's do that now.") % str(p)
if not hgCmds.isAncestor(repoDir, startRepo, p) and \
not hgCmds.isAncestor(repoDir, endRepo, p):
print ('We did not test rev %s because it is not a descendant of either ' +
'%s or %s.') % (str(p), startRepo, endRepo)
# Note this in case we later decide the bisect result is wrong.
missedCommonAncestor = True
label = testRev(p)
labels[p] = label
print label[0] + " (" + label[1] + ") "
print "As expected, the parent's label is the opposite of the blamed rev's label."
# Check that the parent's label is the opposite of the blamed merge's label.
if labels[p][0] == "skip":
print "Parent rev %s was marked as 'skip', so the regression window includes it." % str(p)
elif labels[p][0] == blamedGoodOrBad:
print "Bisect lied to us! Parent rev %s was also %s!" % (str(p), blamedGoodOrBad)
bisectLied = True
else:
assert labels[p][0] == {'good': 'bad', 'bad': 'good'}[blamedGoodOrBad]
# Explain why bisect blamed the merge.
if bisectLied:
if missedCommonAncestor:
ca = hgCmds.findCommonAncestor(repoDir, parents[0], parents[1])
print ""
print "Bisect blamed the merge because our initial range did not include one"
print "of the parents."
print "The common ancestor of %s and %s is %s." % (parents[0], parents[1], ca)
label = testRev(ca)
print label[0] + " (" + label[1] + ") "
print "Consider re-running autoBisect with -s %s -e %s" % (ca, blamedRev)
print "in a configuration where earliestWorking is before the common ancestor."
else:
print ""
print "Most likely, bisect's result was unhelpful because one of the"
print "tested revisions was marked as 'good' or 'bad' for the wrong reason."
print "I don't know which revision was incorrectly marked. Sorry."
else:
print ""
print "The bug was introduced by a merge (it was not present on either parent)."
print "I don't know which patches from each side of the merge contributed to the bug. Sorry."
def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo):
"""If bisect blamed a merge, try to figure out why."""
bisectLied = False
missedCommonAncestor = False
parents = sps.captureStdout(["hg", "-R", repoDir] + ["parent", '--template={node|short},',
"-r", blamedRev])[0].split(",")[:-1]
if len(parents) == 1:
return
for p in parents:
# Ensure we actually tested the parent.
if labels.get(p) is None:
print ""
print ("Oops! We didn't test rev %s, a parent of the blamed revision! " +
"Let's do that now.") % str(p)
if not hgCmds.isAncestor(repoDir, startRepo, p) and \
not hgCmds.isAncestor(repoDir, endRepo, p):
print ('We did not test rev %s because it is not a descendant of either ' +
'%s or %s.') % (str(p), startRepo, endRepo)
# Note this in case we later decide the bisect result is wrong.
missedCommonAncestor = True
label = testRev(p)
labels[p] = label
print label[0] + " (" + label[1] + ") "
print "As expected, the parent's label is the opposite of the blamed rev's label."
# Check that the parent's label is the opposite of the blamed merge's label.
if labels[p][0] == "skip":
print "Parent rev %s was marked as 'skip', so the regression window includes it." % str(p)
elif labels[p][0] == blamedGoodOrBad:
print "Bisect lied to us! Parent rev %s was also %s!" % (str(p), blamedGoodOrBad)
bisectLied = True
else:
assert labels[p][0] == {'good': 'bad', 'bad': 'good'}[blamedGoodOrBad]
# Explain why bisect blamed the merge.
if bisectLied:
if missedCommonAncestor:
ca = hgCmds.findCommonAncestor(repoDir, parents[0], parents[1])
print ""
print "Bisect blamed the merge because our initial range did not include one"
print "of the parents."
print "The common ancestor of %s and %s is %s." % (parents[0], parents[1], ca)
label = testRev(ca)
print label[0] + " (" + label[1] + ") "
print "Consider re-running autoBisect with -s %s -e %s" % (ca, blamedRev)
print "in a configuration where earliestWorking is before the common ancestor."
else:
print ""
print "Most likely, bisect's result was unhelpful because one of the"
print "tested revisions was marked as 'good' or 'bad' for the wrong reason."
print "I don't know which revision was incorrectly marked. Sorry."
else:
print ""
print "The bug was introduced by a merge (it was not present on either parent)."
print "I don't know which patches from each side of the merge contributed to the bug. Sorry."
def isAncestor(repoDir, a, b):
"""Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist."""
return (
sps.captureStdout(
["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"]
)[0]
!= ""
)
def downloadURL(url, dest):
"""Read in a URL and downloads it to a destination."""
inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', dest, url]
out, retVal = sps.captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
if retVal != 0:
print out
raise Exception('Return code is not 0, but is: ' + str(retVal))
return dest
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
"""Tell hg what we learned about the revision."""
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix +
['bisect', '-U', '--' +
hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
if options.buildOptions:
repoDir = options.buildOptions.repoDir
if re.compile(
"Due to skipped revisions, the first (good|bad) revision could be any of:"
).match(outputLines[0]):
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print()
print()
print(
"autoBisect shows this is probably related to the following changeset:"
)
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print("Resetting to default revision...")
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def testIsHardFpShellARM(s):
'''Tests if the ARM shell is compiled with hardfp support.'''
readelfBin = '/usr/bin/readelf'
if os.path.exists(readelfBin):
newEnv = envVars.envWithPath(os.path.dirname(os.path.abspath(s)))
readelfOutput = sps.captureStdout([readelfBin, '-A', s], env=newEnv)[0]
return 'Tag_ABI_VFP_args: VFP registers' in readelfOutput
else:
raise Exception('readelf is not found.')
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def testIsHardFpShellARM(s):
'''Tests if the ARM shell is compiled with hardfp support.'''
readelfBin = '/usr/bin/readelf'
if os.path.exists(readelfBin):
newEnv = envVars.envWithPath(os.path.dirname(os.path.abspath(s)))
readelfOutput = sps.captureStdout([readelfBin, '-A', s], env=newEnv)[0]
return 'Tag_ABI_VFP_args: VFP registers' in readelfOutput
else:
raise Exception('readelf is not found.')
def existsAndIsAncestor(repoDir, a, b):
"""Return true iff |a| exists and is an ancestor of |b|."""
# Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error
out = sps.captureStdout(
["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"],
combineStderr=True,
ignoreExitCode=True,
)[0]
return out != "" and out.find("abort: unknown revision") < 0
def testBinary(shellPath, args, useValgrind):
'''Tests the given shell with the given args.'''
testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
sps.vdump('The testing command is: ' + sps.shellify(testCmd))
out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True,
ignoreExitCode=True, env=envVars.envWithPath(
os.path.dirname(os.path.abspath(shellPath))))
sps.vdump('The exit code is: ' + str(rCode))
return out, rCode
def diffFiles(f1, f2):
"""Return a command to diff two files, along with the diff output (if it's short)."""
diffcmd = ["diff", "-u", f1, f2]
s = ' '.join(diffcmd) + "\n\n"
diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0]
if len(diff) < 10000:
s += diff + "\n\n"
else:
s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n"
return s
def existsAndIsAncestor(repoDir, a, b):
"""Returns true iff |a| exists and is an ancestor of |b|."""
# Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error
out = sps.captureStdout([
'hg', '-R', repoDir, 'log', '-r',
a + ' and ancestor(' + a + ',' + b + ')', '--template={node|short}'
],
combineStderr=True,
ignoreExitCode=True)[0]
return out != "" and out.find("abort: unknown revision") < 0
def hgHashAddToFuzzPath(fuzzPath, repoDir):
'''
This function finds the mercurial revision and appends it to the directory name.
It also prompts if the user wants to continue, should the repository not be on tip.
'''
hgIdCmdList = ['hg', 'identify', '-i', '-n', '-b', repoDir]
vdump('About to start running `' + ' '.join(hgIdCmdList) + '` ...')
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdFull = captureStdout(hgIdCmdList,
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0]
hgIdChangesetHash = hgIdFull.split(' ')[0]
hgIdLocalNum = hgIdFull.split(' ')[1]
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir)[0]
onDefaultTip = True
if 'tip' not in hgIdBranch:
print 'The repository is at this changeset -', hgIdLocalNum + ':' + hgIdChangesetHash
notOnDefaultTipApproval = str(
raw_input(
'Not on default tip! Are you sure you want to continue? (y/n): '
))
if notOnDefaultTipApproval == ('y' or 'yes'):
onDefaultTip = False
else:
switchToDefaultTipApproval = str(
raw_input('Do you want to switch to the default tip? (y/n): '))
if switchToDefaultTipApproval == ('y' or 'yes'):
subprocess.check_call(['hg', 'up', 'default'], cwd=repoDir)
else:
raise Exception('Not on default tip.')
fuzzPath = '-'.join([fuzzPath, hgIdLocalNum, hgIdChangesetHash])
vdump('Finished running `' + ' '.join(hgIdCmdList) + '`.')
return normExpUserPath(fuzzPath), onDefaultTip
def diffFiles(f1, f2):
"""Return a command to diff two files, along with the diff output (if it's short)."""
diffcmd = ["diff", "-u", f1, f2]
s = ' '.join(diffcmd) + "\n\n"
diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0]
if len(diff) < 10000:
s += diff + "\n\n"
else:
s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n"
return s
def downloadURL(url, dest):
"""Read in a URL and downloads it to a destination."""
inpCmdList = [
'curl', '--output', dest, url
] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', dest, url]
out, retVal = sps.captureStdout(inpCmdList,
combineStderr=True,
ignoreExitCode=True)
if retVal != 0:
print out
raise Exception('Return code is not 0, but is: ' + str(retVal))
return dest
def hgQpopQrmAppliedPatch(patchFile, repoDir):
'''Remove applied patch using `hg qpop` and `hg qdelete`.'''
qpopOutput, qpopRetCode = sps.captureStdout(['hg', '-R', repoDir, 'qpop'],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qpopRetCode != 0:
print '`hg qpop` output is: ' + qpopOutput
raise Exception('Return code from `hg qpop` is: ' + str(qpopRetCode))
print("Patch qpop'ed..."),
subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)])
print("Patch qdelete'd.")
def hgQpopQrmAppliedPatch(patchFile, repoDir):
"""Remove applied patch using `hg qpop` and `hg qdelete`."""
qpopOutput, qpopRetCode = sps.captureStdout(['hg', '-R', repoDir, 'qpop'],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qpopRetCode != 0:
print '`hg qpop` output is: ' + qpopOutput
raise Exception('Return code from `hg qpop` is: ' + str(qpopRetCode))
print "Patch qpop'ed...",
subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)])
print "Patch qdelete'd."
def hgQpopQrmAppliedPatch(patchFile, repoDir):
"""Remove applied patch using `hg qpop` and `hg qdelete`."""
qpopOutput, qpopRetCode = sps.captureStdout(
["hg", "-R", repoDir, "qpop"], combineStderr=True, ignoreStderr=True, ignoreExitCode=True
)
if qpopRetCode != 0:
print "`hg qpop` output is: " + qpopOutput
raise Exception("Return code from `hg qpop` is: " + str(qpopRetCode))
print "Patch qpop'ed...",
subprocess.check_call(["hg", "-R", repoDir, "qdelete", os.path.basename(patchFile)])
print "Patch qdelete'd."
def compileJs(shell):
'''This function compiles and copies a binary.'''
try:
cmdList = [
MAKE_BINARY, '-C',
shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'
]
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and \
('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
# FIXME: Absolute hack to retry after hitting OOM.
print 'Trying once more due to the compiler running out of memory...'
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(),
shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
else:
print out
raise Exception(MAKE_BINARY +
" did not result in a js shell, no exception thrown.")
def compileJs(shell):
"""This function compiles and copies a binary."""
try:
cmdList = [MAKE_BINARY, "-C", shell.getJsObjdir(), "-j" + str(COMPILATION_JOBS), "-s"]
out = sps.captureStdout(
cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull()
)[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and (
"GCC running out of memory" in repr(e) or "Clang running out of memory" in repr(e)
):
# FIXME: Absolute hack to retry after hitting OOM.
print "Trying once more due to the compiler running out of memory..."
out = sps.captureStdout(
cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull(),
)[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print "A shell was compiled even though there was a non-zero exit code. Continuing..."
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
if sps.isLinux:
# Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a)
# files in the objdir or else the stacks from failing testcases will lack symbols.
shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js")))
else:
print out
raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def archOfBinary(binary):
'''This function tests if a binary is 32-bit or 64-bit.'''
unsplitFiletype = sps.captureStdout(['file', binary])[0]
filetype = unsplitFiletype.split(':', 1)[1]
if sps.isWin:
assert 'MS Windows' in filetype
return '32' if 'Intel 80386 32-bit' in filetype else '64'
else:
if 'universal binary' in filetype:
raise Exception("I don't know how to deal with multiple-architecture binaries")
if '32-bit' in filetype or 'i386' in filetype:
assert '64-bit' not in filetype
return '32'
if '64-bit' in filetype:
assert '32-bit' not in filetype
return '64'
def archOfBinary(binary):
'''This function tests if a binary is 32-bit or 64-bit.'''
unsplitFiletype = sps.captureStdout(['file', binary])[0]
filetype = unsplitFiletype.split(':', 1)[1]
if sps.isWin:
assert 'MS Windows' in filetype
return '32' if 'Intel 80386 32-bit' in filetype else '64'
else:
if 'universal binary' in filetype:
raise Exception("I don't know how to deal with multiple-architecture binaries")
if '32-bit' in filetype or 'i386' in filetype:
assert '64-bit' not in filetype
return '32'
if '64-bit' in filetype:
assert '32-bit' not in filetype
return '64'
def hgFindFixParent(self, repoDir, bugNum):
prevRev = None
hgOut = captureStdout(['hg', 'log', '-l', '10000', '--template', '{node} {desc}\n'], ignoreStderr=True, currWorkingDir=repoDir)[0].split("\n")
for line in reversed(hgOut):
line = line.split(' ', 1)
if len(line) < 2:
continue
rev = line[0][0:12]
if (line[1].find(str(bugNum)) != -1):
return prevRev
prevRev = rev
return None
def compileNspr(shell):
'''Compile a NSPR binary.'''
cfgBin(shell, 'nspr')
# Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well.
# Even if we move to parallel compile NSPR in the future, we must beware of breaking old
# build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1,
# and after that, use -jX ?
nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s']
out = sps.captureStdout(nsprCmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getNsprObjdir(), env=shell.getEnvFull())[0]
for compileLib in inspectShell.ALL_COMPILE_LIBS:
if not sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)):
print out
raise Exception(MAKE_BINARY + " did not result in a NSPR binary.")
assert os.path.isdir(sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def undmg(fn, dest, mountpoint):
"""Extract .dmg files to their destination via a mount point."""
if os.path.exists(mountpoint):
# If the mount point already exists, detach it first.
sps.captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
sps.captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
try:
apps = [x for x in os.listdir(mountpoint) if x.endswith('app')]
assert len(apps) == 1
shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
finally:
sps.captureStdout(['hdiutil', 'detach', mountpoint])
def undmg(fn, dest, mountpoint):
"""Extract .dmg files to their destination via a mount point."""
if os.path.exists(mountpoint):
# If the mount point already exists, detach it first.
sps.captureStdout(["hdiutil", "detach", mountpoint, "-force"])
sps.captureStdout(["hdiutil", "attach", "-quiet", "-mountpoint", mountpoint, fn])
try:
apps = [x for x in os.listdir(mountpoint) if x.endswith("app")]
assert len(apps) == 1
shutil.copytree(mountpoint + "/" + apps[0], dest + "/" + apps[0])
finally:
sps.captureStdout(["hdiutil", "detach", mountpoint])
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
'''Tell hg what we learned about the revision.'''
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir
if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]):
print '\n' + sanitizeCsetMsg(outputResult, repoDir) + '\n'
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print '\n\nautoBisect shows this is probably related to the following changeset:\n'
print sanitizeCsetMsg(outputResult, repoDir) + '\n'
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print 'Resetting to default revision...'
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def undmg(fn, dest, mountpoint):
"""Extract .dmg files to their destination via a mount point."""
if os.path.exists(mountpoint):
# If the mount point already exists, detach it first.
sps.captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
sps.captureStdout(
['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
try:
apps = [x for x in os.listdir(mountpoint) if x.endswith('app')]
assert len(apps) == 1
shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
finally:
sps.captureStdout(['hdiutil', 'detach', mountpoint])
def getShell(self, shellCacheDir, archNum, compileType, valgrindSupport, rev, updated, repoDir, buildFlags=None):
shell = None
# This code maps the old "-c dbg / -c opt" configurations to their configurations
haveDebugOptFlags = False
if buildFlags != None:
haveDebugOptFlags = ('--enable-debug' in buildFlags) or ('--disable-debug' in buildFlags) or ('--enable-optimize' in buildFlags) or ('--disable-optimize' in buildFlags)
print "haveDebugOptFlags: %s %s" % (str(haveDebugOptFlags), " ".join(buildFlags))
if compileType == 'dbg':
if buildFlags != None:
if not haveDebugOptFlags:
buildFlags.append('--enable-debug')
buildFlags.append('--enable-optimize')
else:
buildFlags = [ '--enable-debug', '--enable-optimize' ]
elif compileType == 'opt':
if buildFlags != None:
if not haveDebugOptFlags:
buildFlags.append('--disable-debug')
buildFlags.append('--enable-optimize')
else:
buildFlags = [ '--disable-debug', '--enable-optimize' ]
if archNum == "32":
buildFlags.append('--32')
buildOpts = '-R %s' % (repoDir)
if buildFlags != None and len(buildFlags) > 0:
buildOpts += ' %s' % " ".join(buildFlags)
if (shell == None):
if (rev == None):
rev = self.hgUpdate(repoDir, rev)
print "Compiling a new shell for tip (revision " + rev + ")"
else:
print "Compiling a new shell for revision " + rev
shell = captureStdout(['/srv/repos/funfuzz/js/compileShell.py', '-b', buildOpts, '-r', rev])[0].split("\n")[-1]
return (shell, rev)
def hgUpdate(self, repoDir, rev=None):
print "Running hg update..."
if (rev != None):
captureStdout(['hg', 'update', '-r', rev], ignoreStderr=True, currWorkingDir=repoDir)
else:
captureStdout(['hg', 'update'], ignoreStderr=True, currWorkingDir=repoDir)
hgIdCmdList = ['hg', 'identify', repoDir]
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir, ignoreStderr=True)[0]
else:
hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0]
hgIdChangesetHash = hgIdFull.split(' ')[0]
#os.chdir(savedPath)
return hgIdChangesetHash
def compileNspr(shell):
'''Compile a NSPR binary.'''
cfgBin(shell, 'nspr')
# Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well.
# Even if we move to parallel compile NSPR in the future, we must beware of breaking old
# build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1,
# and after that, use -jX ?
nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s']
out = sps.captureStdout(nsprCmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getNsprObjdir(),
env=shell.getEnvFull())[0]
for compileLib in inspectShell.ALL_COMPILE_LIBS:
if not sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib)):
print out
raise Exception(MAKE_BINARY + " did not result in a NSPR binary.")
assert os.path.isdir(
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt[b"AR"] = b"ar"
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS)
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt[b"RANLIB"] = b"ranlib"
cfgEnvDt[b"AS"] = b"$CC"
cfgEnvDt[b"LD"] = b"ld"
cfgEnvDt[b"STRIP"] = b"strip -x -S"
cfgEnvDt[b"CROSS_COMPILE"] = b"1"
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
# Hacked up for new and old Homebrew configs, we can probably just call autoconf213
if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])):
cfgEnvDt[b"AUTOCONF"] = b"autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig"
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str(
"clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str(
"clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG))
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS
cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS
cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib "
b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib "
b"clang_rt.asan_dynamic-x86_64.dll")
cfgEnvDt[b"HOST_CFLAGS"] = b" "
cfgEnvDt[b"HOST_CXXFLAGS"] = b" "
cfgEnvDt[b"HOST_LDFLAGS"] = b" "
cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows"
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS
cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS
cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert b"clang-cl" in cfgEnvDt[b"CC"]
assert b"clang-cl" in cfgEnvDt[b"CXX"]
else:
assert b"clang" in cfgEnvDt[b"CC"]
assert b"clang++" in cfgEnvDt[b"CXX"]
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.disableProfiling:
cfgCmdList.append('--disable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] +
'"' if " " in cfgEnvDt[str(envVar)] else envVar +
"=" + cfgEnvDt[str(envVar)])
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def unzip(fn, dest):
'''
Extracts .zip files to their destination.
'''
sps.captureStdout(['unzip', fn, '-d', dest])
def findBlamedCset(options, repoDir, testRev):
print "%s | Bisecting on: %s" % (sps.dateStr(), repoDir)
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(
repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir,
repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(
fileManipulation.firstLine(
sps.captureStdout(hgPrefix +
['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev,
realStartRepo, realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'],
ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def findCommonAncestor(repoDir, a, b):
return sps.captureStdout([
'hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')',
'--template={node|short}'
])[0]
def compileCopy(archNum,
compileType,
extraID,
usePymake,
repoDir,
destDir,
objDir,
vgSupport,
verbose=False):
'''
This function compiles and copies a binary.
'''
# Replace cpuCount() with cpu_count from the multiprocessing library once Python 2.6 is in
# all build machines.
jobs = (cpuCount() * 3) // 2
compiledNamePath = normExpUserPath(
os.path.join(objDir, 'js' +
('.exe' if platform.system() == 'Windows' else '')))
try:
if usePymake:
out = captureStdout([
'python', '-OO',
os.path.normpath(
os.path.join(repoDir, 'build', 'pymake', 'make.py')),
'-j' + str(jobs), '-s'
],
combineStderr=True,
currWorkingDir=objDir)[0]
# Pymake in builds earlier than revision 232553f741a0 did not support the '-s' option.
if 'no such option: -s' in out:
out = captureStdout([
'python', '-OO',
os.path.normpath(
os.path.join(repoDir, 'build', 'pymake', 'make.py')),
'-j' + str(jobs)
],
combineStderr=True,
currWorkingDir=objDir)[0]
else:
out = captureStdout(['make', '-C', objDir, '-j' + str(jobs), '-s'],
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=objDir)[0]
except Exception:
# Sometimes a non-zero error can be returned during the make process, but eventually a
# shell still gets compiled.
if os.path.exists(compiledNamePath):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
if verbose:
print out
raise Exception("`make` did not result in a js shell, '" +
repr(e) + "' thrown.")
if not os.path.exists(compiledNamePath):
if verbose:
print out
raise Exception(
"`make` did not result in a js shell, no exception thrown.")
else:
newNamePath = normExpUserPath(
os.path.join(destDir,
shellName(archNum, compileType, extraID, vgSupport)))
shutil.copy2(compiledNamePath, newNamePath)
return newNamePath
def updateRepo(repo, rev):
'''Updates repository to the specified revision.'''
# Print *with* a trailing newline to avoid breaking other stuff
print "Updating to rev %s in the %s repository..." % (rev, repo)
sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev],
ignoreStderr=True)
def unzip(fn, dest):
'''
Extracts .zip files to their destination.
'''
sps.captureStdout(['unzip', fn, '-d', dest])
def updateRepo(repo, rev):
'''Updates repository to the specified revision.'''
# Print *with* a trailing newline to avoid breaking other stuff
print "Updating to rev %s in the %s repository..." % (rev, repo)
sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True)
def unzip(fn, dest):
"""Extract .zip files to their destination."""
sps.captureStdout(['unzip', fn, '-d', dest])
def updateRepo(repo, rev):
"""Update repository to the specific revision."""
# Print *with* a trailing newline to avoid breaking other stuff
print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"),
repo.decode("utf-8", errors="replace")))
sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True)
def findBlamedCset(options, repoDir, testRev):
print sps.dateStr()
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine(
sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo,
realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append('--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append('--with-nspr-prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def isAncestor(repoDir, a, b):
"""Returns true iff |a| is an ancestor of |b|. (Throws if |a| or |b| does not exist.)"""
return sps.captureStdout([
'hg', '-R', repoDir, 'log', '-r',
a + ' and ancestor(' + a + ',' + b + ')', '--template={node|short}'
])[0] != ""
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt[
'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt[
'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [
10, 7
] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt[
'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt[
'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt[
'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt[
'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7
] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt[
'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append(
'--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' +
('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append(
'--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag(
) and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append(
'--with-nspr-prefix=' + sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(
), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append(
'--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' +
sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir(
) if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList,
ignoreStderr=True,
currWorkingDir=wDir,
env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def countCsets(revset, rdir):
"""Count the number of changesets in the revsets by outputting ones and counting them."""
listCmd = ['hg', 'log', '-r', revset, '--template=1']
rangeIntersectionOnes = sps.captureStdout(listCmd, currWorkingDir=rdir)
assert rangeIntersectionOnes[1] == 0
return len(rangeIntersectionOnes[0])
def countCsets(revset, rdir):
"""Count the number of changesets in the revsets by outputting ones and counting them."""
listCmd = ['hg', 'log', '-r', revset, '--template=1']
rangeIntersectionOnes = sps.captureStdout(listCmd, currWorkingDir=rdir)
assert rangeIntersectionOnes[1] == 0
return len(rangeIntersectionOnes[0])
def cfgBin(shell):
"""Configure a binary according to required parameters."""
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.10 Yosemite and greater
if sps.isMac:
assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior.
# Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else:
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.10 Yosemite and greater
elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
# cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
# Newer configure.in changes mean that things blow up if unknown/removed configure
# options are entered, so specify it only if it's requested.
if shell.buildOptions.enableArmSimulatorObsolete:
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
# Note: Doesn't seem to work yet.
# if shell.buildOptions.buildWithAsan:
# cfgCmdList.append('--enable-address-sanitizer')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithClang:
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if shell.buildOptions.buildWithAsan:
cfgEnvDt['CC'] += CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithClang:
if sps.isWin:
assert 'clang-cl' in cfgEnvDt['CC']
assert 'clang-cl' in cfgEnvDt['CXX']
else:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
if shell.buildOptions.enableDbg:
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg:
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif shell.buildOptions.disableOpt:
cfgCmdList.append('--disable-optimize')
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions
cfgCmdList.append('--enable-oom-breakpoint')
if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default
cfgCmdList.append('--without-intl-api')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
