def makeShell(shellCacheDir,
sourceDir,
archNum,
compileType,
valgrindSupport,
currRev,
verbose=False):
tempDir = tempfile.mkdtemp(prefix="abc-" + currRev + "-")
compileJsSrcPath = normExpUserPath(
os.path.join(tempDir, 'compilePath', 'js', 'src'))
vdump("Compiling in " + tempDir)
# Copy the js tree.
jsSrcDir = normExpUserPath(os.path.join(sourceDir, 'js', 'src'))
if sys.version_info >= (2, 6):
shutil.copytree(
jsSrcDir,
compileJsSrcPath,
ignore=shutil.ignore_patterns(
# ignore_patterns does not work in Python 2.5.
'jit-test',
'tests',
'trace-test',
'xpconnect'))
else:
shutil.copytree(jsSrcDir, compileJsSrcPath)
jsPubSrcDir = normExpUserPath(os.path.join(sourceDir, 'js', 'public'))
if os.path.isdir(jsPubSrcDir):
shutil.copytree(jsPubSrcDir,
os.path.join(compileJsSrcPath, '..', 'public'))
mfbtSrcDir = normExpUserPath(os.path.join(sourceDir, 'mfbt'))
if os.path.isdir(mfbtSrcDir):
shutil.copytree(mfbtSrcDir,
os.path.join(compileJsSrcPath, '..', '..', 'mfbt'))
# Run autoconf.
autoconfRun(compileJsSrcPath)
# Create objdir within the compileJsSrcPath.
objdir = os.path.join(compileJsSrcPath, compileType + '-objdir')
os.mkdir(objdir)
# Run configure.
threadsafe = False # Let's disable support for threadsafety in the js shell
cfgPath = normExpUserPath(os.path.join(compileJsSrcPath, 'configure'))
cfgJsBin(archNum, compileType, threadsafe, cfgPath, objdir)
# Compile and copy the first binary.
# Only pymake was tested on Windows.
usePymake = True if platform.system() == 'Windows' else False
try:
shell = compileCopy(archNum, compileType, currRev, usePymake,
sourceDir, shellCacheDir, objdir, valgrindSupport,
verbose)
finally:
assert os.path.isdir(tempDir) is True
rmDirInclSubDirs(tempDir)
assert os.path.isdir(tempDir) is False
return shell
def getOneBuild(isJsShell, url, buildType):
'''
Try to get a complete working build.
'''
idNum = getIdFromTboxUrl(url)
tboxCacheFolder = sps.normExpUserPath(os.path.join(compileShell.ensureCacheDir(),
'tboxjs-' + buildType + '-' + idNum))
createTboxCacheFolder(tboxCacheFolder)
incompleteBuildTxtFile = sps.normExpUserPath(os.path.join(tboxCacheFolder, INCOMPLETE_NOTE))
if os.path.isfile(getTboxJsBinPath(tboxCacheFolder)):
return True, idNum, tboxCacheFolder # Cached, complete
if os.path.isfile(incompleteBuildTxtFile):
assert os.listdir(tboxCacheFolder) == [INCOMPLETE_NOTE], 'Only ' + \
'incompleteBuild.txt should be present in ' + tboxCacheFolder
readIncompleteBuildTxtFile(incompleteBuildTxtFile, idNum)
return False, None, None # Cached, incomplete
if downloadBuild.downloadBuild(url, tboxCacheFolder, jsShell=isJsShell):
assert os.listdir(tboxCacheFolder) == ['build'], 'Only ' + \
'the build subdirectory should be present in ' + tboxCacheFolder
return True, idNum, tboxCacheFolder # Downloaded, complete
else:
writeIncompleteBuildTxtFile(url, tboxCacheFolder, incompleteBuildTxtFile, idNum)
return False, None, None # Downloaded, incomplete
def parseShellOptions(inputArgs):
"""Returns a 'buildOptions' object, which is intended to be immutable."""
parser, randomizer = addParserOptions()
buildOptions = parser.parse_args(inputArgs.split())
# Ensures releng machines do not enter the if block and assumes mozilla-central always exists
if os.path.isdir(DEFAULT_TREES_LOCATION):
# Repositories do not get randomized if a repository is specified.
if buildOptions.repoDir is None:
# For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead
if buildOptions.enableRandom and not buildOptions.patchFile:
buildOptions.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION)
else:
buildOptions.repoDir = os.path.realpath(sps.normExpUserPath(
os.path.join(DEFAULT_TREES_LOCATION, 'mozilla-central')))
assert hgCmds.isRepoValid(buildOptions.repoDir)
if buildOptions.patchFile:
hgCmds.ensureMqEnabled()
buildOptions.patchFile = sps.normExpUserPath(buildOptions.patchFile)
assert os.path.isfile(buildOptions.patchFile)
if buildOptions.enableRandom:
buildOptions = generateRandomConfigurations(parser, randomizer)
else:
buildOptions.buildOptionsStr = inputArgs
valid = areArgsValid(buildOptions)
if not valid[0]:
print 'WARNING: This set of build options is not tested well because: ' + valid[1]
return buildOptions
def cfgJsCompile(shell):
"""Configures, compiles and copies a js shell according to required parameters."""
print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff
os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
autoconfRun(shell.getRepoDirJsSrc())
configureTryCount = 0
while True:
try:
cfgBin(shell)
break
except Exception as e:
configureTryCount += 1
if configureTryCount > 3:
print("Configuration of the js binary failed 3 times.")
raise
# This exception message is returned from sps.captureStdout via cfgBin.
# No idea why this is sps.isLinux as well..
if sps.isLinux or (sps.isWin and 'Windows conftest.exe configuration permission' in repr(e)):
print("Trying once more...")
continue
compileJs(shell)
inspectShell.verifyBinary(shell)
compileLog = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(),
shell.getShellNameWithoutExt() + '.fuzzmanagerconf'))
if not os.path.isfile(compileLog):
envDump(shell, compileLog)
def getRandomValidRepo(treeLocation):
validRepos = []
for repo in [
'mozilla-central', 'mozilla-aurora', 'mozilla-beta',
'mozilla-release', 'mozilla-esr31'
]:
if os.path.isfile(
sps.normExpUserPath(
os.path.join(treeLocation, repo, '.hg', 'hgrc'))):
validRepos.append(repo)
# After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
if 'mozilla-aurora' in validRepos and chance(0.4):
validRepos.remove('mozilla-aurora')
if 'mozilla-beta' in validRepos and chance(0.7):
validRepos.remove('mozilla-beta')
if 'mozilla-release' in validRepos and chance(0.9):
validRepos.remove('mozilla-release')
if 'mozilla-esr31' in validRepos and chance(0.8):
validRepos.remove('mozilla-esr31')
validRepos = [
'mozilla-central'
] # FIXME: Let's set to random configurations within m-c for now
return os.path.realpath(
sps.normExpUserPath(
os.path.join(treeLocation, random.choice(validRepos))))
def getOneBuild(isJsShell, url, buildType):
"""Try to get a complete working build."""
idNum = getIdFromTboxUrl(url)
tboxCacheFolder = sps.normExpUserPath(
os.path.join(compileShell.ensureCacheDir(),
'tboxjs-' + buildType + '-' + idNum))
createTboxCacheFolder(tboxCacheFolder)
incompleteBuildTxtFile = sps.normExpUserPath(
os.path.join(tboxCacheFolder, INCOMPLETE_NOTE))
if os.path.isfile(getTboxJsBinPath(tboxCacheFolder)):
return True, idNum, tboxCacheFolder # Cached, complete
if os.path.isfile(incompleteBuildTxtFile):
assert os.listdir(tboxCacheFolder) == [INCOMPLETE_NOTE], 'Only ' + \
'incompleteBuild.txt should be present in ' + tboxCacheFolder
readIncompleteBuildTxtFile(incompleteBuildTxtFile, idNum)
return False, None, None # Cached, incomplete
if downloadBuild.downloadBuild(url, tboxCacheFolder, jsShell=isJsShell):
assert os.listdir(tboxCacheFolder) == ['build'], 'Only ' + \
'the build subdirectory should be present in ' + tboxCacheFolder
return True, idNum, tboxCacheFolder # Downloaded, complete
else:
writeIncompleteBuildTxtFile(url, tboxCacheFolder,
incompleteBuildTxtFile, idNum)
return False, None, None # Downloaded, incomplete
def cfgJsCompile(shell):
'''Configures, compiles and copies a js shell according to required parameters.'''
print "Compiling..." # Print *with* a trailing newline to avoid breaking other stuff
os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
autoconfRun(shell.getRepoDirJsSrc())
configureTryCount = 0
while True:
try:
cfgBin(shell)
break
except Exception as e:
configureTryCount += 1
if configureTryCount > 3:
print 'Configuration of the js binary failed 3 times.'
raise
# This exception message is returned from sps.captureStdout via cfgBin.
# No idea why this is sps.isLinux as well..
if sps.isLinux or (sps.isWin and 'Windows conftest.exe configuration permission' in repr(e)):
print 'Trying once more...'
continue
compileJs(shell)
inspectShell.verifyBinary(shell)
compileLog = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(),
shell.getShellNameWithoutExt() + '.fuzzmanagerconf'))
if not os.path.isfile(compileLog):
envDump(shell, compileLog)
def assertSaneJsBinary(cacheF):
'''
If the cache folder is present, check that the js binary is working properly.
'''
if os.path.isdir(cacheF):
fList = os.listdir(cacheF)
if 'build' in fList:
if INCOMPLETE_NOTE in fList:
print cacheF + ' has subdirectories: ' + str(fList)
raise Exception(
'Downloaded binaries and incompleteBuild.txt should not both be '
+ 'present together in this directory.')
assert os.path.isdir(
sps.normExpUserPath(os.path.join(cacheF, 'build', 'download')))
assert os.path.isdir(
sps.normExpUserPath(os.path.join(cacheF, 'build', 'dist')))
assert os.path.isfile(
sps.normExpUserPath(
os.path.join(cacheF, 'build', 'dist',
'js' + ('.exe' if sps.isWin else ''))))
try:
shellPath = getTboxJsBinPath(cacheF)
# Ensure we don't fail because the shell lacks u+x
if not os.access(shellPath, os.X_OK):
os.chmod(shellPath, stat.S_IXUSR)
# tbpl binaries are always:
# * run without Valgrind (they are not compiled with --enable-valgrind)
retCode = inspectShell.testBinary(shellPath, ['-e', '42'],
False)[1]
# Exit code -1073741515 on Windows shows up when a required DLL is not present.
# This was testable at the time of writing, see bug 953314.
isDllNotPresentWinStartupError = (sps.isWin
and retCode == -1073741515)
# We should have another condition here for non-Windows platforms but we do not yet
# have a situation where we can test broken treeherder js shells on those platforms.
if isDllNotPresentWinStartupError:
raise Exception(
'Shell startup error - a .dll file is probably not present.'
)
elif retCode != 0:
raise Exception('Non-zero return code: ' + str(retCode))
return True # Binary is working correctly
except (OSError, IOError):
raise Exception('Cache folder ' + cacheF +
' is corrupt, please delete it ' +
'and try again.')
elif INCOMPLETE_NOTE in fList:
return True
else:
raise Exception(
'Neither build/ nor INCOMPLETE_NOTE were found in the cache folder.'
)
else:
raise Exception('Cache folder ' + cacheF + ' is not found.')
def getRandomValidRepo(treeLocation):
validRepos = []
for repo in ['mozilla-central', 'mozilla-beta']:
if os.path.isfile(sps.normExpUserPath(os.path.join(
treeLocation, repo, '.hg', 'hgrc'))):
validRepos.append(repo)
# After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
if 'mozilla-beta' in validRepos and chance(0.5):
validRepos.remove('mozilla-beta')
return os.path.realpath(sps.normExpUserPath(
os.path.join(treeLocation, random.choice(validRepos))))
def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num):
"""Write a text file indicating that this particular build is incomplete."""
if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \
os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))):
sps.rmTreeIncludingReadOnly(
sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
assert not os.path.isfile(
txtFile), 'incompleteBuild.txt should not be present.'
with open(txtFile, 'wb') as f:
f.write('This build with numeric ID ' + num + ' is incomplete.')
assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \
' has to be the one we downloaded from ' + url
print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.'
return False # False indicates that this text file has not yet been looked at.
def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num):
'''
Writes a text file indicating that this particular build is incomplete.
'''
if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \
os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
assert not os.path.isfile(txtFile), 'incompleteBuild.txt should not be present.'
with open(txtFile, 'wb') as f:
f.write('This build with numeric ID ' + num + ' is incomplete.')
assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \
' has to be the one we downloaded from ' + url
print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.'
return False # False indicates that this text file has not yet been looked at.
def makeRegressionTestPrologue(repo, regressionTestListFile):
"""Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests"""
# We use json.dumps to escape strings (Windows paths have backslashes).
return """
const regressionTestsRoot = %s;
const libdir = regressionTestsRoot + %s; // needed by jit-tests
var regressionTestList;
try { regressionTestList = read(%s).match(/.+/g); } catch(e) { }
""" % (
json.dumps(sps.normExpUserPath(repo) + os.sep),
json.dumps(os.path.join('js', 'src', 'jit-test', 'lib') + os.sep),
json.dumps(os.path.abspath(sps.normExpUserPath(regressionTestListFile))),
)
def ensureCacheDir():
'''Returns a cache directory for compiled shells to live in, creating one if needed'''
cacheDir = os.path.join(sps.normExpUserPath('~'), 'shell-cache')
ensureDir(cacheDir)
# Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff)
# This has to occur after the shell-cache directory is created
if sps.isWin: # adapted from http://stackoverflow.com/a/3931799
winTmpDir = unicode(cacheDir)
GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
unicodeBuffer = ctypes.create_unicode_buffer(GetLongPathName(winTmpDir, 0, 0))
GetLongPathName(winTmpDir, unicodeBuffer, len(unicodeBuffer))
cacheDir = sps.normExpUserPath(str(unicodeBuffer.value)) # convert back to a str
return cacheDir
def getRandomValidRepo(treeLocation):
validRepos = []
for repo in ['mozilla-central', 'mozilla-aurora', 'mozilla-esr45']:
if os.path.isfile(sps.normExpUserPath(os.path.join(
treeLocation, repo, '.hg', 'hgrc'))):
validRepos.append(repo)
# After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
if 'mozilla-aurora' in validRepos and chance(0.5):
validRepos.remove('mozilla-aurora')
if 'mozilla-esr45' in validRepos and chance(0.9):
validRepos.remove('mozilla-esr45')
return os.path.realpath(sps.normExpUserPath(
os.path.join(treeLocation, random.choice(validRepos))))
def getRepoNameFromHgrc(repoDir):
"""Look in the hgrc file in the .hg directory of the repository and return the name."""
assert isRepoValid(repoDir)
hgCfg = ConfigParser.SafeConfigParser()
hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, ".hg", "hgrc")))
# Not all default entries in [paths] end with "/".
return [i for i in hgCfg.get("paths", "default").split("/") if i][-1]
def compileJs(shell):
'''This function compiles and copies a binary.'''
try:
cmdList = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s']
out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and \
('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
# FIXME: Absolute hack to retry after hitting OOM.
print 'Trying once more due to the compiler running out of memory...'
out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
if sps.isLinux:
# Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a)
# files in the objdir or else the stacks from failing testcases will lack symbols.
shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
else:
print out
raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()):
# We may have passed in the patch with or without the full directory.
patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile))
pname = os.path.basename(patchAbsPath)
assert pname != ''
qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qimportRetCode != 0:
if 'already exists' in qimportOutput:
print "A patch with the same name has already been qpush'ed. Please qremove it first."
raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode))
print("Patch qimport'ed..."),
qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname],
combineStderr=True, ignoreStderr=True)
assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty."
if qpushRetCode != 0:
hgQpopQrmAppliedPatch(patchFile, workingDir)
print 'You may have untracked .rej or .orig files in the repository.'
print '`hg status` output of the repository of interesting files in ' + workingDir + ' :'
subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added',
'--removed', '--deleted'])
raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode))
print("Patch qpush'ed. Continuing..."),
return pname
def getRepoNameFromHgrc(repoDir):
'''Looks in the hgrc file in the .hg directory of the repository and returns the name.'''
assert isRepoValid(repoDir)
hgCfg = ConfigParser.SafeConfigParser()
hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, '.hg', 'hgrc')))
# Not all default entries in [paths] end with "/".
return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1]
def getShellCompiledRunLibsPath(self):
lDir = self.getJsObjdir()
libsList = [
sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
for runLib in inspectShell.ALL_RUN_LIBS
]
return libsList
def getShellCompiledRunLibsPath(self):
lDir = self.getJsObjdir()
libsList = [
sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
for runLib in inspectShell.ALL_RUN_LIBS
]
return libsList
def getShellCompiledRunLibsPath(self):
lDir = self.getJsObjdir() if self.getJsBuildSystemConsidersNspr() else self.getNsprObjdir()
libsList = [
sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
for runLib in inspectShell.ALL_RUN_LIBS
]
return libsList
def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()):
# We may have passed in the patch with or without the full directory.
patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile))
pname = os.path.basename(patchAbsPath)
assert pname != ''
qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath],
combineStderr=True, ignoreStderr=True,
ignoreExitCode=True)
if qimportRetCode != 0:
if 'already exists' in qimportOutput:
print "A patch with the same name has already been qpush'ed. Please qremove it first."
raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode))
print("Patch qimport'ed..."),
qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname],
combineStderr=True, ignoreStderr=True)
assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty."
if qpushRetCode != 0:
hgQpopQrmAppliedPatch(patchFile, workingDir)
print 'You may have untracked .rej or .orig files in the repository.'
print '`hg status` output of the repository of interesting files in ' + workingDir + ' :'
subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added',
'--removed', '--deleted'])
raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode))
print("Patch qpush'ed. Continuing..."),
return pname
def jsFilesIn(repoPathLength, root):
return [
os.path.join(path, filename)[repoPathLength:]
for path, _dirs, files in os.walk(sps.normExpUserPath(root))
for filename in files
if filename.endswith(".js")
]
def getTboxJsBinPath(baseDir):
'''
Returns the path to the treeherder js binary from a download folder.
'''
return sps.normExpUserPath(
os.path.join(baseDir, 'build', 'dist',
'js.exe' if sps.isWin else 'js'))
def getRepoNameFromHgrc(repoDir):
'''Looks in the hgrc file in the .hg directory of the repository and returns the name.'''
assert isRepoValid(repoDir)
hgCfg = ConfigParser.SafeConfigParser()
hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, '.hg', 'hgrc')))
# Not all default entries in [paths] end with "/".
return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1]
def getShellCompiledRunLibsPath(self):
lDir = self.getJsObjdir() if self.getJsBuildSystemConsidersNspr(
) else self.getNsprObjdir()
libsList = [
sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
for runLib in inspectShell.ALL_RUN_LIBS
]
return libsList
def parseOptions():
parser = OptionParser()
parser.add_option('-R', '--repo', dest='rDir',
help='Sets the repository to analyze..')
options, args = parser.parse_args()
assert options.rDir is not None
assert os.path.isdir(sps.normExpUserPath(options.rDir))
return options
def ensureCacheDir():
"""Return a cache directory for compiled shells to live in, and create one if needed."""
cacheDir = os.path.join(sps.normExpUserPath('~'), 'shell-cache')
ensureDir(cacheDir)
# Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff)
# This has to occur after the shell-cache directory is created
if sps.isWin: # adapted from http://stackoverflow.com/a/3931799
winTmpDir = unicode(cacheDir)
GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
unicodeBuffer = ctypes.create_unicode_buffer(
GetLongPathName(winTmpDir, 0, 0))
GetLongPathName(winTmpDir, unicodeBuffer, len(unicodeBuffer))
cacheDir = sps.normExpUserPath(str(
unicodeBuffer.value)) # convert back to a str
return cacheDir
def compileNspr(shell):
'''Compile a NSPR binary.'''
cfgBin(shell, 'nspr')
# Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well.
# Even if we move to parallel compile NSPR in the future, we must beware of breaking old
# build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1,
# and after that, use -jX ?
nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s']
out = sps.captureStdout(nsprCmdList, combineStderr=True, ignoreExitCode=True,
currWorkingDir=shell.getNsprObjdir(), env=shell.getEnvFull())[0]
for compileLib in inspectShell.ALL_COMPILE_LIBS:
if not sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)):
print out
raise Exception(MAKE_BINARY + " did not result in a NSPR binary.")
assert os.path.isdir(sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def parseOptions():
parser = OptionParser()
parser.add_option('-R', '--repo', dest='rDir',
help='Sets the repository to analyze..')
options, _args = parser.parse_args()
assert options.rDir is not None
assert os.path.isdir(sps.normExpUserPath(options.rDir))
return options
def assertSaneJsBinary(cacheF):
'''
If the cache folder is present, check that the js binary is working properly.
'''
if os.path.isdir(cacheF):
fList = os.listdir(cacheF)
if 'build' in fList:
if INCOMPLETE_NOTE in fList:
print cacheF + ' has subdirectories: ' + str(fList)
raise Exception('Downloaded binaries and incompleteBuild.txt should not both be ' +
'present together in this directory.')
assert os.path.isdir(sps.normExpUserPath(os.path.join(cacheF, 'build', 'download')))
assert os.path.isdir(sps.normExpUserPath(os.path.join(cacheF, 'build', 'dist')))
assert os.path.isfile(sps.normExpUserPath(os.path.join(cacheF, 'build', 'dist',
'js' + ('.exe' if sps.isWin else ''))))
try:
shellPath = getTboxJsBinPath(cacheF)
# Ensure we don't fail because the shell lacks u+x
if not os.access(shellPath, os.X_OK):
os.chmod(shellPath, stat.S_IXUSR)
# tbpl binaries are always:
# * run without Valgrind (they are not compiled with --enable-valgrind)
retCode = inspectShell.testBinary(shellPath, ['-e', '42'], False)[1]
# Exit code -1073741515 on Windows shows up when a required DLL is not present.
# This was testable at the time of writing, see bug 953314.
isDllNotPresentWinStartupError = (sps.isWin and retCode == -1073741515)
# We should have another condition here for non-Windows platforms but we do not yet
# have a situation where we can test broken treeherder js shells on those platforms.
if isDllNotPresentWinStartupError:
raise Exception('Shell startup error - a .dll file is probably not present.')
elif retCode != 0:
raise Exception('Non-zero return code: ' + str(retCode))
return True # Binary is working correctly
except (OSError, IOError):
raise Exception('Cache folder ' + cacheF + ' is corrupt, please delete it ' +
'and try again.')
elif INCOMPLETE_NOTE in fList:
return True
else:
raise Exception('Neither build/ nor INCOMPLETE_NOTE were found in the cache folder.')
else:
raise Exception('Cache folder ' + cacheF + ' is not found.')
def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType):
"""Download a build. If the build is incomplete, find a working neighbour, then return results."""
offset = None
skippedChangesetNum = 0
while True:
if offset is None:
offset = 0
elif offset > 16:
print("Failed to find a working build after ~30 tries.")
return None, None, None, None
elif offset > 0:
# Stop once we are testing beyond the start & end entries of the list
if (preferredIndex + offset >=
len(urls)) and (preferredIndex - offset < 0):
print(
"Stop looping because everything within the range was tested."
)
return None, None, None, None
offset = -offset
else:
offset = -offset + 1 # Alternate between positive and negative offsets
newIndex = preferredIndex + offset
if newIndex < 0:
continue
elif newIndex >= len(urls):
continue
isWorking, idNum, tboxCacheFolder = getOneBuild(
isJsShell, urls[newIndex], buildType)
if isWorking:
try:
assertSaneJsBinary(tboxCacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Shell startup error' in repr(e):
writeIncompleteBuildTxtFile(
urls[newIndex], tboxCacheFolder,
sps.normExpUserPath(
os.path.join(tboxCacheFolder, INCOMPLETE_NOTE)),
idNum)
continue
return newIndex, idNum, tboxCacheFolder, skippedChangesetNum
else:
skippedChangesetNum += 1
if len(urls) == 4:
# If we have [good, untested, incomplete, bad], after testing the middle changeset that
# has the "incomplete" result, the offset will push us the boundary changeset with the
# "bad" result. In this case, switch to the beginning changeset so the offset of 1 will
# push us into the untested changeset, avoiding a loop involving
# "incomplete->bad->incomplete->bad->..."
# See https://github.com/MozillaSecurity/funfuzz/issues/18
preferredIndex = 0
def makeRegressionTestPrologue(repo):
"""Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests."""
repo = sps.normExpUserPath(repo) + os.sep
return """
const regressionTestsRoot = %s;
const libdir = regressionTestsRoot + %s; // needed by jit-tests
const regressionTestList = %s;
""" % (json.dumps(repo),
json.dumps(os.path.join('js', 'src', 'jit-test', 'lib') + os.sep),
json.dumps(inTreeRegressionTests(repo)))
def getTimestampAndHashFromTboxFiles(folder):
'''
Returns timestamp and changeset information from the .txt file downloaded from treeherder.
'''
downloadDir = sps.normExpUserPath(os.path.join(folder, 'build', 'download'))
for fn in os.listdir(downloadDir):
if fn.startswith('firefox-') and fn.endswith('.txt'):
with open(os.path.join(downloadDir, fn), 'rb') as f:
fContents = f.read().splitlines()
break
assert len(fContents) == 2, 'Contents of the .txt file should only have 2 lines'
return fContents[0], fContents[1].split('/')[-1]
def extractVersions(objdir):
"""Extract the version from <objdir>/js/src/js.pc and put it into *.fuzzmanagerconf."""
jspcFilename = sps.normExpUserPath(os.path.join(objdir, 'js', 'src', 'js.pc'))
if os.path.isfile(jspcFilename):
with open(jspcFilename, 'rb') as f:
for line in f:
if line.startswith('Version: '):
# Sample line: 'Version: 47.0a2'
version = line.split(': ')[1].rstrip()
majorVersion = version.split('.')[0]
return majorVersion, version
return '', ''
def getRandomValidRepo(treeLocation):
validRepos = []
for repo in ['mozilla-central', 'mozilla-aurora', 'mozilla-beta', 'mozilla-release',
'mozilla-esr31']:
if os.path.isfile(sps.normExpUserPath(os.path.join(
treeLocation, repo, '.hg', 'hgrc'))):
validRepos.append(repo)
# After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
if 'mozilla-aurora' in validRepos and chance(0.4):
validRepos.remove('mozilla-aurora')
if 'mozilla-beta' in validRepos and chance(0.7):
validRepos.remove('mozilla-beta')
if 'mozilla-release' in validRepos and chance(0.9):
validRepos.remove('mozilla-release')
if 'mozilla-esr31' in validRepos and chance(0.8):
validRepos.remove('mozilla-esr31')
validRepos = ['mozilla-central'] # FIXME: Let's set to random configurations within m-c for now
return os.path.realpath(sps.normExpUserPath(
os.path.join(treeLocation, random.choice(validRepos))))
def getTimestampAndHashFromTboxFiles(folder):
'''
Returns timestamp and changeset information from the .txt file downloaded from treeherder.
'''
downloadDir = sps.normExpUserPath(os.path.join(folder, 'build', 'download'))
for fn in os.listdir(downloadDir):
if fn.startswith('firefox-') and fn.endswith('.txt') and '_info' not in fn:
with open(os.path.join(downloadDir, fn), 'rb') as f:
fContents = f.read().splitlines()
break
assert len(fContents) == 2, 'Contents of the .txt file should only have 2 lines'
return fContents[0], fContents[1].split('/')[-1]
def makeRegressionTestPrologue(repo):
"""Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests"""
return """
const regressionTestsRoot = %s;
const libdir = regressionTestsRoot + %s; // needed by jit-tests
const regressionTestList = %s;
""" % (
json.dumps(sps.normExpUserPath(repo) + os.sep),
json.dumps(os.path.join('js', 'src', 'jit-test', 'lib') + os.sep),
json.dumps(inTreeRegressionTests(repo))
)
def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType):
'''
Downloads a build. If the build is incomplete, find a working neighbour, then return results.
'''
offset = None
skippedChangesetNum = 0
while True:
if offset is None:
offset = 0
elif offset > 16:
print 'Failed to find a working build after ~30 tries.'
return None, None, None, None
elif offset > 0:
# Stop once we are testing beyond the start & end entries of the list
if (preferredIndex + offset >= len(urls)) and (preferredIndex - offset < 0):
print 'Stop looping because everything within the range was tested.'
return None, None, None, None
offset = -offset
else:
offset = -offset + 1 # Alternate between positive and negative offsets
newIndex = preferredIndex + offset
if newIndex < 0:
continue
elif newIndex >= len(urls):
continue
isWorking, idNum, tboxCacheFolder = getOneBuild(isJsShell, urls[newIndex], buildType)
if isWorking:
try:
assertSaneJsBinary(tboxCacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Shell startup error' in repr(e):
writeIncompleteBuildTxtFile(urls[newIndex], tboxCacheFolder,
sps.normExpUserPath(os.path.join(tboxCacheFolder,
INCOMPLETE_NOTE)),
idNum)
continue
return newIndex, idNum, tboxCacheFolder, skippedChangesetNum
else:
skippedChangesetNum += 1
if len(urls) == 4:
# If we have [good, untested, incomplete, bad], after testing the middle changeset that
# has the "incomplete" result, the offset will push us the boundary changeset with the
# "bad" result. In this case, switch to the beginning changeset so the offset of 1 will
# push us into the untested changeset, avoiding a loop involving
# "incomplete->bad->incomplete->bad->..."
# See https://github.com/MozillaSecurity/funfuzz/issues/18
preferredIndex = 0
def extractVersions(objdir):
"""Extract the version from <objdir>/js/src/js.pc and put it into *.fuzzmanagerconf."""
jspcFilename = sps.normExpUserPath(
os.path.join(objdir, 'js', 'src', 'js.pc'))
if os.path.isfile(jspcFilename):
with open(jspcFilename, 'rb') as f:
for line in f:
if line.startswith('Version: '):
# Sample line: 'Version: 47.0a2'
version = line.split(': ')[1].rstrip()
majorVersion = version.split('.')[0]
return majorVersion, version
return '', ''
def compileJs(shell):
"""Compile and copy a binary."""
try:
cmdList = [
MAKE_BINARY, '-C',
shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'
]
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
except Exception as e:
# This exception message is returned from sps.captureStdout via cmdList.
if (sps.isLinux or sps.isMac) and \
('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
# FIXME: Absolute hack to retry after hitting OOM.
print 'Trying once more due to the compiler running out of memory...'
out = sps.captureStdout(cmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getJsObjdir(),
env=shell.getEnvFull())[0]
# A non-zero error can be returned during make, but eventually a shell still gets compiled.
if os.path.exists(shell.getShellCompiledPath()):
print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
else:
print MAKE_BINARY + " did not result in a js shell:"
raise
if os.path.exists(shell.getShellCompiledPath()):
shutil.copy2(shell.getShellCompiledPath(),
shell.getShellCacheFullPath())
for runLib in shell.getShellCompiledRunLibsPath():
if os.path.isfile(runLib):
shutil.copy2(runLib, shell.getShellCacheDir())
majorVersion, version = extractVersions(shell.getJsObjdir())
shell.setMajorVersion(majorVersion)
shell.setVersion(version)
if sps.isLinux:
# Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a)
# files in the objdir or else the stacks from failing testcases will lack symbols.
shutil.rmtree(
sps.normExpUserPath(
os.path.join(shell.getShellCacheDir(), 'objdir-js')))
else:
print out
raise Exception(MAKE_BINARY +
" did not result in a js shell, no exception thrown.")
def compileNspr(shell):
'''Compile a NSPR binary.'''
cfgBin(shell, 'nspr')
# Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well.
# Even if we move to parallel compile NSPR in the future, we must beware of breaking old
# build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1,
# and after that, use -jX ?
nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s']
out = sps.captureStdout(nsprCmdList,
combineStderr=True,
ignoreExitCode=True,
currWorkingDir=shell.getNsprObjdir(),
env=shell.getEnvFull())[0]
for compileLib in inspectShell.ALL_COMPILE_LIBS:
if not sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib)):
print out
raise Exception(MAKE_BINARY + " did not result in a NSPR binary.")
assert os.path.isdir(
sps.normExpUserPath(
os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def createTboxCacheFolder(cacheFolder):
'''
Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check
that its binaries are working properly.
'''
try:
os.mkdir(cacheFolder)
except OSError:
assertSaneJsBinary(cacheFolder)
try:
ensureCacheDirHasCorrectIdNum(cacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
def createTboxCacheFolder(cacheFolder):
'''
Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check
that its binaries are working properly.
'''
try:
os.mkdir(cacheFolder)
except OSError:
assertSaneJsBinary(cacheFolder)
try:
ensureCacheDirHasCorrectIdNum(cacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
def ensureCacheDirHasCorrectIdNum(cacheFolder):
'''
Ensures that the cache folder is named with the correct numeric ID.
'''
srcUrlPath = sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download', 'source-url.txt'))
if os.path.isfile(srcUrlPath):
with open(srcUrlPath, 'rb') as f:
fContents = f.read().splitlines()
idNumFolderName = cacheFolder.split('-')[-1]
idNumSourceUrl = fContents[0].split('/')[-2]
if idNumFolderName != idNumSourceUrl:
print '\nWARNING: Numeric ID in folder name (current value: ' + \
idNumFolderName + ') is not equal to the numeric ID from source URL ' + \
'(current value: ' + idNumSourceUrl + ')\n'
raise Exception('Folder name numeric ID not equal to source URL numeric ID.')
def ensureCacheDirHasCorrectIdNum(cacheFolder):
"""Ensure that the cache folder is named with the correct numeric ID."""
srcUrlPath = sps.normExpUserPath(
os.path.join(cacheFolder, 'build', 'download', 'source-url.txt'))
if os.path.isfile(srcUrlPath):
with open(srcUrlPath, 'rb') as f:
fContents = f.read().splitlines()
idNumFolderName = cacheFolder.split('-')[-1]
idNumSourceUrl = fContents[0].split('/')[-2]
if idNumFolderName != idNumSourceUrl:
print '\nWARNING: Numeric ID in folder name (current value: ' + \
idNumFolderName + ') is not equal to the numeric ID from source URL ' + \
'(current value: ' + idNumSourceUrl + ')\n'
raise Exception(
'Folder name numeric ID not equal to source URL numeric ID.')
def mtrArgsCreation(options, cshell):
'''Create many_timed_run arguments for compiled builds'''
manyTimedRunArgs = []
manyTimedRunArgs.append('--repo=' + sps.normExpUserPath(options.buildOptions.repoDir))
manyTimedRunArgs.append("--build=" + options.buildOptions.buildOptionsStr)
if options.buildOptions.runWithVg:
manyTimedRunArgs.append('--valgrind')
if options.buildOptions.enableMoreDeterministic:
# Treeherder shells not using compareJIT:
# They are not built with --enable-more-deterministic - bug 751700
manyTimedRunArgs.append('--comparejit')
manyTimedRunArgs.append('--random-flags')
# Ordering of elements in manyTimedRunArgs is important.
manyTimedRunArgs.append(str(options.timeout))
manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory
manyTimedRunArgs.append(cshell.getShellCacheFullPath())
return manyTimedRunArgs
def mtrArgsCreation(options, cshell):
"""Create many_timed_run arguments for compiled builds."""
manyTimedRunArgs = []
manyTimedRunArgs.append('--repo=' + sps.normExpUserPath(options.buildOptions.repoDir))
manyTimedRunArgs.append("--build=" + options.buildOptions.buildOptionsStr)
if options.buildOptions.runWithVg:
manyTimedRunArgs.append('--valgrind')
if options.buildOptions.enableMoreDeterministic:
# Treeherder shells not using compareJIT:
# They are not built with --enable-more-deterministic - bug 751700
manyTimedRunArgs.append('--comparejit')
manyTimedRunArgs.append('--random-flags')
# Ordering of elements in manyTimedRunArgs is important.
manyTimedRunArgs.append(str(options.timeout))
manyTimedRunArgs.append(cshell.getRepoName()) # known bugs' directory
manyTimedRunArgs.append(cshell.getShellCacheFullPath())
return manyTimedRunArgs
def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType):
'''
Downloads a build. If the build is incomplete, find a working neighbour, then return results.
'''
offset = None
while True:
if offset is None:
offset = 0
elif offset > 16:
print 'Failed to find a working build after ~30 tries.'
return None, None, None
elif offset > 0:
# Stop once we are testing beyond the start & end entries of the list
if (preferredIndex + offset >=
len(urls)) and (preferredIndex - offset < 0):
print 'Stop looping because everything within the range was tested.'
return None, None, None
offset = -offset
else:
offset = -offset + 1 # Alternate between positive and negative offsets
newIndex = preferredIndex + offset
if newIndex < 0:
continue
elif newIndex >= len(urls):
continue
isWorking, idNum, tboxCacheFolder = getOneBuild(
isJsShell, urls[newIndex], buildType)
if isWorking:
try:
assertSaneJsBinary(tboxCacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Shell startup error' in repr(e):
writeIncompleteBuildTxtFile(
urls[newIndex], tboxCacheFolder,
sps.normExpUserPath(
os.path.join(tboxCacheFolder, INCOMPLETE_NOTE)),
idNum)
continue
return newIndex, idNum, tboxCacheFolder
def hgHashAddToFuzzPath(fuzzPath, repoDir):
'''
This function finds the mercurial revision and appends it to the directory name.
It also prompts if the user wants to continue, should the repository not be on tip.
'''
hgIdCmdList = ['hg', 'identify', '-i', '-n', '-b', repoDir]
vdump('About to start running `' + ' '.join(hgIdCmdList) + '` ...')
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdFull = captureStdout(hgIdCmdList,
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0]
hgIdChangesetHash = hgIdFull.split(' ')[0]
hgIdLocalNum = hgIdFull.split(' ')[1]
# In Windows, this throws up a warning about failing to set color mode to win32.
if platform.system() == 'Windows':
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir,
ignoreStderr=True)[0]
else:
hgIdBranch = captureStdout(['hg', 'id', '-t'],
currWorkingDir=repoDir)[0]
onDefaultTip = True
if 'tip' not in hgIdBranch:
print 'The repository is at this changeset -', hgIdLocalNum + ':' + hgIdChangesetHash
notOnDefaultTipApproval = str(
raw_input(
'Not on default tip! Are you sure you want to continue? (y/n): '
))
if notOnDefaultTipApproval == ('y' or 'yes'):
onDefaultTip = False
else:
switchToDefaultTipApproval = str(
raw_input('Do you want to switch to the default tip? (y/n): '))
if switchToDefaultTipApproval == ('y' or 'yes'):
subprocess.check_call(['hg', 'up', 'default'], cwd=repoDir)
else:
raise Exception('Not on default tip.')
fuzzPath = '-'.join([fuzzPath, hgIdLocalNum, hgIdChangesetHash])
vdump('Finished running `' + ' '.join(hgIdCmdList) + '`.')
return normExpUserPath(fuzzPath), onDefaultTip
def extractVersions(objdir):
"""Extract the version from js.pc and put it into *.fuzzmanagerconf."""
jspcDir = sps.normExpUserPath(os.path.join(objdir, 'js', 'src'))
jspcFilename = os.path.join(jspcDir, 'js.pc')
# Moved to <objdir>/js/src/build/, see bug 1262241, Fx55 rev 2159959522f4
jspcNewDir = os.path.join(jspcDir, 'build')
jspcNewFilename = os.path.join(jspcNewDir, 'js.pc')
def fixateVer(pcfile):
"""Returns the current version number (47.0a2)."""
with io.open(pcfile, mode='r', encoding="utf-8", errors="replace") as f:
for line in f:
if line.startswith('Version: '):
# Sample line: 'Version: 47.0a2'
return line.split(': ')[1].rstrip()
if os.path.isfile(jspcFilename):
return fixateVer(jspcFilename)
elif os.path.isfile(jspcNewFilename):
return fixateVer(jspcNewFilename)
def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType):
'''
Downloads a build. If the build is incomplete, find a working neighbour, then return results.
'''
offset = None
while True:
if offset is None:
offset = 0
elif offset > 16:
print 'Failed to find a working build after ~30 tries.'
return None, None, None
elif offset > 0:
# Stop once we are testing beyond the start & end entries of the list
if (preferredIndex + offset >= len(urls)) and (preferredIndex - offset < 0):
print 'Stop looping because everything within the range was tested.'
return None, None, None
offset = -offset
else:
offset = -offset + 1 # Alternate between positive and negative offsets
newIndex = preferredIndex + offset
if newIndex < 0:
continue
elif newIndex >= len(urls):
continue
isWorking, idNum, tboxCacheFolder = getOneBuild(isJsShell, urls[newIndex], buildType)
if isWorking:
try:
assertSaneJsBinary(tboxCacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Shell startup error' in repr(e):
writeIncompleteBuildTxtFile(urls[newIndex], tboxCacheFolder,
sps.normExpUserPath(os.path.join(tboxCacheFolder,
INCOMPLETE_NOTE)),
idNum)
continue
return newIndex, idNum, tboxCacheFolder
def autoconfRun(cwDir):
"""Run autoconf binaries corresponding to the platform."""
if sps.isMac:
autoconf213MacBin = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' \
if sps.isProgramInstalled('brew') else 'autoconf213'
# Total hack to support new and old Homebrew configs, we can probably just call autoconf213
if not os.path.isfile(sps.normExpUserPath(autoconf213MacBin)):
autoconf213MacBin = 'autoconf213'
subprocess.check_call([autoconf213MacBin], cwd=cwDir)
elif sps.isLinux:
# FIXME: We should use a method that is similar to the client.mk one, as per
# https://github.com/MozillaSecurity/funfuzz/issues/9
try:
# Ubuntu
subprocess.check_call(['autoconf2.13'], cwd=cwDir)
except OSError:
# Fedora has a different name
subprocess.check_call(['autoconf-2.13'], cwd=cwDir)
elif sps.isWin:
# Windows needs to call sh to be able to find autoconf.
subprocess.check_call(['sh', 'autoconf-2.13'], cwd=cwDir)
def rmOldLocalCachedDirs(cacheDir):
'''Removes old local cached directories, which were created four weeks ago.'''
# This is in autoBisect because it has a lock so we do not race while removing directories
# Adapted from http://stackoverflow.com/a/11337407
SECONDS_IN_A_DAY = 24 * 60 * 60
s3CacheObj = s3cache.S3Cache(compileShell.S3_SHELL_CACHE_DIRNAME)
if s3CacheObj.connect():
NUMBER_OF_DAYS = 1 # EC2 VMs generally have less disk space for local shell caches
elif sps.isARMv7l:
NUMBER_OF_DAYS = 3 # native ARM boards usually have less disk space
else:
NUMBER_OF_DAYS = 28
cacheDir = sps.normExpUserPath(cacheDir)
names = [os.path.join(cacheDir, fname) for fname in os.listdir(cacheDir)]
for name in names:
if os.path.isdir(name):
timediff = time.mktime(time.gmtime()) - os.stat(name).st_atime
if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS:
shutil.rmtree(name)
def rmOldLocalCachedDirs(cacheDir):
"""Remove old local cached directories, which were created four weeks ago."""
# This is in autoBisect because it has a lock so we do not race while removing directories
# Adapted from http://stackoverflow.com/a/11337407
SECONDS_IN_A_DAY = 24 * 60 * 60
s3CacheObj = s3cache.S3Cache(compileShell.S3_SHELL_CACHE_DIRNAME)
if s3CacheObj.connect():
NUMBER_OF_DAYS = 1 # EC2 VMs generally have less disk space for local shell caches
elif sps.isARMv7l:
NUMBER_OF_DAYS = 3 # native ARM boards usually have less disk space
else:
NUMBER_OF_DAYS = 28
cacheDir = sps.normExpUserPath(cacheDir)
names = [os.path.join(cacheDir, fname) for fname in os.listdir(cacheDir)]
for name in names:
if os.path.isdir(name):
timediff = time.mktime(time.gmtime()) - os.stat(name).st_atime
if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS:
shutil.rmtree(name)
def parseOptions(inputArgs):
"""Returns a 'buildOptions' object, which is intended to be immutable."""
parser = optparse.OptionParser()
# http://docs.python.org/library/optparse.html#optparse.OptionParser.disable_interspersed_args
parser.disable_interspersed_args()
parser.set_defaults(
repoDir = sps.normExpUserPath(os.path.join('~', 'trees', 'mozilla-central')),
objDir = None,
mozconfig = None
)
parser.add_option('-R', '--repoDir', dest='repoDir',
help='Sets the source repository. Defaults to "%default".')
parser.add_option('-o', '--objDir', dest='objDir',
help='The obj dir that will be created by the given mozconfig. May get clobbered.')
parser.add_option('-c', '--mozconfig', dest='mozconfig',
help='A mozconfig file.')
(options, args) = parser.parse_args(inputArgs)
if len(args) > 0:
parser.print_help()
raise Exception("buildBrowser.py: extra arguments")
# All are required for now
if not (options.objDir and options.repoDir and options.mozconfig):
print "buildBrowser requires you to specify a repoDir, objDir, and mozconfig"
parser.print_help()
raise Exception("buildBrowser.py: usage")
options.objDir = os.path.expanduser(options.objDir)
options.repoDir = os.path.expanduser(options.repoDir)
options.mozconfig = os.path.expanduser(options.mozconfig)
assert os.path.exists(options.repoDir)
assert os.path.exists(options.mozconfig)
return options
def cfgBin(shell, binToBeCompiled):
'''This function configures a binary according to required parameters.'''
cfgCmdList = []
cfgEnvDt = copy.deepcopy(os.environ)
origCfgEnvDt = copy.deepcopy(os.environ)
cfgEnvDt['AR'] = 'ar'
# Check for determinism to prevent LLVM compilation from happening on releng machines,
# since releng machines only test non-deterministic builds.
if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic:
llvmPath = envVars.findLlvmBinPath()
assert llvmPath is not None
CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang'))
CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++'))
if sps.isARMv7l:
# 32-bit shell on ARM boards, e.g. odroid boards.
# This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed.
assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.'
if not shell.buildOptions.enableHardFp:
cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# From mjrosenb: things might go wrong if these three lines are not present for
# compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself.
# cfgCmdList.append('--target=arm-linux-gnueabi')
# cfgCmdList.append('--with-arch=armv7-a')
# cfgCmdList.append('--with-thumb')
if not shell.buildOptions.enableHardFp:
cfgCmdList.append('--target=arm-linux-gnueabi')
elif shell.buildOptions.enable32 and os.name == 'posix':
# 32-bit shell on Mac OS X 10.7 Lion and greater
if sps.isMac:
assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior.
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS
else: # Uses system clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS
cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC
cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX
cfgEnvDt['RANLIB'] = 'ranlib'
cfgEnvDt['AS'] = '$CC'
cfgEnvDt['LD'] = 'ld'
cfgEnvDt['STRIP'] = 'strip -x -S'
cfgEnvDt['CROSS_COMPILE'] = '1'
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2
cfgCmdList.append('--enable-macos-target=10.5')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
# 32-bit shell on 32/64-bit x86 Linux
elif sps.isLinux and not sps.isARMv7l:
cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig'
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \
CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG
else: # Uses system clang
# We might still be using GCC on Linux 32-bit, use clang only if we specify ASan
# apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux.
cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS
cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=i686-pc-linux')
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
# 64-bit shell on Mac OS X 10.7 Lion and greater
elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32:
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
else: # Uses system clang
cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS
cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS
if sps.isProgramInstalled('brew'):
cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213'
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
elif sps.isWin:
cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
if shell.buildOptions.enable32:
cfgCmdList.append('--enable-win32-target=WIN95')
else:
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.enable32:
if shell.buildOptions.enableSimulatorArm32:
# --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482
# but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated
cfgCmdList.append('--enable-arm-simulator')
cfgCmdList.append('--enable-simulator=arm')
else:
cfgCmdList.append('--host=x86_64-pc-mingw32')
cfgCmdList.append('--target=x86_64-pc-mingw32')
if shell.buildOptions.enableSimulatorArm64:
cfgCmdList.append('--enable-simulator=arm64')
else:
# We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified
if shell.buildOptions.buildWithAsan: # Uses custom compiled clang
cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS
cfgCmdList.append('sh')
if binToBeCompiled == 'nspr':
cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath()))
cfgCmdList.append('--enable-64bit')
else:
cfgCmdList.append(os.path.normpath(shell.getJsCfgPath()))
if shell.buildOptions.buildWithAsan:
cfgCmdList.append('--enable-address-sanitizer')
if shell.buildOptions.buildWithAsan:
assert 'clang' in cfgEnvDt['CC']
assert 'clang++' in cfgEnvDt['CXX']
cfgCmdList.append('--disable-jemalloc') # See bug 1146895
# For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build.
# They can actually be used independently, but it's not recommended.
# https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options
if shell.buildOptions.enableDbg:
# NSPR configure without options compiles debug by default
if binToBeCompiled == 'js':
cfgCmdList.append('--enable-debug')
elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr':
cfgCmdList.append('--disable-debug')
if shell.buildOptions.enableOpt:
cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else ''))
elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt:
# NSPR configure without options compiles debug by default
cfgCmdList.append('--disable-optimize')
if binToBeCompiled == 'nspr':
cfgCmdList.append('--prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
else:
if shell.buildOptions.enableProfiling:
cfgCmdList.append('--enable-profiling')
if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-nspr-build')
else:
if shell.buildOptions.enableNsprBuild:
cfgCmdList.append('--enable-threadsafe')
if not shell.getJsBuildSystemConsidersNspr():
cfgCmdList.append('--with-nspr-prefix=' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist')))
cfgCmdList.append('--with-nspr-cflags=-I' +
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
cfgCmdList.append('--with-nspr-libs=' + ' '.join([
sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib',
compileLib))
for compileLib in inspectShell.ALL_COMPILE_LIBS
]))
else:
cfgCmdList.append('--disable-threadsafe')
if shell.buildOptions.enableMoreDeterministic:
# Fuzzing tweaks for more useful output, implemented in bug 706433
cfgCmdList.append('--enable-more-deterministic')
if shell.buildOptions.buildWithVg:
cfgCmdList.append('--enable-valgrind')
cfgCmdList.append('--disable-jemalloc')
# We add the following flags by default.
if os.name == 'posix':
cfgCmdList.append('--with-ccache')
cfgCmdList.append('--enable-gczeal')
cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells
cfgCmdList.append('--disable-tests')
if os.name == 'nt':
# FIXME: Replace this with sps.shellify.
counter = 0
for entry in cfgCmdList:
if os.sep in entry:
assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\".
cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//')
counter = counter + 1
# Print whatever we added to the environment
envVarList = []
for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()):
strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \
if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar]
envVarList.append(strToBeAppended)
sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList))
wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir()
assert os.path.isdir(wDir)
if sps.isWin:
changedCfgCmdList = []
for entry in cfgCmdList:
# See bug 986715 comment 6 as to why we need forward slashes for NSPR
# For JS, quoted from :glandium: "the way icu subconfigure is called is what changed.
# but really, the whole thing likes forward slashes way better"
# See bug 1038590 comment 9.
if '\\' in entry:
entry = entry.replace('\\', '/')
changedCfgCmdList.append(entry)
sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
else:
sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt)
shell.setEnvAdded(envVarList)
shell.setEnvFull(cfgEnvDt)
shell.setCfgCmdExclEnv(cfgCmdList)
def getShellCompiledPath(self):
return sps.normExpUserPath(
os.path.join(self.getJsObjdir(), 'dist', 'bin', 'js' + ('.exe' if sps.isWin else '')))