def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo, buildOptionsStr, targetTime, suspiciousLevel): """ Run Lithium and autobisect. itest must be an array of the form [module, ...] where module is an interestingness module. The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ lithArgs = itest + [jsEngine] + engineFlags + [infilename] (lithResult, lithDetails) = strategicReduction(logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) print "\nDone running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:" print sps.shellify([lithiumpy, "--strategy=check-only"] + lithArgs) + '\n' if bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and buildOptionsStr is not None: if platform.uname()[2] == 'XP': print 'Not pinpointing to exact changeset since autoBisect does not work well in WinXP.' elif testJsShellOrXpcshell(jsEngine) != "xpcshell": autobisectCmd = ([sys.executable, autobisectpy] + ["-b", buildOptionsStr] + ["-p", ' '.join(engineFlags + [infilename])] + ["-i"] + itest) print sps.shellify(autobisectCmd) subprocess.call(autobisectCmd, stdout=open(logPrefix + "-autobisect.txt", "w"), stderr=subprocess.STDOUT) print "Done running autobisect. Log: " + logPrefix + "-autobisect.txt" return (lithResult, lithDetails)
def runLithium(lithArgs, logPrefix, targetTime): """ Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. Returns a tuple of (lithlogfn, LITH_*, details). """ deletableLithTemp = None if targetTime: # FIXME: this could be based on whether bot.py has a remoteHost # loopdomfuzz.py or loopjsfuzz.py is being used by bot.py deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium") lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs else: # loopdomfuzz.py or loopjsfuzz.py is being run standalone lithtmp = logPrefix + "-lith-tmp" os.mkdir(lithtmp) lithArgs = ["--tempdir=" + lithtmp] + lithArgs lithlogfn = logPrefix + "-lith-out.txt" print "Preparing to run Lithium, log file " + lithlogfn print sps.shellify(runlithiumpy + lithArgs) subprocess.call(runlithiumpy + lithArgs, stdout=open(lithlogfn, "w"), stderr=subprocess.STDOUT) print "Done running Lithium" if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) subprocess.call(["gzip", "-f", lithlogfn]) return r
def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, bisectRepo, buildOptionsStr, targetTime, suspiciousLevel): """ Run Lithium and autobisect. itest must be an array of the form [module, ...] where module is an interestingness module. The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ lithArgs = itest + [jsEngine] + engineFlags + [infilename] (lithResult, lithDetails) = strategicReduction(logPrefix, infilename, lithArgs, bisectRepo, buildOptionsStr, targetTime, suspiciousLevel) print "\nDone running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:" print sps.shellify([lithiumpy, "--strategy=check-only"] + lithArgs) + '\n' if bisectRepo is not "none" and targetTime >= 3*60*60 and buildOptionsStr is not None: if platform.uname()[2] == 'XP': print 'Not pinpointing to exact changeset since autoBisect does not work well in WinXP.' elif testJsShellOrXpcshell(jsEngine) != "xpcshell": autobisectCmd = ( [sys.executable, autobisectpy] + ["-b", buildOptionsStr] + ["-p", ' '.join(engineFlags + [infilename])] + ["-i"] + itest ) print sps.shellify(autobisectCmd) subprocess.call(autobisectCmd, stdout=open(logPrefix + "-autobisect", "w"), stderr=subprocess.STDOUT) print "Done running autobisect. Log: " + logPrefix + "-autobisect" return (lithResult, lithDetails)
def runLithium(lithArgs, logPrefix, targetTime): """ Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. Returns a tuple of (lithlogfn, LITH_*, details). """ deletableLithTemp = None if targetTime: # FIXME: this could be based on whether bot.py has a remoteHost # loopdomfuzz.py or loopjsfuzz.py is being used by bot.py deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium") lithArgs = [ "--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp ] + lithArgs else: # loopdomfuzz.py or loopjsfuzz.py is being run standalone lithtmp = logPrefix + "-lith-tmp" os.mkdir(lithtmp) lithArgs = ["--tempdir=" + lithtmp] + lithArgs lithlogfn = logPrefix + "-lith-out.txt" print "Preparing to run Lithium, log file " + lithlogfn print sps.shellify(runlithiumpy + lithArgs) subprocess.call(runlithiumpy + lithArgs, stdout=open(lithlogfn, "w"), stderr=subprocess.STDOUT) print "Done running Lithium" if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) subprocess.call(["gzip", "-f", lithlogfn]) return r
def envDump(shell, log): '''Dumps environment to a .fuzzmanagerconf file.''' # Platform and OS detection for the spec, part of which is in: # https://wiki.mozilla.org/Security/CrashSignatures if sps.isARMv7l: fmconfPlatform = 'ARM' elif sps.isARMv7l and not shell.buildOptions.enable32: print 'ARM64 is not supported in .fuzzmanagerconf yet.' fmconfPlatform = 'ARM64' elif shell.buildOptions.enable32: fmconfPlatform = 'x86' else: fmconfPlatform = 'x86-64' if sps.isLinux: fmconfOS = 'linux' elif sps.isMac: fmconfOS = 'macosx' elif sps.isWin: fmconfOS = 'windows' with open(log, 'ab') as f: f.write('# Information about shell:\n# \n') f.write('# Create another shell in shell-cache like this one:\n') f.write('# python -u %s -b "%s" -r %s\n# \n' % ('~/funfuzz/js/compileShell.py', shell.buildOptions.buildOptionsStr, shell.getHgHash())) f.write('# Full environment is:\n') f.write('# %s\n# \n' % str(shell.getEnvFull())) f.write( '# Full configuration command with needed environment variables is:\n' ) f.write('# %s %s\n# \n' % (sps.shellify( shell.getEnvAdded()), sps.shellify(shell.getCfgCmdExclEnv()))) # .fuzzmanagerconf details f.write('\n') f.write('[Main]\n') f.write('platform = %s\n' % fmconfPlatform) f.write('product = %s\n' % shell.getRepoName()) f.write('product_version = %s\n' % shell.getHgHash()) f.write('os = %s\n' % fmconfOS) f.write('\n') f.write('[Metadata]\n') f.write('buildFlags = %s\n' % shell.buildOptions.buildOptionsStr) f.write('majorVersion = %s\n' % shell.getMajorVersion()) f.write('pathPrefix = %s%s\n' % (shell.getRepoDir(), '/' if not shell.getRepoDir().endswith('/') else '')) f.write('version = %s\n' % shell.getVersion())
def envDump(shell, log): '''Dumps environment to a .fuzzmanagerconf file.''' # Platform and OS detection for the spec, part of which is in: # https://wiki.mozilla.org/Security/CrashSignatures if sps.isARMv7l: fmconfPlatform = 'ARM' elif sps.isARMv7l and not shell.buildOptions.enable32: print 'ARM64 is not supported in .fuzzmanagerconf yet.' fmconfPlatform = 'ARM64' elif shell.buildOptions.enable32: fmconfPlatform = 'x86' else: fmconfPlatform = 'x86-64' if sps.isLinux: fmconfOS = 'linux' elif sps.isMac: fmconfOS = 'macosx' elif sps.isWin: fmconfOS = 'windows' with open(log, 'ab') as f: f.write('# Information about shell:\n# \n') f.write('# Create another shell in shell-cache like this one:\n') f.write('# python -u %s -b "%s" -r %s\n# \n' % ('~/funfuzz/js/compileShell.py', shell.buildOptions.buildOptionsStr, shell.getHgHash())) f.write('# Full environment is:\n') f.write('# %s\n# \n' % str(shell.getEnvFull())) f.write( '# Full configuration command with needed environment variables is:\n' ) f.write('# %s %s\n# \n' % (sps.shellify(shell.getEnvAdded()), sps.shellify(shell.getCfgCmdExclEnv()))) # .fuzzmanagerconf details f.write('\n') f.write('[Main]\n') f.write('platform = %s\n' % fmconfPlatform) f.write('product = %s\n' % shell.getRepoName()) f.write('product_version = %s\n' % shell.getHgHash()) f.write('os = %s\n' % fmconfOS) f.write('\n') f.write('[Metadata]\n') f.write('pathPrefix = %s%s\n' % (shell.getRepoDir(), '/' if not shell.getRepoDir().endswith('/') else '')) f.write('buildFlags = %s\n' % shell.buildOptions.buildOptionsStr)
def lithReduceCmd(strategy): """Lithium reduction commands accepting various strategies.""" reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print sps.shellify([lithiumpy] + fullLithArgs) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium(fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc, targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def envDump(shell, log): """Dumps environment to a .fuzzmanagerconf file.""" # Platform and OS detection for the spec, part of which is in: # https://wiki.mozilla.org/Security/CrashSignatures if sps.isARMv7l: fmconfPlatform = "ARM" elif sps.isARMv7l and not shell.buildOptions.enable32: print "ARM64 is not supported in .fuzzmanagerconf yet." fmconfPlatform = "ARM64" elif shell.buildOptions.enable32: fmconfPlatform = "x86" else: fmconfPlatform = "x86-64" if sps.isLinux: fmconfOS = "linux" elif sps.isMac: fmconfOS = "macosx" elif sps.isWin: fmconfOS = "windows" with open(log, "ab") as f: f.write("# Information about shell:\n# \n") f.write("# Create another shell in shell-cache like this one:\n") f.write( '# python -u %s -b "%s" -r %s\n# \n' % ("~/funfuzz/js/compileShell.py", shell.buildOptions.buildOptionsStr, shell.getHgHash()) ) f.write("# Full environment is:\n") f.write("# %s\n# \n" % str(shell.getEnvFull())) f.write("# Full configuration command with needed environment variables is:\n") f.write("# %s %s\n# \n" % (sps.shellify(shell.getEnvAdded()), sps.shellify(shell.getCfgCmdExclEnv()))) # .fuzzmanagerconf details f.write("\n") f.write("[Main]\n") f.write("platform = %s\n" % fmconfPlatform) f.write("product = %s\n" % shell.getRepoName()) f.write("product_version = %s\n" % shell.getHgHash()) f.write("os = %s\n" % fmconfOS) f.write("\n") f.write("[Metadata]\n") f.write("pathPrefix = %s%s\n" % (shell.getRepoDir(), "/" if not shell.getRepoDir().endswith("/") else "")) f.write("buildFlags = %s\n" % shell.buildOptions.buildOptionsStr)
def lithReduceCmd(strategy): '''Lithium reduction commands accepting various strategies.''' reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print sps.shellify([lithiumpy] + fullLithArgs) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium( fullLithArgs, logPrefix + "-" + str(reductionCount[0]) + desc, targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def readFromURL(url): ''' Reads in a URL and returns its contents as a list. ''' inpCmdList = ['curl', '--silent', url ] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', '-', url] p = subprocess.Popen(inpCmdList, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not useCurl and p.returncode == 5: print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \ '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \ 'in ~/.wgetrc' raise Exception( 'Unable to read from URL. Please check your ~/.wgetrc file.') elif p.returncode != 0: print 'inpCmdList is: ' + sps.shellify(inpCmdList) print 'stdout: ' + repr(out) print 'stderr: ' + repr(err) raise Exception('The following exit code was returned: ' + str(p.returncode)) else: # Ignore whatever verbose output wget spewed to stderr. return out
def testBinary(shellPath, args, useValgrind): '''Tests the given shell with the given args.''' testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args sps.vdump('The testing command is: ' + sps.shellify(testCmd)) out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True, env=envVars.envWithPath( os.path.dirname(os.path.abspath(shellPath)))) sps.vdump('The exit code is: ' + str(rCode)) return out, rCode
def readFromURL(url): """Read in a URL and returns its contents as a list.""" inpCmdList = ["curl", "--silent", url] if useCurl else ["wget"] + wgetMaybeNCC + ["-O", "-", url] p = subprocess.Popen(inpCmdList, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not useCurl and p.returncode == 5: print "Unable to read from URL. If you installed wget using MacPorts, you should put " + '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + "in ~/.wgetrc" raise Exception("Unable to read from URL. Please check your ~/.wgetrc file.") elif p.returncode != 0: print "inpCmdList is: " + sps.shellify(inpCmdList) print "stdout: " + repr(out) print "stderr: " + repr(err) raise Exception("The following exit code was returned: " + str(p.returncode)) else: # Ignore whatever verbose output wget spewed to stderr. return out
def lithReduceCmd(strategy): """Lithium reduction commands accepting various strategies.""" reductionCount[0] += 1 fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements print( sps.shellify([sys.executable, "-u", "-m", "lithium"] + fullLithArgs)) desc = '-chars' if strategy == '--char' else '-lines' (lithResult, lithDetails) = runLithium( fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) return lithResult, lithDetails
def readFromURL(url): ''' Reads in a URL and returns its contents as a list. ''' inpCmdList = ['curl', '--silent', url] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', '-', url] p = subprocess.Popen(inpCmdList, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if not useCurl and p.returncode == 5: print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \ '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \ 'in ~/.wgetrc' raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.') elif p.returncode != 0: print 'inpCmdList is: ' + sps.shellify(inpCmdList) print 'stdout: ' + repr(out) print 'stderr: ' + repr(err) raise Exception('The following exit code was returned: ' + str(p.returncode)) else: # Ignore whatever verbose output wget spewed to stderr. return out
def jsfunfuzzLevel(options, logPrefix, quiet=False): (lev, issues, runinfo) = baseLevel(options.jsengineWithArgs, options.timeout, options.knownPath, logPrefix, valgrind=options.valgrind) if lev == JS_FINE: # Check for unexplained exits and for jsfunfuzz saying "Found a bug". understoodExit = False # Read in binary mode, because otherwise Python on Windows will # throw a fit when it encounters certain unicode. Note that this # makes line endings platform-specific. if '-dm-' in options.jsengineWithArgs[0]: # Since this is an --enable-more-deterministic build, we should get messages on stderr # if the shell quit() or terminate() functions are called. # (We use a sketchy filename-matching check because it's faster than inspecting the binary.) with open(logPrefix + "-err.txt", "rb") as f: for line in f: if "terminate called" in line or "quit called" in line: understoodExit = True if "can't allocate region" in line: understoodExit = True else: understoodExit = True with open(logPrefix + "-out.txt", "rb") as f: for line in f: if line.startswith("It's looking good!") or line.startswith( "jsfunfuzz broke its own scripting environment: "): understoodExit = True if line.startswith("Found a bug: "): understoodExit = True if not ("NestTest" in line and oomed(logPrefix)): lev = JS_DECIDED_TO_EXIT issues.append(line.rstrip()) # FIXME: if not quiet: # FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug." if not understoodExit: issues.append("jsfunfuzz didn't finish") lev = JS_DID_NOT_FINISH # FIXME: if not quiet: # FIXME: output the last tryItOut line if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored. sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev)) lev = JS_FINE issues = [] if lev != JS_FINE: # FIXME: compareJIT failures do not generate this -summary file. statusIssueList = [] for i in issues: statusIssueList.append('Status: ' + i) assert len(statusIssueList) != 0 fileManipulation.writeLinesToFile([ 'Number: ' + logPrefix + '\n', 'Command: ' + sps.shellify(options.jsengineWithArgs) + '\n' ] + [i + '\n' for i in statusIssueList], logPrefix + '-summary.txt') if not quiet: print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime) return lev
def cfgBin(shell): """This function configures a binary according to required parameters.""" cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt["AR"] = "ar" # Check for determinism to prevent LLVM compilation from happening on releng machines, # since releng machines only test non-deterministic builds. if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic: llvmPath = envVars.findLlvmBinPath() assert llvmPath is not None CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang")) CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, "clang++")) if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, "arm7vl boards are only 32-bit, armv8 boards will be 64-bit." if not shell.buildOptions.enableHardFp: cfgEnvDt["CC"] = "gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgEnvDt["CXX"] = "g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append("--target=arm-linux-gnueabi") elif shell.buildOptions.enable32 and os.name == "posix": # 32-bit shell on Mac OS X 10.7 Lion and greater if sps.isMac: assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior. if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS else: # Uses system clang cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = "clang" + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = "clang++" + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt["CC"] = cfgEnvDt["CC"] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt["CXX"] = cfgEnvDt["CXX"] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt["RANLIB"] = "ranlib" cfgEnvDt["AS"] = "$CC" cfgEnvDt["LD"] = "ld" cfgEnvDt["STRIP"] = "strip -x -S" cfgEnvDt["CROSS_COMPILE"] = "1" if sps.isProgramInstalled("brew"): cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append("--target=i386-apple-darwin9.2.0") # Leopard 10.5.2 cfgCmdList.append("--enable-macos-target=10.5") if shell.buildOptions.buildWithAsan: cfgCmdList.append("--enable-address-sanitizer") if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append("--enable-arm-simulator") cfgCmdList.append("--enable-simulator=arm") # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt["PKG_CONFIG_LIBDIR"] = "/usr/lib/pkgconfig" if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt["CC"] = cfgEnvDt["HOST_CC"] = ( CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG ) cfgEnvDt["CXX"] = cfgEnvDt["HOST_CXX"] = ( CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG ) else: # Uses system clang # We might still be using GCC on Linux 32-bit, use clang only if we specify ASan # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt["CC"] = "gcc -m32" + SSE2_FLAGS cfgEnvDt["CXX"] = "g++ -m32" + SSE2_FLAGS cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append("--target=i686-pc-linux") if shell.buildOptions.buildWithAsan: cfgCmdList.append("--enable-address-sanitizer") if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append("--enable-arm-simulator") cfgCmdList.append("--enable-simulator=arm") else: cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.7 Lion and greater elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32: if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS else: # Uses system clang cfgEnvDt["CC"] = "clang" + CLANG_PARAMS cfgEnvDt["CXX"] = "clang++" + CLANG_PARAMS if sps.isProgramInstalled("brew"): cfgEnvDt["AUTOCONF"] = "/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append("--target=x86_64-apple-darwin12.5.0") # Mountain Lion 10.8.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append("--enable-address-sanitizer") if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append("--enable-simulator=arm64") elif sps.isWin: cfgEnvDt["MAKE"] = "mozmake" # Workaround for bug 948534 cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append("--enable-arm-simulator") cfgCmdList.append("--enable-simulator=arm") else: cfgCmdList.append("--host=x86_64-pc-mingw32") cfgCmdList.append("--target=x86_64-pc-mingw32") if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append("--enable-simulator=arm64") else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt["CC"] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt["CXX"] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgCmdList.append("sh") cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append("--enable-address-sanitizer") if shell.buildOptions.buildWithAsan: assert "clang" in cfgEnvDt["CC"] assert "clang++" in cfgEnvDt["CXX"] cfgCmdList.append("--disable-jemalloc") # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append("--enable-debug") elif shell.buildOptions.disableDbg: cfgCmdList.append("--disable-debug") if shell.buildOptions.enableOpt: cfgCmdList.append("--enable-optimize" + ("=-O1" if shell.buildOptions.buildWithVg else "")) elif shell.buildOptions.disableOpt: cfgCmdList.append("--disable-optimize") if shell.buildOptions.enableProfiling: cfgCmdList.append("--enable-profiling") if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append("--enable-more-deterministic") if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append("--enable-oom-breakpoint") if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append("--without-intl-api") if shell.buildOptions.buildWithVg: cfgCmdList.append("--enable-valgrind") cfgCmdList.append("--disable-jemalloc") # We add the following flags by default. if os.name == "posix": cfgCmdList.append("--with-ccache") cfgCmdList.append("--enable-gczeal") cfgCmdList.append("--enable-debug-symbols") # gets debug symbols on opt shells cfgCmdList.append("--disable-tests") if os.name == "nt": # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, "//") counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = ( envVar + '="' + cfgEnvDt[envVar] + '"' if " " in cfgEnvDt[envVar] else envVar + "=" + cfgEnvDt[envVar] ) envVarList.append(strToBeAppended) sps.vdump("Command to be run is: " + sps.shellify(envVarList) + " " + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if "\\" in entry: entry = entry.replace("\\", "/") changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell, binToBeCompiled): '''This function configures a binary according to required parameters.''' cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' # Check for determinism to prevent LLVM compilation from happening on releng machines, # since releng machines only test non-deterministic builds. if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic: llvmPath = envVars.findLlvmBinPath() assert llvmPath is not None CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang')) CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++')) if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt[ 'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt[ 'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.7 Lion and greater if sps.isMac: assert sps.macVer() >= [ 10, 7 ] # We no longer support Snow Leopard 10.6 and prior. if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS else: # Uses system clang cfgEnvDt['CC'] = cfgEnvDt[ 'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt[ 'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CC'] = cfgEnvDt[ 'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt[ 'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=i386-apple-darwin9.2.0') # Leopard 10.5.2 cfgCmdList.append('--enable-macos-target=10.5') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # Uses system clang # We might still be using GCC on Linux 32-bit, use clang only if we specify ASan # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.7 Lion and greater elif sps.isMac and sps.macVer() >= [10, 7 ] and not shell.buildOptions.enable32: if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS else: # Uses system clang cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) if shell.buildOptions.enable32: cfgCmdList.append('--enable-win32-target=WIN95') else: cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithAsan: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 # For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build. # They can actually be used independently, but it's not recommended. # https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options if shell.buildOptions.enableDbg: # NSPR configure without options compiles debug by default if binToBeCompiled == 'js': cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr': cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt: # NSPR configure without options compiles debug by default cfgCmdList.append('--disable-optimize') if binToBeCompiled == 'nspr': cfgCmdList.append( '--prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) else: if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.getJsUsesNoThreadsFlag( ) and shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-nspr-build') else: if shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-threadsafe') if not shell.getJsBuildSystemConsidersNspr(): cfgCmdList.append( '--with-nspr-prefix=' + sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist'))) cfgCmdList.append('--with-nspr-cflags=-I' + sps.normExpUserPath( os.path.join(shell.getNsprObjdir( ), 'dist', 'include', 'nspr'))) cfgCmdList.append('--with-nspr-libs=' + ' '.join([ sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)) for compileLib in inspectShell.ALL_COMPILE_LIBS ])) else: cfgCmdList.append('--disable-threadsafe') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append( '--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getNsprObjdir( ) if binToBeCompiled == 'nspr' else shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # See bug 986715 comment 6 as to why we need forward slashes for NSPR # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell): """Configure a binary according to required parameters.""" cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.10 Yosemite and greater if sps.isMac: assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. # Uses system clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.10 Yosemite and greater elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgEnvDt['CC'] += CLANG_ASAN_PARAMS # cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgCmdList.append('--enable-address-sanitizer') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithClang: if sps.isWin: assert 'clang-cl' in cfgEnvDt['CC'] assert 'clang-cl' in cfgEnvDt['CXX'] else: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg: cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif shell.buildOptions.disableOpt: cfgCmdList.append('--disable-optimize') if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append('--enable-oom-breakpoint') if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append('--without-intl-api') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def __init__(self, options, runthis, logPrefix, inCompareJIT): pathToBinary = runthis[0] # This relies on the shell being a local one from compileShell.py: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like './js' pc = ProgramConfiguration.fromBinary( os.path.abspath(pathToBinary).split('.')[0]) pc.addProgramArguments(runthis[1:-1]) if options.valgrind: runthis = (inspectShell.constructVgCmdList( errorCode=VALGRIND_ERROR_EXIT_CODE) + valgrindSuppressions(options.knownPath) + runthis) preexec_fn = ulimitSet if os.name == 'posix' else None runinfo = timed_run.timed_run(runthis, options.timeout, logPrefix, preexec_fn=preexec_fn) lev = JS_FINE issues = [] auxCrashData = [] # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and 'readlines' everything into memory. with open(logPrefix + "-out.txt") as f: out = f.readlines() with open(logPrefix + "-err.txt") as f: err = f.readlines() if options.valgrind and runinfo.return_code == VALGRIND_ERROR_EXIT_CODE: issues.append("valgrind reported an error") lev = max(lev, JS_VG_AMISS) valgrindErrorPrefix = "==" + str(runinfo.pid) + "==" for line in err: if valgrindErrorPrefix and line.startswith( valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timed_run.CRASHED: if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True): with open(logPrefix + "-crash.txt") as f: auxCrashData = [line.strip() for line in f.readlines()] elif detect_malloc_errors.amiss(logPrefix): issues.append("malloc error") lev = max(lev, JS_NEW_ASSERT_OR_CRASH) elif runinfo.return_code == 0 and not inCompareJIT: # We might have(??) run jsfunfuzz directly, so check for special kinds of bugs for line in out: if line.startswith("Found a bug: ") and not ("NestTest" in line and oomed(err)): lev = JS_DECIDED_TO_EXIT issues.append(line.rstrip()) if options.shellIsDeterministic and not understoodJsfunfuzzExit( out, err) and not oomed(err): issues.append("jsfunfuzz didn't finish") lev = JS_DID_NOT_FINISH # Copy non-crash issues to where FuzzManager's "AssertionHelper.py" can see it. if lev != JS_FINE: for issue in issues: err.append("[Non-crash bug] " + issue) # Finally, make a CrashInfo object and parse stack traces for asan/crash/assertion bugs crashInfo = CrashInfo.CrashInfo.fromRawCrashData( out, err, pc, auxCrashData=auxCrashData) createCollector.printCrashInfo(crashInfo) # We only care about crashes and assertion failures on shells with no symbols # Note that looking out for the Assertion failure message is highly SpiderMonkey-specific if not isinstance(crashInfo, CrashInfo.NoCrashInfo) or \ 'Assertion failure: ' in str(crashInfo.rawStderr) or \ 'Segmentation fault' in str(crashInfo.rawStderr) or \ 'Bus error' in str(crashInfo.rawStderr): lev = max(lev, JS_NEW_ASSERT_OR_CRASH) match = options.collector.search(crashInfo) if match[0] is not None: createCollector.printMatchingSignature(match) lev = JS_FINE print("%s | %s" % (logPrefix, summaryString(issues, lev, runinfo.elapsedtime))) if lev != JS_FINE: fileManipulation.writeLinesToFile([ 'Number: ' + logPrefix + '\n', 'Command: ' + sps.shellify(runthis) + '\n' ] + ['Status: ' + i + "\n" for i in issues], logPrefix + '-summary.txt') self.lev = lev self.out = out self.err = err self.issues = issues self.crashInfo = crashInfo self.match = match self.runinfo = runinfo self.return_code = runinfo.return_code
def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDiffs, quickMode): # options dict must be one we can pass to jsInteresting.ShellResult # we also use it directly for knownPath, timeout, and collector # Return: (lev, crashInfo) or (jsInteresting.JS_FINE, None) combos = shellFlags.basicFlagSets(jsEngine) if quickMode: # Only used during initial fuzzing. Allowed to have false negatives. combos = [combos[0]] if len(flags): combos.append(flags) commands = [[jsEngine] + combo + [infilename] for combo in combos] for i in range(0, len(commands)): prefix = logPrefix + "-r" + str(i) command = commands[i] r = jsInteresting.ShellResult(options, command, prefix, True) oom = jsInteresting.oomed(r.err) r.err = ignoreSomeOfStderr(r.err) if (r.rc == 1 or r.rc == 2) and ( anyLineContains(r.out, '[[script] scriptArgs*]') or anyLineContains(r.err, '[scriptfile] [scriptarg...]')): print "Got usage error from:" print " " + sps.shellify(command) assert i > 0 jsInteresting.deleteLogs(prefix) elif r.lev > jsInteresting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh print infilename + " | " + jsInteresting.summaryString( r.issues + ["compareJIT found a more serious bug"], r.lev, r.runinfo.elapsedtime) with open(logPrefix + "-summary.txt", 'wb') as f: f.write('\n'.join(r.issues + [ sps.shellify(command), "compareJIT found a more serious bug" ]) + '\n') print " " + sps.shellify(command) return (r.lev, r.crashInfo) elif r.lev != jsInteresting.JS_FINE or r.rc != 0: print infilename + " | " + jsInteresting.summaryString( r.issues + [ "compareJIT is not comparing output, because the shell exited strangely" ], r.lev, r.runinfo.elapsedtime) print " " + sps.shellify(command) jsInteresting.deleteLogs(prefix) if i == 0: return (jsInteresting.JS_FINE, None) elif oom: # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compareJIT is not comparing output: OOM" print infilename + " | " + jsInteresting.summaryString( r.issues + [message], r.lev, r.runinfo.elapsedtime) jsInteresting.deleteLogs(prefix) if i == 0: return (jsInteresting.JS_FINE, None) elif i == 0: # Stash output from this run (the first one), so for subsequent runs, we can compare against it. (r0, prefix0) = (r, prefix) else: # Compare the output of this run (r.out) to the output of the first run (r0.out), etc. def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg): fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command)) return fpuOptionDisabledAsm and fpuOptionDiffers def optionDisabledAsmOnOneSide(): asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" optionDisabledAsm = anyLineContains( r0.err, asmMsg) or anyLineContains(r.err, asmMsg) optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command)) return optionDisabledAsm and optionDiffers mismatchErr = ( r.err != r0.err and # --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things. # This should only affect asm.js diagnostics on stderr. not fpuOptionDisabledAsmOnOneSide( "asm.js type error: Disabled by lack of floating point support" ) and # And also wasm stuff. See bug 1243031. not fpuOptionDisabledAsmOnOneSide( "WebAssembly is not supported on the current device") and not optionDisabledAsmOnOneSide()) mismatchOut = (r.out != r0.out) if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. rerunCommand = sps.shellify([ '~/funfuzz/js/compareJIT.py', "--flags=" + ' '.join(flags), "--timeout=" + str(options.timeout), options.knownPath, jsEngine, os.path.basename(infilename) ]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = " " + sps.shellify( commands[0]) + "\n " + sps.shellify( command) + "\n\n" + summary with open(logPrefix + "-summary.txt", 'wb') as f: f.write(rerunCommand + "\n\n" + summary) print infilename + " | " + jsInteresting.summaryString( issues, jsInteresting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime) if quickMode: print rerunCommand if showDetailedDiffs: print summary print "" # Create a crashInfo object with empty stdout, and stderr showing diffs pc = ProgramConfiguration.fromBinary(jsEngine) pc.addProgramArguments(flags) crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary, pc) return (jsInteresting.JS_OVERALL_MISMATCH, crashInfo) else: # print "compareJIT: match" jsInteresting.deleteLogs(prefix) # All matched :) jsInteresting.deleteLogs(prefix0) return (jsInteresting.JS_FINE, None)
def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDiffs, quickMode): # options dict must be one we can pass to jsInteresting.ShellResult # we also use it directly for knownPath, timeout, and collector # Return: (lev, crashInfo) or (jsInteresting.JS_FINE, None) combos = shellFlags.basicFlagSets(jsEngine) if quickMode: # Only used during initial fuzzing. Allowed to have false negatives. combos = [combos[0]] if len(flags): combos.append(flags) commands = [[jsEngine] + combo + [infilename] for combo in combos] for i in range(0, len(commands)): prefix = logPrefix + "-r" + str(i) command = commands[i] r = jsInteresting.ShellResult(options, command, prefix, True) oom = jsInteresting.oomed(r.err) r.err = ignoreSomeOfStderr(r.err) if (r.rc == 1 or r.rc == 2) and (anyLineContains(r.out, '[[script] scriptArgs*]') or anyLineContains(r.err, '[scriptfile] [scriptarg...]')): print "Got usage error from:" print " " + sps.shellify(command) assert i > 0 jsInteresting.deleteLogs(prefix) elif r.lev > jsInteresting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh print infilename + " | " + jsInteresting.summaryString(r.issues + ["compareJIT found a more serious bug"], r.lev, r.runinfo.elapsedtime) with open(logPrefix + "-summary.txt", 'wb') as f: f.write('\n'.join(r.issues + [sps.shellify(command), "compareJIT found a more serious bug"]) + '\n') print " " + sps.shellify(command) return (r.lev, r.crashInfo) elif r.lev != jsInteresting.JS_FINE or r.rc != 0: print infilename + " | " + jsInteresting.summaryString(r.issues + ["compareJIT is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime) print " " + sps.shellify(command) jsInteresting.deleteLogs(prefix) if i == 0: return (jsInteresting.JS_FINE, None) elif oom: # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compareJIT is not comparing output: OOM" print infilename + " | " + jsInteresting.summaryString(r.issues + [message], r.lev, r.runinfo.elapsedtime) jsInteresting.deleteLogs(prefix) if i == 0: return (jsInteresting.JS_FINE, None) elif i == 0: # Stash output from this run (the first one), so for subsequent runs, we can compare against it. (r0, prefix0) = (r, prefix) else: # Compare the output of this run (r.out) to the output of the first run (r0.out), etc. def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg): fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command)) return fpuOptionDisabledAsm and fpuOptionDiffers def optionDisabledAsmOnOneSide(): asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" optionDisabledAsm = anyLineContains(r0.err, asmMsg) or anyLineContains(r.err, asmMsg) optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command)) return optionDisabledAsm and optionDiffers mismatchErr = (r.err != r0.err and # --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things. # This should only affect asm.js diagnostics on stderr. not fpuOptionDisabledAsmOnOneSide("asm.js type error: Disabled by lack of floating point support") and # And also wasm stuff. See bug 1243031. not fpuOptionDisabledAsmOnOneSide("WebAssembly is not supported on the current device") and not optionDisabledAsmOnOneSide()) mismatchOut = (r.out != r0.out) if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. rerunCommand = sps.shellify(['~/funfuzz/js/compareJIT.py', "--flags="+' '.join(flags), "--timeout="+str(options.timeout), options.knownPath, jsEngine, os.path.basename(infilename)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = " " + sps.shellify(commands[0]) + "\n " + sps.shellify(command) + "\n\n" + summary with open(logPrefix + "-summary.txt", 'wb') as f: f.write(rerunCommand + "\n\n" + summary) print infilename + " | " + jsInteresting.summaryString(issues, jsInteresting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime) if quickMode: print rerunCommand if showDetailedDiffs: print summary print "" # Create a crashInfo object with empty stdout, and stderr showing diffs pc = ProgramConfiguration.fromBinary(jsEngine) pc.addProgramArguments(flags) crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary, pc) return (jsInteresting.JS_OVERALL_MISMATCH, crashInfo) else: # print "compareJIT: match" jsInteresting.deleteLogs(prefix) # All matched :) jsInteresting.deleteLogs(prefix0) return (jsInteresting.JS_FINE, None)
def __init__(self, options, runthis, logPrefix, inCompareJIT): pathToBinary = runthis[0] # This relies on the shell being a local one from compileShell.py: pc = ProgramConfiguration.fromBinary(pathToBinary.split('.')[0]) # Ignore trailing ".exe" in Win pc.addProgramArguments(runthis[1:-1]) if options.valgrind: runthis = ( inspectShell.constructVgCmdList(errorCode=VALGRIND_ERROR_EXIT_CODE) + valgrindSuppressions(options.knownPath) + runthis) preexec_fn = ulimitSet if os.name == 'posix' else None runinfo = timedRun.timed_run(runthis, options.timeout, logPrefix, preexec_fn=preexec_fn) lev = JS_FINE issues = [] auxCrashData = [] # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and 'readlines' everything into memory. with open(logPrefix + "-out.txt") as f: out = f.readlines() with open(logPrefix + "-err.txt") as f: err = f.readlines() if options.valgrind and runinfo.rc == VALGRIND_ERROR_EXIT_CODE: issues.append("valgrind reported an error") lev = max(lev, JS_VG_AMISS) valgrindErrorPrefix = "==" + str(runinfo.pid) + "==" for line in err: if valgrindErrorPrefix and line.startswith(valgrindErrorPrefix): issues.append(line.rstrip()) elif runinfo.sta == timedRun.CRASHED: if sps.grabCrashLog(runthis[0], runinfo.pid, logPrefix, True): with open(logPrefix + "-crash.txt") as f: auxCrashData = [line.strip() for line in f.readlines()] elif detect_malloc_errors.amiss(logPrefix): issues.append("malloc error") lev = max(lev, JS_NEW_ASSERT_OR_CRASH) elif runinfo.rc == 0 and not inCompareJIT: # We might have(??) run jsfunfuzz directly, so check for special kinds of bugs for line in out: if line.startswith("Found a bug: ") and not ("NestTest" in line and oomed(err)): lev = JS_DECIDED_TO_EXIT issues.append(line.rstrip()) if options.shellIsDeterministic and not understoodJsfunfuzzExit(out, err) and not oomed(err): issues.append("jsfunfuzz didn't finish") lev = JS_DID_NOT_FINISH # Copy non-crash issues to where FuzzManager's "AssertionHelper.py" can see it. if lev != JS_FINE: for issue in issues: err.append("[Non-crash bug] " + issue) # Finally, make a CrashInfo object and parse stack traces for asan/crash/assertion bugs crashInfo = CrashInfo.CrashInfo.fromRawCrashData(out, err, pc, auxCrashData=auxCrashData) createCollector.printCrashInfo(crashInfo) if not isinstance(crashInfo, CrashInfo.NoCrashInfo): lev = max(lev, JS_NEW_ASSERT_OR_CRASH) match = options.collector.search(crashInfo) if match[0] is not None: createCollector.printMatchingSignature(match) lev = JS_FINE print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime) if lev != JS_FINE: fileManipulation.writeLinesToFile( ['Number: ' + logPrefix + '\n', 'Command: ' + sps.shellify(runthis) + '\n'] + ['Status: ' + i + "\n" for i in issues], logPrefix + '-summary.txt') self.lev = lev self.out = out self.err = err self.issues = issues self.crashInfo = crashInfo self.match = match self.runinfo = runinfo self.rc = runinfo.rc
def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout, showDetailedDiffs, quickMode): combos = shellFlags.basicFlagSets(jsEngine) if quickMode: # Only used during initial fuzzing. Allowed to have false negatives. combos = [combos[0]] if len(flags): combos.append(flags) commands = [[jsEngine] + combo + [infilename] for combo in combos] for i in range(0, len(commands)): prefix = logPrefix + "-r" + str(i) command = commands[i] (lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath, prefix) with open(prefix + "-out.txt") as f: r.out = f.read(lengthLimit) with open(prefix + "-err.txt") as f: r.err = f.read(lengthLimit) oom = jsInteresting.hitMemoryLimit(r.err) if (not oom) and (len(r.err) + 5 > lengthLimit): # The output was too long for Python to read it in all at once. Assume the worst. oom = "stderr too long" r.err = ignoreSomeOfStderr(r.err) if (r.rc == 1 or r.rc == 2) and (r.out.find('[[script] scriptArgs*]') != -1 or r.err.find('[scriptfile] [scriptarg...]') != -1): print "Got usage error from:" print " " + sps.shellify(command) assert i > 0 jsInteresting.deleteLogs(prefix) elif lev > jsInteresting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh print infilename + " | " + jsInteresting.summaryString(issues + ["compareJIT found a more serious bug"], lev, r.elapsedtime) with open(logPrefix + "-summary.txt", 'wb') as f: f.write('\n'.join(issues + [sps.shellify(command), "compareJIT found a more serious bug"]) + '\n') print " " + sps.shellify(command) return lev elif lev != jsInteresting.JS_FINE: print infilename + " | " + jsInteresting.summaryString(issues + ["compareJIT is not comparing output, because the shell exited strangely"], lev, r.elapsedtime) print " " + sps.shellify(command) jsInteresting.deleteLogs(prefix) if i == 0: return jsInteresting.JS_FINE elif oom: # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compareJIT is not comparing output: OOM (" + oom + ")" print infilename + " | " + jsInteresting.summaryString(issues + [message], lev, r.elapsedtime) jsInteresting.deleteLogs(prefix) if i == 0: return jsInteresting.JS_FINE elif i == 0: # Stash output from this run (the first one), so for subsequent runs, we can compare against it. (r0, prefix0) = (r, prefix) else: # Compare the output of this run (r.out) to the output of the first run (r0.out), etc. def fpuOptionDisabledAsmOnOneSide(): # --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things. # This should only affect asm.js diagnostics on stderr. fpuAsmMsg = "asm.js type error: Disabled by lack of floating point support" fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command)) return (fpuOptionDisabledAsm and fpuOptionDiffers) def optionDisabledAsmOnOneSide(): asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" optionDisabledAsm = asmMsg in r0.err or asmMsg in r.err optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command)) return (optionDisabledAsm and optionDiffers) mismatchErr = (r.err != r0.err and not fpuOptionDisabledAsmOnOneSide() and not optionDisabledAsmOnOneSide()) mismatchOut = (r.out != r0.out) if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. rerunCommand = sps.shellify(['~/funfuzz/js/compareJIT.py', "--flags="+' '.join(flags), "--timeout="+str(timeout), knownPath, jsEngine, os.path.basename(infilename)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = " " + sps.shellify(commands[0]) + "\n " + sps.shellify(command) + "\n\n" + summary with open(logPrefix + "-summary.txt", 'wb') as f: f.write(rerunCommand + "\n\n" + summary) print infilename + " | " + jsInteresting.summaryString(issues, jsInteresting.JS_OVERALL_MISMATCH, r.elapsedtime) if quickMode: print rerunCommand if showDetailedDiffs: print summary print "" return jsInteresting.JS_OVERALL_MISMATCH else: #print "compareJIT: match" jsInteresting.deleteLogs(prefix) # All matched :) jsInteresting.deleteLogs(prefix0) return jsInteresting.JS_FINE
def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout, showDetailedDiffs, quickMode): combos = shellFlags.basicFlagSets(jsEngine) if quickMode: # Only used during initial fuzzing. Allowed to have false negatives. combos = [combos[0]] if len(flags): combos.append(flags) commands = [[jsEngine] + combo + [infilename] for combo in combos] for i in range(0, len(commands)): prefix = logPrefix + "-r" + str(i) command = commands[i] (lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath, prefix) with open(prefix + "-out.txt") as f: r.out = f.read(lengthLimit) with open(prefix + "-err.txt") as f: r.err = f.read(lengthLimit) oom = jsInteresting.hitMemoryLimit(r.err) if (not oom) and (len(r.err) + 5 > lengthLimit): # The output was too long for Python to read it in all at once. Assume the worst. oom = "stderr too long" r.err = ignoreSomeOfStderr(r.err) if (r.rc == 1 or r.rc == 2) and (r.out.find('[[script] scriptArgs*]') != -1 or r.err.find('[scriptfile] [scriptarg...]') != -1): print "Got usage error from:" print " " + sps.shellify(command) assert i > 0 jsInteresting.deleteLogs(prefix) elif lev > jsInteresting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh print infilename + " | " + jsInteresting.summaryString( issues + ["compareJIT found a more serious bug"], lev, r.elapsedtime) with open(logPrefix + "-summary.txt", 'wb') as f: f.write('\n'.join(issues + [ sps.shellify(command), "compareJIT found a more serious bug" ]) + '\n') print " " + sps.shellify(command) return lev elif lev != jsInteresting.JS_FINE: print infilename + " | " + jsInteresting.summaryString( issues + [ "compareJIT is not comparing output, because the shell exited strangely" ], lev, r.elapsedtime) print " " + sps.shellify(command) jsInteresting.deleteLogs(prefix) if i == 0: return jsInteresting.JS_FINE elif oom: # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. message = "compareJIT is not comparing output: OOM (" + oom + ")" print infilename + " | " + jsInteresting.summaryString( issues + [message], lev, r.elapsedtime) jsInteresting.deleteLogs(prefix) if i == 0: return jsInteresting.JS_FINE elif i == 0: # Stash output from this run (the first one), so for subsequent runs, we can compare against it. (r0, prefix0) = (r, prefix) else: # Compare the output of this run (r.out) to the output of the first run (r0.out), etc. def fpuOptionDisabledAsmOnOneSide(): # --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things. # This should only affect asm.js diagnostics on stderr. fpuAsmMsg = "asm.js type error: Disabled by lack of floating point support" fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command)) return (fpuOptionDisabledAsm and fpuOptionDiffers) def optionDisabledAsmOnOneSide(): asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" optionDisabledAsm = asmMsg in r0.err or asmMsg in r.err optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command)) return (optionDisabledAsm and optionDiffers) mismatchErr = (r.err != r0.err and not fpuOptionDisabledAsmOnOneSide() and not optionDisabledAsmOnOneSide()) mismatchOut = (r.out != r0.out) if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. rerunCommand = sps.shellify([ '~/funfuzz/js/compareJIT.py', "--flags=" + ' '.join(flags), "--timeout=" + str(timeout), knownPath, jsEngine, os.path.basename(infilename) ]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = " " + sps.shellify( commands[0]) + "\n " + sps.shellify( command) + "\n\n" + summary with open(logPrefix + "-summary.txt", 'wb') as f: f.write(rerunCommand + "\n\n" + summary) print infilename + " | " + jsInteresting.summaryString( issues, jsInteresting.JS_OVERALL_MISMATCH, r.elapsedtime) if quickMode: print rerunCommand if showDetailedDiffs: print summary print "" return jsInteresting.JS_OVERALL_MISMATCH else: #print "compareJIT: match" jsInteresting.deleteLogs(prefix) # All matched :) jsInteresting.deleteLogs(prefix0) return jsInteresting.JS_FINE
def cfgBin(shell): """Configure a binary according to required parameters.""" cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt[b"AR"] = b"ar" if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.10 Yosemite and greater if sps.isMac: assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. # Uses system clang cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS) cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS) if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt[b"RANLIB"] = b"ranlib" cfgEnvDt[b"AS"] = b"$CC" cfgEnvDt[b"LD"] = b"ld" cfgEnvDt[b"STRIP"] = b"strip -x -S" cfgEnvDt[b"CROSS_COMPILE"] = b"1" if sps.isProgramInstalled('brew'): cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" # Hacked up for new and old Homebrew configs, we can probably just call autoconf213 if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])): cfgEnvDt[b"AUTOCONF"] = b"autoconf213" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig" if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str( "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str( "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) else: # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.10 Yosemite and greater elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32: cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534 if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib " b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib " b"clang_rt.asan_dynamic-x86_64.dll") cfgEnvDt[b"HOST_CFLAGS"] = b" " cfgEnvDt[b"HOST_CXXFLAGS"] = b" " cfgEnvDt[b"HOST_LDFLAGS"] = b" " cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithClang: if sps.isWin: assert b"clang-cl" in cfgEnvDt[b"CC"] assert b"clang-cl" in cfgEnvDt[b"CXX"] else: assert b"clang" in cfgEnvDt[b"CC"] assert b"clang++" in cfgEnvDt[b"CXX"] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg: cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif shell.buildOptions.disableOpt: cfgCmdList.append('--disable-optimize') if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility cfgCmdList.append('--enable-profiling') if shell.buildOptions.disableProfiling: cfgCmdList.append('--disable-profiling') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append('--enable-oom-breakpoint') if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append('--without-intl-api') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] + '"' if " " in cfgEnvDt[str(envVar)] else envVar + "=" + cfgEnvDt[str(envVar)]) envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def jsfunfuzzLevel(options, logPrefix, quiet=False): (lev, issues, runinfo) = baseLevel( options.jsengineWithArgs, options.timeout, options.knownPath, logPrefix, valgrind=options.valgrind ) if lev == JS_FINE: # Check for unexplained exits and for jsfunfuzz saying "Found a bug". understoodExit = False # Read in binary mode, because otherwise Python on Windows will # throw a fit when it encounters certain unicode. Note that this # makes line endings platform-specific. if "-dm-" in options.jsengineWithArgs[0]: # Since this is an --enable-more-deterministic build, we should get messages on stderr # if the shell quit() or terminate() functions are called. # (We use a sketchy filename-matching check because it's faster than inspecting the binary.) with open(logPrefix + "-err.txt", "rb") as f: for line in f: if "terminate called" in line or "quit called" in line: understoodExit = True if "can't allocate region" in line: understoodExit = True else: understoodExit = True with open(logPrefix + "-out.txt", "rb") as f: for line in f: if line.startswith("It's looking good!") or line.startswith( "jsfunfuzz broke its own scripting environment: " ): understoodExit = True if line.startswith("Found a bug: "): understoodExit = True if not ("NestTest" in line and oomed(logPrefix)): lev = JS_DECIDED_TO_EXIT issues.append(line.rstrip()) # FIXME: if not quiet: # FIXME: output everything between this line and "jsfunfuzz stopping due to finding a bug." if not understoodExit: issues.append("jsfunfuzz didn't finish") lev = JS_DID_NOT_FINISH # FIXME: if not quiet: # FIXME: output the last tryItOut line if lev <= JS_ABNORMAL_EXIT: # JS_ABNORMAL_EXIT and below (inclusive) will be ignored. sps.vdump("jsfunfuzzLevel is ignoring a baseLevel of " + str(lev)) lev = JS_FINE issues = [] if lev != JS_FINE: # FIXME: compareJIT failures do not generate this -summary file. statusIssueList = [] for i in issues: statusIssueList.append("Status: " + i) assert len(statusIssueList) != 0 fileManipulation.writeLinesToFile( ["Number: " + logPrefix + "\n", "Command: " + sps.shellify(options.jsengineWithArgs) + "\n"] + [i + "\n" for i in statusIssueList], logPrefix + "-summary.txt", ) if not quiet: print logPrefix + " | " + summaryString(issues, lev, runinfo.elapsedtime) return lev
def cfgBin(shell, binToBeCompiled): '''This function configures a binary according to required parameters.''' cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' # Check for determinism to prevent LLVM compilation from happening on releng machines, # since releng machines only test non-deterministic builds. if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic: llvmPath = envVars.findLlvmBinPath() assert llvmPath is not None CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang')) CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++')) if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.7 Lion and greater if sps.isMac: assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior. if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS else: # Uses system clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2 cfgCmdList.append('--enable-macos-target=10.5') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # Uses system clang # We might still be using GCC on Linux 32-bit, use clang only if we specify ASan # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.7 Lion and greater elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32: if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS else: # Uses system clang cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) if shell.buildOptions.enable32: cfgCmdList.append('--enable-win32-target=WIN95') else: cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithAsan: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 # For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build. # They can actually be used independently, but it's not recommended. # https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options if shell.buildOptions.enableDbg: # NSPR configure without options compiles debug by default if binToBeCompiled == 'js': cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr': cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt: # NSPR configure without options compiles debug by default cfgCmdList.append('--disable-optimize') if binToBeCompiled == 'nspr': cfgCmdList.append('--prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) else: if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-nspr-build') else: if shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-threadsafe') if not shell.getJsBuildSystemConsidersNspr(): cfgCmdList.append('--with-nspr-prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) cfgCmdList.append('--with-nspr-cflags=-I' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr'))) cfgCmdList.append('--with-nspr-libs=' + ' '.join([ sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)) for compileLib in inspectShell.ALL_COMPILE_LIBS ])) else: cfgCmdList.append('--disable-threadsafe') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # See bug 986715 comment 6 as to why we need forward slashes for NSPR # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def cfgBin(shell): '''This function configures a binary according to required parameters.''' cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt[ 'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt[ 'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.10 Yosemite and greater if sps.isMac: assert sps.macVer() >= [ 10, 10 ] # We no longer support 10.9 Mavericks and prior. # Uses system clang cfgEnvDt['CC'] = cfgEnvDt[ 'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt[ 'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgEnvDt['CC'] = cfgEnvDt[ 'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt[ 'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = cfgEnvDt[ 'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt[ 'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.10 Yosemite and greater elif sps.isMac and sps.macVer() >= [10, 10 ] and not shell.buildOptions.enable32: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgEnvDt['CC'] += CLANG_ASAN_PARAMS # cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgCmdList.append('--enable-address-sanitizer') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithClang: if sps.isWin: assert 'clang-cl' in cfgEnvDt['CC'] assert 'clang-cl' in cfgEnvDt['CXX'] else: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg: cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif shell.buildOptions.disableOpt: cfgCmdList.append('--disable-optimize') if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append('--enable-oom-breakpoint') if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append('--without-intl-api') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append( '--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)