def untarbz2(fn, dest): ''' Extracts .tar.bz2 files to their destination. ''' if not os.path.exists(dest): os.mkdir(dest) sps.captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])
def getRepoHashAndId(repoDir, repoRev="parents() and default"): """Return the repository hash and id, and whether it is on default. It will also ask what the user would like to do, should the repository not be on default. """ # This returns null if the repository is not on default. hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", repoRev, "--template", "{node|short} {rev}"] hgIdFull = sps.captureStdout(hgLogTmplList)[0] onDefault = bool(hgIdFull) if not onDefault: updateDefault = raw_input( "Not on default tip! " + "Would you like to (a)bort, update to (d)efault, or (u)se this rev: " ) if updateDefault == "a": print "Aborting..." sys.exit(0) elif updateDefault == "d": subprocess.check_call(["hg", "-R", repoDir, "update", "default"]) onDefault = True elif updateDefault == "u": hgLogTmplList = ["hg", "-R", repoDir, "log", "-r", "parents()", "--template", "{node|short} {rev}"] else: raise Exception("Invalid choice.") hgIdFull = sps.captureStdout(hgLogTmplList)[0] assert hgIdFull != "" (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(" ") sps.vdump("Finished getting the hash and local id number of the repository.") return hgIdChangesetHash, hgIdLocalNum, onDefault
def printMachineInfo(): # Log information about the machine. print "Platform details: " + " ".join(platform.uname()) print "hg version: " + sps.captureStdout(['hg', '-q', 'version'])[0] # In here temporarily to see if mock Linux slaves on TBPL have gdb installed try: print "gdb version: " + sps.captureStdout(['gdb', '--version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0] except (KeyboardInterrupt, Exception) as e: print('Error involving gdb is: ' + repr(e)) # FIXME: Should have if os.path.exists(path to git) or something #print "git version: " + sps.captureStdout(['git', 'version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0] print "Python version: " + sys.version.split()[0] print "Number of cores visible to OS: " + str(multiprocessing.cpu_count()) print 'Free space (GB): ' + str('%.2f') % sps.getFreeSpace('/', 3) hgrcLocation = os.path.join(path0, '.hg', 'hgrc') if os.path.isfile(hgrcLocation): print 'The hgrc of this repository is:' with open(hgrcLocation, 'rb') as f: hgrcContentList = f.readlines() for line in hgrcContentList: print line.rstrip() if os.name == 'posix': # resource library is only applicable to Linux or Mac platforms. import resource print "Corefile size (soft limit, hard limit) is: " + \ repr(resource.getrlimit(resource.RLIMIT_CORE))
def printMachineInfo(): # Log information about the machine. print "Platform details: " + " ".join(platform.uname()) print "hg version: " + sps.captureStdout(['hg', '-q', 'version'])[0] # In here temporarily to see if mock Linux slaves on TBPL have gdb installed try: print "gdb version: " + sps.captureStdout(['gdb', '--version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0] except (KeyboardInterrupt, Exception) as e: print 'Error involving gdb is: ' + repr(e) # FIXME: Should have if os.path.exists(path to git) or something # print "git version: " + sps.captureStdout(['git', 'version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0] print "Python version: " + sys.version.split()[0] print "Number of cores visible to OS: " + str(multiprocessing.cpu_count()) print 'Free space (GB): ' + str('%.2f') % sps.getFreeSpace('/', 3) hgrcLocation = os.path.join(path0, '.hg', 'hgrc') if os.path.isfile(hgrcLocation): print 'The hgrc of this repository is:' with open(hgrcLocation, 'rb') as f: hgrcContentList = f.readlines() for line in hgrcContentList: print line.rstrip() if os.name == 'posix': # resource library is only applicable to Linux or Mac platforms. import resource print "Corefile size (soft limit, hard limit) is: " + \ repr(resource.getrlimit(resource.RLIMIT_CORE))
def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()): # We may have passed in the patch with or without the full directory. patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile)) pname = os.path.basename(patchAbsPath) assert pname != '' qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath], combineStderr=True, ignoreStderr=True, ignoreExitCode=True) if qimportRetCode != 0: if 'already exists' in qimportOutput: print "A patch with the same name has already been qpush'ed. Please qremove it first." raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode)) print("Patch qimport'ed..."), qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname], combineStderr=True, ignoreStderr=True) assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty." if qpushRetCode != 0: hgQpopQrmAppliedPatch(patchFile, workingDir) print 'You may have untracked .rej or .orig files in the repository.' print '`hg status` output of the repository of interesting files in ' + workingDir + ' :' subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added', '--removed', '--deleted']) raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode)) print("Patch qpush'ed. Continuing..."), return pname
def compileJs(shell): '''This function compiles and copies a binary.''' try: cmdList = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'] out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] except Exception as e: # This exception message is returned from sps.captureStdout via cmdList. if (sps.isLinux or sps.isMac) and \ ('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)): # FIXME: Absolute hack to retry after hitting OOM. print 'Trying once more due to the compiler running out of memory...' out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] # A non-zero error can be returned during make, but eventually a shell still gets compiled. if os.path.exists(shell.getShellCompiledPath()): print 'A shell was compiled even though there was a non-zero exit code. Continuing...' else: print MAKE_BINARY + " did not result in a js shell:" raise if os.path.exists(shell.getShellCompiledPath()): shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) for runLib in shell.getShellCompiledRunLibsPath(): if os.path.isfile(runLib): shutil.copy2(runLib, shell.getShellCacheDir()) else: print out raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def getRepoHashAndId(repoDir, repoRev='parents() and default'): ''' This function returns the repository hash and id, and whether it is on default. It also asks what the user would like to do, should the repository not be on default. ''' # This returns null if the repository is not on default. hgLogTmplList = [ 'hg', '-R', repoDir, 'log', '-r', repoRev, '--template', '{node|short} {rev}' ] hgIdFull = sps.captureStdout(hgLogTmplList)[0] onDefault = bool(hgIdFull) if not onDefault: updateDefault = raw_input('Not on default tip! ' + \ 'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ') if updateDefault == 'a': print 'Aborting...' sys.exit(0) elif updateDefault == 'd': subprocess.check_call(['hg', '-R', repoDir, 'update', 'default']) onDefault = True elif updateDefault == 'u': hgLogTmplList = [ 'hg', '-R', repoDir, 'log', '-r', 'parents()', '--template', '{node|short} {rev}' ] else: raise Exception('Invalid choice.') hgIdFull = sps.captureStdout(hgLogTmplList)[0] assert hgIdFull != '' (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ') sps.vdump( 'Finished getting the hash and local id number of the repository.') return hgIdChangesetHash, hgIdLocalNum, onDefault
def getRepoHashAndId(repoDir, repoRev='parents() and default'): ''' This function returns the repository hash and id, and whether it is on default. It also asks what the user would like to do, should the repository not be on default. ''' # This returns null if the repository is not on default. hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev, '--template', '{node|short} {rev}'] hgIdFull = sps.captureStdout(hgLogTmplList)[0] onDefault = bool(hgIdFull) if not onDefault: updateDefault = raw_input('Not on default tip! ' + \ 'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ') if updateDefault == 'a': print 'Aborting...' sys.exit(0) elif updateDefault == 'd': subprocess.check_call(['hg', '-R', repoDir, 'update', 'default']) onDefault = True elif updateDefault == 'u': hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template', '{node|short} {rev}'] else: raise Exception('Invalid choice.') hgIdFull = sps.captureStdout(hgLogTmplList)[0] assert hgIdFull != '' (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ') sps.vdump('Finished getting the hash and local id number of the repository.') return hgIdChangesetHash, hgIdLocalNum, onDefault
def compileJs(shell): """Compile and copy a binary.""" try: cmdList = [ MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s' ] out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] except Exception as e: # This exception message is returned from sps.captureStdout via cmdList. if (sps.isLinux or sps.isMac) and \ ('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)): # FIXME: Absolute hack to retry after hitting OOM. print 'Trying once more due to the compiler running out of memory...' out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] # A non-zero error can be returned during make, but eventually a shell still gets compiled. if os.path.exists(shell.getShellCompiledPath()): print 'A shell was compiled even though there was a non-zero exit code. Continuing...' else: print MAKE_BINARY + " did not result in a js shell:" raise if os.path.exists(shell.getShellCompiledPath()): shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) for runLib in shell.getShellCompiledRunLibsPath(): if os.path.isfile(runLib): shutil.copy2(runLib, shell.getShellCacheDir()) majorVersion, version = extractVersions(shell.getJsObjdir()) shell.setMajorVersion(majorVersion) shell.setVersion(version) if sps.isLinux: # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) # files in the objdir or else the stacks from failing testcases will lack symbols. shutil.rmtree( sps.normExpUserPath( os.path.join(shell.getShellCacheDir(), 'objdir-js'))) else: print out raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): """If bisect blamed a merge, try to figure out why.""" bisectLied = False missedCommonAncestor = False parents = sps.captureStdout(["hg", "-R", repoDir] + ["parent", '--template={node|short},', "-r", blamedRev])[0].split(",")[:-1] if len(parents) == 1: return for p in parents: # Ensure we actually tested the parent. if labels.get(p) is None: print "" print ("Oops! We didn't test rev %s, a parent of the blamed revision! " + "Let's do that now.") % str(p) if not hgCmds.isAncestor(repoDir, startRepo, p) and \ not hgCmds.isAncestor(repoDir, endRepo, p): print ('We did not test rev %s because it is not a descendant of either ' + '%s or %s.') % (str(p), startRepo, endRepo) # Note this in case we later decide the bisect result is wrong. missedCommonAncestor = True label = testRev(p) labels[p] = label print label[0] + " (" + label[1] + ") " print "As expected, the parent's label is the opposite of the blamed rev's label." # Check that the parent's label is the opposite of the blamed merge's label. if labels[p][0] == "skip": print "Parent rev %s was marked as 'skip', so the regression window includes it." % str(p) elif labels[p][0] == blamedGoodOrBad: print "Bisect lied to us! Parent rev %s was also %s!" % (str(p), blamedGoodOrBad) bisectLied = True else: assert labels[p][0] == {'good': 'bad', 'bad': 'good'}[blamedGoodOrBad] # Explain why bisect blamed the merge. if bisectLied: if missedCommonAncestor: ca = hgCmds.findCommonAncestor(repoDir, parents[0], parents[1]) print "" print "Bisect blamed the merge because our initial range did not include one" print "of the parents." print "The common ancestor of %s and %s is %s." % (parents[0], parents[1], ca) label = testRev(ca) print label[0] + " (" + label[1] + ") " print "Consider re-running autoBisect with -s %s -e %s" % (ca, blamedRev) print "in a configuration where earliestWorking is before the common ancestor." else: print "" print "Most likely, bisect's result was unhelpful because one of the" print "tested revisions was marked as 'good' or 'bad' for the wrong reason." print "I don't know which revision was incorrectly marked. Sorry." else: print "" print "The bug was introduced by a merge (it was not present on either parent)." print "I don't know which patches from each side of the merge contributed to the bug. Sorry."
def isAncestor(repoDir, a, b): """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" return ( sps.captureStdout( ["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"] )[0] != "" )
def downloadURL(url, dest): """Read in a URL and downloads it to a destination.""" inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', dest, url] out, retVal = sps.captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True) if retVal != 0: print out raise Exception('Return code is not 0, but is: ' + str(retVal)) return dest
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): """Tell hg what we learned about the revision.""" assert hgLabel in ("good", "bad", "skip") outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0] outputLines = outputResult.split("\n") if options.buildOptions: repoDir = options.buildOptions.repoDir if re.compile( "Due to skipped revisions, the first (good|bad) revision could be any of:" ).match(outputLines[0]): print() print(sanitizeCsetMsg(outputResult, repoDir)) print() return None, None, None, startRepo, endRepo r = re.compile("The first (good|bad) revision is:") m = r.match(outputLines[0]) if m: print() print() print( "autoBisect shows this is probably related to the following changeset:" ) print() print(sanitizeCsetMsg(outputResult, repoDir)) print() blamedGoodOrBad = m.group(1) blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1]) return blamedGoodOrBad, blamedRev, None, startRepo, endRepo if options.testInitialRevs: return None, None, None, startRepo, endRepo # e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)" sps.vdump(outputLines[0]) currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0]) if currRev is None: print("Resetting to default revision...") subprocess.check_call(hgPrefix + ['update', '-C', 'default']) hgCmds.destroyPyc(repoDir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. start = startRepo end = endRepo if hgLabel == 'bad': end = currRev elif hgLabel == 'good': start = currRev elif hgLabel == 'skip': pass return None, None, currRev, start, end
def testIsHardFpShellARM(s): '''Tests if the ARM shell is compiled with hardfp support.''' readelfBin = '/usr/bin/readelf' if os.path.exists(readelfBin): newEnv = envVars.envWithPath(os.path.dirname(os.path.abspath(s))) readelfOutput = sps.captureStdout([readelfBin, '-A', s], env=newEnv)[0] return 'Tag_ABI_VFP_args: VFP registers' in readelfOutput else: raise Exception('readelf is not found.')
def testBinary(shellPath, args, useValgrind): '''Tests the given shell with the given args.''' testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args sps.vdump('The testing command is: ' + sps.shellify(testCmd)) out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True, env=envVars.envWithPath( os.path.dirname(os.path.abspath(shellPath)))) sps.vdump('The exit code is: ' + str(rCode)) return out, rCode
def existsAndIsAncestor(repoDir, a, b): """Return true iff |a| exists and is an ancestor of |b|.""" # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error out = sps.captureStdout( ["hg", "-R", repoDir, "log", "-r", a + " and ancestor(" + a + "," + b + ")", "--template={node|short}"], combineStderr=True, ignoreExitCode=True, )[0] return out != "" and out.find("abort: unknown revision") < 0
def diffFiles(f1, f2): """Return a command to diff two files, along with the diff output (if it's short).""" diffcmd = ["diff", "-u", f1, f2] s = ' '.join(diffcmd) + "\n\n" diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0] if len(diff) < 10000: s += diff + "\n\n" else: s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n" return s
def existsAndIsAncestor(repoDir, a, b): """Returns true iff |a| exists and is an ancestor of |b|.""" # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error out = sps.captureStdout([ 'hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', '--template={node|short}' ], combineStderr=True, ignoreExitCode=True)[0] return out != "" and out.find("abort: unknown revision") < 0
def hgHashAddToFuzzPath(fuzzPath, repoDir): ''' This function finds the mercurial revision and appends it to the directory name. It also prompts if the user wants to continue, should the repository not be on tip. ''' hgIdCmdList = ['hg', 'identify', '-i', '-n', '-b', repoDir] vdump('About to start running `' + ' '.join(hgIdCmdList) + '` ...') # In Windows, this throws up a warning about failing to set color mode to win32. if platform.system() == 'Windows': hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir, ignoreStderr=True)[0] else: hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0] hgIdChangesetHash = hgIdFull.split(' ')[0] hgIdLocalNum = hgIdFull.split(' ')[1] # In Windows, this throws up a warning about failing to set color mode to win32. if platform.system() == 'Windows': hgIdBranch = captureStdout(['hg', 'id', '-t'], currWorkingDir=repoDir, ignoreStderr=True)[0] else: hgIdBranch = captureStdout(['hg', 'id', '-t'], currWorkingDir=repoDir)[0] onDefaultTip = True if 'tip' not in hgIdBranch: print 'The repository is at this changeset -', hgIdLocalNum + ':' + hgIdChangesetHash notOnDefaultTipApproval = str( raw_input( 'Not on default tip! Are you sure you want to continue? (y/n): ' )) if notOnDefaultTipApproval == ('y' or 'yes'): onDefaultTip = False else: switchToDefaultTipApproval = str( raw_input('Do you want to switch to the default tip? (y/n): ')) if switchToDefaultTipApproval == ('y' or 'yes'): subprocess.check_call(['hg', 'up', 'default'], cwd=repoDir) else: raise Exception('Not on default tip.') fuzzPath = '-'.join([fuzzPath, hgIdLocalNum, hgIdChangesetHash]) vdump('Finished running `' + ' '.join(hgIdCmdList) + '`.') return normExpUserPath(fuzzPath), onDefaultTip
def downloadURL(url, dest): """Read in a URL and downloads it to a destination.""" inpCmdList = [ 'curl', '--output', dest, url ] if useCurl else ['wget'] + wgetMaybeNCC + ['-O', dest, url] out, retVal = sps.captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True) if retVal != 0: print out raise Exception('Return code is not 0, but is: ' + str(retVal)) return dest
def hgQpopQrmAppliedPatch(patchFile, repoDir): '''Remove applied patch using `hg qpop` and `hg qdelete`.''' qpopOutput, qpopRetCode = sps.captureStdout(['hg', '-R', repoDir, 'qpop'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True) if qpopRetCode != 0: print '`hg qpop` output is: ' + qpopOutput raise Exception('Return code from `hg qpop` is: ' + str(qpopRetCode)) print("Patch qpop'ed..."), subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)]) print("Patch qdelete'd.")
def hgQpopQrmAppliedPatch(patchFile, repoDir): """Remove applied patch using `hg qpop` and `hg qdelete`.""" qpopOutput, qpopRetCode = sps.captureStdout(['hg', '-R', repoDir, 'qpop'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True) if qpopRetCode != 0: print '`hg qpop` output is: ' + qpopOutput raise Exception('Return code from `hg qpop` is: ' + str(qpopRetCode)) print "Patch qpop'ed...", subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)]) print "Patch qdelete'd."
def hgQpopQrmAppliedPatch(patchFile, repoDir): """Remove applied patch using `hg qpop` and `hg qdelete`.""" qpopOutput, qpopRetCode = sps.captureStdout( ["hg", "-R", repoDir, "qpop"], combineStderr=True, ignoreStderr=True, ignoreExitCode=True ) if qpopRetCode != 0: print "`hg qpop` output is: " + qpopOutput raise Exception("Return code from `hg qpop` is: " + str(qpopRetCode)) print "Patch qpop'ed...", subprocess.check_call(["hg", "-R", repoDir, "qdelete", os.path.basename(patchFile)]) print "Patch qdelete'd."
def compileJs(shell): '''This function compiles and copies a binary.''' try: cmdList = [ MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s' ] out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] except Exception as e: # This exception message is returned from sps.captureStdout via cmdList. if (sps.isLinux or sps.isMac) and \ ('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)): # FIXME: Absolute hack to retry after hitting OOM. print 'Trying once more due to the compiler running out of memory...' out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] # A non-zero error can be returned during make, but eventually a shell still gets compiled. if os.path.exists(shell.getShellCompiledPath()): print 'A shell was compiled even though there was a non-zero exit code. Continuing...' else: print MAKE_BINARY + " did not result in a js shell:" raise if os.path.exists(shell.getShellCompiledPath()): shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) for runLib in shell.getShellCompiledRunLibsPath(): if os.path.isfile(runLib): shutil.copy2(runLib, shell.getShellCacheDir()) else: print out raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def compileJs(shell): """This function compiles and copies a binary.""" try: cmdList = [MAKE_BINARY, "-C", shell.getJsObjdir(), "-j" + str(COMPILATION_JOBS), "-s"] out = sps.captureStdout( cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull() )[0] except Exception as e: # This exception message is returned from sps.captureStdout via cmdList. if (sps.isLinux or sps.isMac) and ( "GCC running out of memory" in repr(e) or "Clang running out of memory" in repr(e) ): # FIXME: Absolute hack to retry after hitting OOM. print "Trying once more due to the compiler running out of memory..." out = sps.captureStdout( cmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull(), )[0] # A non-zero error can be returned during make, but eventually a shell still gets compiled. if os.path.exists(shell.getShellCompiledPath()): print "A shell was compiled even though there was a non-zero exit code. Continuing..." else: print MAKE_BINARY + " did not result in a js shell:" raise if os.path.exists(shell.getShellCompiledPath()): shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) for runLib in shell.getShellCompiledRunLibsPath(): if os.path.isfile(runLib): shutil.copy2(runLib, shell.getShellCacheDir()) if sps.isLinux: # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) # files in the objdir or else the stacks from failing testcases will lack symbols. shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), "objdir-js"))) else: print out raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")
def archOfBinary(binary): '''This function tests if a binary is 32-bit or 64-bit.''' unsplitFiletype = sps.captureStdout(['file', binary])[0] filetype = unsplitFiletype.split(':', 1)[1] if sps.isWin: assert 'MS Windows' in filetype return '32' if 'Intel 80386 32-bit' in filetype else '64' else: if 'universal binary' in filetype: raise Exception("I don't know how to deal with multiple-architecture binaries") if '32-bit' in filetype or 'i386' in filetype: assert '64-bit' not in filetype return '32' if '64-bit' in filetype: assert '32-bit' not in filetype return '64'
def hgFindFixParent(self, repoDir, bugNum): prevRev = None hgOut = captureStdout(['hg', 'log', '-l', '10000', '--template', '{node} {desc}\n'], ignoreStderr=True, currWorkingDir=repoDir)[0].split("\n") for line in reversed(hgOut): line = line.split(' ', 1) if len(line) < 2: continue rev = line[0][0:12] if (line[1].find(str(bugNum)) != -1): return prevRev prevRev = rev return None
def compileNspr(shell): '''Compile a NSPR binary.''' cfgBin(shell, 'nspr') # Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well. # Even if we move to parallel compile NSPR in the future, we must beware of breaking old # build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1, # and after that, use -jX ? nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s'] out = sps.captureStdout(nsprCmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getNsprObjdir(), env=shell.getEnvFull())[0] for compileLib in inspectShell.ALL_COMPILE_LIBS: if not sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)): print out raise Exception(MAKE_BINARY + " did not result in a NSPR binary.") assert os.path.isdir(sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def undmg(fn, dest, mountpoint): """Extract .dmg files to their destination via a mount point.""" if os.path.exists(mountpoint): # If the mount point already exists, detach it first. sps.captureStdout(['hdiutil', 'detach', mountpoint, '-force']) sps.captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn]) try: apps = [x for x in os.listdir(mountpoint) if x.endswith('app')] assert len(apps) == 1 shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0]) finally: sps.captureStdout(['hdiutil', 'detach', mountpoint])
def undmg(fn, dest, mountpoint): """Extract .dmg files to their destination via a mount point.""" if os.path.exists(mountpoint): # If the mount point already exists, detach it first. sps.captureStdout(["hdiutil", "detach", mountpoint, "-force"]) sps.captureStdout(["hdiutil", "attach", "-quiet", "-mountpoint", mountpoint, fn]) try: apps = [x for x in os.listdir(mountpoint) if x.endswith("app")] assert len(apps) == 1 shutil.copytree(mountpoint + "/" + apps[0], dest + "/" + apps[0]) finally: sps.captureStdout(["hdiutil", "detach", mountpoint])
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): '''Tell hg what we learned about the revision.''' assert hgLabel in ("good", "bad", "skip") outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0] outputLines = outputResult.split("\n") repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]): print '\n' + sanitizeCsetMsg(outputResult, repoDir) + '\n' return None, None, None, startRepo, endRepo r = re.compile("The first (good|bad) revision is:") m = r.match(outputLines[0]) if m: print '\n\nautoBisect shows this is probably related to the following changeset:\n' print sanitizeCsetMsg(outputResult, repoDir) + '\n' blamedGoodOrBad = m.group(1) blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1]) return blamedGoodOrBad, blamedRev, None, startRepo, endRepo if options.testInitialRevs: return None, None, None, startRepo, endRepo # e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)" sps.vdump(outputLines[0]) currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0]) if currRev is None: print 'Resetting to default revision...' subprocess.check_call(hgPrefix + ['update', '-C', 'default']) hgCmds.destroyPyc(repoDir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. start = startRepo end = endRepo if hgLabel == 'bad': end = currRev elif hgLabel == 'good': start = currRev elif hgLabel == 'skip': pass return None, None, currRev, start, end
def undmg(fn, dest, mountpoint): """Extract .dmg files to their destination via a mount point.""" if os.path.exists(mountpoint): # If the mount point already exists, detach it first. sps.captureStdout(['hdiutil', 'detach', mountpoint, '-force']) sps.captureStdout( ['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn]) try: apps = [x for x in os.listdir(mountpoint) if x.endswith('app')] assert len(apps) == 1 shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0]) finally: sps.captureStdout(['hdiutil', 'detach', mountpoint])
def getShell(self, shellCacheDir, archNum, compileType, valgrindSupport, rev, updated, repoDir, buildFlags=None): shell = None # This code maps the old "-c dbg / -c opt" configurations to their configurations haveDebugOptFlags = False if buildFlags != None: haveDebugOptFlags = ('--enable-debug' in buildFlags) or ('--disable-debug' in buildFlags) or ('--enable-optimize' in buildFlags) or ('--disable-optimize' in buildFlags) print "haveDebugOptFlags: %s %s" % (str(haveDebugOptFlags), " ".join(buildFlags)) if compileType == 'dbg': if buildFlags != None: if not haveDebugOptFlags: buildFlags.append('--enable-debug') buildFlags.append('--enable-optimize') else: buildFlags = [ '--enable-debug', '--enable-optimize' ] elif compileType == 'opt': if buildFlags != None: if not haveDebugOptFlags: buildFlags.append('--disable-debug') buildFlags.append('--enable-optimize') else: buildFlags = [ '--disable-debug', '--enable-optimize' ] if archNum == "32": buildFlags.append('--32') buildOpts = '-R %s' % (repoDir) if buildFlags != None and len(buildFlags) > 0: buildOpts += ' %s' % " ".join(buildFlags) if (shell == None): if (rev == None): rev = self.hgUpdate(repoDir, rev) print "Compiling a new shell for tip (revision " + rev + ")" else: print "Compiling a new shell for revision " + rev shell = captureStdout(['/srv/repos/funfuzz/js/compileShell.py', '-b', buildOpts, '-r', rev])[0].split("\n")[-1] return (shell, rev)
def hgUpdate(self, repoDir, rev=None): print "Running hg update..." if (rev != None): captureStdout(['hg', 'update', '-r', rev], ignoreStderr=True, currWorkingDir=repoDir) else: captureStdout(['hg', 'update'], ignoreStderr=True, currWorkingDir=repoDir) hgIdCmdList = ['hg', 'identify', repoDir] # In Windows, this throws up a warning about failing to set color mode to win32. if platform.system() == 'Windows': hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir, ignoreStderr=True)[0] else: hgIdFull = captureStdout(hgIdCmdList, currWorkingDir=repoDir)[0] hgIdChangesetHash = hgIdFull.split(' ')[0] #os.chdir(savedPath) return hgIdChangesetHash
def compileNspr(shell): '''Compile a NSPR binary.''' cfgBin(shell, 'nspr') # Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well. # Even if we move to parallel compile NSPR in the future, we must beware of breaking old # build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1, # and after that, use -jX ? nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s'] out = sps.captureStdout(nsprCmdList, combineStderr=True, ignoreExitCode=True, currWorkingDir=shell.getNsprObjdir(), env=shell.getEnvFull())[0] for compileLib in inspectShell.ALL_COMPILE_LIBS: if not sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)): print out raise Exception(MAKE_BINARY + " did not result in a NSPR binary.") assert os.path.isdir( sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))
def cfgBin(shell): """Configure a binary according to required parameters.""" cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt[b"AR"] = b"ar" if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.10 Yosemite and greater if sps.isMac: assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. # Uses system clang cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS) cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS) if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt[b"RANLIB"] = b"ranlib" cfgEnvDt[b"AS"] = b"$CC" cfgEnvDt[b"LD"] = b"ld" cfgEnvDt[b"STRIP"] = b"strip -x -S" cfgEnvDt[b"CROSS_COMPILE"] = b"1" if sps.isProgramInstalled('brew'): cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" # Hacked up for new and old Homebrew configs, we can probably just call autoconf213 if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])): cfgEnvDt[b"AUTOCONF"] = b"autoconf213" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig" if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str( "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str( "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) else: # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.10 Yosemite and greater elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32: cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534 if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib " b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib " b"clang_rt.asan_dynamic-x86_64.dll") cfgEnvDt[b"HOST_CFLAGS"] = b" " cfgEnvDt[b"HOST_CXXFLAGS"] = b" " cfgEnvDt[b"HOST_LDFLAGS"] = b" " cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithClang: cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithClang: if sps.isWin: assert b"clang-cl" in cfgEnvDt[b"CC"] assert b"clang-cl" in cfgEnvDt[b"CXX"] else: assert b"clang" in cfgEnvDt[b"CC"] assert b"clang++" in cfgEnvDt[b"CXX"] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg: cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif shell.buildOptions.disableOpt: cfgCmdList.append('--disable-optimize') if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility cfgCmdList.append('--enable-profiling') if shell.buildOptions.disableProfiling: cfgCmdList.append('--disable-profiling') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append('--enable-oom-breakpoint') if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append('--without-intl-api') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] + '"' if " " in cfgEnvDt[str(envVar)] else envVar + "=" + cfgEnvDt[str(envVar)]) envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def unzip(fn, dest): ''' Extracts .zip files to their destination. ''' sps.captureStdout(['unzip', fn, '-d', dest])
def findBlamedCset(options, repoDir, testRev): print "%s | Bisecting on: %s" % (sps.dateStr(), repoDir) hgPrefix = ['hg', '-R', repoDir] # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". realStartRepo = sRepo = hgCmds.getRepoHashAndId( repoDir, repoRev=options.startRepo)[0] realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: subprocess.check_call(hgPrefix + ['update', '-C', 'default']) # Throws exit code 255 if purge extension is not enabled in .hgrc: subprocess.check_call(hgPrefix + ['purge', '--all']) # Reset bisect ranges and set skip ranges. sps.captureStdout(hgPrefix + ['bisect', '-r']) if options.skipRevs: sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs]) labels = {} # Specify `hg bisect` ranges. if options.testInitialRevs: currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first. else: labels[sRepo] = ('good', 'assumed start rev is good') labels[eRepo] = ('bad', 'assumed end rev is bad') subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo]) currRev = hgCmds.getCsetHashFromBisectMsg( fileManipulation.firstLine( sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0])) iterNum = 1 if options.testInitialRevs: iterNum -= 2 skipCount = 0 blamedRev = None while currRev is not None: startTime = time.time() label = testRev(currRev) labels[currRev] = label if label[0] == 'skip': skipCount += 1 # If we use "skip", we tell hg bisect to do a linear search to get around the skipping. # If the range is large, doing a bisect to find the start and endpoints of compilation # bustage would be faster. 20 total skips being roughly the time that the pair of # bisections would take. if skipCount > 20: print 'Skipped 20 times, stopping autoBisect.' break print label[0] + " (" + label[1] + ") ", if iterNum <= 0: print 'Finished testing the initial boundary revisions...', else: print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \ str(2**iterNum), "...", (blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \ bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo) if options.testInitialRevs: options.testInitialRevs = False assert currRev is None currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next. iterNum += 1 endTime = time.time() oneRunTime = endTime - startTime print 'This iteration took %.3f seconds to run.' % oneRunTime if blamedRev is not None: checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") subprocess.check_call(hgPrefix + ['bisect', '-U', '-r']) sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True) hgCmds.destroyPyc(repoDir) print sps.dateStr()
def findCommonAncestor(repoDir, a, b): return sps.captureStdout([ 'hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')', '--template={node|short}' ])[0]
def compileCopy(archNum, compileType, extraID, usePymake, repoDir, destDir, objDir, vgSupport, verbose=False): ''' This function compiles and copies a binary. ''' # Replace cpuCount() with cpu_count from the multiprocessing library once Python 2.6 is in # all build machines. jobs = (cpuCount() * 3) // 2 compiledNamePath = normExpUserPath( os.path.join(objDir, 'js' + ('.exe' if platform.system() == 'Windows' else ''))) try: if usePymake: out = captureStdout([ 'python', '-OO', os.path.normpath( os.path.join(repoDir, 'build', 'pymake', 'make.py')), '-j' + str(jobs), '-s' ], combineStderr=True, currWorkingDir=objDir)[0] # Pymake in builds earlier than revision 232553f741a0 did not support the '-s' option. if 'no such option: -s' in out: out = captureStdout([ 'python', '-OO', os.path.normpath( os.path.join(repoDir, 'build', 'pymake', 'make.py')), '-j' + str(jobs) ], combineStderr=True, currWorkingDir=objDir)[0] else: out = captureStdout(['make', '-C', objDir, '-j' + str(jobs), '-s'], combineStderr=True, ignoreExitCode=True, currWorkingDir=objDir)[0] except Exception: # Sometimes a non-zero error can be returned during the make process, but eventually a # shell still gets compiled. if os.path.exists(compiledNamePath): print 'A shell was compiled even though there was a non-zero exit code. Continuing...' else: if verbose: print out raise Exception("`make` did not result in a js shell, '" + repr(e) + "' thrown.") if not os.path.exists(compiledNamePath): if verbose: print out raise Exception( "`make` did not result in a js shell, no exception thrown.") else: newNamePath = normExpUserPath( os.path.join(destDir, shellName(archNum, compileType, extraID, vgSupport))) shutil.copy2(compiledNamePath, newNamePath) return newNamePath
def updateRepo(repo, rev): '''Updates repository to the specified revision.''' # Print *with* a trailing newline to avoid breaking other stuff print "Updating to rev %s in the %s repository..." % (rev, repo) sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True)
def unzip(fn, dest): """Extract .zip files to their destination.""" sps.captureStdout(['unzip', fn, '-d', dest])
def updateRepo(repo, rev): """Update repository to the specific revision.""" # Print *with* a trailing newline to avoid breaking other stuff print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), repo.decode("utf-8", errors="replace"))) sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True)
def findBlamedCset(options, repoDir, testRev): print sps.dateStr() hgPrefix = ['hg', '-R', repoDir] # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: subprocess.check_call(hgPrefix + ['update', '-C', 'default']) # Throws exit code 255 if purge extension is not enabled in .hgrc: subprocess.check_call(hgPrefix + ['purge', '--all']) # Reset bisect ranges and set skip ranges. sps.captureStdout(hgPrefix + ['bisect', '-r']) if options.skipRevs: sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs]) labels = {} # Specify `hg bisect` ranges. if options.testInitialRevs: currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first. else: labels[sRepo] = ('good', 'assumed start rev is good') labels[eRepo] = ('bad', 'assumed end rev is bad') subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo]) currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine( sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0])) iterNum = 1 if options.testInitialRevs: iterNum -= 2 skipCount = 0 blamedRev = None while currRev is not None: startTime = time.time() label = testRev(currRev) labels[currRev] = label if label[0] == 'skip': skipCount += 1 # If we use "skip", we tell hg bisect to do a linear search to get around the skipping. # If the range is large, doing a bisect to find the start and endpoints of compilation # bustage would be faster. 20 total skips being roughly the time that the pair of # bisections would take. if skipCount > 20: print 'Skipped 20 times, stopping autoBisect.' break print label[0] + " (" + label[1] + ") ", if iterNum <= 0: print 'Finished testing the initial boundary revisions...', else: print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \ str(2**iterNum), "...", (blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \ bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo) if options.testInitialRevs: options.testInitialRevs = False assert currRev is None currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next. iterNum += 1 endTime = time.time() oneRunTime = endTime - startTime print 'This iteration took %.3f seconds to run.' % oneRunTime if blamedRev is not None: checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") subprocess.check_call(hgPrefix + ['bisect', '-U', '-r']) sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True) hgCmds.destroyPyc(repoDir) print sps.dateStr()
def cfgBin(shell, binToBeCompiled): '''This function configures a binary according to required parameters.''' cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' # Check for determinism to prevent LLVM compilation from happening on releng machines, # since releng machines only test non-deterministic builds. if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic: llvmPath = envVars.findLlvmBinPath() assert llvmPath is not None CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang')) CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++')) if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.7 Lion and greater if sps.isMac: assert sps.macVer() >= [10, 7] # We no longer support Snow Leopard 10.6 and prior. if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS else: # Uses system clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin9.2.0') # Leopard 10.5.2 cfgCmdList.append('--enable-macos-target=10.5') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # Uses system clang # We might still be using GCC on Linux 32-bit, use clang only if we specify ASan # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.7 Lion and greater elif sps.isMac and sps.macVer() >= [10, 7] and not shell.buildOptions.enable32: if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS else: # Uses system clang cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) if shell.buildOptions.enable32: cfgCmdList.append('--enable-win32-target=WIN95') else: cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithAsan: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 # For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build. # They can actually be used independently, but it's not recommended. # https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options if shell.buildOptions.enableDbg: # NSPR configure without options compiles debug by default if binToBeCompiled == 'js': cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr': cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt: # NSPR configure without options compiles debug by default cfgCmdList.append('--disable-optimize') if binToBeCompiled == 'nspr': cfgCmdList.append('--prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) else: if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.getJsUsesNoThreadsFlag() and shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-nspr-build') else: if shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-threadsafe') if not shell.getJsBuildSystemConsidersNspr(): cfgCmdList.append('--with-nspr-prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) cfgCmdList.append('--with-nspr-cflags=-I' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr'))) cfgCmdList.append('--with-nspr-libs=' + ' '.join([ sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)) for compileLib in inspectShell.ALL_COMPILE_LIBS ])) else: cfgCmdList.append('--disable-threadsafe') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getNsprObjdir() if binToBeCompiled == 'nspr' else shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # See bug 986715 comment 6 as to why we need forward slashes for NSPR # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def isAncestor(repoDir, a, b): """Returns true iff |a| is an ancestor of |b|. (Throws if |a| or |b| does not exist.)""" return sps.captureStdout([ 'hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', '--template={node|short}' ])[0] != ""
def cfgBin(shell, binToBeCompiled): '''This function configures a binary according to required parameters.''' cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' # Check for determinism to prevent LLVM compilation from happening on releng machines, # since releng machines only test non-deterministic builds. if shell.buildOptions.buildWithAsan and shell.buildOptions.enableMoreDeterministic: llvmPath = envVars.findLlvmBinPath() assert llvmPath is not None CLANG_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang')) CLANGPP_PATH = sps.normExpUserPath(os.path.join(llvmPath, 'clang++')) if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt[ 'CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt[ 'CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.7 Lion and greater if sps.isMac: assert sps.macVer() >= [ 10, 7 ] # We no longer support Snow Leopard 10.6 and prior. if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS else: # Uses system clang cfgEnvDt['CC'] = cfgEnvDt[ 'HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt[ 'HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CC'] = cfgEnvDt[ 'CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt[ 'CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=i386-apple-darwin9.2.0') # Leopard 10.5.2 cfgCmdList.append('--enable-macos-target=10.5') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = CLANG_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = CLANGPP_PATH + CLANG_PARAMS + \ CLANG_ASAN_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # Uses system clang # We might still be using GCC on Linux 32-bit, use clang only if we specify ASan # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.7 Lion and greater elif sps.isMac and sps.macVer() >= [10, 7 ] and not shell.buildOptions.enable32: if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS else: # Uses system clang cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt[ 'AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append( '--target=x86_64-apple-darwin12.5.0') # Mountain Lion 10.8.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) if shell.buildOptions.enable32: cfgCmdList.append('--enable-win32-target=WIN95') else: cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithAsan: # Uses custom compiled clang cfgEnvDt['CC'] = CLANG_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgEnvDt['CXX'] = CLANGPP_PATH + CLANG_PARAMS + CLANG_ASAN_PARAMS cfgCmdList.append('sh') if binToBeCompiled == 'nspr': cfgCmdList.append(os.path.normpath(shell.getNsprCfgPath())) cfgCmdList.append('--enable-64bit') else: cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithAsan: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 # For NSPR, specify "--disable-debug --enable-optimize" to generate an opt build. # They can actually be used independently, but it's not recommended. # https://developer.mozilla.org/en-US/docs/NSPR_build_instructions#Configure_options if shell.buildOptions.enableDbg: # NSPR configure without options compiles debug by default if binToBeCompiled == 'js': cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg or binToBeCompiled == 'nspr': cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif binToBeCompiled == 'js' and shell.buildOptions.disableOpt: # NSPR configure without options compiles debug by default cfgCmdList.append('--disable-optimize') if binToBeCompiled == 'nspr': cfgCmdList.append( '--prefix=' + sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist'))) else: if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.getJsUsesNoThreadsFlag( ) and shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-nspr-build') else: if shell.buildOptions.enableNsprBuild: cfgCmdList.append('--enable-threadsafe') if not shell.getJsBuildSystemConsidersNspr(): cfgCmdList.append( '--with-nspr-prefix=' + sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist'))) cfgCmdList.append('--with-nspr-cflags=-I' + sps.normExpUserPath( os.path.join(shell.getNsprObjdir( ), 'dist', 'include', 'nspr'))) cfgCmdList.append('--with-nspr-libs=' + ' '.join([ sps.normExpUserPath( os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)) for compileLib in inspectShell.ALL_COMPILE_LIBS ])) else: cfgCmdList.append('--disable-threadsafe') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append( '--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getNsprObjdir( ) if binToBeCompiled == 'nspr' else shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # See bug 986715 comment 6 as to why we need forward slashes for NSPR # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)
def countCsets(revset, rdir): """Count the number of changesets in the revsets by outputting ones and counting them.""" listCmd = ['hg', 'log', '-r', revset, '--template=1'] rangeIntersectionOnes = sps.captureStdout(listCmd, currWorkingDir=rdir) assert rangeIntersectionOnes[1] == 0 return len(rangeIntersectionOnes[0])
def cfgBin(shell): """Configure a binary according to required parameters.""" cfgCmdList = [] cfgEnvDt = copy.deepcopy(os.environ) origCfgEnvDt = copy.deepcopy(os.environ) cfgEnvDt['AR'] = 'ar' if sps.isARMv7l: # 32-bit shell on ARM boards, e.g. odroid boards. # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' if not shell.buildOptions.enableHardFp: cfgEnvDt['CC'] = 'gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgEnvDt['CXX'] = 'g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # From mjrosenb: things might go wrong if these three lines are not present for # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. # cfgCmdList.append('--target=arm-linux-gnueabi') # cfgCmdList.append('--with-arch=armv7-a') # cfgCmdList.append('--with-thumb') if not shell.buildOptions.enableHardFp: cfgCmdList.append('--target=arm-linux-gnueabi') elif shell.buildOptions.enable32 and os.name == 'posix': # 32-bit shell on Mac OS X 10.10 Yosemite and greater if sps.isMac: assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. # Uses system clang cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgEnvDt['CC'] = cfgEnvDt['CC'] + CLANG_X86_FLAG # only needed for CC, not HOST_CC cfgEnvDt['CXX'] = cfgEnvDt['CXX'] + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX cfgEnvDt['RANLIB'] = 'ranlib' cfgEnvDt['AS'] = '$CC' cfgEnvDt['LD'] = 'ld' cfgEnvDt['STRIP'] = 'strip -x -S' cfgEnvDt['CROSS_COMPILE'] = '1' if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') # 32-bit shell on 32/64-bit x86 Linux elif sps.isLinux and not sps.isARMv7l: cfgEnvDt['PKG_CONFIG_LIBDIR'] = '/usr/lib/pkgconfig' if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = cfgEnvDt['HOST_CC'] = 'clang' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG cfgEnvDt['CXX'] = cfgEnvDt['HOST_CXX'] = 'clang++' + CLANG_PARAMS + SSE2_FLAGS + CLANG_X86_FLAG else: # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. cfgEnvDt['CC'] = 'gcc -m32' + SSE2_FLAGS cfgEnvDt['CXX'] = 'g++ -m32' + SSE2_FLAGS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=i686-pc-linux') if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) # 64-bit shell on Mac OS X 10.10 Yosemite and greater elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS if sps.isProgramInstalled('brew'): cfgEnvDt['AUTOCONF'] = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') elif sps.isWin: cfgEnvDt['MAKE'] = 'mozmake' # Workaround for bug 948534 if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang-cl.exe' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang-cl.exe' + CLANG_PARAMS # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgEnvDt['CC'] += CLANG_ASAN_PARAMS # cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.enable32: if shell.buildOptions.enableSimulatorArm32: # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated # Newer configure.in changes mean that things blow up if unknown/removed configure # options are entered, so specify it only if it's requested. if shell.buildOptions.enableArmSimulatorObsolete: cfgCmdList.append('--enable-arm-simulator') cfgCmdList.append('--enable-simulator=arm') else: cfgCmdList.append('--host=x86_64-pc-mingw32') cfgCmdList.append('--target=x86_64-pc-mingw32') if shell.buildOptions.enableSimulatorArm64: cfgCmdList.append('--enable-simulator=arm64') # Note: Doesn't seem to work yet. # if shell.buildOptions.buildWithAsan: # cfgCmdList.append('--enable-address-sanitizer') else: # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified if shell.buildOptions.buildWithClang: cfgEnvDt['CC'] = 'clang' + CLANG_PARAMS cfgEnvDt['CXX'] = 'clang++' + CLANG_PARAMS if shell.buildOptions.buildWithAsan: cfgEnvDt['CC'] += CLANG_ASAN_PARAMS cfgEnvDt['CXX'] += CLANG_ASAN_PARAMS cfgCmdList.append('sh') cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) if shell.buildOptions.buildWithAsan: cfgCmdList.append('--enable-address-sanitizer') if shell.buildOptions.buildWithClang: if sps.isWin: assert 'clang-cl' in cfgEnvDt['CC'] assert 'clang-cl' in cfgEnvDt['CXX'] else: assert 'clang' in cfgEnvDt['CC'] assert 'clang++' in cfgEnvDt['CXX'] cfgCmdList.append('--disable-jemalloc') # See bug 1146895 if shell.buildOptions.enableDbg: cfgCmdList.append('--enable-debug') elif shell.buildOptions.disableDbg: cfgCmdList.append('--disable-debug') if shell.buildOptions.enableOpt: cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) elif shell.buildOptions.disableOpt: cfgCmdList.append('--disable-optimize') if shell.buildOptions.enableProfiling: cfgCmdList.append('--enable-profiling') if shell.buildOptions.enableMoreDeterministic: # Fuzzing tweaks for more useful output, implemented in bug 706433 cfgCmdList.append('--enable-more-deterministic') if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions cfgCmdList.append('--enable-oom-breakpoint') if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default cfgCmdList.append('--without-intl-api') if shell.buildOptions.buildWithVg: cfgCmdList.append('--enable-valgrind') cfgCmdList.append('--disable-jemalloc') # We add the following flags by default. if os.name == 'posix': cfgCmdList.append('--with-ccache') cfgCmdList.append('--enable-gczeal') cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells cfgCmdList.append('--disable-tests') if os.name == 'nt': # FIXME: Replace this with sps.shellify. counter = 0 for entry in cfgCmdList: if os.sep in entry: assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') counter = counter + 1 # Print whatever we added to the environment envVarList = [] for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): strToBeAppended = envVar + '="' + cfgEnvDt[envVar] + '"' \ if ' ' in cfgEnvDt[envVar] else envVar + '=' + cfgEnvDt[envVar] envVarList.append(strToBeAppended) sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) wDir = shell.getJsObjdir() assert os.path.isdir(wDir) if sps.isWin: changedCfgCmdList = [] for entry in cfgCmdList: # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. # but really, the whole thing likes forward slashes way better" # See bug 1038590 comment 9. if '\\' in entry: entry = entry.replace('\\', '/') changedCfgCmdList.append(entry) sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) else: sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) shell.setEnvAdded(envVarList) shell.setEnvFull(cfgEnvDt) shell.setCfgCmdExclEnv(cfgCmdList)