def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): """Tell hg what we learned about the revision.""" assert hgLabel in ("good", "bad", "skip") outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0] outputLines = outputResult.split("\n") if options.buildOptions: repoDir = options.buildOptions.repoDir if re.compile( "Due to skipped revisions, the first (good|bad) revision could be any of:" ).match(outputLines[0]): print() print(sanitizeCsetMsg(outputResult, repoDir)) print() return None, None, None, startRepo, endRepo r = re.compile("The first (good|bad) revision is:") m = r.match(outputLines[0]) if m: print() print() print( "autoBisect shows this is probably related to the following changeset:" ) print() print(sanitizeCsetMsg(outputResult, repoDir)) print() blamedGoodOrBad = m.group(1) blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1]) return blamedGoodOrBad, blamedRev, None, startRepo, endRepo if options.testInitialRevs: return None, None, None, startRepo, endRepo # e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)" sps.vdump(outputLines[0]) currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0]) if currRev is None: print("Resetting to default revision...") subprocess.check_call(hgPrefix + ['update', '-C', 'default']) hgCmds.destroyPyc(repoDir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. start = startRepo end = endRepo if hgLabel == 'bad': end = currRev elif hgLabel == 'good': start = currRev elif hgLabel == 'skip': pass return None, None, currRev, start, end
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): '''Tell hg what we learned about the revision.''' assert hgLabel in ("good", "bad", "skip") outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0] outputLines = outputResult.split("\n") repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]): print '\n' + sanitizeCsetMsg(outputResult, repoDir) + '\n' return None, None, None, startRepo, endRepo r = re.compile("The first (good|bad) revision is:") m = r.match(outputLines[0]) if m: print '\n\nautoBisect shows this is probably related to the following changeset:\n' print sanitizeCsetMsg(outputResult, repoDir) + '\n' blamedGoodOrBad = m.group(1) blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1]) return blamedGoodOrBad, blamedRev, None, startRepo, endRepo if options.testInitialRevs: return None, None, None, startRepo, endRepo # e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)" sps.vdump(outputLines[0]) currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0]) if currRev is None: print 'Resetting to default revision...' subprocess.check_call(hgPrefix + ['update', '-C', 'default']) hgCmds.destroyPyc(repoDir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. start = startRepo end = endRepo if hgLabel == 'bad': end = currRev elif hgLabel == 'good': start = currRev elif hgLabel == 'skip': pass return None, None, currRev, start, end
def obtainShell(shell, updateToRev=None): '''Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.''' assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print 'Found cached shell...' # Assuming that since the binary is present, everything else (e.g. symbols) is also present return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print 'Found a cache dir without a successful/failed shell...' sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted', shell.getShellCacheFullPath() + '.busted'): raise Exception('Found a .busted file for rev ' + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', shell.getS3TarballWithExtFullPath()): print 'Extracting shell...' with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False): """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print("Found cached shell...") # Assuming that since the binary is present, everything else (e.g. symbols) is also present verifyFullWinPageHeap(shell.getShellCacheFullPath()) return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print("Found a cache dir without a successful/failed shell...") sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted', shell.getShellCacheFullPath() + '.busted'): raise Exception('Found a .busted file for rev ' + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', shell.getS3TarballWithExtFullPath()): print("Extracting shell...") with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) verifyFullWinPageHeap(shell.getShellCacheFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) verifyFullWinPageHeap(shell.getShellCacheFullPath()) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. txtInfo = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt' s3CacheObj.uploadStrToS3('', txtInfo, shell.getS3TarballWithExt()) os.remove(shell.getS3TarballWithExtFullPath())
def findBlamedCset(options, repoDir, testRev): print "%s | Bisecting on: %s" % (sps.dateStr(), repoDir) hgPrefix = ['hg', '-R', repoDir] # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". realStartRepo = sRepo = hgCmds.getRepoHashAndId( repoDir, repoRev=options.startRepo)[0] realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: subprocess.check_call(hgPrefix + ['update', '-C', 'default']) # Throws exit code 255 if purge extension is not enabled in .hgrc: subprocess.check_call(hgPrefix + ['purge', '--all']) # Reset bisect ranges and set skip ranges. sps.captureStdout(hgPrefix + ['bisect', '-r']) if options.skipRevs: sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs]) labels = {} # Specify `hg bisect` ranges. if options.testInitialRevs: currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first. else: labels[sRepo] = ('good', 'assumed start rev is good') labels[eRepo] = ('bad', 'assumed end rev is bad') subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo]) currRev = hgCmds.getCsetHashFromBisectMsg( fileManipulation.firstLine( sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0])) iterNum = 1 if options.testInitialRevs: iterNum -= 2 skipCount = 0 blamedRev = None while currRev is not None: startTime = time.time() label = testRev(currRev) labels[currRev] = label if label[0] == 'skip': skipCount += 1 # If we use "skip", we tell hg bisect to do a linear search to get around the skipping. # If the range is large, doing a bisect to find the start and endpoints of compilation # bustage would be faster. 20 total skips being roughly the time that the pair of # bisections would take. if skipCount > 20: print 'Skipped 20 times, stopping autoBisect.' break print label[0] + " (" + label[1] + ") ", if iterNum <= 0: print 'Finished testing the initial boundary revisions...', else: print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \ str(2**iterNum), "...", (blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \ bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo) if options.testInitialRevs: options.testInitialRevs = False assert currRev is None currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next. iterNum += 1 endTime = time.time() oneRunTime = endTime - startTime print 'This iteration took %.3f seconds to run.' % oneRunTime if blamedRev is not None: checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") subprocess.check_call(hgPrefix + ['bisect', '-U', '-r']) sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True) hgCmds.destroyPyc(repoDir) print sps.dateStr()
def findBlamedCset(options, repoDir, testRev): print sps.dateStr() hgPrefix = ['hg', '-R', repoDir] # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. if options.resetRepoFirst: subprocess.check_call(hgPrefix + ['update', '-C', 'default']) # Throws exit code 255 if purge extension is not enabled in .hgrc: subprocess.check_call(hgPrefix + ['purge', '--all']) # Reset bisect ranges and set skip ranges. sps.captureStdout(hgPrefix + ['bisect', '-r']) if options.skipRevs: sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs]) labels = {} # Specify `hg bisect` ranges. if options.testInitialRevs: currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first. else: labels[sRepo] = ('good', 'assumed start rev is good') labels[eRepo] = ('bad', 'assumed end rev is bad') subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo]) currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine( sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0])) iterNum = 1 if options.testInitialRevs: iterNum -= 2 skipCount = 0 blamedRev = None while currRev is not None: startTime = time.time() label = testRev(currRev) labels[currRev] = label if label[0] == 'skip': skipCount += 1 # If we use "skip", we tell hg bisect to do a linear search to get around the skipping. # If the range is large, doing a bisect to find the start and endpoints of compilation # bustage would be faster. 20 total skips being roughly the time that the pair of # bisections would take. if skipCount > 20: print 'Skipped 20 times, stopping autoBisect.' break print label[0] + " (" + label[1] + ") ", if iterNum <= 0: print 'Finished testing the initial boundary revisions...', else: print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \ str(2**iterNum), "...", (blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \ bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo) if options.testInitialRevs: options.testInitialRevs = False assert currRev is None currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next. iterNum += 1 endTime = time.time() oneRunTime = endTime - startTime print 'This iteration took %.3f seconds to run.' % oneRunTime if blamedRev is not None: checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo, realEndRepo) sps.vdump("Resetting bisect") subprocess.check_call(hgPrefix + ['bisect', '-U', '-r']) sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True) hgCmds.destroyPyc(repoDir) print sps.dateStr()
def obtainShell(shell, updateToRev=None, updateLatestTxt=False): """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print "Found cached shell..." # Assuming that since the binary is present, everything else (e.g. symbols) is also present return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print "Found a cache dir without a successful/failed shell..." sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile( shell.getShellNameWithoutExt() + ".busted", shell.getShellCacheFullPath() + ".busted" ): raise Exception("Found a .busted file for rev " + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", shell.getS3TarballWithExtFullPath()): print "Extracting shell..." with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted") raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. txtInfo = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt" s3CacheObj.uploadStrToS3("", txtInfo, shell.getS3TarballWithExt()) os.remove(shell.getS3TarballWithExtFullPath())