def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
"""Tell hg what we learned about the revision."""
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix +
['bisect', '-U', '--' +
hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
if options.buildOptions:
repoDir = options.buildOptions.repoDir
if re.compile(
"Due to skipped revisions, the first (good|bad) revision could be any of:"
).match(outputLines[0]):
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print()
print()
print(
"autoBisect shows this is probably related to the following changeset:"
)
print()
print(sanitizeCsetMsg(outputResult, repoDir))
print()
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print("Resetting to default revision...")
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo):
'''Tell hg what we learned about the revision.'''
assert hgLabel in ("good", "bad", "skip")
outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0]
outputLines = outputResult.split("\n")
repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir
if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]):
print '\n' + sanitizeCsetMsg(outputResult, repoDir) + '\n'
return None, None, None, startRepo, endRepo
r = re.compile("The first (good|bad) revision is:")
m = r.match(outputLines[0])
if m:
print '\n\nautoBisect shows this is probably related to the following changeset:\n'
print sanitizeCsetMsg(outputResult, repoDir) + '\n'
blamedGoodOrBad = m.group(1)
blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1])
return blamedGoodOrBad, blamedRev, None, startRepo, endRepo
if options.testInitialRevs:
return None, None, None, startRepo, endRepo
# e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)"
sps.vdump(outputLines[0])
currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0])
if currRev is None:
print 'Resetting to default revision...'
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
hgCmds.destroyPyc(repoDir)
raise Exception("hg did not suggest a changeset to test!")
# Update the startRepo/endRepo values.
start = startRepo
end = endRepo
if hgLabel == 'bad':
end = currRev
elif hgLabel == 'good':
start = currRev
elif hgLabel == 'skip':
pass
return None, None, currRev, start, end
def obtainShell(shell, updateToRev=None):
'''Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.'''
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print 'Found cached shell...'
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print 'Found a cache dir without a successful/failed shell...'
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted',
shell.getShellCacheFullPath() + '.busted'):
raise Exception('Found a .busted file for rev ' + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2',
shell.getS3TarballWithExtFullPath()):
print 'Extracting shell...'
with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted')
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False):
"""Obtain a js shell. Keep the objdir for now, especially .a files, for symbols."""
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print("Found cached shell...")
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
verifyFullWinPageHeap(shell.getShellCacheFullPath())
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print("Found a cache dir without a successful/failed shell...")
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted',
shell.getShellCacheFullPath() + '.busted'):
raise Exception('Found a .busted file for rev ' + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2',
shell.getS3TarballWithExtFullPath()):
print("Extracting shell...")
with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
verifyFullWinPageHeap(shell.getShellCacheFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
verifyFullWinPageHeap(shell.getShellCacheFullPath())
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted')
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
if updateLatestTxt:
# So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with
# js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents.
txtInfo = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt'
s3CacheObj.uploadStrToS3('', txtInfo, shell.getS3TarballWithExt())
os.remove(shell.getS3TarballWithExtFullPath())
def findBlamedCset(options, repoDir, testRev):
print "%s | Bisecting on: %s" % (sps.dateStr(), repoDir)
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(
repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir,
repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(
fileManipulation.firstLine(
sps.captureStdout(hgPrefix +
['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev,
realStartRepo, realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'],
ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def findBlamedCset(options, repoDir, testRev):
print sps.dateStr()
hgPrefix = ['hg', '-R', repoDir]
# Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8".
realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0]
realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0]
sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo)
# Refresh source directory (overwrite all local changes) to default tip if required.
if options.resetRepoFirst:
subprocess.check_call(hgPrefix + ['update', '-C', 'default'])
# Throws exit code 255 if purge extension is not enabled in .hgrc:
subprocess.check_call(hgPrefix + ['purge', '--all'])
# Reset bisect ranges and set skip ranges.
sps.captureStdout(hgPrefix + ['bisect', '-r'])
if options.skipRevs:
sps.captureStdout(hgPrefix + ['bisect', '--skip', options.skipRevs])
labels = {}
# Specify `hg bisect` ranges.
if options.testInitialRevs:
currRev = eRepo # If testInitialRevs mode is set, compile and test the latest rev first.
else:
labels[sRepo] = ('good', 'assumed start rev is good')
labels[eRepo] = ('bad', 'assumed end rev is bad')
subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo])
currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine(
sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0]))
iterNum = 1
if options.testInitialRevs:
iterNum -= 2
skipCount = 0
blamedRev = None
while currRev is not None:
startTime = time.time()
label = testRev(currRev)
labels[currRev] = label
if label[0] == 'skip':
skipCount += 1
# If we use "skip", we tell hg bisect to do a linear search to get around the skipping.
# If the range is large, doing a bisect to find the start and endpoints of compilation
# bustage would be faster. 20 total skips being roughly the time that the pair of
# bisections would take.
if skipCount > 20:
print 'Skipped 20 times, stopping autoBisect.'
break
print label[0] + " (" + label[1] + ") ",
if iterNum <= 0:
print 'Finished testing the initial boundary revisions...',
else:
print "Bisecting for the n-th round where n is", iterNum, "and 2^n is", \
str(2**iterNum), "...",
(blamedGoodOrBad, blamedRev, currRev, sRepo, eRepo) = \
bisectLabel(hgPrefix, options, label[0], currRev, sRepo, eRepo)
if options.testInitialRevs:
options.testInitialRevs = False
assert currRev is None
currRev = sRepo # If options.testInitialRevs is set, test earliest possible rev next.
iterNum += 1
endTime = time.time()
oneRunTime = endTime - startTime
print 'This iteration took %.3f seconds to run.' % oneRunTime
if blamedRev is not None:
checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, realStartRepo,
realEndRepo)
sps.vdump("Resetting bisect")
subprocess.check_call(hgPrefix + ['bisect', '-U', '-r'])
sps.vdump("Resetting working directory")
sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True)
hgCmds.destroyPyc(repoDir)
print sps.dateStr()
def obtainShell(shell, updateToRev=None, updateLatestTxt=False):
"""Obtain a js shell. Keep the objdir for now, especially .a files, for symbols."""
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print "Found cached shell..."
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print "Found a cache dir without a successful/failed shell..."
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(
shell.getShellNameWithoutExt() + ".busted", shell.getShellCacheFullPath() + ".busted"
):
raise Exception("Found a .busted file for rev " + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", shell.getS3TarballWithExtFullPath()):
print "Extracting shell..."
with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted")
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
if updateLatestTxt:
# So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with
# js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents.
txtInfo = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt"
s3CacheObj.uploadStrToS3("", txtInfo, shell.getS3TarballWithExt())
os.remove(shell.getS3TarballWithExtFullPath())
