def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num):
'''
Writes a text file indicating that this particular build is incomplete.
'''
if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \
os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
assert not os.path.isfile(txtFile), 'incompleteBuild.txt should not be present.'
with open(txtFile, 'wb') as f:
f.write('This build with numeric ID ' + num + ' is incomplete.')
assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \
' has to be the one we downloaded from ' + url
print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.'
return False # False indicates that this text file has not yet been looked at.
def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num):
"""Write a text file indicating that this particular build is incomplete."""
if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \
os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))):
sps.rmTreeIncludingReadOnly(
sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
assert not os.path.isfile(
txtFile), 'incompleteBuild.txt should not be present.'
with open(txtFile, 'wb') as f:
f.write('This build with numeric ID ' + num + ' is incomplete.')
assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \
' has to be the one we downloaded from ' + url
print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.'
return False # False indicates that this text file has not yet been looked at.
def inner(shellFilename, hgHash):
conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList
tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash)
tempPrefix = os.path.join(tempDir, 't')
if hasattr(conditionScript, "init"):
# Since we're changing the js shell name, call init() again!
conditionScript.init(conditionArgs)
if conditionScript.interesting(conditionArgs, tempPrefix):
innerResult = ('bad', 'interesting')
else:
innerResult = ('good', 'not interesting')
if os.path.isdir(tempDir):
sps.rmTreeIncludingReadOnly(tempDir)
return innerResult
def inner(shellFilename, hgHash):
conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList
tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash)
tempPrefix = os.path.join(tempDir, 't')
if hasattr(conditionScript, "init"):
# Since we're changing the js shell name, call init() again!
conditionScript.init(conditionArgs)
if conditionScript.interesting(conditionArgs, tempPrefix):
innerResult = ('bad', 'interesting')
else:
innerResult = ('good', 'not interesting')
if os.path.isdir(tempDir):
sps.rmTreeIncludingReadOnly(tempDir)
return innerResult
def createTboxCacheFolder(cacheFolder):
'''
Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check
that its binaries are working properly.
'''
try:
os.mkdir(cacheFolder)
except OSError:
assertSaneJsBinary(cacheFolder)
try:
ensureCacheDirHasCorrectIdNum(cacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
def createTboxCacheFolder(cacheFolder):
'''
Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check
that its binaries are working properly.
'''
try:
os.mkdir(cacheFolder)
except OSError:
assertSaneJsBinary(cacheFolder)
try:
ensureCacheDirHasCorrectIdNum(cacheFolder)
except (KeyboardInterrupt, Exception) as e:
if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e):
sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
def obtainShell(shell, updateToRev=None):
'''Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.'''
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print 'Found cached shell...'
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print 'Found a cache dir without a successful/failed shell...'
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted',
shell.getShellCacheFullPath() + '.busted'):
raise Exception('Found a .busted file for rev ' + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2',
shell.getS3TarballWithExtFullPath()):
print 'Extracting shell...'
with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted')
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False):
"""Obtain a js shell. Keep the objdir for now, especially .a files, for symbols."""
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print("Found cached shell...")
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
verifyFullWinPageHeap(shell.getShellCacheFullPath())
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print("Found a cache dir without a successful/failed shell...")
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted',
shell.getShellCacheFullPath() + '.busted'):
raise Exception('Found a .busted file for rev ' + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2',
shell.getS3TarballWithExtFullPath()):
print("Extracting shell...")
with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
verifyFullWinPageHeap(shell.getShellCacheFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
verifyFullWinPageHeap(shell.getShellCacheFullPath())
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted')
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
if updateLatestTxt:
# So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with
# js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents.
txtInfo = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt'
s3CacheObj.uploadStrToS3('', txtInfo, shell.getS3TarballWithExt())
os.remove(shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False):
"""Obtain a js shell. Keep the objdir for now, especially .a files, for symbols."""
assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir))
cachedNoShell = shell.getShellCacheFullPath() + ".busted"
if os.path.isfile(shell.getShellCacheFullPath()):
# Don't remove the comma at the end of this line, and thus remove the newline printed.
# We would break JSBugMon.
print "Found cached shell..."
# Assuming that since the binary is present, everything else (e.g. symbols) is also present
return
elif os.path.isfile(cachedNoShell):
raise Exception("Found a cached shell that failed compilation...")
elif os.path.isdir(shell.getShellCacheDir()):
print "Found a cache dir without a successful/failed shell..."
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
hgCmds.destroyPyc(shell.buildOptions.repoDir)
s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME)
useS3Cache = s3CacheObj.connect()
if useS3Cache:
if s3CacheObj.downloadFile(
shell.getShellNameWithoutExt() + ".busted", shell.getShellCacheFullPath() + ".busted"
):
raise Exception("Found a .busted file for rev " + shell.getHgHash())
if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", shell.getS3TarballWithExtFullPath()):
print "Extracting shell..."
with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as z:
z.extractall(shell.getShellCacheDir())
# Delete tarball after downloading from S3
os.remove(shell.getS3TarballWithExtFullPath())
return
try:
if updateToRev:
updateRepo(shell.buildOptions.repoDir, updateToRev)
if shell.buildOptions.patchFile:
hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir())
cfgJsCompile(shell)
except KeyboardInterrupt:
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
raise
except Exception as e:
# Remove the cache dir, but recreate it with only the .busted file.
sps.rmTreeIncludingReadOnly(shell.getShellCacheDir())
os.mkdir(shell.getShellCacheDir())
createBustedFile(cachedNoShell, e)
if useS3Cache:
s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted")
raise
finally:
if shell.buildOptions.patchFile:
hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir())
if useS3Cache:
s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
if updateLatestTxt:
# So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with
# js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents.
txtInfo = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt"
s3CacheObj.uploadStrToS3("", txtInfo, shell.getS3TarballWithExt())
os.remove(shell.getS3TarballWithExtFullPath())