def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num): ''' Writes a text file indicating that this particular build is incomplete. ''' if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \ os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))): sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build'))) assert not os.path.isfile(txtFile), 'incompleteBuild.txt should not be present.' with open(txtFile, 'wb') as f: f.write('This build with numeric ID ' + num + ' is incomplete.') assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \ ' has to be the one we downloaded from ' + url print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.' return False # False indicates that this text file has not yet been looked at.
def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num): """Write a text file indicating that this particular build is incomplete.""" if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \ os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))): sps.rmTreeIncludingReadOnly( sps.normExpUserPath(os.path.join(cacheFolder, 'build'))) assert not os.path.isfile( txtFile), 'incompleteBuild.txt should not be present.' with open(txtFile, 'wb') as f: f.write('This build with numeric ID ' + num + ' is incomplete.') assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \ ' has to be the one we downloaded from ' + url print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.' return False # False indicates that this text file has not yet been looked at.
def inner(shellFilename, hgHash): conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) tempPrefix = os.path.join(tempDir, 't') if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) if conditionScript.interesting(conditionArgs, tempPrefix): innerResult = ('bad', 'interesting') else: innerResult = ('good', 'not interesting') if os.path.isdir(tempDir): sps.rmTreeIncludingReadOnly(tempDir) return innerResult
def createTboxCacheFolder(cacheFolder): ''' Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check that its binaries are working properly. ''' try: os.mkdir(cacheFolder) except OSError: assertSaneJsBinary(cacheFolder) try: ensureCacheDirHasCorrectIdNum(cacheFolder) except (KeyboardInterrupt, Exception) as e: if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e): sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
def obtainShell(shell, updateToRev=None): '''Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.''' assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print 'Found cached shell...' # Assuming that since the binary is present, everything else (e.g. symbols) is also present return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print 'Found a cache dir without a successful/failed shell...' sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted', shell.getShellCacheFullPath() + '.busted'): raise Exception('Found a .busted file for rev ' + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', shell.getS3TarballWithExtFullPath()): print 'Extracting shell...' with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False): """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print("Found cached shell...") # Assuming that since the binary is present, everything else (e.g. symbols) is also present verifyFullWinPageHeap(shell.getShellCacheFullPath()) return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print("Found a cache dir without a successful/failed shell...") sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted', shell.getShellCacheFullPath() + '.busted'): raise Exception('Found a .busted file for rev ' + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', shell.getS3TarballWithExtFullPath()): print("Extracting shell...") with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) verifyFullWinPageHeap(shell.getShellCacheFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) verifyFullWinPageHeap(shell.getShellCacheFullPath()) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. txtInfo = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt' s3CacheObj.uploadStrToS3('', txtInfo, shell.getS3TarballWithExt()) os.remove(shell.getS3TarballWithExtFullPath())
def obtainShell(shell, updateToRev=None, updateLatestTxt=False): """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) cachedNoShell = shell.getShellCacheFullPath() + ".busted" if os.path.isfile(shell.getShellCacheFullPath()): # Don't remove the comma at the end of this line, and thus remove the newline printed. # We would break JSBugMon. print "Found cached shell..." # Assuming that since the binary is present, everything else (e.g. symbols) is also present return elif os.path.isfile(cachedNoShell): raise Exception("Found a cached shell that failed compilation...") elif os.path.isdir(shell.getShellCacheDir()): print "Found a cache dir without a successful/failed shell..." sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) hgCmds.destroyPyc(shell.buildOptions.repoDir) s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) useS3Cache = s3CacheObj.connect() if useS3Cache: if s3CacheObj.downloadFile( shell.getShellNameWithoutExt() + ".busted", shell.getShellCacheFullPath() + ".busted" ): raise Exception("Found a .busted file for rev " + shell.getHgHash()) if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + ".tar.bz2", shell.getS3TarballWithExtFullPath()): print "Extracting shell..." with tarfile.open(shell.getS3TarballWithExtFullPath(), "r") as z: z.extractall(shell.getShellCacheDir()) # Delete tarball after downloading from S3 os.remove(shell.getS3TarballWithExtFullPath()) return try: if updateToRev: updateRepo(shell.buildOptions.repoDir, updateToRev) if shell.buildOptions.patchFile: hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) cfgJsCompile(shell) except KeyboardInterrupt: sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) raise except Exception as e: # Remove the cache dir, but recreate it with only the .busted file. sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) os.mkdir(shell.getShellCacheDir()) createBustedFile(cachedNoShell, e) if useS3Cache: s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + ".busted") raise finally: if shell.buildOptions.patchFile: hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) if useS3Cache: s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) if updateLatestTxt: # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. txtInfo = "-".join(shell.getS3TarballWithExt().split("-")[:-1] + ["latest"]) + ".txt" s3CacheObj.uploadStrToS3("", txtInfo, shell.getS3TarballWithExt()) os.remove(shell.getS3TarballWithExtFullPath())