def archive(options, args): build_dir, _ = chromium_utils.ConvertBuildDirToLegacy( options.build_dir, use_out=chromium_utils.IsLinux()) build_dir = os.path.join(build_dir, options.target) src_dir = os.path.abspath(os.path.dirname(options.build_dir)) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir = None # TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not # ready with the server-side change. if chromium_utils.IsMac(): subdir = '%s-%s' % (chromium_utils.PlatformName(), options.target.lower()) prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def archive_layout(options, args): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s:%(lineno)-3d' ' %(levelname)s %(message)s', datefmt='%y%m%d %H:%M:%S') chrome_dir = os.path.abspath(options.build_dir) results_dir_basename = os.path.basename(options.results_dir) if options.results_dir is not None: options.results_dir = os.path.abspath( os.path.join(options.build_dir, options.results_dir)) else: options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR) print 'Archiving results from %s' % options.results_dir staging_dir = slave_utils.GetStagingDir(chrome_dir) print 'Staging in %s' % staging_dir (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir) zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename, actual_file_list, options.results_dir)[1] full_results_json = os.path.join(options.results_dir, 'full_results.json') # Extract the build name of this slave (e.g., 'chrome-release') from its # configuration file if not provided as a param. build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir) build_name = re.sub('[ .()]', '_', build_name) last_change = str(slave_utils.SubversionRevision(chrome_dir)) print 'last change: %s' % last_change print 'build name: %s' % build_name print 'host name: %s' % socket.gethostname() # Where to save layout test results. dest_parent_dir = os.path.join(config.Archive.www_dir_base, results_dir_basename.replace('-', '_'), build_name) dest_dir = os.path.join(dest_parent_dir, last_change) gs_bucket = options.factory_properties.get('gs_bucket', None) if gs_bucket: gs_base = '/'.join([gs_bucket, build_name, last_change]) gs_acl = options.factory_properties.get('gs_acl', None) slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl) slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl) else: slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir) slave_utils.CopyFileToArchiveHost(zip_file, dest_dir) slave_utils.CopyFileToArchiveHost(full_results_json, dest_dir) # Not supported on Google Storage yet. _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list, options) return 0
def main(argv): o3d_dir = os.path.join(os.getcwd(), 'o3d') staging_dir = slave_utils.GetStagingDir(o3d_dir) # Find builder name and revision #s. builder_name = slave_utils.SlaveBuildName(o3d_dir) o3d_rev = str(slave_utils.SubversionRevision(o3d_dir)) platform = chromium_utils.PlatformName() # Upload zip. local_zip = os.path.join(staging_dir, 'full-build-' + platform + '_' + o3d_rev + '.zip') remote_zip = 'snapshots/o3d/' + o3d_rev + '/' + builder_name + '.zip' archive_file.UploadFile(local_zip, remote_zip) return 0
def Archive(options): src_dir = os.path.abspath(options.src_dir) build_dir = GetRealBuildDirectory(options.build_dir, options.target, options.factory_properties) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) webkit_dir, webkit_revision = None, None if options.webkit_dir: webkit_dir = os.path.join(src_dir, options.webkit_dir) webkit_revision = slave_utils.SubversionRevision(webkit_dir) print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Build the list of files to archive. root_files = os.listdir(build_dir) path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list) if webkit_revision: (zip_base, zip_ext) = MakeWebKitVersionedArchive(zip_file, build_revision, webkit_revision, options) else: (zip_base, zip_ext) = MakeVersionedArchive(zip_file, build_revision, options) PruneOldArchives(staging_dir, zip_base, zip_ext) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. WriteRevisionFile(staging_dir, build_revision) return 0
def archive(options, args): src_dir = os.path.abspath(os.path.dirname(options.build_dir)) build_dir = os.path.join(src_dir, 'out', options.target) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] prefix = options.factory_properties.get('asan_archive_name', 'asan') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def main(options, args): # Create some variables src_dir = os.path.abspath(options.src_dir) build_dir = os.path.dirname(options.build_dir) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) build_version = str(build_revision) if chromium_utils.IsMac() or chromium_utils.IsLinux(): # Files are created umask 077 by default, we need to make sure the staging # dir can be fetch from, do this by recursively chmoding back up to the root # before pushing to web server. a_path = staging_dir while a_path != '/': current_permissions = os.stat(a_path)[0] if current_permissions & 0555 == 0555: break print 'Fixing permissions (%o) for \'%s\'' % (current_permissions, a_path) os.chmod(a_path, current_permissions | 0555) a_path = os.path.dirname(a_path)
def Archive(options): build_dir = build_directory.GetBuildOutputDirectory( options.src_dir, options.cros_board) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = slave_utils.GetStagingDir(options.src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.master_name, options.build_number, options.parent_build_number, build_revision, webkit_revision, use_try_buildnumber=(not options.append_deps_patch_sha)) if options.append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) # Remove initial\chrome.ilk. The filtering is only done on toplevel files, # and we can't exclude everything in initial since initial\chrome.dll.pdb is # needed in the archive. (And we can't delete it on disk because that would # slow down the next incremental build). if 'initial' in root_files: # Expand 'initial' directory by its contents, so that initial\chrome.ilk # will be filtered out by the blacklist. index = root_files.index('initial') root_files[index:index + 1] = [ os.path.join('initial', f) for f in os.listdir(os.path.join(build_dir, 'initial')) ] path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Once we have swarming support ready, we could use it to # archive run time dependencies of tests and remove this step. mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py']) print 'Include mojom files: %s' % mojom_files zip_file_list.extend(mojom_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = 10 if options.build_url.startswith('gs://'): # Don't keep builds lying around when uploading them to google storage. prune_limit = 3 PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) urls = {} if options.build_url.startswith('gs://'): zip_url = UploadToGoogleStorage(versioned_file, revision_file, options.build_url, options.gs_acl) storage_url = ('https://storage.googleapis.com/%s/%s' % (options.build_url[len('gs://'):], os.path.basename(versioned_file))) urls['storage_url'] = storage_url else: staging_path = (os.path.splitdrive(versioned_file)[1].replace( os.path.sep, '/')) zip_url = 'http://' + options.slave_name + staging_path urls['zip_url'] = zip_url return urls
def __init__(self, options, build_revision): """Sets a number of file and directory paths for convenient use.""" self.options = options self._src_dir = os.path.abspath(options.src_dir) self._chrome_dir = os.path.join(self._src_dir, 'chrome') build_dir = build_directory.GetBuildOutputDirectory() self._build_dir = os.path.join(build_dir, options.target) if chromium_utils.IsWindows(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'win') elif chromium_utils.IsLinux(): # On Linux, we might have built for chromeos. Archive the same. if (options.factory_properties.get('chromeos', None) or slave_utils.GypFlagIsOn(options, 'chromeos')): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'chromeos') # Or, we might have built for Android. elif options.factory_properties.get('target_os') == 'android': self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'android') else: self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'linux') elif chromium_utils.IsMac(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'mac') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) self._staging_dir = (options.staging_dir or slave_utils.GetStagingDir(self._src_dir)) if not os.path.exists(self._staging_dir): os.makedirs(self._staging_dir) self._symbol_dir_base = options.dirs['symbol_dir_base'] self._www_dir_base = options.dirs['www_dir_base'] if options.build_name: self._build_name = options.build_name else: self._build_name = slave_utils.SlaveBuildName(self._src_dir) self._symbol_dir_base = os.path.join(self._symbol_dir_base, self._build_name) self._www_dir_base = os.path.join(self._www_dir_base, self._build_name) self._version_file = os.path.join(self._chrome_dir, 'VERSION') self._chromium_revision = chromium_utils.GetBuildSortKey(options)[1] self._v8_revision = chromium_utils.GetBuildSortKey(options, project='v8')[1] self._v8_revision_git = chromium_utils.GetGitCommit(options, project='v8') self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE') # The REVISIONS file will record the revisions information of the main # components Chromium/WebKit/V8. self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS') self._build_revision = build_revision self._build_path_component = str(self._build_revision) # Will be initialized in GetLastBuildRevision. self.last_chromium_revision = None self.last_v8_revision = None self._files_file = os.path.join(self._tool_dir, archive_utils.FILES_FILENAME) self._test_files = self.BuildOldFilesList(TEST_FILE_NAME) self._dual_upload = options.factory_properties.get( 'dual_upload', False) self._archive_files = None
def __init__(self, options, build_revision): """Sets a number of file and directory paths for convenient use.""" self.options = options self._src_dir = os.path.abspath(options.src_dir) self._chrome_dir = os.path.join(self._src_dir, 'chrome') # TODO: This scode should not be grabbing so deeply into WebKit. # Worse, this code ends up looking at top-of-tree WebKit # instead of the revision in DEPS. self._webkit_dir = os.path.join(self._src_dir, 'third_party', 'WebKit', 'Source') self._v8_dir = os.path.join(self._src_dir, 'v8') build_dir = build_directory.GetBuildOutputDirectory() self._build_dir = os.path.join(build_dir, options.target) if chromium_utils.IsWindows(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'win') elif chromium_utils.IsLinux(): # On Linux, we might have built for chromeos. Archive the same. if (options.factory_properties.get('chromeos', None) or slave_utils.GypFlagIsOn(options, 'chromeos')): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'chromeos') # Or, we might have built for Android. elif options.factory_properties.get('target_os') == 'android': self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'android') else: self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'linux') elif chromium_utils.IsMac(): self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'mac') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) self._staging_dir = slave_utils.GetStagingDir(self._src_dir) self._symbol_dir_base = options.dirs['symbol_dir_base'] self._www_dir_base = options.dirs['www_dir_base'] if options.build_name: self._build_name = options.build_name else: self._build_name = slave_utils.SlaveBuildName(self._src_dir) self._symbol_dir_base = os.path.join(self._symbol_dir_base, self._build_name) self._www_dir_base = os.path.join(self._www_dir_base, self._build_name) self._version_file = os.path.join(self._chrome_dir, 'VERSION') if options.default_chromium_revision: self._chromium_revision = options.default_chromium_revision else: self._chromium_revision = slave_utils.GetHashOrRevision( os.path.dirname( self._chrome_dir)) # src/ instead of src/chrome if options.default_webkit_revision: self._webkit_revision = options.default_webkit_revision else: self._webkit_revision = slave_utils.GetHashOrRevision( os.path.dirname( self._webkit_dir)) # WebKit/ instead of WebKit/Source if options.default_v8_revision: self._v8_revision = options.default_v8_revision else: self._v8_revision = slave_utils.GetHashOrRevision(self._v8_dir) self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE') # The REVISIONS file will record the revisions information of the main # components Chromium/WebKit/V8. self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS') self._build_revision = build_revision # Will be initialized in GetLastBuildRevision. self.last_chromium_revision = None self.last_webkit_revision = None self.last_v8_revision = None self._files_file = os.path.join(self._tool_dir, archive_utils.FILES_FILENAME) self._test_files = self.BuildOldFilesList(TEST_FILE_NAME) self._dual_upload = options.factory_properties.get( 'dual_upload', False) self._archive_files = None
def archive(options, args): # Disable 'unused argument' warning for 'args' | pylint: disable=W0613 build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') primary_project = chromium_utils.GetPrimaryProject(options) build_sortkey_branch, build_sortkey_value = GetBuildSortKey( options, primary_project) build_git_commit = GetGitCommit(options, primary_project) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') sortkey_path = chromium_utils.GetSortableUploadPathForSortKey( build_sortkey_branch, build_sortkey_value) zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, sortkey_path) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) gs_metadata = { GS_COMMIT_POSITION_NUMBER_KEY: build_sortkey_value, } if build_sortkey_branch: gs_metadata[ GS_COMMIT_POSITION_KEY] = chromium_utils.BuildCommitPosition( build_sortkey_branch, build_sortkey_value) if build_git_commit: gs_metadata[GS_GIT_COMMIT_KEY] = build_git_commit status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl, metadata=gs_metadata) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def archive_layout(options, args): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s:%(lineno)-3d' ' %(levelname)s %(message)s', datefmt='%y%m%d %H:%M:%S') chrome_dir = os.path.abspath(options.build_dir) results_dir_basename = os.path.basename(options.results_dir) if options.results_dir is not None: options.results_dir = os.path.abspath(os.path.join(options.build_dir, options.results_dir)) else: options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR) print 'Archiving results from %s' % options.results_dir staging_dir = options.staging_dir or slave_utils.GetStagingDir(chrome_dir) print 'Staging in %s' % staging_dir if not os.path.exists(staging_dir): os.makedirs(staging_dir) (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir) zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename, actual_file_list, options.results_dir)[1] # TODO(ojan): Stop separately uploading full_results.json once garden-o-matic # switches to using failing_results.json. full_results_json = os.path.join(options.results_dir, 'full_results.json') failing_results_json = os.path.join(options.results_dir, 'failing_results.json') # Extract the build name of this slave (e.g., 'chrome-release') from its # configuration file if not provided as a param. build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir) build_name = re.sub('[ .()]', '_', build_name) wc_dir = os.path.dirname(chrome_dir) last_change = slave_utils.GetHashOrRevision(wc_dir) # TODO(dpranke): Is it safe to assume build_number is not blank? Should we # assert() this ? build_number = str(options.build_number) print 'last change: %s' % last_change print 'build name: %s' % build_name print 'build number: %s' % build_number print 'host name: %s' % socket.gethostname() if options.gs_bucket: # Create a file containing last_change revision. This file will be uploaded # after all layout test results are uploaded so the client can check this # file to see if the upload for the revision is complete. # See crbug.com/574272 for more details. last_change_file = os.path.join(staging_dir, 'LAST_CHANGE') with open(last_change_file, 'w') as f: f.write(last_change) # Copy the results to a directory archived by build number. gs_base = '/'.join([options.gs_bucket, build_name, build_number]) gs_acl = options.gs_acl # These files never change, cache for a year. cache_control = "public, max-age=31556926" slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl, cache_control=cache_control) # TODO(dpranke): Remove these two lines once clients are fetching the # files from the layout-test-results dir. slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyFile(failing_results_json, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyFile(last_change_file, gs_base + '/' + results_dir_basename, gs_acl=gs_acl, cache_control=cache_control) # And also to the 'results' directory to provide the 'latest' results # and make sure they are not cached at all (Cloud Storage defaults to # caching w/ a max-age=3600). gs_base = '/'.join([options.gs_bucket, build_name, 'results']) cache_control = 'no-cache' slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyFile(last_change_file, gs_base + '/' + results_dir_basename, gs_acl=gs_acl, cache_control=cache_control) else: # Where to save layout test results. dest_parent_dir = os.path.join(archive_utils.Config.www_dir_base, results_dir_basename.replace('-', '_'), build_name) dest_dir = os.path.join(dest_parent_dir, last_change) _MaybeMakeDirectoryOnArchiveHost(dest_dir) _CopyFileToArchiveHost(zip_file, dest_dir) _CopyFileToArchiveHost(full_results_json, dest_dir) _CopyFileToArchiveHost(failing_results_json, dest_dir) # Not supported on Google Storage yet. _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list, options) return 0
def Archive(options): build_dir = build_directory.GetBuildOutputDirectory( options.src_dir, options.cros_board) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = (options.staging_dir or slave_utils.GetStagingDir(options.src_dir)) if not os.path.exists(staging_dir): os.makedirs(staging_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.master_name, options.build_number, options.parent_build_number, build_revision, webkit_revision, use_try_buildnumber=(not options.append_deps_patch_sha)) # TODO(robertocn): Remove this if no one other than bisect uses it. if options.append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) path_filter = PathMatcher(options) # Expand one level deep so that secondary toolchains can be filtered. for i in xrange(len(root_files) - 1, -1, -1): path = root_files[i] # Don't expand directories that will be filtered out. if not path_filter.Match(path): continue abs_path = os.path.join(build_dir, path) if os.path.isdir(abs_path): root_files[i:i + 1] = [ os.path.join(path, f) for f in os.listdir(abs_path) ] print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Switch layout tests to use files from 'gen/layout_test_data' # and remove this. mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py']) print 'Include mojom files: %s' % mojom_files zip_file_list.extend(mojom_files) layout_test_data_files = _LayoutTestFiles(build_dir) print 'Include layout test data: %s' % layout_test_data_files zip_file_list.extend(layout_test_data_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name, strip_files=options.strip_files) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = 10 if options.build_url.startswith('gs://'): # Don't keep builds lying around when uploading them to google storage. prune_limit = 3 PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) urls = {} if options.build_url.startswith('gs://'): zip_url = UploadToGoogleStorage(versioned_file, revision_file, options.build_url, options.gs_acl, options.gsutil_py_path) storage_url = ('https://storage.googleapis.com/%s/%s' % (options.build_url[len('gs://'):], os.path.basename(versioned_file))) urls['storage_url'] = storage_url else: staging_path = (os.path.splitdrive(versioned_file)[1].replace( os.path.sep, '/')) zip_url = 'http://' + options.slave_name + staging_path urls['zip_url'] = zip_url return urls
def __init__(self, options, build_revision): """Sets a number of file and directory paths for convenient use.""" self.options = options self._src_dir = os.path.abspath(options.src_dir) self._chrome_dir = os.path.join(self._src_dir, 'chrome') # TODO: This scode should not be grabbing so deeply into WebKit. # Worse, this code ends up looking at top-of-tree WebKit # instead of the revision in DEPS. self._webkit_dir = os.path.join(self._src_dir, 'third_party', 'WebKit', 'Source', 'WebCore') self._v8_dir = os.path.join(self._src_dir, 'v8') # TODO: need to get the build *output* directory passed in instead so Linux # and Mac don't have to walk up a directory to get to the right directory. if chromium_utils.IsWindows(): self._build_dir = os.path.join(options.build_dir, options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'win') elif chromium_utils.IsLinux(): self._build_dir = os.path.join(os.path.dirname(options.build_dir), 'out', options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'linux') elif chromium_utils.IsMac(): self._build_dir = os.path.join(os.path.dirname(options.build_dir), 'xcodebuild', options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'mac') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) self._staging_dir = slave_utils.GetStagingDir(self._src_dir) self._symbol_dir_base = options.dirs['symbol_dir_base'] self._www_dir_base = options.dirs['www_dir_base'] self._build_name = slave_utils.SlaveBuildName(self._src_dir) self._symbol_dir_base = os.path.join(self._symbol_dir_base, self._build_name) self._www_dir_base = os.path.join(self._www_dir_base, self._build_name) self._version_file = os.path.join(self._chrome_dir, 'VERSION') if options.default_chromium_revision: self._chromium_revision = options.default_chromium_revision else: self._chromium_revision = slave_utils.SubversionRevision( self._chrome_dir) if options.default_webkit_revision: self._webkit_revision = options.default_webkit_revision else: self._webkit_revision = slave_utils.SubversionRevision( self._webkit_dir) if options.default_v8_revision: self._v8_revision = options.default_v8_revision else: self._v8_revision = slave_utils.SubversionRevision(self._v8_dir) self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE') # The REVISIONS file will record the revisions information of the main # components Chromium/WebKit/V8. self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS') self._build_revision = build_revision # Will be initialized in GetLastBuildRevision. self.last_chromium_revision = None self.last_webkit_revision = None self.last_v8_revision = None self._files_file = os.path.join(self._tool_dir, archive_utils.FILES_FILENAME) self._test_files = self.BuildOldFilesList(TEST_FILE_NAME) self._dual_upload = options.factory_properties.get( 'dual_upload', False) self._archive_files = None
def archive(options, args): build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') (build_revision, _) = slave_utils.GetBuildRevisions( src_dir, None, revision_dir) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target)] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def Archive(options): build_dir = build_directory.GetBuildOutputDirectory(options.src_dir) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = slave_utils.GetStagingDir(options.src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision append_deps_patch_sha = options.factory_properties.get( 'append_deps_patch_sha') unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.build_properties, build_revision, webkit_revision, use_try_buildnumber=(not append_deps_patch_sha)) if append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) # Remove initial\chrome.ilk. The filtering is only done on toplevel files, # and we can't exclude everything in initial since initial\chrome.dll.pdb is # needed in the archive. (And we can't delete it on disk because that would # slow down the next incremental build). if 'initial' in root_files: # Expand 'initial' directory by its contents, so that initial\chrome.ilk # will be filtered out by the blacklist. index = root_files.index('initial') root_files[index:index + 1] = [ os.path.join('initial', f) for f in os.listdir(os.path.join(build_dir, 'initial')) ] path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Once we have swarming support ready, we could use it to # archive run time dependencies of tests and remove this step. mojom_js_files = MojomJSFiles(build_dir) print 'Include mojom JavaScript files: %s' % mojom_js_files zip_file_list.extend(mojom_js_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name, options.path_filter) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = max(0, int(options.factory_properties.get('prune_limit', 10))) PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) build_url = (options.build_url or options.factory_properties.get('build_url', '')) if build_url.startswith('gs://'): gs_acl = options.factory_properties.get('gs_acl') UploadToGoogleStorage(versioned_file, revision_file, build_url, gs_acl) return 0