def WriteRevisionFile(dirname, build_revision): """Writes a file containing revision number to given directory. Replaces the target file in place. Args: dirname: Directory to write the file in. build_revision: Revision number or hash. Returns: The path of the written file. """ try: # Script only works on python 2.6 # pylint: disable=E1123 tmp_revision_file = tempfile.NamedTemporaryFile(mode='w', dir=dirname, delete=False) tmp_revision_file.write('%s' % build_revision) tmp_revision_file.close() chromium_utils.MakeWorldReadable(tmp_revision_file.name) dest_path = os.path.join(dirname, chromium_utils.FULL_BUILD_REVISION_FILENAME) shutil.move(tmp_revision_file.name, dest_path) return dest_path except IOError: print 'Writing to revision file in %s failed.' % dirname
def MakeVersionedArchive(zip_file, file_suffix, options): """Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz, and copies (or hardlinks) the file to /foo/bar_baz.zip. Returns: A tuple containing three elements: the base filename, the extension and the full versioned filename.""" zip_template = os.path.basename(zip_file) zip_base, zip_ext = os.path.splitext(zip_template) # Create a versioned copy of the file. versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext) # Allow for overriding the name of the file based on the given upload url. if options.use_build_url_name: new_build_url, new_archive_name = options.build_url.rsplit('/', 1) if new_archive_name and new_archive_name.endswith('.zip'): options.build_url = new_build_url versioned_file = zip_file.replace(zip_template, new_archive_name) if os.path.exists(versioned_file): # This file already exists. Maybe we are doing a clobber build at the same # revision. We can move this file away. old_file = versioned_file.replace(zip_ext, '_old' + zip_ext) chromium_utils.MoveFile(versioned_file, old_file) if chromium_utils.IsWindows(): shutil.copyfile(zip_file, versioned_file) else: os.link(zip_file, versioned_file) chromium_utils.MakeWorldReadable(versioned_file) print 'Created versioned archive', versioned_file return (zip_base, zip_ext, versioned_file)
def MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, zip_file_name, strip_files=None): """Creates an unversioned full build archive. Returns the path of the created archive.""" # Prevents having zip_file_list to contain duplicates zip_file_list = list(set(zip_file_list)) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True, strip_files=strip_files) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big and # can cause bot failures from timeouts during downloads to testers. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size return zip_file
def archive(options, args): build_dir, _ = chromium_utils.ConvertBuildDirToLegacy( options.build_dir, use_out=chromium_utils.IsLinux()) build_dir = os.path.join(build_dir, options.target) src_dir = os.path.abspath(os.path.dirname(options.build_dir)) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir = None # TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not # ready with the server-side change. if chromium_utils.IsMac(): subdir = '%s-%s' % (chromium_utils.PlatformName(), options.target.lower()) prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def CreateCoverageFileAndUpload(options): """Create coverage file with bbcov2lcov binary and upload to www dir.""" # Assert log files exist log_files = glob.glob(os.path.join(options.dynamorio_log_dir, '*.log')) if not log_files: print 'No coverage log files found.' return 1 if (options.browser_shard_index and options.test_to_upload in options.sharded_tests): coverage_info = os.path.join( options.build_dir, 'coverage_%s.info' % options.browser_shard_index) else: coverage_info = os.path.join(options.build_dir, COVERAGE_INFO) coverage_info = os.path.normpath(coverage_info) if os.path.isfile(coverage_info): os.remove(coverage_info) bbcov2lcov_binary = GetExecutableName( os.path.join(options.dynamorio_dir, 'tools', 'bin32', 'bbcov2lcov')) cmd = [ bbcov2lcov_binary, '--dir', options.dynamorio_log_dir, '--output', coverage_info ] RunCmd(cmd) # Delete log files. log_files = glob.glob(os.path.join(options.dynamorio_log_dir, '*.log')) for log_file in log_files: os.remove(log_file) # Assert coverage.info file exist if not os.path.isfile(coverage_info): print 'Failed to create coverage.info file.' return 1 # Upload coverage file. cov_dir = options.test_to_upload.replace('_', '') + COVERAGE_DIR_POSTFIX dest = os.path.join(options.www_dir, options.platform, options.build_id, cov_dir) dest = os.path.normpath(dest) if chromium_utils.IsWindows(): print('chromium_utils.CopyFileToDir(%s, %s)' % (coverage_info, dest)) chromium_utils.MaybeMakeDirectory(dest) chromium_utils.CopyFileToDir(coverage_info, dest) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): print 'SshCopyFiles(%s, %s, %s)' % (coverage_info, options.host, dest) chromium_utils.SshMakeDirectory(options.host, dest) chromium_utils.MakeWorldReadable(coverage_info) chromium_utils.SshCopyFiles(coverage_info, options.host, dest) os.unlink(coverage_info) else: raise NotImplementedError('Platform "%s" is not currently supported.' % sys.platform) return 0
def archive(options, args): src_dir = os.path.abspath(os.path.dirname(options.build_dir)) build_dir = os.path.join(src_dir, 'out', options.target) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] prefix = options.factory_properties.get('asan_archive_name', 'asan') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def CopyFileToArchiveHost(src, dest_dir): """A wrapper method to copy files to the archive host. It calls CopyFileToDir on Windows and SshCopyFiles on Linux/Mac. TODO: we will eventually want to change the code to upload the data to appengine. Args: src: full path to the src file. dest_dir: destination directory on the host. """ host = config.Archive.archive_host if not os.path.exists(src): raise chromium_utils.ExternalError('Source path "%s" does not exist' % src) chromium_utils.MakeWorldReadable(src) if chromium_utils.IsWindows(): chromium_utils.CopyFileToDir(src, dest_dir) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): chromium_utils.SshCopyFiles(src, host, dest_dir) else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform)
def _MakeVersionedArchive(zip_file, file_suffix, options): """Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz, and copies the file to /foo/bar_baz.zip.""" zip_template = os.path.basename(zip_file) zip_base, zip_ext = os.path.splitext(zip_template) # Create a versioned copy of the file. versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext) if os.path.exists(versioned_file): # This file already exists. Maybe we are doing a clobber build at the same # revision. We can move this file away. old_file = versioned_file.replace(zip_ext, '_old' + zip_ext) chromium_utils.MoveFile(versioned_file, old_file) shutil.copyfile(zip_file, versioned_file) chromium_utils.MakeWorldReadable(versioned_file) # For chromium.perf, upload the versioned file to a GS bucket. if (options.build_properties.get('mastername') == 'chromium.perf' and options.build_properties.get('buildername') == 'Win Builder'): print 'Uploading to Google Storage...' slave_utils.GSUtilCopyFile(versioned_file, 'gs://chrome-perf/', options.build_properties['buildername']) print 'Created versioned archive', versioned_file return (zip_base, zip_ext)
def MyMakeWorldReadable(self, destination, gs_base): if not gs_base or self._dual_upload: chromium_utils.MakeWorldReadable(destination)
def archive(options, args): # Disable 'unused argument' warning for 'args' | pylint: disable=W0613 build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') primary_project = chromium_utils.GetPrimaryProject(options) build_sortkey_branch, build_sortkey_value = GetBuildSortKey( options, primary_project) build_git_commit = GetGitCommit(options, primary_project) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') sortkey_path = chromium_utils.GetSortableUploadPathForSortKey( build_sortkey_branch, build_sortkey_value) zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, sortkey_path) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) gs_metadata = { GS_COMMIT_POSITION_NUMBER_KEY: build_sortkey_value, } if build_sortkey_branch: gs_metadata[ GS_COMMIT_POSITION_KEY] = chromium_utils.BuildCommitPosition( build_sortkey_branch, build_sortkey_value) if build_git_commit: gs_metadata[GS_GIT_COMMIT_KEY] = build_git_commit status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl, metadata=gs_metadata) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def MakeSourceWorldReadable(from_dir): """Makes the source tree world-readable.""" for (dirpath, dirnames, filenames) in os.walk(from_dir): for node in dirnames + filenames: chromium_utils.MakeWorldReadable(os.path.join(dirpath, node))
def archive(options, args): build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') (build_revision, _) = slave_utils.GetBuildRevisions( src_dir, None, revision_dir) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target)] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status