def FetchCoverageFiles(options): """Fetch coverage files from www dir to be processed.""" target_dir = os.path.join(options.build_dir, options.target) for test in options.tests: cov_dir = test.replace('_', '') + COVERAGE_DIR_POSTFIX output_path = os.path.join(target_dir, cov_dir) chromium_utils.MaybeMakeDirectory(output_path) cov_src = ( '/%s/%s/%s/%s/' % (options.upload_dir, options.platform, options.build_id, cov_dir)) if options.sharded_tests and test in options.sharded_tests: # Copy all Lvoc files. for shard_count in xrange(1, options.browser_total_shards + 1): src = cov_src + 'coverage_%s.info' % shard_count chromium_utils.CopyFileToDir(src, output_path) else: chromium_utils.CopyFileToDir(cov_src + COVERAGE_INFO, output_path)
def _MaybeMakeDirectoryOnArchiveHost(dest_dir): """A wrapper method to create a directory on the archive host. It calls MaybeMakeDirectory on Windows and SshMakeDirectory on Linux/Mac. Args: dest_dir: destination directory on the host. """ host = archive_utils.Config.archive_host if chromium_utils.IsWindows(): chromium_utils.MaybeMakeDirectory(dest_dir) print 'saving results to %s' % dest_dir elif chromium_utils.IsLinux() or chromium_utils.IsMac(): chromium_utils.SshMakeDirectory(host, dest_dir) print 'saving results to "%s" on "%s"' % (dest_dir, host) else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform)
def ArchiveBuild(self): """Zips build files and uploads them, their symbols, and a change log.""" result = 0 if self._build_revision is None: raise archive_utils.StagingError('No build revision was provided') print 'Staging in %s' % self._staging_dir fparser = archive_utils.FilesCfgParser(self._files_file, self.options.mode, self.options.arch) files_list = fparser.ParseLegacyList() self._archive_files = archive_utils.ExpandWildcards( self._build_dir, files_list) archives_list = fparser.ParseArchiveLists() # Check files and revision numbers. all_files_list = self._archive_files + [ item['filename'] for sublist in archives_list.values() for item in sublist ] all_files_list.append(self._version_file) not_found = archive_utils.VerifyFiles(all_files_list, self._build_dir, self.options.ignore) not_found_optional = [] for bad_fn in not_found[:]: if fparser.IsOptional(bad_fn): not_found_optional.append(bad_fn) not_found.remove(bad_fn) # Remove it from all file lists so we don't try to process it. if bad_fn in self._archive_files: self._archive_files.remove(bad_fn) for archive_list in archives_list.values(): archive_list[:] = [ x for x in archive_list if bad_fn != x['filename'] ] # TODO(mmoss): Now that we can declare files optional in FILES.cfg, should # we only allow not_found_optional, and fail on any leftover not_found # files? print 'last change: %s' % self._build_revision previous_revision = self.GetLastBuildRevision() # TODO(agable): This conditional only works for svn because git can't easily # compare revisions. if (slave_utils.GitOrSubversion(self._src_dir) == 'svn' and self._build_revision <= previous_revision): # If there have been no changes, report it but don't raise an exception. # Someone might have pushed the "force build" button. print 'No changes since last build (r%s <= r%s)' % ( self._build_revision, previous_revision) return 0 print 'build name: %s' % self._build_name archive_name = 'chrome-%s.zip' % self.TargetPlatformName() archive_file = self.CreateArchiveFile(archive_name, self._archive_files)[1] # Handle any custom archives. # TODO(mmoss): Largely copied from stage_build.py. Maybe refactor more of # this into archive_utils.py. archive_files = [archive_file] for archive_name in archives_list: # The list might be empty if it was all 'not_found' optional files. if not archives_list[archive_name]: continue if fparser.IsDirectArchive(archives_list[archive_name]): fileobj = archives_list[archive_name][0] # Copy the file to the path specified in archive_name, which might be # different than the dirname or basename in 'filename' (allowed by # 'direct_archive'). stage_subdir = os.path.dirname(archive_name) stage_fn = os.path.basename(archive_name) chromium_utils.MaybeMakeDirectory( os.path.join(self._staging_dir, stage_subdir)) print 'chromium_utils.CopyFileToDir(%s, %s, dest_fn=%s)' % ( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), stage_fn) if not self.options.dry_run: chromium_utils.CopyFileToDir( os.path.join(self._build_dir, fileobj['filename']), os.path.join(self._staging_dir, stage_subdir), dest_fn=stage_fn) archive_files.append( os.path.join(self._staging_dir, archive_name)) else: custom_archive = self.CreateArchiveFile( archive_name, [f['filename'] for f in archives_list[archive_name]])[1] print 'Adding %s to be archived.' % (custom_archive) archive_files.append(custom_archive) # Generate a revisions file which contains the Chromium/WebKit/V8's # revision information. self.GenerateRevisionFile() www_dir = os.path.join(self._www_dir_base, self._build_path_component) gs_bucket = self.options.factory_properties.get('gs_bucket', None) gs_acl = self.options.factory_properties.get('gs_acl', None) gs_base = None if gs_bucket: gs_base = '/'.join( [gs_bucket, self._build_name, self._build_path_component]) self._UploadBuild(www_dir, self.revisions_path, archive_files, gs_base, gs_acl) # Archive Linux packages (if any -- only created for Chrome builds). if chromium_utils.IsLinux(): linux_packages = (glob.glob( os.path.join(self._build_dir, '*-r%s_*.deb' % self._chromium_revision))) linux_packages.extend( glob.glob( os.path.join(self._build_dir, '*-%s.*.rpm' % self._chromium_revision))) for package_file in linux_packages: print 'SshCopyFiles(%s, %s, %s)' % ( package_file, self.options.archive_host, www_dir) if not self.options.dry_run: print 'SshMakeDirectory(%s, %s)' % (self.options.archive_host, www_dir) self.MySshMakeDirectory(self.options.archive_host, www_dir, gs_base) for package_file in linux_packages: self.MyMakeWorldReadable(package_file, gs_base) self.MySshCopyFiles(package_file, self.options.archive_host, www_dir, gs_base, gs_acl=gs_acl) # Cleanup archived packages, otherwise they keep accumlating since # they have different filenames with each build. os.unlink(package_file) self.UploadTests(www_dir, gs_base, gs_acl) if not self.options.dry_run: # Save the current build revision locally so we can compute a changelog # next time self.SaveBuildRevisionToSpecifiedFile(self.last_change_file) # Record the latest revision in the developer archive directory. latest_file_path = os.path.join(self._www_dir_base, 'LATEST') if chromium_utils.IsWindows(): print 'Saving revision to %s' % latest_file_path if gs_base: self.CopyFileToGS(self.last_change_file, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) if not gs_base or self._dual_upload: self.SaveBuildRevisionToSpecifiedFile(latest_file_path) elif chromium_utils.IsLinux() or chromium_utils.IsMac(): # Files are created umask 077 by default, so make it world-readable # before pushing to web server. self.MyMakeWorldReadable(self.last_change_file, gs_base) print 'Saving revision to %s:%s' % (self.options.archive_host, latest_file_path) self.MySshCopyFiles(self.last_change_file, self.options.archive_host, latest_file_path, gs_base, '..', mimetype='text/plain', gs_acl=gs_acl) else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) if len(not_found_optional): sys.stderr.write('\n\nINFO: Optional File(s) not found: %s\n' % ', '.join(not_found_optional)) if len(not_found): sys.stderr.write('\n\nWARNING: File(s) not found: %s\n' % ', '.join(not_found)) return result
def MyMaybeMakeDirectory(self, destination, gs_base): if not gs_base or self._dual_upload: chromium_utils.MaybeMakeDirectory(destination)