def FetchCoverageFiles(options):
"""Fetch coverage files from www dir to be processed."""
target_dir = os.path.join(options.build_dir, options.target)
for test in options.tests:
cov_dir = test.replace('_', '') + COVERAGE_DIR_POSTFIX
output_path = os.path.join(target_dir, cov_dir)
chromium_utils.MaybeMakeDirectory(output_path)
cov_src = (
'/%s/%s/%s/%s/' %
(options.upload_dir, options.platform, options.build_id, cov_dir))
if options.sharded_tests and test in options.sharded_tests:
# Copy all Lvoc files.
for shard_count in xrange(1, options.browser_total_shards + 1):
src = cov_src + 'coverage_%s.info' % shard_count
chromium_utils.CopyFileToDir(src, output_path)
else:
chromium_utils.CopyFileToDir(cov_src + COVERAGE_INFO, output_path)
def _MaybeMakeDirectoryOnArchiveHost(dest_dir):
"""A wrapper method to create a directory on the archive host.
It calls MaybeMakeDirectory on Windows and SshMakeDirectory on Linux/Mac.
Args:
dest_dir: destination directory on the host.
"""
host = archive_utils.Config.archive_host
if chromium_utils.IsWindows():
chromium_utils.MaybeMakeDirectory(dest_dir)
print 'saving results to %s' % dest_dir
elif chromium_utils.IsLinux() or chromium_utils.IsMac():
chromium_utils.SshMakeDirectory(host, dest_dir)
print 'saving results to "%s" on "%s"' % (dest_dir, host)
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
def ArchiveBuild(self):
"""Zips build files and uploads them, their symbols, and a change log."""
result = 0
if self._build_revision is None:
raise archive_utils.StagingError('No build revision was provided')
print 'Staging in %s' % self._staging_dir
fparser = archive_utils.FilesCfgParser(self._files_file,
self.options.mode,
self.options.arch)
files_list = fparser.ParseLegacyList()
self._archive_files = archive_utils.ExpandWildcards(
self._build_dir, files_list)
archives_list = fparser.ParseArchiveLists()
# Check files and revision numbers.
all_files_list = self._archive_files + [
item['filename'] for sublist in archives_list.values()
for item in sublist
]
all_files_list.append(self._version_file)
not_found = archive_utils.VerifyFiles(all_files_list, self._build_dir,
self.options.ignore)
not_found_optional = []
for bad_fn in not_found[:]:
if fparser.IsOptional(bad_fn):
not_found_optional.append(bad_fn)
not_found.remove(bad_fn)
# Remove it from all file lists so we don't try to process it.
if bad_fn in self._archive_files:
self._archive_files.remove(bad_fn)
for archive_list in archives_list.values():
archive_list[:] = [
x for x in archive_list if bad_fn != x['filename']
]
# TODO(mmoss): Now that we can declare files optional in FILES.cfg, should
# we only allow not_found_optional, and fail on any leftover not_found
# files?
print 'last change: %s' % self._build_revision
previous_revision = self.GetLastBuildRevision()
# TODO(agable): This conditional only works for svn because git can't easily
# compare revisions.
if (slave_utils.GitOrSubversion(self._src_dir) == 'svn'
and self._build_revision <= previous_revision):
# If there have been no changes, report it but don't raise an exception.
# Someone might have pushed the "force build" button.
print 'No changes since last build (r%s <= r%s)' % (
self._build_revision, previous_revision)
return 0
print 'build name: %s' % self._build_name
archive_name = 'chrome-%s.zip' % self.TargetPlatformName()
archive_file = self.CreateArchiveFile(archive_name,
self._archive_files)[1]
# Handle any custom archives.
# TODO(mmoss): Largely copied from stage_build.py. Maybe refactor more of
# this into archive_utils.py.
archive_files = [archive_file]
for archive_name in archives_list:
# The list might be empty if it was all 'not_found' optional files.
if not archives_list[archive_name]:
continue
if fparser.IsDirectArchive(archives_list[archive_name]):
fileobj = archives_list[archive_name][0]
# Copy the file to the path specified in archive_name, which might be
# different than the dirname or basename in 'filename' (allowed by
# 'direct_archive').
stage_subdir = os.path.dirname(archive_name)
stage_fn = os.path.basename(archive_name)
chromium_utils.MaybeMakeDirectory(
os.path.join(self._staging_dir, stage_subdir))
print 'chromium_utils.CopyFileToDir(%s, %s, dest_fn=%s)' % (
os.path.join(self._build_dir, fileobj['filename']),
os.path.join(self._staging_dir, stage_subdir), stage_fn)
if not self.options.dry_run:
chromium_utils.CopyFileToDir(
os.path.join(self._build_dir, fileobj['filename']),
os.path.join(self._staging_dir, stage_subdir),
dest_fn=stage_fn)
archive_files.append(
os.path.join(self._staging_dir, archive_name))
else:
custom_archive = self.CreateArchiveFile(
archive_name,
[f['filename'] for f in archives_list[archive_name]])[1]
print 'Adding %s to be archived.' % (custom_archive)
archive_files.append(custom_archive)
# Generate a revisions file which contains the Chromium/WebKit/V8's
# revision information.
self.GenerateRevisionFile()
www_dir = os.path.join(self._www_dir_base, self._build_path_component)
gs_bucket = self.options.factory_properties.get('gs_bucket', None)
gs_acl = self.options.factory_properties.get('gs_acl', None)
gs_base = None
if gs_bucket:
gs_base = '/'.join(
[gs_bucket, self._build_name, self._build_path_component])
self._UploadBuild(www_dir, self.revisions_path, archive_files, gs_base,
gs_acl)
# Archive Linux packages (if any -- only created for Chrome builds).
if chromium_utils.IsLinux():
linux_packages = (glob.glob(
os.path.join(self._build_dir,
'*-r%s_*.deb' % self._chromium_revision)))
linux_packages.extend(
glob.glob(
os.path.join(self._build_dir,
'*-%s.*.rpm' % self._chromium_revision)))
for package_file in linux_packages:
print 'SshCopyFiles(%s, %s, %s)' % (
package_file, self.options.archive_host, www_dir)
if not self.options.dry_run:
print 'SshMakeDirectory(%s, %s)' % (self.options.archive_host,
www_dir)
self.MySshMakeDirectory(self.options.archive_host, www_dir,
gs_base)
for package_file in linux_packages:
self.MyMakeWorldReadable(package_file, gs_base)
self.MySshCopyFiles(package_file,
self.options.archive_host,
www_dir,
gs_base,
gs_acl=gs_acl)
# Cleanup archived packages, otherwise they keep accumlating since
# they have different filenames with each build.
os.unlink(package_file)
self.UploadTests(www_dir, gs_base, gs_acl)
if not self.options.dry_run:
# Save the current build revision locally so we can compute a changelog
# next time
self.SaveBuildRevisionToSpecifiedFile(self.last_change_file)
# Record the latest revision in the developer archive directory.
latest_file_path = os.path.join(self._www_dir_base, 'LATEST')
if chromium_utils.IsWindows():
print 'Saving revision to %s' % latest_file_path
if gs_base:
self.CopyFileToGS(self.last_change_file,
gs_base,
'..',
mimetype='text/plain',
gs_acl=gs_acl)
if not gs_base or self._dual_upload:
self.SaveBuildRevisionToSpecifiedFile(latest_file_path)
elif chromium_utils.IsLinux() or chromium_utils.IsMac():
# Files are created umask 077 by default, so make it world-readable
# before pushing to web server.
self.MyMakeWorldReadable(self.last_change_file, gs_base)
print 'Saving revision to %s:%s' % (self.options.archive_host,
latest_file_path)
self.MySshCopyFiles(self.last_change_file,
self.options.archive_host,
latest_file_path,
gs_base,
'..',
mimetype='text/plain',
gs_acl=gs_acl)
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
if len(not_found_optional):
sys.stderr.write('\n\nINFO: Optional File(s) not found: %s\n' %
', '.join(not_found_optional))
if len(not_found):
sys.stderr.write('\n\nWARNING: File(s) not found: %s\n' %
', '.join(not_found))
return result
def MyMaybeMakeDirectory(self, destination, gs_base):
if not gs_base or self._dual_upload:
chromium_utils.MaybeMakeDirectory(destination)