def WriteRevisionFile(dirname, build_revision):
"""Writes a file containing revision number to given directory.
Replaces the target file in place.
Args:
dirname: Directory to write the file in.
build_revision: Revision number or hash.
Returns: The path of the written file.
"""
try:
# Script only works on python 2.6
# pylint: disable=E1123
tmp_revision_file = tempfile.NamedTemporaryFile(mode='w',
dir=dirname,
delete=False)
tmp_revision_file.write('%s' % build_revision)
tmp_revision_file.close()
chromium_utils.MakeWorldReadable(tmp_revision_file.name)
dest_path = os.path.join(dirname,
chromium_utils.FULL_BUILD_REVISION_FILENAME)
shutil.move(tmp_revision_file.name, dest_path)
return dest_path
except IOError:
print 'Writing to revision file in %s failed.' % dirname
def MakeVersionedArchive(zip_file, file_suffix, options):
"""Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz,
and copies (or hardlinks) the file to /foo/bar_baz.zip.
Returns: A tuple containing three elements: the base filename, the extension
and the full versioned filename."""
zip_template = os.path.basename(zip_file)
zip_base, zip_ext = os.path.splitext(zip_template)
# Create a versioned copy of the file.
versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext)
# Allow for overriding the name of the file based on the given upload url.
if options.use_build_url_name:
new_build_url, new_archive_name = options.build_url.rsplit('/', 1)
if new_archive_name and new_archive_name.endswith('.zip'):
options.build_url = new_build_url
versioned_file = zip_file.replace(zip_template, new_archive_name)
if os.path.exists(versioned_file):
# This file already exists. Maybe we are doing a clobber build at the same
# revision. We can move this file away.
old_file = versioned_file.replace(zip_ext, '_old' + zip_ext)
chromium_utils.MoveFile(versioned_file, old_file)
if chromium_utils.IsWindows():
shutil.copyfile(zip_file, versioned_file)
else:
os.link(zip_file, versioned_file)
chromium_utils.MakeWorldReadable(versioned_file)
print 'Created versioned archive', versioned_file
return (zip_base, zip_ext, versioned_file)
def MakeUnversionedArchive(build_dir,
staging_dir,
zip_file_list,
zip_file_name,
strip_files=None):
"""Creates an unversioned full build archive.
Returns the path of the created archive."""
# Prevents having zip_file_list to contain duplicates
zip_file_list = list(set(zip_file_list))
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True,
strip_files=strip_files)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big and
# can cause bot failures from timeouts during downloads to testers.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
return zip_file
def archive(options, args):
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir, use_out=chromium_utils.IsLinux())
build_dir = os.path.join(build_dir, options.target)
src_dir = os.path.abspath(os.path.dirname(options.build_dir))
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
subdir = None
# TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not
# ready with the server-side change.
if chromium_utils.IsMac():
subdir = '%s-%s' % (chromium_utils.PlatformName(),
options.target.lower())
prefix = options.factory_properties.get('cf_archive_name', 'cf_archive')
zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file,
gs_bucket,
subdir=subdir,
gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def CreateCoverageFileAndUpload(options):
"""Create coverage file with bbcov2lcov binary and upload to www dir."""
# Assert log files exist
log_files = glob.glob(os.path.join(options.dynamorio_log_dir, '*.log'))
if not log_files:
print 'No coverage log files found.'
return 1
if (options.browser_shard_index
and options.test_to_upload in options.sharded_tests):
coverage_info = os.path.join(
options.build_dir,
'coverage_%s.info' % options.browser_shard_index)
else:
coverage_info = os.path.join(options.build_dir, COVERAGE_INFO)
coverage_info = os.path.normpath(coverage_info)
if os.path.isfile(coverage_info):
os.remove(coverage_info)
bbcov2lcov_binary = GetExecutableName(
os.path.join(options.dynamorio_dir, 'tools', 'bin32', 'bbcov2lcov'))
cmd = [
bbcov2lcov_binary, '--dir', options.dynamorio_log_dir, '--output',
coverage_info
]
RunCmd(cmd)
# Delete log files.
log_files = glob.glob(os.path.join(options.dynamorio_log_dir, '*.log'))
for log_file in log_files:
os.remove(log_file)
# Assert coverage.info file exist
if not os.path.isfile(coverage_info):
print 'Failed to create coverage.info file.'
return 1
# Upload coverage file.
cov_dir = options.test_to_upload.replace('_', '') + COVERAGE_DIR_POSTFIX
dest = os.path.join(options.www_dir, options.platform, options.build_id,
cov_dir)
dest = os.path.normpath(dest)
if chromium_utils.IsWindows():
print('chromium_utils.CopyFileToDir(%s, %s)' % (coverage_info, dest))
chromium_utils.MaybeMakeDirectory(dest)
chromium_utils.CopyFileToDir(coverage_info, dest)
elif chromium_utils.IsLinux() or chromium_utils.IsMac():
print 'SshCopyFiles(%s, %s, %s)' % (coverage_info, options.host, dest)
chromium_utils.SshMakeDirectory(options.host, dest)
chromium_utils.MakeWorldReadable(coverage_info)
chromium_utils.SshCopyFiles(coverage_info, options.host, dest)
os.unlink(coverage_info)
else:
raise NotImplementedError('Platform "%s" is not currently supported.' %
sys.platform)
return 0
def archive(options, args):
src_dir = os.path.abspath(os.path.dirname(options.build_dir))
build_dir = os.path.join(src_dir, 'out', options.target)
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
prefix = options.factory_properties.get('asan_archive_name', 'asan')
zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def CopyFileToArchiveHost(src, dest_dir):
"""A wrapper method to copy files to the archive host.
It calls CopyFileToDir on Windows and SshCopyFiles on Linux/Mac.
TODO: we will eventually want to change the code to upload the
data to appengine.
Args:
src: full path to the src file.
dest_dir: destination directory on the host.
"""
host = config.Archive.archive_host
if not os.path.exists(src):
raise chromium_utils.ExternalError('Source path "%s" does not exist' % src)
chromium_utils.MakeWorldReadable(src)
if chromium_utils.IsWindows():
chromium_utils.CopyFileToDir(src, dest_dir)
elif chromium_utils.IsLinux() or chromium_utils.IsMac():
chromium_utils.SshCopyFiles(src, host, dest_dir)
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
def _MakeVersionedArchive(zip_file, file_suffix, options):
"""Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz,
and copies the file to /foo/bar_baz.zip."""
zip_template = os.path.basename(zip_file)
zip_base, zip_ext = os.path.splitext(zip_template)
# Create a versioned copy of the file.
versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext)
if os.path.exists(versioned_file):
# This file already exists. Maybe we are doing a clobber build at the same
# revision. We can move this file away.
old_file = versioned_file.replace(zip_ext, '_old' + zip_ext)
chromium_utils.MoveFile(versioned_file, old_file)
shutil.copyfile(zip_file, versioned_file)
chromium_utils.MakeWorldReadable(versioned_file)
# For chromium.perf, upload the versioned file to a GS bucket.
if (options.build_properties.get('mastername') == 'chromium.perf'
and options.build_properties.get('buildername') == 'Win Builder'):
print 'Uploading to Google Storage...'
slave_utils.GSUtilCopyFile(versioned_file, 'gs://chrome-perf/',
options.build_properties['buildername'])
print 'Created versioned archive', versioned_file
return (zip_base, zip_ext)
def MyMakeWorldReadable(self, destination, gs_base):
if not gs_base or self._dual_upload:
chromium_utils.MakeWorldReadable(destination)
def archive(options, args):
# Disable 'unused argument' warning for 'args' | pylint: disable=W0613
build_dir = build_directory.GetBuildOutputDirectory()
src_dir = os.path.abspath(os.path.dirname(build_dir))
build_dir = os.path.join(build_dir, options.target)
revision_dir = options.factory_properties.get('revision_dir')
primary_project = chromium_utils.GetPrimaryProject(options)
build_sortkey_branch, build_sortkey_value = GetBuildSortKey(
options, primary_project)
build_git_commit = GetGitCommit(options, primary_project)
staging_dir = slave_utils.GetStagingDir(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix',
'')
pieces = [chromium_utils.PlatformName(), options.target.lower()]
if subdir_suffix:
pieces.append(subdir_suffix)
subdir = '-'.join(pieces)
# Components like v8 get a <name>-v8-component-<revision> infix.
component = ''
if revision_dir:
component = '-%s-component' % revision_dir
prefix = options.factory_properties.get('cf_archive_name', 'cf_archive')
sortkey_path = chromium_utils.GetSortableUploadPathForSortKey(
build_sortkey_branch, build_sortkey_value)
zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), component,
sortkey_path)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
gs_metadata = {
GS_COMMIT_POSITION_NUMBER_KEY: build_sortkey_value,
}
if build_sortkey_branch:
gs_metadata[
GS_COMMIT_POSITION_KEY] = chromium_utils.BuildCommitPosition(
build_sortkey_branch, build_sortkey_value)
if build_git_commit:
gs_metadata[GS_GIT_COMMIT_KEY] = build_git_commit
status = slave_utils.GSUtilCopyFile(zip_file,
gs_bucket,
subdir=subdir,
gs_acl=gs_acl,
metadata=gs_metadata)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def MakeSourceWorldReadable(from_dir):
"""Makes the source tree world-readable."""
for (dirpath, dirnames, filenames) in os.walk(from_dir):
for node in dirnames + filenames:
chromium_utils.MakeWorldReadable(os.path.join(dirpath, node))
def archive(options, args):
build_dir = build_directory.GetBuildOutputDirectory()
src_dir = os.path.abspath(os.path.dirname(build_dir))
build_dir = os.path.join(build_dir, options.target)
revision_dir = options.factory_properties.get('revision_dir')
(build_revision, _) = slave_utils.GetBuildRevisions(
src_dir, None, revision_dir)
staging_dir = slave_utils.GetStagingDir(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)]
subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix',
'')
pieces = [chromium_utils.PlatformName(), options.target.lower()]
if subdir_suffix:
pieces.append(subdir_suffix)
subdir = '-'.join(pieces)
# Components like v8 get a <name>-v8-component-<revision> infix.
component = ''
if revision_dir:
component = '-%s-component' % revision_dir
prefix = options.factory_properties.get('cf_archive_name', 'cf_archive')
zip_file_name = '%s-%s-%s%s-%s' % (prefix,
chromium_utils.PlatformName(),
options.target.lower(),
component,
build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir,
gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file,
gs_bucket,
status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
