def MakeVersionedArchive(zip_file, file_suffix, options):
"""Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz,
and copies (or hardlinks) the file to /foo/bar_baz.zip.
Returns: A tuple containing three elements: the base filename, the extension
and the full versioned filename."""
zip_template = os.path.basename(zip_file)
zip_base, zip_ext = os.path.splitext(zip_template)
# Create a versioned copy of the file.
versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext)
# Allow for overriding the name of the file based on the given upload url.
if options.use_build_url_name:
new_build_url, new_archive_name = options.build_url.rsplit('/', 1)
if new_archive_name and new_archive_name.endswith('.zip'):
options.build_url = new_build_url
versioned_file = zip_file.replace(zip_template, new_archive_name)
if os.path.exists(versioned_file):
# This file already exists. Maybe we are doing a clobber build at the same
# revision. We can move this file away.
old_file = versioned_file.replace(zip_ext, '_old' + zip_ext)
chromium_utils.MoveFile(versioned_file, old_file)
if chromium_utils.IsWindows():
shutil.copyfile(zip_file, versioned_file)
else:
os.link(zip_file, versioned_file)
chromium_utils.MakeWorldReadable(versioned_file)
print 'Created versioned archive', versioned_file
return (zip_base, zip_ext, versioned_file)
def CreateArchive(build_dir,
staging_dir,
files_list,
archive_name,
allow_missing=True):
"""Put files into an archive dir as well as a zip of said archive dir.
This method takes the list of files to archive, then prunes non-existing
files from that list.
archive_name is the desired name for the output zip file. It is also used as
the basis for the directory that the files are zipped into. For instance,
'foo.zip' creates the file foo.zip with the hierarchy foo/*. 'some_archive'
creates the file 'some_archive' with the hierarhy some_archive_unzipped/*
(the directory name is different to prevent name conflicts when extracting to
the directory containing 'some_archive').
If files_list is empty or has no existing CreateArchive returns ('', '').
Otherwise, this method returns the archive directory the files are
copied to and the full path of the zip file in a tuple.
"""
print 'Creating archive %s ...' % archive_name
if allow_missing:
# Filter out files that don't exist.
filtered_file_list = [
f.strip() for f in files_list
if os.path.exists(os.path.join(build_dir, f.strip()))
]
else:
filtered_file_list = list(files_list)
if not filtered_file_list:
# We have no files to archive, don't create an empty zip file.
print 'WARNING: No files to archive.'
return ('', '')
if archive_name.endswith('.zip'):
archive_dirname = archive_name[:-4]
else:
archive_dirname = archive_name + '_unzipped'
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
archive_dirname,
filtered_file_list,
build_dir,
raise_error=not allow_missing)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
if os.path.basename(zip_file) != archive_name:
orig_zip = zip_file
zip_file = os.path.join(os.path.dirname(orig_zip), archive_name)
print 'Renaming archive: "%s" -> "%s"' % (orig_zip, zip_file)
chromium_utils.MoveFile(orig_zip, zip_file)
return (zip_dir, zip_file)
def get_build_db(filename):
"""Open the build_db file.
filename: the filename of the build db.
"""
build_db = gen_db()
if os.path.isfile(filename):
print 'loading build_db from', filename
try:
with open(filename) as f:
build_db = load_from_json(f)
except BadConf as e:
new_fn = '%s.old' % filename
logging.warn('error loading %s: %s, moving to %s' % (
filename, e, new_fn))
chromium_utils.MoveFile(filename, new_fn)
return build_db
def _MakeVersionedArchive(zip_file, file_suffix, options):
"""Takes a file name, e.g. /foo/bar.zip and an extra suffix, e.g. _baz,
and copies the file to /foo/bar_baz.zip."""
zip_template = os.path.basename(zip_file)
zip_base, zip_ext = os.path.splitext(zip_template)
# Create a versioned copy of the file.
versioned_file = zip_file.replace(zip_ext, file_suffix + zip_ext)
if os.path.exists(versioned_file):
# This file already exists. Maybe we are doing a clobber build at the same
# revision. We can move this file away.
old_file = versioned_file.replace(zip_ext, '_old' + zip_ext)
chromium_utils.MoveFile(versioned_file, old_file)
shutil.copyfile(zip_file, versioned_file)
chromium_utils.MakeWorldReadable(versioned_file)
# For chromium.perf, upload the versioned file to a GS bucket.
if (options.build_properties.get('mastername') == 'chromium.perf'
and options.build_properties.get('buildername') == 'Win Builder'):
print 'Uploading to Google Storage...'
slave_utils.GSUtilCopyFile(versioned_file, 'gs://chrome-perf/',
options.build_properties['buildername'])
print 'Created versioned archive', versioned_file
return (zip_base, zip_ext)
raise StagingError('Failed to make zip package %s' % zip_file)
if chromium_utils.IsMac() or chromium_utils.IsLinux():
os.chmod(zip_file, 0644)
# Report the size of the zip file to help catch when it gets too big and
# can cause bot failures from timeouts during downloads to testers.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
# Create a versioned copy of the file.
versioned_file = zip_file.replace('.zip', '_%d.zip' % build_revision)
if os.path.exists(versioned_file):
# This file already exists. Maybe we are doing a clobber build at the same
# revision. We can move this file away.
old_file = versioned_file.replace('.zip', '_old.zip')
chromium_utils.MoveFile(versioned_file, old_file)
shutil.copyfile(zip_file, versioned_file)
if chromium_utils.IsMac() or chromium_utils.IsLinux():
os.chmod(versioned_file, 0644)
# Now before we finish, trim out old builds to make sure we don't
# fill the disk completely.
zip_template = os.path.basename(zip_file)
stage_dir = os.path.dirname(zip_file)
regexp = re.compile(zip_template.replace('.zip', '_([0-9]+)(_old)?.zip'))
zip_list = glob.glob(
os.path.join(stage_dir, zip_template.replace('.zip', '_*.zip')))
# Build an ordered list of build numbers we have zip files for.
build_list = []
for x in zip_list:
