def Archive(options): build_dir = build_directory.GetBuildOutputDirectory( options.src_dir, options.cros_board) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = slave_utils.GetStagingDir(options.src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.master_name, options.build_number, options.parent_build_number, build_revision, webkit_revision, use_try_buildnumber=(not options.append_deps_patch_sha)) if options.append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) # Remove initial\chrome.ilk. The filtering is only done on toplevel files, # and we can't exclude everything in initial since initial\chrome.dll.pdb is # needed in the archive. (And we can't delete it on disk because that would # slow down the next incremental build). if 'initial' in root_files: # Expand 'initial' directory by its contents, so that initial\chrome.ilk # will be filtered out by the blacklist. index = root_files.index('initial') root_files[index:index + 1] = [ os.path.join('initial', f) for f in os.listdir(os.path.join(build_dir, 'initial')) ] path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Once we have swarming support ready, we could use it to # archive run time dependencies of tests and remove this step. mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py']) print 'Include mojom files: %s' % mojom_files zip_file_list.extend(mojom_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = 10 if options.build_url.startswith('gs://'): # Don't keep builds lying around when uploading them to google storage. prune_limit = 3 PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) urls = {} if options.build_url.startswith('gs://'): zip_url = UploadToGoogleStorage(versioned_file, revision_file, options.build_url, options.gs_acl) storage_url = ('https://storage.googleapis.com/%s/%s' % (options.build_url[len('gs://'):], os.path.basename(versioned_file))) urls['storage_url'] = storage_url else: staging_path = (os.path.splitdrive(versioned_file)[1].replace( os.path.sep, '/')) zip_url = 'http://' + options.slave_name + staging_path urls['zip_url'] = zip_url return urls
def testWebKitDir(self): (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( _BUILD_DIR, webkit_dir=_BUILD_DIR) self.assertTrue(build_revision > 0) self.assertTrue(webkit_revision > 0)
def testRevisionDir(self): (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( _BUILD_DIR, revision_dir=_BUILD_DIR) self.assertTrue(build_revision > 0) self.assertEquals(None, webkit_revision)
def real_main(options): """ Download a build, extract it to build\BuildDir\full-build-win32 and rename it to build\BuildDir\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision base_url, url = GetBuildUrl(options, build_revision, webkit_revision) archive_name = os.path.basename(base_url) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # Check if the url exists. if not handler.is_present(): print '%s is not found' % url failure = True # When 'halt_on_missing_build' is present in factory_properties and if # 'revision' is set in build properties, we assume the build is # triggered automatically and so we halt on a missing build zip. The # other case is if the build is forced, in which case we keep trying # later by looking for the latest build that's available. if (options.factory_properties.get('halt_on_missing_build', False) and 'revision' in options.build_properties and options.build_properties['revision'] != ''): return slave_utils.ERROR_EXIT_CODE # If the url is valid, we download the file. if not failure: if not handler.download(): failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://'): continue else: print 'Fetching latest build at %s' % base_url handler.url = base_url if not handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read( ) build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def Archive(options): build_dir = build_directory.GetBuildOutputDirectory( options.src_dir, options.cros_board) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = (options.staging_dir or slave_utils.GetStagingDir(options.src_dir)) if not os.path.exists(staging_dir): os.makedirs(staging_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.master_name, options.build_number, options.parent_build_number, build_revision, webkit_revision, use_try_buildnumber=(not options.append_deps_patch_sha)) # TODO(robertocn): Remove this if no one other than bisect uses it. if options.append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) path_filter = PathMatcher(options) # Expand one level deep so that secondary toolchains can be filtered. for i in xrange(len(root_files) - 1, -1, -1): path = root_files[i] # Don't expand directories that will be filtered out. if not path_filter.Match(path): continue abs_path = os.path.join(build_dir, path) if os.path.isdir(abs_path): root_files[i:i + 1] = [ os.path.join(path, f) for f in os.listdir(abs_path) ] print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Switch layout tests to use files from 'gen/layout_test_data' # and remove this. mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py']) print 'Include mojom files: %s' % mojom_files zip_file_list.extend(mojom_files) layout_test_data_files = _LayoutTestFiles(build_dir) print 'Include layout test data: %s' % layout_test_data_files zip_file_list.extend(layout_test_data_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name, strip_files=options.strip_files) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = 10 if options.build_url.startswith('gs://'): # Don't keep builds lying around when uploading them to google storage. prune_limit = 3 PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) urls = {} if options.build_url.startswith('gs://'): zip_url = UploadToGoogleStorage(versioned_file, revision_file, options.build_url, options.gs_acl, options.gsutil_py_path) storage_url = ('https://storage.googleapis.com/%s/%s' % (options.build_url[len('gs://'):], os.path.basename(versioned_file))) urls['storage_url'] = storage_url else: staging_path = (os.path.splitdrive(versioned_file)[1].replace( os.path.sep, '/')) zip_url = 'http://' + options.slave_name + staging_path urls['zip_url'] = zip_url return urls
def real_main(options): """ Download a build, extract it to build\\BuildDir\\full-build-win32 and rename it to build\\BuildDir\\Target """ abs_build_dir = os.path.abspath( build_directory.GetBuildOutputDirectory(options.src_dir)) target_build_output_dir = os.path.join(abs_build_dir, options.target) # Generic name for the archive. archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName() # Just take the zip off the name for the output directory name. output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', '')) src_dir = os.path.dirname(abs_build_dir) if not options.build_revision and not options.build_archive_url: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision url, archive_name = GetBuildUrl(options, build_revision, webkit_revision) if archive_name is None: archive_name = 'build.zip' base_url = None else: base_url = '/'.join(url.split('/')[:-1] + [archive_name]) if url.startswith('gs://'): handler = GSHandler(url=url, archive_name=archive_name) else: handler = WebHandler(url=url, archive_name=archive_name) # We try to download and extract 3 times. for tries in range(1, 4): print 'Try %d: Fetching build from %s...' % (tries, url) failure = False # If the url is valid, we download the file. if not failure: if not handler.download(): if options.halt_on_missing_build: return slave_utils.ERROR_EXIT_CODE failure = True # If the versioned url failed, we try to get the latest build. if failure: if url.startswith('gs://') or not base_url: continue else: print 'Fetching latest build at %s' % base_url base_handler = handler.__class__(base_url, handler.archive_name) if not base_handler.download(): continue print 'Extracting build %s to %s...' % (archive_name, abs_build_dir) try: chromium_utils.RemoveDirectory(target_build_output_dir) chromium_utils.ExtractZip(archive_name, abs_build_dir) # For Chrome builds, the build will be stored in chrome-win32. if 'full-build-win32' in output_dir: chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32') if os.path.exists(chrome_dir): output_dir = chrome_dir print 'Moving build from %s to %s' % (output_dir, target_build_output_dir) shutil.move(output_dir, target_build_output_dir) except (OSError, IOError, chromium_utils.ExternalError): print 'Failed to extract the build.' # Print out the traceback in a nice format traceback.print_exc() # Try again... continue # If we got the latest build, then figure out its revision number. if failure: print "Trying to determine the latest build's revision number..." try: build_revision_file_name = os.path.join( target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME) build_revision_file = open(build_revision_file_name, 'r') print 'Latest build is revision: %s' % build_revision_file.read() build_revision_file.close() except IOError: print "Could not determine the latest build's revision number" if failure: # We successfully extracted the archive, but it was the generic one. return slave_utils.WARNING_EXIT_CODE return 0 # If we get here, that means that it failed 3 times. We return a failure. return slave_utils.ERROR_EXIT_CODE
def archive(options, args): build_dir = build_directory.GetBuildOutputDirectory() src_dir = os.path.abspath(os.path.dirname(build_dir)) build_dir = os.path.join(build_dir, options.target) revision_dir = options.factory_properties.get('revision_dir') (build_revision, _) = slave_utils.GetBuildRevisions( src_dir, None, revision_dir) staging_dir = slave_utils.GetStagingDir(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target)] subdir_suffix = options.factory_properties.get('cf_archive_subdir_suffix', '') pieces = [chromium_utils.PlatformName(), options.target.lower()] if subdir_suffix: pieces.append(subdir_suffix) subdir = '-'.join(pieces) # Components like v8 get a <name>-v8-component-<revision> infix. component = '' if revision_dir: component = '-%s-component' % revision_dir prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s%s-%s' % (prefix, chromium_utils.PlatformName(), options.target.lower(), component, build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def Archive(options): build_dir = build_directory.GetBuildOutputDirectory(options.src_dir) build_dir = os.path.abspath(os.path.join(build_dir, options.target)) staging_dir = slave_utils.GetStagingDir(options.src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) if not options.build_revision: (build_revision, webkit_revision) = slave_utils.GetBuildRevisions( options.src_dir, options.webkit_dir, options.revision_dir) else: build_revision = options.build_revision webkit_revision = options.webkit_revision append_deps_patch_sha = options.factory_properties.get( 'append_deps_patch_sha') unversioned_base_name, version_suffix = slave_utils.GetZipFileNames( options.build_properties, build_revision, webkit_revision, use_try_buildnumber=(not append_deps_patch_sha)) if append_deps_patch_sha: deps_sha = os.path.join('src', 'DEPS.sha') if os.path.exists(deps_sha): sha = open(deps_sha).read() version_suffix = '%s_%s' % (version_suffix, sha.strip()) print 'Appending sha of the patch: %s' % sha else: print 'DEPS.sha file not found, not appending sha.' print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Copy the crt files if necessary. if options.target == 'Debug' and chromium_utils.IsWindows(): CopyDebugCRT(build_dir) # Build the list of files to archive. root_files = os.listdir(build_dir) # Remove initial\chrome.ilk. The filtering is only done on toplevel files, # and we can't exclude everything in initial since initial\chrome.dll.pdb is # needed in the archive. (And we can't delete it on disk because that would # slow down the next incremental build). if 'initial' in root_files: # Expand 'initial' directory by its contents, so that initial\chrome.ilk # will be filtered out by the blacklist. index = root_files.index('initial') root_files[index:index + 1] = [ os.path.join('initial', f) for f in os.listdir(os.path.join(build_dir, 'initial')) ] path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] # TODO(yzshen): Once we have swarming support ready, we could use it to # archive run time dependencies of tests and remove this step. mojom_js_files = MojomJSFiles(build_dir) print 'Include mojom JavaScript files: %s' % mojom_js_files zip_file_list.extend(mojom_js_files) zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list, unversioned_base_name, options.path_filter) zip_base, zip_ext, versioned_file = MakeVersionedArchive( zip_file, version_suffix, options) prune_limit = max(0, int(options.factory_properties.get('prune_limit', 10))) PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. revision_file = WriteRevisionFile(staging_dir, build_revision) build_url = (options.build_url or options.factory_properties.get('build_url', '')) if build_url.startswith('gs://'): gs_acl = options.factory_properties.get('gs_acl') UploadToGoogleStorage(versioned_file, revision_file, build_url, gs_acl) return 0