def __init__(self, options): """Constructor. Args: options: Command-line option object from optparse. """ self.options = options # Do platform-specific config if sys.platform in ('win32', 'cygwin'): self.is_posix = False elif sys.platform.startswith('darwin'): self.is_posix = True elif sys.platform.startswith('linux'): self.is_posix = True else: print 'Unknown/unsupported platform.' sys.exit(1) # Extract the build name of this slave (e.g., 'chrome-release') from its # configuration file. chrome_dir = os.path.abspath(options.build_dir) print 'chrome_dir: %s' % chrome_dir build_name = slave_utils.SlaveBuildName(chrome_dir) print 'build name: %s' % build_name # The 'last change:' line MUST appear for the buildbot output-parser to # construct the 'view coverage' link. (See # scripts/master/log_parser/archive_command.py) wc_dir = os.path.dirname(chrome_dir) self.last_change = str(slave_utils.SubversionRevision(wc_dir)) print 'last change: %s' % self.last_change host_name = socket.gethostname() print 'host name: %s' % host_name archive_config = config.Archive() if options.internal: archive_config.Internal() self.archive_host = archive_config.archive_host if self.is_posix: # Always ssh/scp to the archive host as chrome-bot. self.archive_host = 'chrome-bot@' + self.archive_host print 'archive host: %s' % self.archive_host if options.perf_subdir: self.perf_subdir = options.perf_subdir else: self.perf_subdir = build_name if options.build_number: self.perf_subdir = os.path.join(self.perf_subdir, options.build_number) print 'build number: %s' % options.build_number print 'perf subdir: %s' % self.perf_subdir self.archive_path = os.path.join(archive_config.www_dir_base, 'coverage', self.perf_subdir)
def archive(options, args): build_dir, _ = chromium_utils.ConvertBuildDirToLegacy( options.build_dir, use_out=chromium_utils.IsLinux()) build_dir = os.path.join(build_dir, options.target) src_dir = os.path.abspath(os.path.dirname(options.build_dir)) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] subdir = None # TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not # ready with the server-side change. if chromium_utils.IsMac(): subdir = '%s-%s' % (chromium_utils.PlatformName(), options.target.lower()) prefix = options.factory_properties.get('cf_archive_name', 'cf_archive') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, subdir=subdir, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def Archive(options): src_dir = os.path.abspath(options.src_dir) build_dir = GetRealBuildDirectory(options.build_dir, options.target, options.factory_properties) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) webkit_dir, webkit_revision = None, None if options.webkit_dir: webkit_dir = os.path.join(src_dir, options.webkit_dir) webkit_revision = slave_utils.SubversionRevision(webkit_dir) print 'Full Staging in %s' % staging_dir print 'Build Directory %s' % build_dir # Include the revision file in tarballs WriteRevisionFile(build_dir, build_revision) # Build the list of files to archive. root_files = os.listdir(build_dir) path_filter = PathMatcher(options) print path_filter print('\nActually excluded: %s' % [f for f in root_files if not path_filter.Match(f)]) zip_file_list = [f for f in root_files if path_filter.Match(f)] zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list) if webkit_revision: (zip_base, zip_ext) = MakeWebKitVersionedArchive(zip_file, build_revision, webkit_revision, options) else: (zip_base, zip_ext) = MakeVersionedArchive(zip_file, build_revision, options) PruneOldArchives(staging_dir, zip_base, zip_ext) # Update the latest revision file in the staging directory # to allow testers to figure out the latest packaged revision # without downloading tarballs. WriteRevisionFile(staging_dir, build_revision) return 0
def archive_layout(options, args): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s:%(lineno)-3d' ' %(levelname)s %(message)s', datefmt='%y%m%d %H:%M:%S') chrome_dir = os.path.abspath(options.build_dir) results_dir_basename = os.path.basename(options.results_dir) if options.results_dir is not None: options.results_dir = os.path.abspath( os.path.join(options.build_dir, options.results_dir)) else: options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR) print 'Archiving results from %s' % options.results_dir staging_dir = slave_utils.GetStagingDir(chrome_dir) print 'Staging in %s' % staging_dir (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir) zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename, actual_file_list, options.results_dir)[1] full_results_json = os.path.join(options.results_dir, 'full_results.json') # Extract the build name of this slave (e.g., 'chrome-release') from its # configuration file if not provided as a param. build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir) build_name = re.sub('[ .()]', '_', build_name) last_change = str(slave_utils.SubversionRevision(chrome_dir)) print 'last change: %s' % last_change print 'build name: %s' % build_name print 'host name: %s' % socket.gethostname() # Where to save layout test results. dest_parent_dir = os.path.join(config.Archive.www_dir_base, results_dir_basename.replace('-', '_'), build_name) dest_dir = os.path.join(dest_parent_dir, last_change) gs_bucket = options.factory_properties.get('gs_bucket', None) if gs_bucket: gs_base = '/'.join([gs_bucket, build_name, last_change]) gs_acl = options.factory_properties.get('gs_acl', None) slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl) slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl) else: slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir) slave_utils.CopyFileToArchiveHost(zip_file, dest_dir) slave_utils.CopyFileToArchiveHost(full_results_json, dest_dir) # Not supported on Google Storage yet. _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list, options) return 0
def GetBuildUrl(abs_build_dir, options): """Compute the url to download the build from. This will use as a base string, in order of preference: 1) options.build_url 2) options.factory_properties.build_url 3) build url constructed from build_properties Args: abs_build_dir: Full path to source directory. options: options object as specified by parser below. """ url = options.build_url or options.factory_properties.get('build_url') if not url: url = 'http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % ( options.build_properties.parent_slavename, options.build_properties.parent_builddir, chromium_utils.PlatformName()) if 'parentslavename' in url: parentslavename = options.build_properties.get('parentslavename', '') url = url % {'parentslavename': parentslavename} base_url = url versioned_url = url if options.webkit_dir: webkit_revision = slave_utils.SubversionRevision( os.path.join(abs_build_dir, '..', options.webkit_dir)) versioned_url = versioned_url.replace('.zip', '_wk%d.zip' % webkit_revision) # Find the revision that we need to download. chromium_revision = slave_utils.SubversionRevision(abs_build_dir) versioned_url = versioned_url.replace('.zip', '_%d.zip' % chromium_revision) return base_url, versioned_url
def main(argv): o3d_dir = os.path.join(os.getcwd(), 'o3d') staging_dir = slave_utils.GetStagingDir(o3d_dir) # Find builder name and revision #s. builder_name = slave_utils.SlaveBuildName(o3d_dir) o3d_rev = str(slave_utils.SubversionRevision(o3d_dir)) platform = chromium_utils.PlatformName() # Upload zip. local_zip = os.path.join(staging_dir, 'full-build-' + platform + '_' + o3d_rev + '.zip') remote_zip = 'snapshots/o3d/' + o3d_rev + '/' + builder_name + '.zip' archive_file.UploadFile(local_zip, remote_zip) return 0
def archive(options, args): src_dir = os.path.abspath(os.path.dirname(options.build_dir)) build_dir = os.path.join(src_dir, 'out', options.target) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir) print 'Staging in %s' % build_dir # Build the list of files to archive. zip_file_list = [ f for f in os.listdir(build_dir) if ShouldPackageFile(f, options.target) ] prefix = options.factory_properties.get('asan_archive_name', 'asan') zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(), options.target.lower(), build_revision) (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir, zip_file_name, zip_file_list, build_dir, raise_error=True) chromium_utils.RemoveDirectory(zip_dir) if not os.path.exists(zip_file): raise StagingError('Failed to make zip package %s' % zip_file) chromium_utils.MakeWorldReadable(zip_file) # Report the size of the zip file to help catch when it gets too big. zip_size = os.stat(zip_file)[stat.ST_SIZE] print 'Zip file is %ld bytes' % zip_size gs_bucket = options.factory_properties.get('gs_bucket', None) gs_acl = options.factory_properties.get('gs_acl', None) status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl) if status: raise StagingError('Failed to upload %s to %s. Error %d' % (zip_file, gs_bucket, status)) else: # Delete the file, it is not needed anymore. os.remove(zip_file) return status
def main(options, args): # Create some variables src_dir = os.path.abspath(options.src_dir) build_dir = os.path.dirname(options.build_dir) staging_dir = slave_utils.GetStagingDir(src_dir) build_revision = slave_utils.SubversionRevision(src_dir) build_version = str(build_revision) if chromium_utils.IsMac() or chromium_utils.IsLinux(): # Files are created umask 077 by default, we need to make sure the staging # dir can be fetch from, do this by recursively chmoding back up to the root # before pushing to web server. a_path = staging_dir while a_path != '/': current_permissions = os.stat(a_path)[0] if current_permissions & 0555 == 0555: break print 'Fixing permissions (%o) for \'%s\'' % (current_permissions, a_path) os.chmod(a_path, current_permissions | 0555) a_path = os.path.dirname(a_path)
def archive_layout(options, args): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(filename)s:%(lineno)-3d' ' %(levelname)s %(message)s', datefmt='%y%m%d %H:%M:%S') chrome_dir = os.path.abspath(options.build_dir) results_dir_basename = os.path.basename(options.results_dir) if options.results_dir is not None: options.results_dir = os.path.abspath( os.path.join(options.build_dir, options.results_dir)) else: options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR) print 'Archiving results from %s' % options.results_dir staging_dir = slave_utils.GetStagingDir(chrome_dir) print 'Staging in %s' % staging_dir (actual_file_list, diff_file_list) = _CollectArchiveFiles(options.results_dir) zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename, actual_file_list, options.results_dir)[1] # TODO(ojan): Stop separately uploading full_results.json once garden-o-matic # switches to using failing_results.json. full_results_json = os.path.join(options.results_dir, 'full_results.json') failing_results_json = os.path.join(options.results_dir, 'failing_results.json') # Extract the build name of this slave (e.g., 'chrome-release') from its # configuration file if not provided as a param. build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir) build_name = re.sub('[ .()]', '_', build_name) wc_dir = os.path.dirname(chrome_dir) last_change = str(slave_utils.SubversionRevision(wc_dir)) # TODO(dpranke): Is it safe to assume build_number is not blank? Should we # assert() this ? build_number = str(options.build_number) print 'last change: %s' % last_change print 'build name: %s' % build_name print 'build number: %s' % build_number print 'host name: %s' % socket.gethostname() # Where to save layout test results. dest_parent_dir = os.path.join(archive_utils.Config.www_dir_base, results_dir_basename.replace('-', '_'), build_name) dest_dir = os.path.join(dest_parent_dir, last_change) if options.gs_bucket: # Copy the results to a directory archived by build number. gs_base = '/'.join([options.gs_bucket, build_name, build_number]) gs_acl = options.gs_acl # These files never change, cache for a year. cache_control = "public, max-age=31556926" slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl, cache_control=cache_control) # TODO(dpranke): Remove these two lines once clients are fetching the # files from the layout-test-results dir. slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyFile(failing_results_json, gs_base, gs_acl=gs_acl, cache_control=cache_control) # And also to the 'results' directory to provide the 'latest' results. gs_base = '/'.join([options.gs_bucket, build_name, 'results']) slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_base, gs_acl=gs_acl, cache_control=cache_control) slave_utils.GSUtilCopyDir(options.results_dir, gs_base, gs_acl=gs_acl, cache_control=cache_control) else: _MaybeMakeDirectoryOnArchiveHost(dest_dir) _CopyFileToArchiveHost(zip_file, dest_dir) _CopyFileToArchiveHost(full_results_json, dest_dir) _CopyFileToArchiveHost(failing_results_json, dest_dir) # Not supported on Google Storage yet. _ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir, diff_file_list, options) return 0
def dom_perf(options, args): """Using the target build configuration, run the dom perf test.""" build_dir = os.path.abspath(options.build_dir) if chromium_utils.IsWindows(): test_exe_name = 'performance_ui_tests.exe' else: test_exe_name = 'performance_ui_tests' if chromium_utils.IsMac(): is_make_or_ninja = (options.factory_properties.get( "gclient_env", {}).get('GYP_GENERATORS', '') in ('ninja', 'make')) if is_make_or_ninja: build_dir = os.path.join(os.path.dirname(build_dir), 'out') else: build_dir = os.path.join(os.path.dirname(build_dir), 'xcodebuild') elif chromium_utils.IsLinux(): build_dir = os.path.join(os.path.dirname(build_dir), 'sconsbuild') test_exe_path = os.path.join(build_dir, options.target, test_exe_name) if not os.path.exists(test_exe_path): raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path) # Find the current revision to pass to the test. build_revision = slave_utils.SubversionRevision(build_dir) # Compute the path to the test data. src_dir = os.path.dirname(build_dir) data_dir = os.path.join(src_dir, 'data') dom_perf_dir = os.path.join(data_dir, 'dom_perf') iterations = '' # Default if options.target == 'Debug': iterations = '&minIterations=1' def run_and_print(use_refbuild): # Windows used to write to the root of C:, but that doesn't work # on Vista so we write into the build folder instead. suffix = '' if (use_refbuild): suffix = '_ref' output_file = os.path.join( build_dir, options.target, 'dom_perf_result_%s%s.txt' % (build_revision, suffix)) result = 0 compiled_data = [] for test in TESTS: url = URL % (dom_perf_dir, test, iterations, build_revision) url_flag = '--url=%s' % url command = [ test_exe_path, '--wait_cookie_name=__domperf_finished', '--jsvar=__domperf_result', '--jsvar_output=%s' % output_file, '--gtest_filter=UrlFetchTest.UrlFetch', url_flag ] if use_refbuild: command.append('--reference_build') print "Executing: " print command result |= chromium_utils.RunCommand(command) # Open the resulting file and display it. data = json.load(open(output_file, 'r')) for suite in data['BenchmarkSuites']: # Skip benchmarks that we didn't actually run this time around. if len(suite['Benchmarks']) == 0 and suite['score'] == 0: continue compiled_data.append(suite) # Now give the geometric mean as the total for the combined runs. total = geometric_mean([s['score'] for s in compiled_data]) print_result(True, 'Total', total, use_refbuild) for suite in compiled_data: print_result(False, suite['name'], suite['score'], use_refbuild) return result try: if chromium_utils.IsLinux(): xvfb.StartVirtualX(options.target, os.path.join(build_dir, options.target)) result = run_and_print(False) result |= run_and_print(True) finally: if chromium_utils.IsLinux(): xvfb.StopVirtualX(options.target) return result
def __init__(self, options, build_revision): """Sets a number of file and directory paths for convenient use.""" self.options = options self._src_dir = os.path.abspath(options.src_dir) self._chrome_dir = os.path.join(self._src_dir, 'chrome') # TODO: This scode should not be grabbing so deeply into WebKit. # Worse, this code ends up looking at top-of-tree WebKit # instead of the revision in DEPS. self._webkit_dir = os.path.join(self._src_dir, 'third_party', 'WebKit', 'Source', 'WebCore') self._v8_dir = os.path.join(self._src_dir, 'v8') # TODO: need to get the build *output* directory passed in instead so Linux # and Mac don't have to walk up a directory to get to the right directory. if chromium_utils.IsWindows(): self._build_dir = os.path.join(options.build_dir, options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'win') elif chromium_utils.IsLinux(): self._build_dir = os.path.join(os.path.dirname(options.build_dir), 'out', options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'linux') elif chromium_utils.IsMac(): self._build_dir = os.path.join(os.path.dirname(options.build_dir), 'xcodebuild', options.target) self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build', 'mac') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform) self._staging_dir = slave_utils.GetStagingDir(self._src_dir) self._symbol_dir_base = options.dirs['symbol_dir_base'] self._www_dir_base = options.dirs['www_dir_base'] self._build_name = slave_utils.SlaveBuildName(self._src_dir) self._symbol_dir_base = os.path.join(self._symbol_dir_base, self._build_name) self._www_dir_base = os.path.join(self._www_dir_base, self._build_name) self._version_file = os.path.join(self._chrome_dir, 'VERSION') if options.default_chromium_revision: self._chromium_revision = options.default_chromium_revision else: self._chromium_revision = slave_utils.SubversionRevision( self._chrome_dir) if options.default_webkit_revision: self._webkit_revision = options.default_webkit_revision else: self._webkit_revision = slave_utils.SubversionRevision( self._webkit_dir) if options.default_v8_revision: self._v8_revision = options.default_v8_revision else: self._v8_revision = slave_utils.SubversionRevision(self._v8_dir) self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE') # The REVISIONS file will record the revisions information of the main # components Chromium/WebKit/V8. self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS') self._build_revision = build_revision # Will be initialized in GetLastBuildRevision. self.last_chromium_revision = None self.last_webkit_revision = None self.last_v8_revision = None self._files_file = os.path.join(self._tool_dir, archive_utils.FILES_FILENAME) self._test_files = self.BuildOldFilesList(TEST_FILE_NAME) self._dual_upload = options.factory_properties.get( 'dual_upload', False) self._archive_files = None
def main(argv): if len(argv) != 3: print 'Usage: prepare_selenium_tests.py <o3d_src_root> <destination>' print 'Exiting...' return 1 # Make given directories absolute before changing the working directory. src_root = os.path.abspath(argv[1]) o3d_dir = os.path.join(src_root, 'o3d') o3d_internal_dir = os.path.join(src_root, 'o3d-internal') destination = os.path.abspath(argv[2]) config_dir = os.path.abspath(os.path.dirname(__file__)) config_file = os.path.join(config_dir, ARCHIVE_CONFIG_NAME) print 'O3D source root:', src_root print 'Destination:', destination print 'Config file:', config_file # Change umask on linux so that outputs (latest file and zip) are readable. if utils.IsLinux(): mask = os.umask(0022) # Build ChangeResolution project. BuildChangeResolution(src_root) # Create test archive. files = GetO3DArchiveFiles(src_root, config_file) zip_name = 'o3d' utils.MakeZip(destination, zip_name, files, src_root) zip_path = os.path.join(destination, zip_name + '.zip') print 'Zip archive created: %s' % zip_path # Find builder name and revision #s. builder_name = slave_utils.SlaveBuildName(o3d_dir) o3d_rev = str(slave_utils.SubversionRevision(o3d_dir)) o3d_internal_rev = str(slave_utils.SubversionRevision(o3d_internal_dir)) package_name = 'test_' + builder_name + '.zip' package_dir = o3d_rev + '_' + o3d_internal_rev package_path = package_dir + '/' + package_name print 'Builder name:', builder_name print 'O3D revision:', o3d_rev print 'O3D-internal revision:', o3d_internal_rev print 'Package path:', package_path # Create latest file. latest_path = os.path.join(destination, 'latest_' + builder_name) file(latest_path, 'w').write(package_path) # Upload files. package_dst = ('snapshots/o3d/test_packages/o3d/' + package_dir + '/' + package_name) latest_dst = 'snapshots/o3d/test_packages/o3d/latest_' + builder_name UploadFile(zip_path, package_dst) UploadFile(latest_path, latest_dst) # Reset the umask on linux. if utils.IsLinux(): os.umask(mask) return 0