def __init__(self, options):
"""Constructor.
Args:
options: Command-line option object from optparse.
"""
self.options = options
# Do platform-specific config
if sys.platform in ('win32', 'cygwin'):
self.is_posix = False
elif sys.platform.startswith('darwin'):
self.is_posix = True
elif sys.platform.startswith('linux'):
self.is_posix = True
else:
print 'Unknown/unsupported platform.'
sys.exit(1)
# Extract the build name of this slave (e.g., 'chrome-release') from its
# configuration file.
chrome_dir = os.path.abspath(options.build_dir)
print 'chrome_dir: %s' % chrome_dir
build_name = slave_utils.SlaveBuildName(chrome_dir)
print 'build name: %s' % build_name
# The 'last change:' line MUST appear for the buildbot output-parser to
# construct the 'view coverage' link. (See
# scripts/master/log_parser/archive_command.py)
wc_dir = os.path.dirname(chrome_dir)
self.last_change = str(slave_utils.SubversionRevision(wc_dir))
print 'last change: %s' % self.last_change
host_name = socket.gethostname()
print 'host name: %s' % host_name
archive_config = config.Archive()
if options.internal:
archive_config.Internal()
self.archive_host = archive_config.archive_host
if self.is_posix:
# Always ssh/scp to the archive host as chrome-bot.
self.archive_host = 'chrome-bot@' + self.archive_host
print 'archive host: %s' % self.archive_host
if options.perf_subdir:
self.perf_subdir = options.perf_subdir
else:
self.perf_subdir = build_name
if options.build_number:
self.perf_subdir = os.path.join(self.perf_subdir,
options.build_number)
print 'build number: %s' % options.build_number
print 'perf subdir: %s' % self.perf_subdir
self.archive_path = os.path.join(archive_config.www_dir_base,
'coverage', self.perf_subdir)
def archive(options, args):
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir, use_out=chromium_utils.IsLinux())
build_dir = os.path.join(build_dir, options.target)
src_dir = os.path.abspath(os.path.dirname(options.build_dir))
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
subdir = None
# TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not
# ready with the server-side change.
if chromium_utils.IsMac():
subdir = '%s-%s' % (chromium_utils.PlatformName(),
options.target.lower())
prefix = options.factory_properties.get('cf_archive_name', 'cf_archive')
zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file,
gs_bucket,
subdir=subdir,
gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def Archive(options):
src_dir = os.path.abspath(options.src_dir)
build_dir = GetRealBuildDirectory(options.build_dir, options.target,
options.factory_properties)
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
webkit_dir, webkit_revision = None, None
if options.webkit_dir:
webkit_dir = os.path.join(src_dir, options.webkit_dir)
webkit_revision = slave_utils.SubversionRevision(webkit_dir)
print 'Full Staging in %s' % staging_dir
print 'Build Directory %s' % build_dir
# Include the revision file in tarballs
WriteRevisionFile(build_dir, build_revision)
# Build the list of files to archive.
root_files = os.listdir(build_dir)
path_filter = PathMatcher(options)
print path_filter
print('\nActually excluded: %s' %
[f for f in root_files if not path_filter.Match(f)])
zip_file_list = [f for f in root_files if path_filter.Match(f)]
zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list)
if webkit_revision:
(zip_base,
zip_ext) = MakeWebKitVersionedArchive(zip_file, build_revision,
webkit_revision, options)
else:
(zip_base, zip_ext) = MakeVersionedArchive(zip_file, build_revision,
options)
PruneOldArchives(staging_dir, zip_base, zip_ext)
# Update the latest revision file in the staging directory
# to allow testers to figure out the latest packaged revision
# without downloading tarballs.
WriteRevisionFile(staging_dir, build_revision)
return 0
def archive_layout(options, args):
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s:%(lineno)-3d'
' %(levelname)s %(message)s',
datefmt='%y%m%d %H:%M:%S')
chrome_dir = os.path.abspath(options.build_dir)
results_dir_basename = os.path.basename(options.results_dir)
if options.results_dir is not None:
options.results_dir = os.path.abspath(
os.path.join(options.build_dir, options.results_dir))
else:
options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
print 'Archiving results from %s' % options.results_dir
staging_dir = slave_utils.GetStagingDir(chrome_dir)
print 'Staging in %s' % staging_dir
(actual_file_list,
diff_file_list) = _CollectArchiveFiles(options.results_dir)
zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename,
actual_file_list, options.results_dir)[1]
full_results_json = os.path.join(options.results_dir, 'full_results.json')
# Extract the build name of this slave (e.g., 'chrome-release') from its
# configuration file if not provided as a param.
build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir)
build_name = re.sub('[ .()]', '_', build_name)
last_change = str(slave_utils.SubversionRevision(chrome_dir))
print 'last change: %s' % last_change
print 'build name: %s' % build_name
print 'host name: %s' % socket.gethostname()
# Where to save layout test results.
dest_parent_dir = os.path.join(config.Archive.www_dir_base,
results_dir_basename.replace('-', '_'),
build_name)
dest_dir = os.path.join(dest_parent_dir, last_change)
gs_bucket = options.factory_properties.get('gs_bucket', None)
if gs_bucket:
gs_base = '/'.join([gs_bucket, build_name, last_change])
gs_acl = options.factory_properties.get('gs_acl', None)
slave_utils.GSUtilCopyFile(zip_file, gs_base, gs_acl=gs_acl)
slave_utils.GSUtilCopyFile(full_results_json, gs_base, gs_acl=gs_acl)
else:
slave_utils.MaybeMakeDirectoryOnArchiveHost(dest_dir)
slave_utils.CopyFileToArchiveHost(zip_file, dest_dir)
slave_utils.CopyFileToArchiveHost(full_results_json, dest_dir)
# Not supported on Google Storage yet.
_ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir,
diff_file_list, options)
return 0
def GetBuildUrl(abs_build_dir, options):
"""Compute the url to download the build from. This will use as a base
string, in order of preference:
1) options.build_url
2) options.factory_properties.build_url
3) build url constructed from build_properties
Args:
abs_build_dir: Full path to source directory.
options: options object as specified by parser below.
"""
url = options.build_url or options.factory_properties.get('build_url')
if not url:
url = 'http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % (
options.build_properties.parent_slavename,
options.build_properties.parent_builddir,
chromium_utils.PlatformName())
if 'parentslavename' in url:
parentslavename = options.build_properties.get('parentslavename', '')
url = url % {'parentslavename': parentslavename}
base_url = url
versioned_url = url
if options.webkit_dir:
webkit_revision = slave_utils.SubversionRevision(
os.path.join(abs_build_dir, '..', options.webkit_dir))
versioned_url = versioned_url.replace('.zip',
'_wk%d.zip' % webkit_revision)
# Find the revision that we need to download.
chromium_revision = slave_utils.SubversionRevision(abs_build_dir)
versioned_url = versioned_url.replace('.zip',
'_%d.zip' % chromium_revision)
return base_url, versioned_url
def main(argv):
o3d_dir = os.path.join(os.getcwd(), 'o3d')
staging_dir = slave_utils.GetStagingDir(o3d_dir)
# Find builder name and revision #s.
builder_name = slave_utils.SlaveBuildName(o3d_dir)
o3d_rev = str(slave_utils.SubversionRevision(o3d_dir))
platform = chromium_utils.PlatformName()
# Upload zip.
local_zip = os.path.join(staging_dir,
'full-build-' + platform + '_' + o3d_rev + '.zip')
remote_zip = 'snapshots/o3d/' + o3d_rev + '/' + builder_name + '.zip'
archive_file.UploadFile(local_zip, remote_zip)
return 0
def archive(options, args):
src_dir = os.path.abspath(os.path.dirname(options.build_dir))
build_dir = os.path.join(src_dir, 'out', options.target)
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
prefix = options.factory_properties.get('asan_archive_name', 'asan')
zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file, gs_bucket, gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def main(options, args):
# Create some variables
src_dir = os.path.abspath(options.src_dir)
build_dir = os.path.dirname(options.build_dir)
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
build_version = str(build_revision)
if chromium_utils.IsMac() or chromium_utils.IsLinux():
# Files are created umask 077 by default, we need to make sure the staging
# dir can be fetch from, do this by recursively chmoding back up to the root
# before pushing to web server.
a_path = staging_dir
while a_path != '/':
current_permissions = os.stat(a_path)[0]
if current_permissions & 0555 == 0555:
break
print 'Fixing permissions (%o) for \'%s\'' % (current_permissions,
a_path)
os.chmod(a_path, current_permissions | 0555)
a_path = os.path.dirname(a_path)
def archive_layout(options, args):
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s:%(lineno)-3d'
' %(levelname)s %(message)s',
datefmt='%y%m%d %H:%M:%S')
chrome_dir = os.path.abspath(options.build_dir)
results_dir_basename = os.path.basename(options.results_dir)
if options.results_dir is not None:
options.results_dir = os.path.abspath(
os.path.join(options.build_dir, options.results_dir))
else:
options.results_dir = chromium_utils.FindUpward(chrome_dir, RESULT_DIR)
print 'Archiving results from %s' % options.results_dir
staging_dir = slave_utils.GetStagingDir(chrome_dir)
print 'Staging in %s' % staging_dir
(actual_file_list,
diff_file_list) = _CollectArchiveFiles(options.results_dir)
zip_file = chromium_utils.MakeZip(staging_dir, results_dir_basename,
actual_file_list, options.results_dir)[1]
# TODO(ojan): Stop separately uploading full_results.json once garden-o-matic
# switches to using failing_results.json.
full_results_json = os.path.join(options.results_dir, 'full_results.json')
failing_results_json = os.path.join(options.results_dir,
'failing_results.json')
# Extract the build name of this slave (e.g., 'chrome-release') from its
# configuration file if not provided as a param.
build_name = options.builder_name or slave_utils.SlaveBuildName(chrome_dir)
build_name = re.sub('[ .()]', '_', build_name)
wc_dir = os.path.dirname(chrome_dir)
last_change = str(slave_utils.SubversionRevision(wc_dir))
# TODO(dpranke): Is it safe to assume build_number is not blank? Should we
# assert() this ?
build_number = str(options.build_number)
print 'last change: %s' % last_change
print 'build name: %s' % build_name
print 'build number: %s' % build_number
print 'host name: %s' % socket.gethostname()
# Where to save layout test results.
dest_parent_dir = os.path.join(archive_utils.Config.www_dir_base,
results_dir_basename.replace('-', '_'),
build_name)
dest_dir = os.path.join(dest_parent_dir, last_change)
if options.gs_bucket:
# Copy the results to a directory archived by build number.
gs_base = '/'.join([options.gs_bucket, build_name, build_number])
gs_acl = options.gs_acl
# These files never change, cache for a year.
cache_control = "public, max-age=31556926"
slave_utils.GSUtilCopyFile(zip_file,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyDir(options.results_dir,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
# TODO(dpranke): Remove these two lines once clients are fetching the
# files from the layout-test-results dir.
slave_utils.GSUtilCopyFile(full_results_json,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyFile(failing_results_json,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
# And also to the 'results' directory to provide the 'latest' results.
gs_base = '/'.join([options.gs_bucket, build_name, 'results'])
slave_utils.GSUtilCopyFile(zip_file,
gs_base,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
slave_utils.GSUtilCopyDir(options.results_dir,
gs_base,
gs_acl=gs_acl,
cache_control=cache_control)
else:
_MaybeMakeDirectoryOnArchiveHost(dest_dir)
_CopyFileToArchiveHost(zip_file, dest_dir)
_CopyFileToArchiveHost(full_results_json, dest_dir)
_CopyFileToArchiveHost(failing_results_json, dest_dir)
# Not supported on Google Storage yet.
_ArchiveFullLayoutTestResults(staging_dir, dest_parent_dir,
diff_file_list, options)
return 0
def dom_perf(options, args):
"""Using the target build configuration, run the dom perf test."""
build_dir = os.path.abspath(options.build_dir)
if chromium_utils.IsWindows():
test_exe_name = 'performance_ui_tests.exe'
else:
test_exe_name = 'performance_ui_tests'
if chromium_utils.IsMac():
is_make_or_ninja = (options.factory_properties.get(
"gclient_env", {}).get('GYP_GENERATORS', '') in ('ninja', 'make'))
if is_make_or_ninja:
build_dir = os.path.join(os.path.dirname(build_dir), 'out')
else:
build_dir = os.path.join(os.path.dirname(build_dir), 'xcodebuild')
elif chromium_utils.IsLinux():
build_dir = os.path.join(os.path.dirname(build_dir), 'sconsbuild')
test_exe_path = os.path.join(build_dir, options.target, test_exe_name)
if not os.path.exists(test_exe_path):
raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path)
# Find the current revision to pass to the test.
build_revision = slave_utils.SubversionRevision(build_dir)
# Compute the path to the test data.
src_dir = os.path.dirname(build_dir)
data_dir = os.path.join(src_dir, 'data')
dom_perf_dir = os.path.join(data_dir, 'dom_perf')
iterations = '' # Default
if options.target == 'Debug':
iterations = '&minIterations=1'
def run_and_print(use_refbuild):
# Windows used to write to the root of C:, but that doesn't work
# on Vista so we write into the build folder instead.
suffix = ''
if (use_refbuild):
suffix = '_ref'
output_file = os.path.join(
build_dir, options.target,
'dom_perf_result_%s%s.txt' % (build_revision, suffix))
result = 0
compiled_data = []
for test in TESTS:
url = URL % (dom_perf_dir, test, iterations, build_revision)
url_flag = '--url=%s' % url
command = [
test_exe_path, '--wait_cookie_name=__domperf_finished',
'--jsvar=__domperf_result',
'--jsvar_output=%s' % output_file,
'--gtest_filter=UrlFetchTest.UrlFetch', url_flag
]
if use_refbuild:
command.append('--reference_build')
print "Executing: "
print command
result |= chromium_utils.RunCommand(command)
# Open the resulting file and display it.
data = json.load(open(output_file, 'r'))
for suite in data['BenchmarkSuites']:
# Skip benchmarks that we didn't actually run this time around.
if len(suite['Benchmarks']) == 0 and suite['score'] == 0:
continue
compiled_data.append(suite)
# Now give the geometric mean as the total for the combined runs.
total = geometric_mean([s['score'] for s in compiled_data])
print_result(True, 'Total', total, use_refbuild)
for suite in compiled_data:
print_result(False, suite['name'], suite['score'], use_refbuild)
return result
try:
if chromium_utils.IsLinux():
xvfb.StartVirtualX(options.target,
os.path.join(build_dir, options.target))
result = run_and_print(False)
result |= run_and_print(True)
finally:
if chromium_utils.IsLinux():
xvfb.StopVirtualX(options.target)
return result
def __init__(self, options, build_revision):
"""Sets a number of file and directory paths for convenient use."""
self.options = options
self._src_dir = os.path.abspath(options.src_dir)
self._chrome_dir = os.path.join(self._src_dir, 'chrome')
# TODO: This scode should not be grabbing so deeply into WebKit.
# Worse, this code ends up looking at top-of-tree WebKit
# instead of the revision in DEPS.
self._webkit_dir = os.path.join(self._src_dir, 'third_party', 'WebKit',
'Source', 'WebCore')
self._v8_dir = os.path.join(self._src_dir, 'v8')
# TODO: need to get the build *output* directory passed in instead so Linux
# and Mac don't have to walk up a directory to get to the right directory.
if chromium_utils.IsWindows():
self._build_dir = os.path.join(options.build_dir, options.target)
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'win')
elif chromium_utils.IsLinux():
self._build_dir = os.path.join(os.path.dirname(options.build_dir),
'out', options.target)
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'linux')
elif chromium_utils.IsMac():
self._build_dir = os.path.join(os.path.dirname(options.build_dir),
'xcodebuild', options.target)
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'mac')
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
self._staging_dir = slave_utils.GetStagingDir(self._src_dir)
self._symbol_dir_base = options.dirs['symbol_dir_base']
self._www_dir_base = options.dirs['www_dir_base']
self._build_name = slave_utils.SlaveBuildName(self._src_dir)
self._symbol_dir_base = os.path.join(self._symbol_dir_base,
self._build_name)
self._www_dir_base = os.path.join(self._www_dir_base, self._build_name)
self._version_file = os.path.join(self._chrome_dir, 'VERSION')
if options.default_chromium_revision:
self._chromium_revision = options.default_chromium_revision
else:
self._chromium_revision = slave_utils.SubversionRevision(
self._chrome_dir)
if options.default_webkit_revision:
self._webkit_revision = options.default_webkit_revision
else:
self._webkit_revision = slave_utils.SubversionRevision(
self._webkit_dir)
if options.default_v8_revision:
self._v8_revision = options.default_v8_revision
else:
self._v8_revision = slave_utils.SubversionRevision(self._v8_dir)
self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE')
# The REVISIONS file will record the revisions information of the main
# components Chromium/WebKit/V8.
self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS')
self._build_revision = build_revision
# Will be initialized in GetLastBuildRevision.
self.last_chromium_revision = None
self.last_webkit_revision = None
self.last_v8_revision = None
self._files_file = os.path.join(self._tool_dir,
archive_utils.FILES_FILENAME)
self._test_files = self.BuildOldFilesList(TEST_FILE_NAME)
self._dual_upload = options.factory_properties.get(
'dual_upload', False)
self._archive_files = None
def main(argv):
if len(argv) != 3:
print 'Usage: prepare_selenium_tests.py <o3d_src_root> <destination>'
print 'Exiting...'
return 1
# Make given directories absolute before changing the working directory.
src_root = os.path.abspath(argv[1])
o3d_dir = os.path.join(src_root, 'o3d')
o3d_internal_dir = os.path.join(src_root, 'o3d-internal')
destination = os.path.abspath(argv[2])
config_dir = os.path.abspath(os.path.dirname(__file__))
config_file = os.path.join(config_dir, ARCHIVE_CONFIG_NAME)
print 'O3D source root:', src_root
print 'Destination:', destination
print 'Config file:', config_file
# Change umask on linux so that outputs (latest file and zip) are readable.
if utils.IsLinux():
mask = os.umask(0022)
# Build ChangeResolution project.
BuildChangeResolution(src_root)
# Create test archive.
files = GetO3DArchiveFiles(src_root, config_file)
zip_name = 'o3d'
utils.MakeZip(destination, zip_name, files, src_root)
zip_path = os.path.join(destination, zip_name + '.zip')
print 'Zip archive created: %s' % zip_path
# Find builder name and revision #s.
builder_name = slave_utils.SlaveBuildName(o3d_dir)
o3d_rev = str(slave_utils.SubversionRevision(o3d_dir))
o3d_internal_rev = str(slave_utils.SubversionRevision(o3d_internal_dir))
package_name = 'test_' + builder_name + '.zip'
package_dir = o3d_rev + '_' + o3d_internal_rev
package_path = package_dir + '/' + package_name
print 'Builder name:', builder_name
print 'O3D revision:', o3d_rev
print 'O3D-internal revision:', o3d_internal_rev
print 'Package path:', package_path
# Create latest file.
latest_path = os.path.join(destination, 'latest_' + builder_name)
file(latest_path, 'w').write(package_path)
# Upload files.
package_dst = ('snapshots/o3d/test_packages/o3d/' + package_dir + '/' +
package_name)
latest_dst = 'snapshots/o3d/test_packages/o3d/latest_' + builder_name
UploadFile(zip_path, package_dst)
UploadFile(latest_path, latest_dst)
# Reset the umask on linux.
if utils.IsLinux():
os.umask(mask)
return 0