def archive(options, args):
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir, use_out=chromium_utils.IsLinux())
build_dir = os.path.join(build_dir, options.target)
src_dir = os.path.abspath(os.path.dirname(options.build_dir))
staging_dir = slave_utils.GetStagingDir(src_dir)
build_revision = slave_utils.SubversionRevision(src_dir)
chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
print 'Staging in %s' % build_dir
# Build the list of files to archive.
zip_file_list = [
f for f in os.listdir(build_dir)
if ShouldPackageFile(f, options.target)
]
subdir = None
# TODO(nsylvain): We need to move linux to a subdir as well, but aarya is not
# ready with the server-side change.
if chromium_utils.IsMac():
subdir = '%s-%s' % (chromium_utils.PlatformName(),
options.target.lower())
prefix = options.factory_properties.get('cf_archive_name', 'cf_archive')
zip_file_name = '%s-%s-%s-%d' % (prefix, chromium_utils.PlatformName(),
options.target.lower(), build_revision)
(zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
zip_file_name,
zip_file_list,
build_dir,
raise_error=True)
chromium_utils.RemoveDirectory(zip_dir)
if not os.path.exists(zip_file):
raise StagingError('Failed to make zip package %s' % zip_file)
chromium_utils.MakeWorldReadable(zip_file)
# Report the size of the zip file to help catch when it gets too big.
zip_size = os.stat(zip_file)[stat.ST_SIZE]
print 'Zip file is %ld bytes' % zip_size
gs_bucket = options.factory_properties.get('gs_bucket', None)
gs_acl = options.factory_properties.get('gs_acl', None)
status = slave_utils.GSUtilCopyFile(zip_file,
gs_bucket,
subdir=subdir,
gs_acl=gs_acl)
if status:
raise StagingError('Failed to upload %s to %s. Error %d' %
(zip_file, gs_bucket, status))
else:
# Delete the file, it is not needed anymore.
os.remove(zip_file)
return status
def run_benchmark(options, use_refbuild, benchmark_results):
build_dir, bad = chromium_utils.ConvertBuildDirToLegacy(options.build_dir)
build_dir = os.path.abspath(options.build_dir)
result = slave_utils.WARNING_EXIT_CODE if bad else 0
if not use_refbuild:
build_dir = os.path.join(build_dir, options.target)
else:
build_dir = os.path.join(os.path.dirname(build_dir), 'chrome', 'tools',
'test', 'reference_build')
if chromium_utils.IsMac():
build_dir = os.path.join(build_dir, 'chrome_mac')
elif chromium_utils.IsLinux():
build_dir = os.path.join(build_dir, 'chrome_linux')
else:
build_dir = os.path.join(build_dir, 'chrome_win')
if chromium_utils.IsWindows():
chrome_exe_name = 'chrome.exe'
elif chromium_utils.IsLinux():
chrome_exe_name = 'chrome'
else:
chrome_exe_name = 'Chromium'
chrome_exe_path = os.path.join(build_dir, chrome_exe_name)
if not os.path.exists(chrome_exe_path):
raise chromium_utils.PathNotFound('Unable to find %s' %
chrome_exe_path)
temp_dir = tempfile.mkdtemp()
command = [
chrome_exe_path,
'--user-data-dir=%s' % temp_dir, '--no-first-run',
'--no-default-browser-check', START_URL
]
print "Executing: "
print command
browser_process = subprocess.Popen(command)
benchmark_results['ready'].wait()
if benchmark_results['ready'].isSet():
results = json.loads(benchmark_results['results'])[0]
print_result(True, 'Total', results['score'], use_refbuild)
for child in results['children']:
print_result(False, child['name'], child['score'], use_refbuild)
benchmark_results['ready'].clear()
if chromium_utils.IsWindows():
subprocess.call('taskkill /f /pid %i /t' % browser_process.pid)
else:
os.system('kill -15 %i' % browser_process.pid)
browser_process.wait()
shutil.rmtree(temp_dir)
return result
def main_linux(options, args):
"""Print appropriate size information about built Linux targets.
Returns the first non-zero exit status of any command it executes,
or zero on success.
"""
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(options.build_dir)
target_dir = os.path.join(build_dir, options.target)
binaries = [
'chrome',
'nacl_helper',
'nacl_helper_bootstrap',
'libffmpegsumo.so',
'libgcflashplayer.so',
'libpdf.so',
'libppGoogleNaClPluginChrome.so',
]
result = 0
totals = {}
for binary in binaries:
this_result, this_sizes = check_linux_binary(target_dir, binary,
options)
if result == 0:
result = this_result
for name, identifier, totals_id, value, units in this_sizes:
print 'RESULT %s: %s= %s %s' % (name, identifier, value, units)
totals_id = totals_id or identifier, units
totals[totals_id] = totals.get(totals_id, 0) + int(value)
files = [
'chrome.pak',
'nacl_irt_x86_64.nexe',
]
for filename in files:
path = os.path.join(target_dir, filename)
try:
size = get_size(path)
except OSError, e:
if e.errno == errno.ENOENT:
continue # Don't print anything for missing files.
raise
print 'RESULT %s: %s= %s bytes' % (filename, filename, size)
totals['size', 'bytes'] += size
def AddIsolateTest(self, test_name, using_ninja):
if not self._target:
log.msg('No target specified, unable to find isolated files')
return
isolated_directory, _ = chromium_utils.ConvertBuildDirToLegacy(
self._build_dir,
target_platform=self._target_platform,
use_out=(using_ninja or self._target_platform.startswith('linux')))
isolated_directory = self.PathJoin(isolated_directory, self._target)
isolated_file = self.PathJoin(isolated_directory,
test_name + '.isolated')
script_path = self.PathJoin(self._swarming_client_dir, 'isolate.py')
args = ['run', '--isolated', isolated_file, '--', '--no-cr']
wrapper_args = ['--annotate=gtest', '--test-type=%s' % test_name]
command = self.GetPythonTestCommand(script_path,
arg_list=args,
wrapper_args=wrapper_args)
self.AddTestStep(chromium_step.AnnotatedCommand, test_name, command)
def main():
option_parser = optparse.OptionParser()
option_parser.add_option('--target',
help='build target to archive (Debug or Release)')
option_parser.add_option(
'--build-dir',
help='path to main build directory (the parent of '
'the Release or Debug directory)')
option_parser.add_option('--build-url',
help='url where to find the build to extract')
option_parser.add_option('--build-output-dir',
help='Output path relative to --build-dir.')
option_parser.add_option('--revision', help='Revision number to download.')
chromium_utils.AddPropertiesOptions(option_parser)
options, args = option_parser.parse_args()
if args:
print 'Unknown options: %s' % args
return 1
if options.build_output_dir:
options.build_output_dir = os.path.join(options.build_dir,
options.build_output_dir)
else:
options.build_output_dir, bad = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir)
if bad:
return slave_utils.WARNING_EXIT_CODE
options.build_output_dir = os.path.abspath(options.build_output_dir)
options.build_url = (options.build_url
or options.factory_properties.get('build_url'))
del options.factory_properties
del options.build_properties
return real_main(options)
def __init__(self, options, build_revision):
"""Sets a number of file and directory paths for convenient use."""
self.options = options
self._src_dir = os.path.abspath(options.src_dir)
self._chrome_dir = os.path.join(self._src_dir, 'chrome')
# TODO: This scode should not be grabbing so deeply into WebKit.
# Worse, this code ends up looking at top-of-tree WebKit
# instead of the revision in DEPS.
self._webkit_dir = os.path.join(self._src_dir, 'third_party', 'WebKit',
'Source')
self._v8_dir = os.path.join(self._src_dir, 'v8')
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir, use_out=chromium_utils.IsLinux())
self._build_dir = os.path.join(build_dir, options.target)
if chromium_utils.IsWindows():
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'win')
elif chromium_utils.IsLinux():
# On Linux, we might have built for chromeos. Archive the same.
if (options.factory_properties.get('chromeos', None)
or slave_utils.GypFlagIsOn(options, 'chromeos')):
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'chromeos')
# Or, we might have built for Android.
elif options.factory_properties.get('target_os') == 'android':
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'android')
else:
self._tool_dir = os.path.join(self._chrome_dir, 'tools',
'build', 'linux')
elif chromium_utils.IsMac():
self._tool_dir = os.path.join(self._chrome_dir, 'tools', 'build',
'mac')
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
self._staging_dir = slave_utils.GetStagingDir(self._src_dir)
self._symbol_dir_base = options.dirs['symbol_dir_base']
self._www_dir_base = options.dirs['www_dir_base']
self._build_name = slave_utils.SlaveBuildName(self._src_dir)
self._symbol_dir_base = os.path.join(self._symbol_dir_base,
self._build_name)
self._www_dir_base = os.path.join(self._www_dir_base, self._build_name)
self._version_file = os.path.join(self._chrome_dir, 'VERSION')
if options.default_chromium_revision:
self._chromium_revision = options.default_chromium_revision
else:
self._chromium_revision = slave_utils.GetHashOrRevision(
os.path.dirname(
self._chrome_dir)) # src/ instead of src/chrome
if options.default_webkit_revision:
self._webkit_revision = options.default_webkit_revision
else:
self._webkit_revision = slave_utils.GetHashOrRevision(
os.path.dirname(
self._webkit_dir)) # WebKit/ instead of WebKit/Source
if options.default_v8_revision:
self._v8_revision = options.default_v8_revision
else:
self._v8_revision = slave_utils.GetHashOrRevision(self._v8_dir)
self.last_change_file = os.path.join(self._staging_dir, 'LAST_CHANGE')
# The REVISIONS file will record the revisions information of the main
# components Chromium/WebKit/V8.
self.revisions_path = os.path.join(self._staging_dir, 'REVISIONS')
self._build_revision = build_revision
# Will be initialized in GetLastBuildRevision.
self.last_chromium_revision = None
self.last_webkit_revision = None
self.last_v8_revision = None
self._files_file = os.path.join(self._tool_dir,
archive_utils.FILES_FILENAME)
self._test_files = self.BuildOldFilesList(TEST_FILE_NAME)
self._dual_upload = options.factory_properties.get(
'dual_upload', False)
self._archive_files = None
def real_main(options):
""" Download a build, extract it to build\BuildDir\full-build-win32
and rename it to build\BuildDir\Target
"""
normal_retval = 0
if options.build_output_dir:
build_output_dir = os.path.join(options.build_dir, options.build_output_dir)
else:
build_output_dir, bad = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir)
if bad:
normal_retval = slave_utils.WARNING_EXIT_CODE
abs_build_dir = os.path.abspath(options.build_dir)
abs_build_output_dir = os.path.abspath(build_output_dir)
target_build_output_dir = os.path.join(abs_build_output_dir, options.target)
# Generic name for the archive.
archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName()
# Just take the zip off the name for the output directory name.
output_dir = os.path.join(abs_build_output_dir,
archive_name.replace('.zip', ''))
base_url, url = GetBuildUrl(abs_build_dir, options)
archive_name = os.path.basename(base_url)
if url.startswith('gs://'):
handler = GSHandler(url=url, archive_name=archive_name)
else:
handler = WebHandler(url=url, archive_name=archive_name)
# We try to download and extract 3 times.
for tries in range(1, 4):
print 'Try %d: Fetching build from %s...' % (tries, url)
failure = False
# Check if the url exists.
if not handler.is_present():
print '%s is not found' % url
failure = True
# When 'halt_on_missing_build' is present in factory_properties and if
# 'revision' is set in build properties, we assume the build is
# triggered automatically and so we halt on a missing build zip. The
# other case is if the build is forced, in which case we keep trying
# later by looking for the latest build that's available.
if (options.factory_properties.get('halt_on_missing_build', False) and
'revision' in options.build_properties and
options.build_properties['revision'] != ''):
return slave_utils.ERROR_EXIT_CODE
# If the url is valid, we download the file.
if not failure:
if not handler.download():
failure = True
# If the versioned url failed, we try to get the latest build.
if failure:
if url.startswith('gs://'):
continue
else:
print 'Fetching latest build at %s' % base_url
handler.url = base_url
if not handler.download():
continue
print 'Extracting build %s to %s...' % (archive_name, abs_build_output_dir)
try:
chromium_utils.RemoveDirectory(target_build_output_dir)
chromium_utils.ExtractZip(archive_name, abs_build_output_dir)
# For Chrome builds, the build will be stored in chrome-win32.
if 'full-build-win32' in output_dir:
chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32')
if os.path.exists(chrome_dir):
output_dir = chrome_dir
print 'Moving build from %s to %s' % (output_dir, target_build_output_dir)
shutil.move(output_dir, target_build_output_dir)
except (OSError, IOError, chromium_utils.ExternalError):
print 'Failed to extract the build.'
# Print out the traceback in a nice format
traceback.print_exc()
# Try again...
continue
# If we got the latest build, then figure out its revision number.
if failure:
print "Trying to determine the latest build's revision number..."
try:
build_revision_file_name = os.path.join(
target_build_output_dir,
chromium_utils.FULL_BUILD_REVISION_FILENAME)
build_revision_file = open(build_revision_file_name, 'r')
print 'Latest build is revision: %s' % build_revision_file.read()
build_revision_file.close()
except IOError:
print "Could not determine the latest build's revision number"
if failure:
# We successfully extracted the archive, but it was the generic one.
return slave_utils.WARNING_EXIT_CODE
return normal_retval
# If we get here, that means that it failed 3 times. We return a failure.
return slave_utils.ERROR_EXIT_CODE
def main_mac(options, args):
"""Print appropriate size information about built Mac targets.
Returns the first non-zero exit status of any command it executes,
or zero on success.
"""
out_dir_path = os.path.join(os.path.dirname(options.build_dir), 'out')
build_dir, _ = chromium_utils.ConvertBuildDirToLegacy(
options.build_dir, use_out=os.path.exists(out_dir_path))
target_dir = os.path.join(build_dir, options.target)
result = 0
# Work with either build type.
base_names = ('Chromium', 'Google Chrome')
for base_name in base_names:
app_bundle = base_name + '.app'
framework_name = base_name + ' Framework'
framework_bundle = framework_name + '.framework'
chromium_app_dir = os.path.join(target_dir, app_bundle)
chromium_executable = os.path.join(chromium_app_dir, 'Contents',
'MacOS', base_name)
chromium_framework_dir = os.path.join(target_dir, framework_bundle)
chromium_framework_executable = os.path.join(chromium_framework_dir,
framework_name)
if os.path.exists(chromium_executable):
print_dict = {
# Remove spaces in the names so any downstream processing is less
# likely to choke.
'app_name': re.sub('\s', '', base_name),
'app_bundle': re.sub('\s', '', app_bundle),
'framework_name': re.sub('\s', '', framework_name),
'framework_bundle': re.sub('\s', '', framework_bundle),
'app_size': get_size(chromium_executable),
'framework_size': get_size(chromium_framework_executable)
}
# Collect the segment info out of the App
p = subprocess.Popen(['size', chromium_executable],
stdout=subprocess.PIPE)
stdout = p.communicate()[0]
print_dict['app_text'], print_dict['app_data'], print_dict['app_objc'] = \
re.search('(\d+)\s+(\d+)\s+(\d+)', stdout).groups()
if result == 0:
result = p.returncode
# Collect the segment info out of the Framework
p = subprocess.Popen(['size', chromium_framework_executable],
stdout=subprocess.PIPE)
stdout = p.communicate()[0]
print_dict['framework_text'], print_dict['framework_data'], \
print_dict['framework_objc'] = \
re.search('(\d+)\s+(\d+)\s+(\d+)', stdout).groups()
if result == 0:
result = p.returncode
# Collect the whole size of the App bundle on disk (include the framework)
p = subprocess.Popen(['du', '-s', '-k', chromium_app_dir],
stdout=subprocess.PIPE)
stdout = p.communicate()[0]
du_s = re.search('(\d+)', stdout).group(1)
if result == 0:
result = p.returncode
print_dict['app_bundle_size'] = (int(du_s) * 1024)
# Count the number of files with at least one static initializer.
pipes = [['otool', '-l', chromium_framework_executable],
['grep', '__mod_init_func', '-C', '5'], ['grep', 'size']]
last_stdout = None
for pipe in pipes:
p = subprocess.Popen(pipe,
stdin=last_stdout,
stdout=subprocess.PIPE)
last_stdout = p.stdout
stdout = p.communicate()[0]
initializers_s = re.search('0x([0-9a-f]+)', stdout).group(1)
if result == 0:
result = p.returncode
word_size = 4 # Assume 32 bit
print_dict['initializers'] = int(initializers_s, 16) / word_size
print("""RESULT %(app_name)s: %(app_name)s= %(app_size)s bytes
RESULT %(app_name)s-__TEXT: __TEXT= %(app_text)s bytes
RESULT %(app_name)s-__DATA: __DATA= %(app_data)s bytes
RESULT %(app_name)s-__OBJC: __OBJC= %(app_objc)s bytes
RESULT %(framework_name)s: %(framework_name)s= %(framework_size)s bytes
RESULT %(framework_name)s-__TEXT: __TEXT= %(framework_text)s bytes
RESULT %(framework_name)s-__DATA: __DATA= %(framework_data)s bytes
RESULT %(framework_name)s-__OBJC: __OBJC= %(framework_objc)s bytes
RESULT %(app_bundle)s: %(app_bundle)s= %(app_bundle_size)s bytes
RESULT chrome-si: initializers= %(initializers)d files
""") % (print_dict)
# Found a match, don't check the other base_names.
return result
# If no base_names matched, fail script.
return 66
