def _collect_metrics(self, func, command_name, *args, **kwargs):
self.add('command', command_name)
try:
start = time.time()
func(*args, **kwargs)
exception = None
# pylint: disable=bare-except
except:
exception = sys.exc_info()
finally:
self.add('execution_time', time.time() - start)
# Print the exception before the metrics notice, so that the notice is
# clearly visible even if gclient fails.
if exception and not isinstance(exception[1], SystemExit):
traceback.print_exception(*exception)
exit_code = metrics_utils.return_code_from_exception(exception)
self.add('exit_code', exit_code)
# Print the metrics notice only if the user has not explicitly opted in
# or out.
if self.config.opted_in is None:
metrics_utils.print_notice(self.config.countdown)
# Add metrics regarding environment information.
self.add('timestamp', metrics_utils.seconds_to_weeks(time.time()))
self.add('python_version', metrics_utils.get_python_version())
self.add('host_os', gclient_utils.GetMacWinOrLinux())
self.add('host_arch', detect_host_arch.HostArch())
self.add('depot_tools_age', metrics_utils.get_repo_timestamp(DEPOT_TOOLS))
self._upload_metrics_data()
sys.exit(exit_code)
def DetectArch(gyp_defines):
# Check for optional target_arch and only install for that architecture.
# If target_arch is not specified, then only install for the host
# architecture.
if 'target_arch=x64' in gyp_defines:
return 'amd64'
elif 'target_arch=ia32' in gyp_defines:
return 'i386'
elif 'target_arch=arm' in gyp_defines:
return 'arm'
elif 'target_arch=mipsel' in gyp_defines:
return 'mips'
# Figure out host arch using build/detect_host_arch.py and
# set target_arch to host arch
build_dir = os.path.dirname(os.path.dirname(os.path.join(SCRIPT_DIR)))
sys.path.append(build_dir)
import detect_host_arch
detected_host_arch = detect_host_arch.HostArch()
if detected_host_arch == 'x64':
return 'amd64'
elif detected_host_arch == 'ia32':
return 'i386'
elif detected_host_arch == 'arm':
return 'arm'
elif detected_host_arch == 'mips':
return 'mips'
else:
print "Unknown host arch: %s" % detected_host_arch
return None
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--ignore-if-arch',
metavar='ARCH',
action='append',
default=[],
help='Do nothing on host architecture ARCH')
options = parser.parse_args(args)
if not sys.platform.startswith('linux'):
return 0
arch = detect_host_arch.HostArch()
if arch in options.ignore_if_arch:
return 0
if arch == 'x64':
return FetchAndExtract(arch)
if arch == 'ia32':
ret = FetchAndExtract(arch)
if ret != 0:
return ret
# Fetch the x64 toolchain as well for official bots with 64-bit kernels.
return FetchAndExtract('x64')
print "Host architecture %s is not supported." % arch
return 1
def DetectHostArch():
# Figure out host arch using build/detect_host_arch.py and
# set target_arch to host arch
detected_host_arch = detect_host_arch.HostArch()
if detected_host_arch == 'x64':
return 'amd64'
elif detected_host_arch == 'ia32':
return 'i386'
elif detected_host_arch == 'arm':
return 'arm'
elif detected_host_arch == 'mips':
return 'mips'
raise Error('Unrecognized host arch: %s' % detected_host_arch)
def _collect_metrics(self, func, command_name, *args, **kwargs):
# If we're already collecting metrics, just execute the function.
# e.g. git-cl split invokes git-cl upload several times to upload each
# split CL.
if self.collecting_metrics:
# Don't collect metrics for this function.
# e.g. Don't record the arguments git-cl split passes to git-cl upload.
with self.pause_metrics_collection():
return func(*args, **kwargs)
self._collecting_metrics = True
self.add('metrics_version', metrics_utils.CURRENT_VERSION)
self.add('command', command_name)
try:
start = time.time()
result = func(*args, **kwargs)
exception = None
# pylint: disable=bare-except
except:
exception = sys.exc_info()
finally:
self.add('execution_time', time.time() - start)
exit_code = metrics_utils.return_code_from_exception(exception)
self.add('exit_code', exit_code)
# Add metrics regarding environment information.
self.add('timestamp', int(time.time()))
self.add('python_version', metrics_utils.get_python_version())
self.add('host_os', gclient_utils.GetMacWinAixOrLinux())
self.add('host_arch', detect_host_arch.HostArch())
depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS)
if depot_tools_age is not None:
self.add('depot_tools_age', int(depot_tools_age))
git_version = metrics_utils.get_git_version()
if git_version:
self.add('git_version', git_version)
bot_metrics = metrics_utils.get_bot_metrics()
if bot_metrics:
self.add('bot_metrics', bot_metrics)
self._upload_metrics_data()
if exception:
gclient_utils.reraise(exception[0], exception[1], exception[2])
return result
def testHostArch(self):
test_cases = [
('ia86', '', [''], 'x86'),
('i86pc', '', [''], 'x86'),
('x86_64', '', [''], 'x64'),
('amd64', '', [''], 'x64'),
('x86_64', '', ['32bit'], 'x86'),
('amd64', '', ['32bit'], 'x86'),
('arm', '', [''], 'arm'),
('aarch64', '', [''], 'arm64'),
('aarch64', '', ['32bit'], 'arm'),
('mips64', '', [''], 'mips64'),
('mips', '', [''], 'mips'),
('ppc', '', [''], 'ppc'),
('foo', 'powerpc', [''], 'ppc'),
('s390', '', [''], 's390'),
]
for machine, processor, arch, expected in test_cases:
platform.machine.return_value = machine
platform.processor.return_value = processor
platform.architecture.return_value = arch
self.assertEqual(expected, detect_host_arch.HostArch())
def DetectArch(gyp_defines, is_android):
# Check for optional target_arch and only install for that architecture.
# If target_arch is not specified, then only install for the host
# architecture.
target_arch = gyp_defines.get('target_arch')
if target_arch == 'x64':
return 'amd64'
elif target_arch == 'ia32':
return 'i386'
elif target_arch == 'arm':
return 'arm'
elif target_arch == 'arm64':
return 'arm64'
elif target_arch == 'mipsel':
return 'mips'
elif target_arch:
raise Exception('Unrecognized target_arch: %s' % target_arch)
if is_android:
return 'arm'
# Figure out host arch using build/detect_host_arch.py and
# set target_arch to host arch
detected_host_arch = detect_host_arch.HostArch()
if detected_host_arch == 'x64':
return 'amd64'
elif detected_host_arch == 'ia32':
return 'i386'
elif detected_host_arch == 'arm':
return 'arm'
elif detected_host_arch == 'mips':
return 'mips'
else:
print "Unknown host arch: %s" % detected_host_arch
return None
def BuildAndTest(options):
# Refuse to run under cygwin.
if sys.platform == 'cygwin':
raise Exception('I do not work under cygwin, sorry.')
# By default, use the version of Python is being used to run this script.
python = sys.executable
if sys.platform == 'darwin':
# Mac 10.5 bots tend to use a particularlly old version of Python, look for
# a newer version.
macpython27 = '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
if os.path.exists(macpython27):
python = macpython27
script_dir = os.path.dirname(os.path.abspath(__file__))
src_dir = os.path.dirname(os.path.dirname(os.path.dirname(script_dir)))
nacl_dir = os.path.join(src_dir, 'native_client')
# Decide platform specifics.
if options.browser_path:
chrome_filename = options.browser_path
else:
chrome_filename = find_chrome.FindChrome(src_dir, [options.mode])
if chrome_filename is None:
raise Exception('Cannot find a chrome binary - specify one with '
'--browser_path?')
env = dict(os.environ)
if sys.platform in ['win32', 'cygwin']:
if options.bits == 64:
bits = 64
elif options.bits == 32:
bits = 32
elif '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
'64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
bits = 64
else:
bits = 32
msvs_path = ';'.join([
r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
r'c:\Program Files\Microsoft Visual Studio 8\VC',
r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
])
env['PATH'] += ';' + msvs_path
scons = [python, 'scons.py']
elif sys.platform == 'darwin':
if options.bits == 64:
bits = 64
elif options.bits == 32:
bits = 32
else:
p = subprocess.Popen(['file', chrome_filename],
stdout=subprocess.PIPE)
(p_stdout, _) = p.communicate()
assert p.returncode == 0
if p_stdout.find('executable x86_64') >= 0:
bits = 64
else:
bits = 32
scons = [python, 'scons.py']
else:
if options.bits == 64:
bits = 64
elif options.bits == 32:
bits = 32
elif '64' in detect_host_arch.HostArch():
bits = 64
else:
bits = 32
# xvfb-run has a 2-second overhead per invocation, so it is cheaper to wrap
# the entire build step rather than each test (browser_headless=1).
# We also need to make sure that there are at least 24 bits per pixel.
# https://code.google.com/p/chromium/issues/detail?id=316687
scons = [
'xvfb-run',
'--auto-servernum',
'--server-args',
'-screen 0 1024x768x24',
python,
'scons.py',
]
if options.jobs > 1:
scons.append('-j%d' % options.jobs)
scons.append('disable_tests=%s' % options.disable_tests)
if options.buildbot is not None:
scons.append('buildbot=%s' % (options.buildbot, ))
# Clean the output of the previous build.
# Incremental builds can get wedged in weird ways, so we're trading speed
# for reliability.
shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True)
# check that the HOST (not target) is 64bit
# this is emulating what msvs_env.bat is doing
if '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
'64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
# 64bit HOST
env['VS90COMNTOOLS'] = (
'c:\\Program Files (x86)\\'
'Microsoft Visual Studio 9.0\\Common7\\Tools\\')
env['VS80COMNTOOLS'] = (
'c:\\Program Files (x86)\\'
'Microsoft Visual Studio 8.0\\Common7\\Tools\\')
else:
# 32bit HOST
env['VS90COMNTOOLS'] = (
'c:\\Program Files\\Microsoft Visual Studio 9.0\\'
'Common7\\Tools\\')
env['VS80COMNTOOLS'] = (
'c:\\Program Files\\Microsoft Visual Studio 8.0\\'
'Common7\\Tools\\')
# Run nacl/chrome integration tests.
# Note that we have to add nacl_irt_test to --mode in order to get
# inbrowser_test_runner to run.
# TODO(mseaborn): Change it so that inbrowser_test_runner is not a
# special case.
cmd = scons + [
'--verbose',
'-k',
'platform=x86-%d' % bits,
'--mode=opt-host,nacl,nacl_irt_test',
'chrome_browser_path=%s' % chrome_filename,
]
if not options.integration_bot and not options.morenacl_bot:
cmd.append('disable_flaky_tests=1')
cmd.append('chrome_browser_tests')
# Propagate path to JSON output if present.
# Note that RunCommand calls sys.exit on errors, so potential errors
# from one command won't be overwritten by another one. Overwriting
# a successful results file with either success or failure is fine.
if options.json_build_results_output_file:
cmd.append('json_build_results_output_file=%s' %
options.json_build_results_output_file)
# Download the toolchain(s).
pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version')
RunCommand([
python,
os.path.join(pkg_ver_dir, 'package_version.py'), '--exclude',
'arm_trusted', '--exclude', 'pnacl_newlib', '--exclude',
'nacl_arm_newlib', 'sync', '--extract'
], nacl_dir, os.environ)
CleanTempDir()
if options.enable_newlib:
RunTests('nacl-newlib', cmd, nacl_dir, env)
if options.enable_glibc:
RunTests('nacl-glibc', cmd + ['--nacl_glibc'], nacl_dir, env)
# Always include common.gypi.
args.extend(['-I' + os.path.join(src_dir, 'build', 'common.gypi')])
# Set these default GYP_DEFINES if user does not set the value explicitly.
_DEFAULT_DEFINES = {
'test_isolation_mode': 'noop',
'use_glib': 0,
'use_openssl': 1,
'use_x11': 0,
'linux_use_gold_binary': 0,
'linux_use_gold_flags': 0
}
# Disable clang on 32 bit systems by default, which is not supported.
if detect_host_arch.HostArch() == 'ia32':
_DEFAULT_DEFINES['clang'] = 0
gyp_defines = (os.environ['GYP_DEFINES']
if os.environ.get('GYP_DEFINES') else '')
for key in _DEFAULT_DEFINES:
if key not in gyp_defines:
gyp_defines += ' {0}={1}'.format(key, _DEFAULT_DEFINES[key])
os.environ['GYP_DEFINES'] = gyp_defines.strip()
# Default to ninja, but only if no generator has explicitly been set.
if not os.environ.get('GYP_GENERATORS'):
os.environ['GYP_GENERATORS'] = 'ninja'
# There shouldn't be a circular dependency relationship between .gyp files,
# but in Chromium's .gyp files, on non-Mac platforms, circular relationships
def main(args):
if options.arch not in ['amd64', 'i386']:
print 'Unknown architecture: %s' % options.arch
return 1
if options.linux_only:
# This argument is passed when run from the gclient hooks.
# In this case we return early on non-linux platforms.
if not sys.platform.startswith('linux'):
return 0
# Only install the sysroot for an Official Chrome Linux build.
defined = ['branding=Chrome', 'buildtype=Official']
undefined = ['chromeos=1']
gyp_defines = os.environ.get('GYP_DEFINES', '')
for option in defined:
if option not in gyp_defines:
return 0
for option in undefined:
if option in gyp_defines:
return 0
# Check for optional target_arch and only install for that architecture.
# If target_arch is not specified, then only install for the host
# architecture.
host_arch = ''
if 'target_arch=x64' in gyp_defines:
host_arch = 'amd64'
elif 'target_arch=ia32' in gyp_defines:
host_arch = 'i386'
else:
# Figure out host arch using build/detect_host_arch.py.
SRC_DIR = os.path.abspath(
os.path.join(SCRIPT_DIR, '..', '..', '..', '..'))
sys.path.append(os.path.join(SRC_DIR, 'build'))
import detect_host_arch
detected_host_arch = detect_host_arch.HostArch()
if detected_host_arch == 'x64':
host_arch = 'amd64'
elif detected_host_arch == 'ia32':
host_arch = 'i386'
if host_arch != options.arch:
return 0
# The sysroot directory should match the one specified in build/common.gypi.
# TODO(thestig) Consider putting this else where to avoid having to recreate
# it on every build.
linux_dir = os.path.dirname(SCRIPT_DIR)
if options.arch == 'amd64':
sysroot = os.path.join(linux_dir, SYSROOT_DIR_AMD64)
tarball_filename = TARBALL_AMD64
tarball_sha1sum = TARBALL_AMD64_SHA1SUM
else:
sysroot = os.path.join(linux_dir, SYSROOT_DIR_I386)
tarball_filename = TARBALL_I386
tarball_sha1sum = TARBALL_I386_SHA1SUM
url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, REVISION, tarball_filename)
stamp = os.path.join(sysroot, '.stamp')
if os.path.exists(stamp):
with open(stamp) as s:
if s.read() == url:
print 'Debian Wheezy %s root image already up-to-date: %s' % \
(options.arch, sysroot)
return 0
print 'Installing Debian Wheezy %s root image: %s' % (options.arch, sysroot)
if os.path.isdir(sysroot):
shutil.rmtree(sysroot)
os.mkdir(sysroot)
tarball = os.path.join(sysroot, tarball_filename)
subprocess.check_call(['curl', '-L', url, '-o', tarball])
sha1sum = get_sha1(tarball)
if sha1sum != tarball_sha1sum:
print 'Tarball sha1sum is wrong.'
print 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)
return 1
subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
os.remove(tarball)
with open(stamp, 'w') as s:
s.write(url)
return 0
def DetectHostArch(): # Figure out host arch using build/detect_host_arch.py and # set target_arch to host arch detected_host_arch = detect_host_arch.HostArch() if detected_host_arch == 'x64':
