def start_http_server(platform, build_dir, test_exe_path, document_root):
# pylint: disable=F0401
import google.httpd_utils
import google.platform_utils
platform_util = google.platform_utils.PlatformUtility(build_dir)
# Name the output directory for the exe, without its path or suffix.
# e.g., chrome-release/httpd_logs/unit_tests/
test_exe_name = os.path.splitext(os.path.basename(test_exe_path))[0]
output_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir),
'httpd_logs', test_exe_name)
# Sanity checks for httpd2_linux.conf.
if platform == 'linux':
for ssl_file in ['ssl.conf', 'ssl.load']:
ssl_path = os.path.join('/etc/apache/mods-enabled', ssl_file)
if not os.path.exists(ssl_path):
sys.stderr.write(
'WARNING: %s missing, http server may not start\n' %
ssl_path)
if not os.access('/var/run/apache2', os.W_OK):
sys.stderr.write('WARNING: cannot write to /var/run/apache2, '
'http server may not start\n')
apache_config_dir = google.httpd_utils.ApacheConfigDir(build_dir)
httpd_conf_path = os.path.join(apache_config_dir, HTTPD_CONF[platform])
mime_types_path = os.path.join(apache_config_dir, 'mime.types')
document_root = os.path.abspath(document_root)
start_cmd = platform_util.GetStartHttpdCommand(output_dir, httpd_conf_path,
mime_types_path,
document_root)
stop_cmd = platform_util.GetStopHttpdCommand()
http_server = google.httpd_utils.ApacheHttpd(start_cmd, stop_cmd, [8000])
try:
http_server.StartServer()
except google.httpd_utils.HttpdNotStarted, e:
raise google.httpd_utils.HttpdNotStarted('%s. See log file in %s' %
(e, output_dir))
def main_ninja(options, args):
"""Interprets options, clobbers object files, and calls ninja."""
# ninja is different from all the other build systems in that it requires
# most configuration to be done at gyp time. This is why this function does
# less than the other comparable functions in this file.
src_dir = os.path.join(
slave_utils.SlaveBaseDir(os.path.abspath(options.build_dir)), 'build',
'src')
print 'chdir to %s' % src_dir
os.chdir(src_dir)
output_dir = os.path.join('out', options.target)
command = ['ninja', '-C', output_dir]
if options.clobber:
print('Removing %s' % output_dir)
# Deleting output_dir would also delete all the .ninja files necessary to
# build. Clobbering should run before runhooks (which creates .ninja files).
# For now, only delete all non-.ninja files. TODO(thakis): Make "clobber" a
# step that runs before "runhooks". Once the master has been restarted,
# remove all clobber handling from compile.py.
def delete_objects(_, directory, files):
for f in files:
if f.endswith('.ninja') or f == 'gyp-mac-tool':
continue
f = os.path.join(directory, f)
if not os.path.isdir(f):
os.unlink(f)
os.path.walk(output_dir, delete_objects, None)
if options.verbose:
command.append('-v')
command.extend(options.build_args)
command.extend(args)
# Prepare environment.
env = EchoDict(os.environ)
if chromium_utils.IsMac() and options.disable_aslr:
# Disallow dyld to randomize the load addresses of executables.
# If any of them is compiled with ASan it will hang otherwise.
env['DYLD_NO_PIE'] = '1'
if options.compiler in ('goma', 'goma-clang'):
goma_key = os.path.join(options.goma_dir, 'goma.key')
if os.path.exists(goma_key):
env['GOMA_API_KEY_FILE'] = goma_key
if sys.platform != 'win32':
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
# If using the Goma compiler, first call goma_ctl with ensure_start
# (or restart in clobber mode) to ensure the proxy is available.
env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true'
if options.clobber:
chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env)
else:
chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'],
env=env)
else:
goma_ctl_cmd = [
sys.executable,
os.path.join(options.goma_dir, 'goma_ctl.py')
]
chromium_util.RunCommand(goma_ctl_cmd + ['start'], env=env)
# CC and CXX are set at gyp time for ninja. PATH still needs to be adjusted.
print 'using', options.compiler
if options.compiler == 'goma':
env['PATH'] = os.pathsep.join([options.goma_dir, env['PATH']])
elif options.compiler == 'goma-clang':
clang_dir = os.path.abspath(
os.path.join('third_party', 'llvm-build', 'Release+Asserts',
'bin'))
env['PATH'] = os.pathsep.join(
[options.goma_dir, clang_dir, env['PATH']])
goma_jobs = 100 if not chromium_utils.IsMac() else 50
command.append('-j%d' % goma_jobs)
if chromium_utils.IsMac() and options.clobber:
env['GOMA_USE_LOCAL'] = '0'
# Run the build.
env.print_overrides()
# TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH.
# At the moment of writing, ninja.bat in depot_tools wraps
# third_party\ninja.exe, which requires shell=True so it is found correctly.
result = chromium_utils.RunCommand(command,
env=env,
shell=sys.platform == 'win32')
if sys.platform != 'win32':
return result
else:
chromium_util.RunCommand(goma_ctl_cmd + ['stop'], env=env)
return result
def main_make(options, args):
"""Interprets options, clobbers object files, and calls make.
"""
options.build_dir = os.path.abspath(options.build_dir)
src_dir = os.path.join(slave_utils.SlaveBaseDir(options.build_dir),
'build', 'src')
# TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is
# passed to all builders. This can be taken out once the master scripts are
# updated to only pass platform-appropriate --solution values.
if options.solution and os.path.splitext(
options.solution)[1] != '.Makefile':
options.solution = None
command = ['make']
if options.solution:
command.extend(['-f', options.solution])
working_dir = options.build_dir
else:
# If no solution file (i.e. sub-project *.Makefile) is specified, try to
# build from <build_dir>/Makefile, or if that doesn't exist, from
# the top-level Makefile.
if os.path.isfile(os.path.join(options.build_dir, 'Makefile')):
working_dir = options.build_dir
else:
working_dir = src_dir
# Lots of test-execution scripts hard-code 'sconsbuild' as the output
# directory. Accomodate them.
# TODO: remove when build_dir is properly parameterized in tests.
sconsbuild = os.path.join(working_dir, 'sconsbuild')
if os.path.islink(sconsbuild):
if os.readlink(sconsbuild) != 'out':
os.remove(sconsbuild)
elif os.path.exists(sconsbuild):
dead = sconsbuild + '.dead'
if os.path.isdir(dead):
shutil.rmtree(dead)
elif os.path.isfile(dead):
os.remove(dead)
os.rename(sconsbuild, sconsbuild + '.dead')
if not os.path.lexists(sconsbuild):
os.symlink('out', sconsbuild)
os.chdir(working_dir)
env = EchoDict(os.environ)
common_make_settings(command, options, env, options.crosstool,
options.compiler)
# V=1 prints the actual executed command
if options.verbose:
command.extend(['V=1'])
command.extend(options.build_args + args)
# If using the Goma compiler, first call goma_ctl with ensure_start
# (or restart in clobber mode) to ensure the proxy is available.
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
if options.compiler in ('goma', 'goma-clang', 'jsonclang'):
goma_key = os.path.join(options.goma_dir, 'goma.key')
env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true'
if os.path.exists(goma_key):
env['GOMA_API_KEY_FILE'] = goma_key
if options.clobber:
chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env)
else:
chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'], env=env)
# Run the build.
env.print_overrides()
result = 0
def clobber(target):
build_output_dir = os.path.join(working_dir, 'out', target)
print('Removing %s' % build_output_dir)
chromium_utils.RemoveDirectory(build_output_dir)
for target in options.target.split(','):
if options.clobber:
clobber(target)
target_command = command + ['BUILDTYPE=' + target]
this_result = chromium_utils.RunCommand(target_command, env=env)
if this_result and not options.clobber:
clobber(target)
# Keep the first non-zero return code as overall result.
if this_result and not result:
result = this_result
if options.compiler in ('goma', 'goma-clang', 'jsonclang'):
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
return result
def common_make_settings(command, options, env, crosstool=None, compiler=None):
"""
Sets desirable environment variables and command-line options
that are common to the Make and SCons builds. Used on Linux
and for the mac make build.
"""
assert compiler in (None, 'clang', 'goma', 'goma-clang', 'tsan_gcc',
'jsonclang')
if options.mode == 'google_chrome' or options.mode == 'official':
env['CHROMIUM_BUILD'] = '_google_chrome'
if options.mode == 'official':
# Official builds are always Google Chrome.
env['OFFICIAL_BUILD'] = '1'
env['CHROME_BUILD_TYPE'] = '_official'
# Don't stop at the first error.
command.append('-k')
# Set jobs parallelization based on number of cores.
jobs = os.sysconf('SC_NPROCESSORS_ONLN')
# Test if we can use ccache.
ccache = ''
if chromium_utils.IsLinux():
if os.path.exists('/usr/bin/ccache'):
# The default CCACHE_DIR is $HOME/.ccache which, on some of our
# bots, is over NFS. This is intentional. Talk to thestig or
# mmoss if you have questions.
ccache = 'ccache '
# Setup crosstool environment variables.
if crosstool:
env['AR'] = crosstool + '-ar'
env['AS'] = crosstool + '-as'
env['CC'] = ccache + crosstool + '-gcc'
env['CXX'] = ccache + crosstool + '-g++'
env['LD'] = crosstool + '-ld'
env['RANLIB'] = crosstool + '-ranlib'
command.append('-j%d' % jobs)
# Don't use build-in rules.
command.append('-r')
return
if chromium_utils.IsMac() and options.disable_aslr:
# Disallow dyld to randomize the load addresses of executables.
# If any of them is compiled with ASan it will hang otherwise.
env['DYLD_NO_PIE'] = '1'
if compiler in ('goma', 'goma-clang', 'jsonclang'):
print 'using', compiler
if compiler == 'goma':
env['CC'] = 'gcc'
env['CXX'] = 'g++'
env['PATH'] = ':'.join([options.goma_dir, env['PATH']])
elif compiler == 'goma-clang':
env['CC'] = 'clang'
env['CXX'] = 'clang++'
clang_dir = os.path.abspath(
os.path.join(slave_utils.SlaveBaseDir(options.build_dir),
'build', 'src', 'third_party', 'llvm-build',
'Release+Asserts', 'bin'))
env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']])
else: # jsonclang
env['CC'] = os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
'jsonclang')
env['CXX'] = os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
'jsonclang++')
command.append('-r')
command.append('-k')
# 'jsonclang' assumes the clang binary is in the path.
clang_dir = os.path.abspath(
os.path.join(slave_utils.SlaveBaseDir(options.build_dir),
'build', 'src', 'third_party', 'llvm-build',
'Release+Asserts', 'bin'))
env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']])
command.append('CC.host=' + env['CC'])
command.append('CXX.host=' + env['CXX'])
if chromium_utils.IsMac():
# The default process limit on 10.6 is 266 (`sysctl kern.maxprocperuid`),
# and about 100 processes are used by the system. The webkit bindings
# generation scripts open a preprocessor child process, so building at
# -j100 runs into the process limit. For now, just build with -j50.
goma_jobs = 50
if options.clobber:
# Disable compiles on local machine. When the goma server-side object
# file cache is warm, this can speed up clobber builds by up to 30%.
env['GOMA_USE_LOCAL'] = '0'
else:
goma_jobs = 100
if jobs < goma_jobs:
jobs = goma_jobs
command.append('-j%d' % jobs)
return
if compiler == 'clang':
clang_dir = os.path.abspath(
os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build',
'src', 'third_party', 'llvm-build', 'Release+Asserts',
'bin'))
env['CC'] = os.path.join(clang_dir, 'clang')
env['CXX'] = os.path.join(clang_dir, 'clang++')
command.append('CC.host=' + env['CC'])
command.append('CXX.host=' + env['CXX'])
command.append('-r')
if compiler == 'tsan_gcc':
# See
# http://dev.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/gcc-tsan
# for build instructions.
tsan_base = os.path.abspath(
os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build',
'src', 'third_party', 'compiler-tsan'))
tsan_gcc_bin = os.path.abspath(
os.path.join(tsan_base, 'gcc-tsan', 'scripts'))
gcctsan_gcc_dir = os.path.abspath(
os.path.join(tsan_base, 'gcc-current'))
if not os.path.isdir(gcctsan_gcc_dir):
# Extract gcc from the tarball.
extract_gcc_sh = os.path.abspath(
os.path.join(tsan_base, 'extract_gcc.sh'))
assert (os.path.exists(extract_gcc_sh))
chromium_utils.RunCommand([extract_gcc_sh])
assert (os.path.isdir(gcctsan_gcc_dir))
env['CC'] = os.path.join(tsan_gcc_bin, 'gcc')
env['CXX'] = os.path.join(tsan_gcc_bin, 'g++')
env['LD'] = os.path.join(tsan_gcc_bin, 'ld')
# GCCTSAN_GCC_DIR and GCCTSAN_GCC_VER point to the symlinks to the current
# versions of the compiler and the instrumentation plugin created by
# extract_gcc.sh
env['GCCTSAN_GCC_DIR'] = gcctsan_gcc_dir
env['GCCTSAN_GCC_VER'] = 'current'
env['GCCTSAN_IGNORE'] = os.path.abspath(
os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build',
'src', 'tools', 'valgrind', 'tsan', 'ignores.txt'))
env['GCCTSAN_ARGS'] = (
'-DADDRESS_SANITIZER -DWTF_USE_DYNAMIC_ANNOTATIONS=1 '
'-DWTF_USE_DYNAMIC_ANNOTATIONS_NOIMPL=1')
command.append('CC=' + env['CC'])
command.append('CXX=' + env['CXX'])
command.append('LD=' + env['LD'])
command.append('-r')
command.append('-j%d' % jobs)
def real_main():
option_parser = optparse.OptionParser()
option_parser.add_option('', '--clobber', action='store_true', default=False,
help='delete the output directory before compiling')
option_parser.add_option('', '--clobber-post-fail', action='store_true',
default=False,
help='delete the output directory after compiling '
'only if it failed. Do not affect ninja.')
option_parser.add_option('', '--keep-version-file', action='store_true',
default=False,
help='do not delete the chrome_dll_version.rc file '
'before compiling (ignored if --clobber is '
'used')
option_parser.add_option('', '--target', default='Release',
help='build target (Debug or Release)')
option_parser.add_option('', '--arch', default=None,
help='target architecture (ia32, x64, ...')
option_parser.add_option('', '--solution', default=None,
help='name of solution/sub-project to build')
option_parser.add_option('', '--project', default=None,
help='name of project to build')
option_parser.add_option('', '--build-dir', default='build',
help='path to directory containing solution and in '
'which the build output will be placed')
option_parser.add_option('', '--src-dir', default=None,
help='path to the root of the source tree')
option_parser.add_option('', '--mode', default='dev',
help='build mode (dev or official) controlling '
'environment variables set during build')
option_parser.add_option('', '--build-tool', default=None,
help='specify build tool (ib, vs, scons, xcode)')
option_parser.add_option('', '--build-args', action='append', default=[],
help='arguments to pass to the build tool')
option_parser.add_option('', '--compiler', default=None,
help='specify alternative compiler (e.g. clang)')
if chromium_utils.IsWindows():
# Windows only.
option_parser.add_option('', '--no-ib', action='store_true', default=False,
help='use Visual Studio instead of IncrediBuild')
option_parser.add_option('', '--msvs_version',
help='VisualStudio version to use')
# For linux to arm cross compile.
option_parser.add_option('', '--crosstool', default=None,
help='optional path to crosstool toolset')
option_parser.add_option('', '--llvm-tsan', action='store_true',
default=False,
help='build with LLVM\'s ThreadSanitizer')
if chromium_utils.IsMac():
# Mac only.
option_parser.add_option('', '--xcode-target', default=None,
help='Target from the xcodeproj file')
option_parser.add_option('', '--goma-dir',
default=os.path.join(BUILD_DIR, 'goma'),
help='specify goma directory')
option_parser.add_option('--verbose', action='store_true')
options, args = option_parser.parse_args()
options.build_dir = os.path.abspath(options.build_dir)
if not options.src_dir:
options.src_dir = os.path.join(slave_utils.SlaveBaseDir(
os.path.abspath(options.build_dir)), 'build', 'src')
if options.build_tool is None:
if chromium_utils.IsWindows():
main = main_win
options.build_tool = 'msvs'
elif chromium_utils.IsMac():
main = main_xcode
options.build_tool = 'xcode'
elif chromium_utils.IsLinux():
# We're in the process of moving to ninja by default on Linux, see
# http://crbug.com/239257
# Builders for different branches will use either make or ninja depending
# on the release channel for a while. Until all release channels are on
# ninja, use build file mtime to figure out which build system to use.
# TODO(thakis): Just use main_ninja once the transition is complete.
make_stat, ninja_stat = 0, 0
ninja_path = os.path.join(
options.src_dir, 'out', options.target, 'build.ninja')
try:
ninja_stat = os.path.getmtime(ninja_path)
except os.error:
pass
make_path = os.path.join(options.src_dir, 'Makefile')
try:
make_stat = os.path.getmtime(make_path)
except os.error:
pass
if ninja_stat > make_stat:
main = main_ninja
options.build_tool = 'ninja'
else:
main = main_make
options.build_tool = 'make'
else:
print('Please specify --build-tool.')
return 1
else:
build_tool_map = {
'ib' : main_win,
'vs' : main_win,
'make' : main_make,
'make-android' : main_make_android,
'ninja' : main_ninja,
'scons' : main_scons,
'xcode' : main_xcode,
}
main = build_tool_map.get(options.build_tool)
if not main:
sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool))
return 2
options.target_output_dir = get_target_build_dir(options.build_tool,
options.src_dir, options.target, 'iphoneos' in args)
options.clobber = (options.clobber or
landmines_triggered(options.target_output_dir))
return main(options, args)
def main_linux(options, args):
if len(args) < 1:
raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
build_dir = os.path.normpath(os.path.abspath(options.build_dir))
slave_name = slave_utils.SlaveBuildName(build_dir)
# If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir),
# look for the test binaries there, otherwise look for the top-level build
# output.
# This assumes we never pass a build_dir which might contain build output that
# we're not trying to test. This is currently a safe assumption since we don't
# have any builders that do both sub-project and top-level builds (only
# Modules builders do sub-project builds), so they shouldn't ever have both
# 'build_dir/sconsbuild' and 'build_dir/../sconsbuild'.
outdir = None
if os.path.exists(os.path.join(build_dir, 'sconsbuild')):
outdir = 'sconsbuild'
elif os.path.exists(os.path.join(build_dir, 'out')):
outdir = 'out'
if outdir:
bin_dir = os.path.join(build_dir, outdir, options.target)
src_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir), 'build',
'src')
os.environ['CR_SOURCE_ROOT'] = src_dir
else:
if os.path.exists(os.path.join(build_dir, '..', 'sconsbuild')):
bin_dir = os.path.join(build_dir, '..', 'sconsbuild',
options.target)
else:
bin_dir = os.path.join(build_dir, '..', 'out', options.target)
# Figure out what we want for a special frame buffer directory.
special_xvfb_dir = None
if options.special_xvfb == 'auto':
fp_special_xvfb = options.factory_properties.get('special_xvfb', None)
fp_chromeos = options.factory_properties.get('chromeos', None)
if fp_special_xvfb or (
fp_special_xvfb is None and
(fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura')
or slave_utils.GypFlagIsOn(options, 'chromeos'))):
special_xvfb_dir = options.special_xvfb_dir
elif options.special_xvfb:
special_xvfb_dir = options.special_xvfb_dir
test_exe = args[0]
test_exe_path = os.path.join(bin_dir, test_exe)
if not os.path.exists(test_exe_path):
if options.factory_properties.get('succeed_on_missing_exe', False):
print '%s missing but succeed_on_missing_exe used, exiting' % (
test_exe_path)
return 0
msg = 'Unable to find %s' % test_exe_path
raise chromium_utils.PathNotFound(msg)
# Decide whether to enable the suid sandbox for Chrome.
if should_enable_sandbox(CHROME_SANDBOX_PATH):
print 'Enabling sandbox. Setting environment variable:'
print ' CHROME_DEVEL_SANDBOX="%s"' % CHROME_SANDBOX_PATH
os.environ['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH
else:
print 'Disabling sandbox. Setting environment variable:'
print ' CHROME_DEVEL_SANDBOX=""'
os.environ['CHROME_DEVEL_SANDBOX'] = ''
# Nuke anything that appears to be stale chrome items in the temporary
# directory from previous test runs (i.e.- from crashes or unittest leaks).
slave_utils.RemoveChromeTemporaryFiles()
os.environ['LD_LIBRARY_PATH'] = '%s:%s/lib:%s/lib.target' % (
bin_dir, bin_dir, bin_dir)
# Figure out what we want for a special llvmpipe directory.
if (options.llvmpipe_dir and os.path.exists(options.llvmpipe_dir)):
os.environ['LD_LIBRARY_PATH'] += ':' + options.llvmpipe_dir
if options.parallel:
command = _BuildParallelCommand(build_dir, test_exe_path, options)
elif options.run_shell_script:
command = ['bash', test_exe_path]
elif options.run_python_script:
command = [sys.executable, test_exe]
else:
command = [test_exe_path]
command.extend(args[1:])
results_tracker = None
if options.generate_json_file:
results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
if os.path.exists(options.test_output_xml):
# remove the old XML output file.
os.remove(options.test_output_xml)
try:
http_server = None
if options.document_root:
http_server = start_http_server(
'linux',
build_dir=build_dir,
test_exe_path=test_exe_path,
document_root=options.document_root)
if options.xvfb:
xvfb.StartVirtualX(slave_name,
bin_dir,
with_wm=options.factory_properties.get(
'window_manager', True),
server_dir=special_xvfb_dir)
if options.factory_properties.get('asan', False):
symbolize = os.path.abspath(
os.path.join('src', 'tools', 'valgrind', 'asan',
'asan_symbolize.py'))
pipes = [[sys.executable, symbolize], ['c++filt']]
result = _RunGTestCommand(command, pipes=pipes)
else:
result = _RunGTestCommand(command, results_tracker)
finally:
if http_server:
http_server.StopServer()
if options.xvfb:
xvfb.StopVirtualX(slave_name)
if options.generate_json_file:
_GenerateJSONForTestResults(options, results_tracker)
return result