def start_http_server(platform, build_dir, test_exe_path, document_root): # pylint: disable=F0401 import google.httpd_utils import google.platform_utils platform_util = google.platform_utils.PlatformUtility(build_dir) # Name the output directory for the exe, without its path or suffix. # e.g., chrome-release/httpd_logs/unit_tests/ test_exe_name = os.path.splitext(os.path.basename(test_exe_path))[0] output_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir), 'httpd_logs', test_exe_name) # Sanity checks for httpd2_linux.conf. if platform == 'linux': for ssl_file in ['ssl.conf', 'ssl.load']: ssl_path = os.path.join('/etc/apache/mods-enabled', ssl_file) if not os.path.exists(ssl_path): sys.stderr.write( 'WARNING: %s missing, http server may not start\n' % ssl_path) if not os.access('/var/run/apache2', os.W_OK): sys.stderr.write('WARNING: cannot write to /var/run/apache2, ' 'http server may not start\n') apache_config_dir = google.httpd_utils.ApacheConfigDir(build_dir) httpd_conf_path = os.path.join(apache_config_dir, HTTPD_CONF[platform]) mime_types_path = os.path.join(apache_config_dir, 'mime.types') document_root = os.path.abspath(document_root) start_cmd = platform_util.GetStartHttpdCommand(output_dir, httpd_conf_path, mime_types_path, document_root) stop_cmd = platform_util.GetStopHttpdCommand() http_server = google.httpd_utils.ApacheHttpd(start_cmd, stop_cmd, [8000]) try: http_server.StartServer() except google.httpd_utils.HttpdNotStarted, e: raise google.httpd_utils.HttpdNotStarted('%s. See log file in %s' % (e, output_dir))
def main_ninja(options, args): """Interprets options, clobbers object files, and calls ninja.""" # ninja is different from all the other build systems in that it requires # most configuration to be done at gyp time. This is why this function does # less than the other comparable functions in this file. src_dir = os.path.join( slave_utils.SlaveBaseDir(os.path.abspath(options.build_dir)), 'build', 'src') print 'chdir to %s' % src_dir os.chdir(src_dir) output_dir = os.path.join('out', options.target) command = ['ninja', '-C', output_dir] if options.clobber: print('Removing %s' % output_dir) # Deleting output_dir would also delete all the .ninja files necessary to # build. Clobbering should run before runhooks (which creates .ninja files). # For now, only delete all non-.ninja files. TODO(thakis): Make "clobber" a # step that runs before "runhooks". Once the master has been restarted, # remove all clobber handling from compile.py. def delete_objects(_, directory, files): for f in files: if f.endswith('.ninja') or f == 'gyp-mac-tool': continue f = os.path.join(directory, f) if not os.path.isdir(f): os.unlink(f) os.path.walk(output_dir, delete_objects, None) if options.verbose: command.append('-v') command.extend(options.build_args) command.extend(args) # Prepare environment. env = EchoDict(os.environ) if chromium_utils.IsMac() and options.disable_aslr: # Disallow dyld to randomize the load addresses of executables. # If any of them is compiled with ASan it will hang otherwise. env['DYLD_NO_PIE'] = '1' if options.compiler in ('goma', 'goma-clang'): goma_key = os.path.join(options.goma_dir, 'goma.key') if os.path.exists(goma_key): env['GOMA_API_KEY_FILE'] = goma_key if sys.platform != 'win32': goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] # If using the Goma compiler, first call goma_ctl with ensure_start # (or restart in clobber mode) to ensure the proxy is available. env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true' if options.clobber: chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env) else: chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'], env=env) else: goma_ctl_cmd = [ sys.executable, os.path.join(options.goma_dir, 'goma_ctl.py') ] chromium_util.RunCommand(goma_ctl_cmd + ['start'], env=env) # CC and CXX are set at gyp time for ninja. PATH still needs to be adjusted. print 'using', options.compiler if options.compiler == 'goma': env['PATH'] = os.pathsep.join([options.goma_dir, env['PATH']]) elif options.compiler == 'goma-clang': clang_dir = os.path.abspath( os.path.join('third_party', 'llvm-build', 'Release+Asserts', 'bin')) env['PATH'] = os.pathsep.join( [options.goma_dir, clang_dir, env['PATH']]) goma_jobs = 100 if not chromium_utils.IsMac() else 50 command.append('-j%d' % goma_jobs) if chromium_utils.IsMac() and options.clobber: env['GOMA_USE_LOCAL'] = '0' # Run the build. env.print_overrides() # TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH. # At the moment of writing, ninja.bat in depot_tools wraps # third_party\ninja.exe, which requires shell=True so it is found correctly. result = chromium_utils.RunCommand(command, env=env, shell=sys.platform == 'win32') if sys.platform != 'win32': return result else: chromium_util.RunCommand(goma_ctl_cmd + ['stop'], env=env) return result
def main_make(options, args): """Interprets options, clobbers object files, and calls make. """ options.build_dir = os.path.abspath(options.build_dir) src_dir = os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src') # TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is # passed to all builders. This can be taken out once the master scripts are # updated to only pass platform-appropriate --solution values. if options.solution and os.path.splitext( options.solution)[1] != '.Makefile': options.solution = None command = ['make'] if options.solution: command.extend(['-f', options.solution]) working_dir = options.build_dir else: # If no solution file (i.e. sub-project *.Makefile) is specified, try to # build from <build_dir>/Makefile, or if that doesn't exist, from # the top-level Makefile. if os.path.isfile(os.path.join(options.build_dir, 'Makefile')): working_dir = options.build_dir else: working_dir = src_dir # Lots of test-execution scripts hard-code 'sconsbuild' as the output # directory. Accomodate them. # TODO: remove when build_dir is properly parameterized in tests. sconsbuild = os.path.join(working_dir, 'sconsbuild') if os.path.islink(sconsbuild): if os.readlink(sconsbuild) != 'out': os.remove(sconsbuild) elif os.path.exists(sconsbuild): dead = sconsbuild + '.dead' if os.path.isdir(dead): shutil.rmtree(dead) elif os.path.isfile(dead): os.remove(dead) os.rename(sconsbuild, sconsbuild + '.dead') if not os.path.lexists(sconsbuild): os.symlink('out', sconsbuild) os.chdir(working_dir) env = EchoDict(os.environ) common_make_settings(command, options, env, options.crosstool, options.compiler) # V=1 prints the actual executed command if options.verbose: command.extend(['V=1']) command.extend(options.build_args + args) # If using the Goma compiler, first call goma_ctl with ensure_start # (or restart in clobber mode) to ensure the proxy is available. goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] if options.compiler in ('goma', 'goma-clang', 'jsonclang'): goma_key = os.path.join(options.goma_dir, 'goma.key') env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true' if os.path.exists(goma_key): env['GOMA_API_KEY_FILE'] = goma_key if options.clobber: chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env) else: chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'], env=env) # Run the build. env.print_overrides() result = 0 def clobber(target): build_output_dir = os.path.join(working_dir, 'out', target) print('Removing %s' % build_output_dir) chromium_utils.RemoveDirectory(build_output_dir) for target in options.target.split(','): if options.clobber: clobber(target) target_command = command + ['BUILDTYPE=' + target] this_result = chromium_utils.RunCommand(target_command, env=env) if this_result and not options.clobber: clobber(target) # Keep the first non-zero return code as overall result. if this_result and not result: result = this_result if options.compiler in ('goma', 'goma-clang', 'jsonclang'): # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) return result
def common_make_settings(command, options, env, crosstool=None, compiler=None): """ Sets desirable environment variables and command-line options that are common to the Make and SCons builds. Used on Linux and for the mac make build. """ assert compiler in (None, 'clang', 'goma', 'goma-clang', 'tsan_gcc', 'jsonclang') if options.mode == 'google_chrome' or options.mode == 'official': env['CHROMIUM_BUILD'] = '_google_chrome' if options.mode == 'official': # Official builds are always Google Chrome. env['OFFICIAL_BUILD'] = '1' env['CHROME_BUILD_TYPE'] = '_official' # Don't stop at the first error. command.append('-k') # Set jobs parallelization based on number of cores. jobs = os.sysconf('SC_NPROCESSORS_ONLN') # Test if we can use ccache. ccache = '' if chromium_utils.IsLinux(): if os.path.exists('/usr/bin/ccache'): # The default CCACHE_DIR is $HOME/.ccache which, on some of our # bots, is over NFS. This is intentional. Talk to thestig or # mmoss if you have questions. ccache = 'ccache ' # Setup crosstool environment variables. if crosstool: env['AR'] = crosstool + '-ar' env['AS'] = crosstool + '-as' env['CC'] = ccache + crosstool + '-gcc' env['CXX'] = ccache + crosstool + '-g++' env['LD'] = crosstool + '-ld' env['RANLIB'] = crosstool + '-ranlib' command.append('-j%d' % jobs) # Don't use build-in rules. command.append('-r') return if chromium_utils.IsMac() and options.disable_aslr: # Disallow dyld to randomize the load addresses of executables. # If any of them is compiled with ASan it will hang otherwise. env['DYLD_NO_PIE'] = '1' if compiler in ('goma', 'goma-clang', 'jsonclang'): print 'using', compiler if compiler == 'goma': env['CC'] = 'gcc' env['CXX'] = 'g++' env['PATH'] = ':'.join([options.goma_dir, env['PATH']]) elif compiler == 'goma-clang': env['CC'] = 'clang' env['CXX'] = 'clang++' clang_dir = os.path.abspath( os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src', 'third_party', 'llvm-build', 'Release+Asserts', 'bin')) env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) else: # jsonclang env['CC'] = os.path.join(SLAVE_SCRIPTS_DIR, 'chromium', 'jsonclang') env['CXX'] = os.path.join(SLAVE_SCRIPTS_DIR, 'chromium', 'jsonclang++') command.append('-r') command.append('-k') # 'jsonclang' assumes the clang binary is in the path. clang_dir = os.path.abspath( os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src', 'third_party', 'llvm-build', 'Release+Asserts', 'bin')) env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) command.append('CC.host=' + env['CC']) command.append('CXX.host=' + env['CXX']) if chromium_utils.IsMac(): # The default process limit on 10.6 is 266 (`sysctl kern.maxprocperuid`), # and about 100 processes are used by the system. The webkit bindings # generation scripts open a preprocessor child process, so building at # -j100 runs into the process limit. For now, just build with -j50. goma_jobs = 50 if options.clobber: # Disable compiles on local machine. When the goma server-side object # file cache is warm, this can speed up clobber builds by up to 30%. env['GOMA_USE_LOCAL'] = '0' else: goma_jobs = 100 if jobs < goma_jobs: jobs = goma_jobs command.append('-j%d' % jobs) return if compiler == 'clang': clang_dir = os.path.abspath( os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src', 'third_party', 'llvm-build', 'Release+Asserts', 'bin')) env['CC'] = os.path.join(clang_dir, 'clang') env['CXX'] = os.path.join(clang_dir, 'clang++') command.append('CC.host=' + env['CC']) command.append('CXX.host=' + env['CXX']) command.append('-r') if compiler == 'tsan_gcc': # See # http://dev.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/gcc-tsan # for build instructions. tsan_base = os.path.abspath( os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src', 'third_party', 'compiler-tsan')) tsan_gcc_bin = os.path.abspath( os.path.join(tsan_base, 'gcc-tsan', 'scripts')) gcctsan_gcc_dir = os.path.abspath( os.path.join(tsan_base, 'gcc-current')) if not os.path.isdir(gcctsan_gcc_dir): # Extract gcc from the tarball. extract_gcc_sh = os.path.abspath( os.path.join(tsan_base, 'extract_gcc.sh')) assert (os.path.exists(extract_gcc_sh)) chromium_utils.RunCommand([extract_gcc_sh]) assert (os.path.isdir(gcctsan_gcc_dir)) env['CC'] = os.path.join(tsan_gcc_bin, 'gcc') env['CXX'] = os.path.join(tsan_gcc_bin, 'g++') env['LD'] = os.path.join(tsan_gcc_bin, 'ld') # GCCTSAN_GCC_DIR and GCCTSAN_GCC_VER point to the symlinks to the current # versions of the compiler and the instrumentation plugin created by # extract_gcc.sh env['GCCTSAN_GCC_DIR'] = gcctsan_gcc_dir env['GCCTSAN_GCC_VER'] = 'current' env['GCCTSAN_IGNORE'] = os.path.abspath( os.path.join(slave_utils.SlaveBaseDir(options.build_dir), 'build', 'src', 'tools', 'valgrind', 'tsan', 'ignores.txt')) env['GCCTSAN_ARGS'] = ( '-DADDRESS_SANITIZER -DWTF_USE_DYNAMIC_ANNOTATIONS=1 ' '-DWTF_USE_DYNAMIC_ANNOTATIONS_NOIMPL=1') command.append('CC=' + env['CC']) command.append('CXX=' + env['CXX']) command.append('LD=' + env['LD']) command.append('-r') command.append('-j%d' % jobs)
def real_main(): option_parser = optparse.OptionParser() option_parser.add_option('', '--clobber', action='store_true', default=False, help='delete the output directory before compiling') option_parser.add_option('', '--clobber-post-fail', action='store_true', default=False, help='delete the output directory after compiling ' 'only if it failed. Do not affect ninja.') option_parser.add_option('', '--keep-version-file', action='store_true', default=False, help='do not delete the chrome_dll_version.rc file ' 'before compiling (ignored if --clobber is ' 'used') option_parser.add_option('', '--target', default='Release', help='build target (Debug or Release)') option_parser.add_option('', '--arch', default=None, help='target architecture (ia32, x64, ...') option_parser.add_option('', '--solution', default=None, help='name of solution/sub-project to build') option_parser.add_option('', '--project', default=None, help='name of project to build') option_parser.add_option('', '--build-dir', default='build', help='path to directory containing solution and in ' 'which the build output will be placed') option_parser.add_option('', '--src-dir', default=None, help='path to the root of the source tree') option_parser.add_option('', '--mode', default='dev', help='build mode (dev or official) controlling ' 'environment variables set during build') option_parser.add_option('', '--build-tool', default=None, help='specify build tool (ib, vs, scons, xcode)') option_parser.add_option('', '--build-args', action='append', default=[], help='arguments to pass to the build tool') option_parser.add_option('', '--compiler', default=None, help='specify alternative compiler (e.g. clang)') if chromium_utils.IsWindows(): # Windows only. option_parser.add_option('', '--no-ib', action='store_true', default=False, help='use Visual Studio instead of IncrediBuild') option_parser.add_option('', '--msvs_version', help='VisualStudio version to use') # For linux to arm cross compile. option_parser.add_option('', '--crosstool', default=None, help='optional path to crosstool toolset') option_parser.add_option('', '--llvm-tsan', action='store_true', default=False, help='build with LLVM\'s ThreadSanitizer') if chromium_utils.IsMac(): # Mac only. option_parser.add_option('', '--xcode-target', default=None, help='Target from the xcodeproj file') option_parser.add_option('', '--goma-dir', default=os.path.join(BUILD_DIR, 'goma'), help='specify goma directory') option_parser.add_option('--verbose', action='store_true') options, args = option_parser.parse_args() options.build_dir = os.path.abspath(options.build_dir) if not options.src_dir: options.src_dir = os.path.join(slave_utils.SlaveBaseDir( os.path.abspath(options.build_dir)), 'build', 'src') if options.build_tool is None: if chromium_utils.IsWindows(): main = main_win options.build_tool = 'msvs' elif chromium_utils.IsMac(): main = main_xcode options.build_tool = 'xcode' elif chromium_utils.IsLinux(): # We're in the process of moving to ninja by default on Linux, see # http://crbug.com/239257 # Builders for different branches will use either make or ninja depending # on the release channel for a while. Until all release channels are on # ninja, use build file mtime to figure out which build system to use. # TODO(thakis): Just use main_ninja once the transition is complete. make_stat, ninja_stat = 0, 0 ninja_path = os.path.join( options.src_dir, 'out', options.target, 'build.ninja') try: ninja_stat = os.path.getmtime(ninja_path) except os.error: pass make_path = os.path.join(options.src_dir, 'Makefile') try: make_stat = os.path.getmtime(make_path) except os.error: pass if ninja_stat > make_stat: main = main_ninja options.build_tool = 'ninja' else: main = main_make options.build_tool = 'make' else: print('Please specify --build-tool.') return 1 else: build_tool_map = { 'ib' : main_win, 'vs' : main_win, 'make' : main_make, 'make-android' : main_make_android, 'ninja' : main_ninja, 'scons' : main_scons, 'xcode' : main_xcode, } main = build_tool_map.get(options.build_tool) if not main: sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool)) return 2 options.target_output_dir = get_target_build_dir(options.build_tool, options.src_dir, options.target, 'iphoneos' in args) options.clobber = (options.clobber or landmines_triggered(options.target_output_dir)) return main(options, args)
def main_linux(options, args): if len(args) < 1: raise chromium_utils.MissingArgument('Usage: %s' % USAGE) build_dir = os.path.normpath(os.path.abspath(options.build_dir)) slave_name = slave_utils.SlaveBuildName(build_dir) # If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir), # look for the test binaries there, otherwise look for the top-level build # output. # This assumes we never pass a build_dir which might contain build output that # we're not trying to test. This is currently a safe assumption since we don't # have any builders that do both sub-project and top-level builds (only # Modules builders do sub-project builds), so they shouldn't ever have both # 'build_dir/sconsbuild' and 'build_dir/../sconsbuild'. outdir = None if os.path.exists(os.path.join(build_dir, 'sconsbuild')): outdir = 'sconsbuild' elif os.path.exists(os.path.join(build_dir, 'out')): outdir = 'out' if outdir: bin_dir = os.path.join(build_dir, outdir, options.target) src_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir), 'build', 'src') os.environ['CR_SOURCE_ROOT'] = src_dir else: if os.path.exists(os.path.join(build_dir, '..', 'sconsbuild')): bin_dir = os.path.join(build_dir, '..', 'sconsbuild', options.target) else: bin_dir = os.path.join(build_dir, '..', 'out', options.target) # Figure out what we want for a special frame buffer directory. special_xvfb_dir = None if options.special_xvfb == 'auto': fp_special_xvfb = options.factory_properties.get('special_xvfb', None) fp_chromeos = options.factory_properties.get('chromeos', None) if fp_special_xvfb or ( fp_special_xvfb is None and (fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura') or slave_utils.GypFlagIsOn(options, 'chromeos'))): special_xvfb_dir = options.special_xvfb_dir elif options.special_xvfb: special_xvfb_dir = options.special_xvfb_dir test_exe = args[0] test_exe_path = os.path.join(bin_dir, test_exe) if not os.path.exists(test_exe_path): if options.factory_properties.get('succeed_on_missing_exe', False): print '%s missing but succeed_on_missing_exe used, exiting' % ( test_exe_path) return 0 msg = 'Unable to find %s' % test_exe_path raise chromium_utils.PathNotFound(msg) # Decide whether to enable the suid sandbox for Chrome. if should_enable_sandbox(CHROME_SANDBOX_PATH): print 'Enabling sandbox. Setting environment variable:' print ' CHROME_DEVEL_SANDBOX="%s"' % CHROME_SANDBOX_PATH os.environ['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH else: print 'Disabling sandbox. Setting environment variable:' print ' CHROME_DEVEL_SANDBOX=""' os.environ['CHROME_DEVEL_SANDBOX'] = '' # Nuke anything that appears to be stale chrome items in the temporary # directory from previous test runs (i.e.- from crashes or unittest leaks). slave_utils.RemoveChromeTemporaryFiles() os.environ['LD_LIBRARY_PATH'] = '%s:%s/lib:%s/lib.target' % ( bin_dir, bin_dir, bin_dir) # Figure out what we want for a special llvmpipe directory. if (options.llvmpipe_dir and os.path.exists(options.llvmpipe_dir)): os.environ['LD_LIBRARY_PATH'] += ':' + options.llvmpipe_dir if options.parallel: command = _BuildParallelCommand(build_dir, test_exe_path, options) elif options.run_shell_script: command = ['bash', test_exe_path] elif options.run_python_script: command = [sys.executable, test_exe] else: command = [test_exe_path] command.extend(args[1:]) results_tracker = None if options.generate_json_file: results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker() if os.path.exists(options.test_output_xml): # remove the old XML output file. os.remove(options.test_output_xml) try: http_server = None if options.document_root: http_server = start_http_server( 'linux', build_dir=build_dir, test_exe_path=test_exe_path, document_root=options.document_root) if options.xvfb: xvfb.StartVirtualX(slave_name, bin_dir, with_wm=options.factory_properties.get( 'window_manager', True), server_dir=special_xvfb_dir) if options.factory_properties.get('asan', False): symbolize = os.path.abspath( os.path.join('src', 'tools', 'valgrind', 'asan', 'asan_symbolize.py')) pipes = [[sys.executable, symbolize], ['c++filt']] result = _RunGTestCommand(command, pipes=pipes) else: result = _RunGTestCommand(command, results_tracker) finally: if http_server: http_server.StopServer() if options.xvfb: xvfb.StopVirtualX(slave_name) if options.generate_json_file: _GenerateJSONForTestResults(options, results_tracker) return result