def _PostParseCheck(options): """Perform some usage validation (after we've parsed the arguments). Args: options: The options object returned by the cli parser. """ if options.local_pkg_path and not os.path.isfile(options.local_pkg_path): cros_build_lib.Die('%s is not a file.', options.local_pkg_path) if not options.gyp_defines: gyp_env = os.getenv('GYP_DEFINES') if gyp_env is not None: options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env) logging.info('GYP_DEFINES taken from environment: %s', options.gyp_defines) if not options.gn_args: gn_env = os.getenv('GN_ARGS') if gn_env is not None: options.gn_args = gn_helpers.FromGNArgs(gn_env) logging.info('GN_ARGS taken from environment: %s', options.gn_args) if not options.staging_flags: use_env = os.getenv('USE') if use_env is not None: options.staging_flags = ' '.join( set(use_env.split()).intersection(chrome_util.STAGING_FLAGS)) logging.info('Staging flags taken from USE in environment: %s', options.staging_flags)
def _StaleGnArgs(self, gn_args, gn_args_file_path): """Returns True if args.gn needs to be updated.""" if not os.path.exists(gn_args_file_path): logging.warning('No args.gn file: %s', gn_args_file_path) return True new_gn_args = self._StripGnArgs(gn_args) old_gn_args = self._StripGnArgs( gn_helpers.FromGNArgs(osutils.ReadFile(gn_args_file_path))) if new_gn_args == old_gn_args: return False logging.warning('Stale args.gn file: %s', gn_args_file_path) self._LogArgsDiff(old_gn_args, new_gn_args) return True
def testGnArgsStalenessExtraArgs(self): """Verifies the GN extra args regenerate gn.""" with cros_test_lib.LoggingCapturer() as logs: self.SetupCommandMock( extra_args=['--gn-extra-args=dcheck_always_on=true']) self.cmd_mock.inst.Run() out_dir = 'out_%s' % SDKFetcherMock.BOARD build_label = 'Release' gn_args_file_dir = os.path.join(self.chrome_src_dir, out_dir, build_label) gn_args_file_path = os.path.join(gn_args_file_dir, 'args.gn') osutils.SafeMakedirs(gn_args_file_dir) gn_args_dict = gn_helpers.FromGNArgs(self.cmd_mock.env['GN_ARGS']) osutils.WriteFile(gn_args_file_path, gn_helpers.ToGNString(gn_args_dict)) self.cmd_mock.inst.Run() self.AssertLogsContain(logs, 'Stale args.gn file', inverted=True)
def testGnArgsStalenessIgnoreStripped(self): """Verifies the GN args ignore stripped args.""" with cros_test_lib.LoggingCapturer() as logs: self.SetupCommandMock() self.cmd_mock.inst.Run() out_dir = 'out_%s' % SDKFetcherMock.BOARD build_label = 'Release' gn_args_file_dir = os.path.join(self.chrome_src_dir, out_dir, build_label) gn_args_file_path = os.path.join(gn_args_file_dir, 'args.gn') osutils.SafeMakedirs(gn_args_file_dir) gn_args_dict = gn_helpers.FromGNArgs(self.cmd_mock.env['GN_ARGS']) # 'dcheck_always_on' should be ignored. gn_args_dict['dcheck_always_on'] = True osutils.WriteFile(gn_args_file_path, gn_helpers.ToGNString(gn_args_dict)) self.cmd_mock.inst.Run() self.AssertLogsContain(logs, 'Stale args.gn file', inverted=True)
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None, goma_port=None): """Sets environment variables to export to the SDK shell.""" if options.chroot: sysroot = os.path.join(options.chroot, 'build', board) if not os.path.isdir(sysroot) and not options.cmd: logging.warning( "Because --chroot is set, expected a sysroot to be at " "%s, but couldn't find one.", sysroot) else: sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path environment = os.path.join( sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment') if options.chroot: # Override with the environment from the chroot if available (i.e. # build_packages or emerge chromeos-chrome has been run for |board|). env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base', 'chromeos-chrome-*') env_glob = glob.glob(env_path) if len(env_glob) != 1: logging.warning( 'Multiple Chrome versions in %s. This can be resolved' ' by running "eclean-$BOARD -d packages". Using' ' environment from: %s', env_path, environment) elif not os.path.isdir(env_glob[0]): logging.warning( 'Environment path not found: %s. Using enviroment from:' ' %s.', env_path, environment) else: chroot_env_file = os.path.join(env_glob[0], 'environment.bz2') if os.path.isfile(chroot_env_file): # Log a warning here since this is new behavior that is not obvious. logging.notice('Environment fetched from: %s', chroot_env_file) # Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment. chroot_cache = os.path.join(self.options.cache_dir, COMMAND_NAME, 'chroot') osutils.SafeMakedirs(chroot_cache) environment = os.path.join(chroot_cache, 'environment_%s' % board) cros_build_lib.UncompressFile(chroot_env_file, environment) env = osutils.SourceEnvironment(environment, self.EBUILD_ENV) self._SetupTCEnvironment(sdk_ctx, options, env) # Add managed components to the PATH. env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH']) env['PATH'] = '%s:%s' % (os.path.dirname( self.sdk.gs_ctx.gsutil_bin), env['PATH']) # Export internally referenced variables. os.environ[self.sdk.SDK_BOARD_ENV] = board if self.options.sdk_path: os.environ[self.sdk.SDK_PATH_ENV] = self.options.sdk_path os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version # Export the board/version info in a more accessible way, so developers can # reference them in their chrome_sdk.bashrc files, as well as within the # chrome-sdk shell. for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]: env[var.lstrip('%')] = os.environ[var] # Export Goma information. if goma_dir: env[self.SDK_GOMA_DIR_ENV] = goma_dir env[self.SDK_GOMA_PORT_ENV] = goma_port # SYSROOT is necessary for Goma and the sysroot wrapper. env['SYSROOT'] = sysroot gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES']) gn_args = gn_helpers.FromGNArgs(env['GN_ARGS']) gyp_dict['sysroot'] = sysroot gn_args['target_sysroot'] = sysroot gyp_dict.pop('pkg-config', None) gn_args.pop('pkg_config', None) if options.clang: gyp_dict['clang'] = 1 gn_args['is_clang'] = True if options.internal: gyp_dict['branding'] = 'Chrome' gn_args['is_chrome_branded'] = True gyp_dict['buildtype'] = 'Official' gn_args['is_official_build'] = True else: gyp_dict.pop('branding', None) gn_args.pop('is_chrome_branded', None) gyp_dict.pop('buildtype', None) gn_args.pop('is_official_build', None) gyp_dict.pop('internal_gles2_conform_tests', None) gn_args.pop('internal_gles2_conform_tests', None) if options.component: gyp_dict['component'] = 'shared_library' gn_args['is_component_build'] = True if options.fastbuild: gyp_dict['fastbuild'] = 1 gyp_dict.pop('release_extra_cflags', None) # symbol_level corresponds to GYP's fastbuild (https://goo.gl/ZC4fUO). gn_args['symbol_level'] = 1 else: # Enable debug fission for GN. gn_args['use_debug_fission'] = True # For SimpleChrome, we use the binutils that comes bundled within Chrome. # We should not use the binutils from the host system. gn_args['linux_use_bundled_binutils'] = True gyp_dict['host_clang'] = 1 # Need to reset these after the env vars have been fixed by # _SetupTCEnvironment. gn_args['cros_host_is_clang'] = True gn_args['cros_target_cc'] = env['CC'] gn_args['cros_target_cxx'] = env['CXX'] gn_args['cros_target_ld'] = env['LD'] # We need to reset extra C/CXX flags to remove references to # EBUILD_CFLAGS, EBUILD_CXXFLAGS gn_args['cros_target_extra_cflags'] = env['CFLAGS'] gn_args['cros_target_extra_cxxflags'] = env['CXXFLAGS'] gn_args['cros_host_cc'] = env['CC_host'] gn_args['cros_host_cxx'] = env['CXX_host'] gn_args['cros_host_ld'] = env['LD_host'] gn_args['cros_host_ar'] = env['AR_host'] gn_args['cros_v8_snapshot_cc'] = env['CC_host'] gn_args['cros_v8_snapshot_cxx'] = env['CXX_host'] gn_args['cros_v8_snapshot_ld'] = env['LD_host'] gn_args['cros_v8_snapshot_ar'] = env['AR_host'] # No need to adjust CFLAGS and CXXFLAGS for GN since the only # adjustment made in _SetupTCEnvironment is for split debug which # is done with 'use_debug_fission'. # Enable goma if requested. if goma_dir: gyp_dict['use_goma'] = 1 gn_args['use_goma'] = True gyp_dict['gomadir'] = goma_dir gn_args['goma_dir'] = goma_dir gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080 env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict) env['GN_ARGS'] = gn_helpers.ToGNString(gn_args) # PS1 sets the command line prompt and xterm window caption. full_version = sdk_ctx.version if full_version != CUSTOM_VERSION: full_version = self.sdk.GetFullVersion(sdk_ctx.version) env['PS1'] = self._CreatePS1(self.board, full_version, chroot=options.chroot) out_dir = 'out_%s' % self.board env['builddir_name'] = out_dir env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir env['GYP_CROSSCOMPILE'] = '1' # deploy_chrome relies on the 'gn' USE flag to locate .so (and potentially # other) files. Set this by default if GYP_CHROMIUM_NO_ACTION=1. # TODO(stevenjb): Maybe figure out a better way to set this by default. if os.environ.get('GYP_CHROMIUM_NO_ACTION', '') == '1': env['USE'] = 'gn' logging.notice( 'GYP_CHROMIUM_NO_ACTION=1, setting USE="gn" for deploy_chrome.' ) return env
def ValidateGnArgs(value): """Convert GN_ARGS-formatted string to dictionary.""" return gn_helpers.FromGNArgs(value)
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None, goma_port=None): """Sets environment variables to export to the SDK shell.""" if options.chroot: sysroot = os.path.join(options.chroot, 'build', board) if not os.path.isdir(sysroot) and not options.cmd: logging.warning("Because --chroot is set, expected a sysroot to be at " "%s, but couldn't find one.", sysroot) else: sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path environment = os.path.join(sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment') if options.chroot: # Override with the environment from the chroot if available (i.e. # build_packages or emerge chromeos-chrome has been run for |board|). env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base', 'chromeos-chrome-*') env_glob = glob.glob(env_path) if len(env_glob) != 1: logging.warning('Multiple Chrome versions in %s. This can be resolved' ' by running "eclean-$BOARD -d packages". Using' ' environment from: %s', env_path, environment) elif not os.path.isdir(env_glob[0]): logging.warning('Environment path not found: %s. Using enviroment from:' ' %s.', env_path, environment) else: chroot_env_file = os.path.join(env_glob[0], 'environment.bz2') if os.path.isfile(chroot_env_file): # Log a warning here since this is new behavior that is not obvious. logging.notice('Environment fetched from: %s', chroot_env_file) # Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment. chroot_cache = os.path.join(options.cache_dir, COMMAND_NAME, 'chroot') osutils.SafeMakedirs(chroot_cache) environment = os.path.join(chroot_cache, 'environment_%s' % board) cros_build_lib.UncompressFile(chroot_env_file, environment) env = osutils.SourceEnvironment(environment, self.EBUILD_ENV) gn_args = gn_helpers.FromGNArgs(env['GN_ARGS']) self._SetupTCEnvironment(sdk_ctx, options, env, gn_args['is_clang']) # Add managed components to the PATH. env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH']) env['PATH'] = '%s:%s' % (os.path.dirname(self.sdk.gs_ctx.gsutil_bin), env['PATH']) # Export internally referenced variables. os.environ[self.sdk.SDK_BOARD_ENV] = board if options.sdk_path: os.environ[self.sdk.SDK_PATH_ENV] = options.sdk_path os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version # Add board and sdk version as gn args so that tests can bind them in # test wrappers generated at compile time. gn_args['cros_board'] = board gn_args['cros_sdk_version'] = sdk_ctx.version # Export the board/version info in a more accessible way, so developers can # reference them in their chrome_sdk.bashrc files, as well as within the # chrome-sdk shell. for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]: env[var.lstrip('%')] = os.environ[var] # Export Goma information. if goma_dir: env[self.SDK_GOMA_DIR_ENV] = goma_dir if goma_port: env[self.SDK_GOMA_PORT_ENV] = goma_port # SYSROOT is necessary for Goma and the sysroot wrapper. env['SYSROOT'] = sysroot # Deprecated options warnings. TODO(stevenjb): Eliminate these entirely # once removed from any builders. if options.component: logging.warning('--component is deprecated, ignoring') if options.fastbuild: logging.warning('--fastbuild is deprecated, ignoring') gn_args['target_sysroot'] = sysroot gn_args.pop('pkg_config', None) # pkg_config only affects the target and comes from the sysroot. # host_pkg_config is used for programs compiled for use later in the build. gn_args['host_pkg_config'] = 'pkg-config' if options.clang: gn_args['is_clang'] = True if options.internal: gn_args['is_chrome_branded'] = True gn_args['is_official_build'] = True else: gn_args.pop('is_chrome_branded', None) gn_args.pop('is_official_build', None) gn_args.pop('internal_gles2_conform_tests', None) # For SimpleChrome, we use the binutils that comes bundled within Chrome. # We should not use the binutils from the host system. gn_args['linux_use_bundled_binutils'] = True # Need to reset these after the env vars have been fixed by # _SetupTCEnvironment. gn_args['cros_host_is_clang'] = True # v8 snapshot is built on the host, so we need to set this. # See crosbug/618346. gn_args['cros_v8_snapshot_is_clang'] = True # gn_args['cros_target_cc'] = env['CC'] gn_args['cros_target_cxx'] = env['CXX'] gn_args['cros_target_ld'] = env['LD'] gn_args['cros_target_extra_cflags'] = env.get('CFLAGS', '') gn_args['cros_target_extra_cxxflags'] = env.get('CXXFLAGS', '') gn_args['cros_host_cc'] = env['CC_host'] gn_args['cros_host_cxx'] = env['CXX_host'] gn_args['cros_host_ld'] = env['LD_host'] gn_args['cros_host_ar'] = env['AR_host'] gn_args['cros_v8_snapshot_cc'] = env['CC_host'] gn_args['cros_v8_snapshot_cxx'] = env['CXX_host'] gn_args['cros_v8_snapshot_ld'] = env['LD_host'] gn_args['cros_v8_snapshot_ar'] = env['AR_host'] # No need to adjust CFLAGS and CXXFLAGS for GN since the only # adjustment made in _SetupTCEnvironment is for split debug which # is done with 'use_debug_fission'. # Enable goma if requested. if goma_dir: gn_args['use_goma'] = True gn_args['goma_dir'] = goma_dir elif not options.goma: # If --nogoma option is explicitly set, disable goma, even if it is # used in the original GN_ARGS. gn_args['use_goma'] = False gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080 # Disable ThinLTO and CFI for simplechrome. Tryjob machines do not have # enough file descriptors to use. crbug.com/789607 if 'use_thin_lto' in gn_args: gn_args['use_thin_lto'] = False if 'is_cfi' in gn_args: gn_args['is_cfi'] = False if 'use_cfi_cast' in gn_args: gn_args['use_cfi_cast'] = False # We need to remove the flag below from cros_target_extra_ldflags. # The format of ld flags is something like # '-Wl,-O1 -Wl,-O2 -Wl,--as-needed -stdlib=libc++' extra_thinlto_flag = '-Wl,-plugin-opt,-import-instr-limit=30' extra_ldflags = gn_args.get('cros_target_extra_ldflags', '') if extra_thinlto_flag in extra_ldflags: gn_args['cros_target_extra_ldflags'] = extra_ldflags.replace( extra_thinlto_flag, '') # We removed webcore debug symbols on release builds on arm. # See crbug.com/792999. However, we want to keep the symbols # for simplechrome builds. # TODO: remove the 'remove_webcore_debug_symbols' once we # change the ebuild file. gn_args['remove_webcore_debug_symbols'] = False gn_args['blink_symbol_level'] = -1 if options.gn_extra_args: gn_args.update(gn_helpers.FromGNArgs(options.gn_extra_args)) gn_args_env = gn_helpers.ToGNString(gn_args) env['GN_ARGS'] = gn_args_env # PS1 sets the command line prompt and xterm window caption. full_version = sdk_ctx.version if full_version != CUSTOM_VERSION: full_version = self.sdk.GetFullVersion(sdk_ctx.version) env['PS1'] = self._CreatePS1(self.board, full_version, chroot=options.chroot) # Set the useful part of PS1 for users with a custom PROMPT_COMMAND. env['CROS_PS1_PREFIX'] = self._PS1Prefix(self.board, full_version, chroot=options.chroot) out_dir = 'out_%s' % self.board env['builddir_name'] = out_dir build_label = 'Release' # This is used by landmines.py to prevent collisions when building both # chromeos and android from shared source. # For context, see crbug.com/407417 env['CHROMIUM_OUT_DIR'] = os.path.join(options.chrome_src, out_dir) self._UpdateGnArgsIfStale(out_dir, build_label, gn_args, env['SDK_BOARD']) return env