def copy_profile(gs_path, local_path): assert local_path.endswith('.afdo'), local_path assert not gs_path.endswith('.afdo'), gs_path compression_suffix = os.path.splitext(gs_path)[1] temp_path = local_path + compression_suffix gs_context.Copy(gs_path, temp_path) cros_build_lib.UncompressFile(temp_path, local_path)
def UncompressAFDOFile(to_decompress, buildroot): """Decompress file used by AFDO process. Args: to_decompress: File to decompress. buildroot: buildroot where to store the decompressed data. """ local_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot} basename = os.path.basename(to_decompress) dest_basename = basename.rsplit('.', 1)[0] dest = os.path.join(local_dir, dest_basename) cros_build_lib.UncompressFile(to_decompress, dest) return dest
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None, goma_port=None): """Sets environment variables to export to the SDK shell.""" if options.chroot: sysroot = os.path.join(options.chroot, 'build', board) if not os.path.isdir(sysroot) and not options.cmd: logging.warning( "Because --chroot is set, expected a sysroot to be at " "%s, but couldn't find one.", sysroot) else: sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path environment = os.path.join( sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment') if options.chroot: # Override with the environment from the chroot if available (i.e. # build_packages or emerge chromeos-chrome has been run for |board|). env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base', 'chromeos-chrome-*') env_glob = glob.glob(env_path) if len(env_glob) != 1: logging.warning( 'Multiple Chrome versions in %s. This can be resolved' ' by running "eclean-$BOARD -d packages". Using' ' environment from: %s', env_path, environment) elif not os.path.isdir(env_glob[0]): logging.warning( 'Environment path not found: %s. Using enviroment from:' ' %s.', env_path, environment) else: chroot_env_file = os.path.join(env_glob[0], 'environment.bz2') if os.path.isfile(chroot_env_file): # Log a warning here since this is new behavior that is not obvious. logging.notice('Environment fetched from: %s', chroot_env_file) # Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment. chroot_cache = os.path.join(self.options.cache_dir, COMMAND_NAME, 'chroot') osutils.SafeMakedirs(chroot_cache) environment = os.path.join(chroot_cache, 'environment_%s' % board) cros_build_lib.UncompressFile(chroot_env_file, environment) env = osutils.SourceEnvironment(environment, self.EBUILD_ENV) self._SetupTCEnvironment(sdk_ctx, options, env) # Add managed components to the PATH. env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH']) env['PATH'] = '%s:%s' % (os.path.dirname( self.sdk.gs_ctx.gsutil_bin), env['PATH']) # Export internally referenced variables. os.environ[self.sdk.SDK_BOARD_ENV] = board if self.options.sdk_path: os.environ[self.sdk.SDK_PATH_ENV] = self.options.sdk_path os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version # Export the board/version info in a more accessible way, so developers can # reference them in their chrome_sdk.bashrc files, as well as within the # chrome-sdk shell. for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]: env[var.lstrip('%')] = os.environ[var] # Export Goma information. if goma_dir: env[self.SDK_GOMA_DIR_ENV] = goma_dir env[self.SDK_GOMA_PORT_ENV] = goma_port # SYSROOT is necessary for Goma and the sysroot wrapper. env['SYSROOT'] = sysroot gyp_dict = chrome_util.ProcessGypDefines(env['GYP_DEFINES']) gn_args = gn_helpers.FromGNArgs(env['GN_ARGS']) gyp_dict['sysroot'] = sysroot gn_args['target_sysroot'] = sysroot gyp_dict.pop('pkg-config', None) gn_args.pop('pkg_config', None) if options.clang: gyp_dict['clang'] = 1 gn_args['is_clang'] = True if options.internal: gyp_dict['branding'] = 'Chrome' gn_args['is_chrome_branded'] = True gyp_dict['buildtype'] = 'Official' gn_args['is_official_build'] = True else: gyp_dict.pop('branding', None) gn_args.pop('is_chrome_branded', None) gyp_dict.pop('buildtype', None) gn_args.pop('is_official_build', None) gyp_dict.pop('internal_gles2_conform_tests', None) gn_args.pop('internal_gles2_conform_tests', None) if options.component: gyp_dict['component'] = 'shared_library' gn_args['is_component_build'] = True if options.fastbuild: gyp_dict['fastbuild'] = 1 gyp_dict.pop('release_extra_cflags', None) # symbol_level corresponds to GYP's fastbuild (https://goo.gl/ZC4fUO). gn_args['symbol_level'] = 1 else: # Enable debug fission for GN. gn_args['use_debug_fission'] = True # For SimpleChrome, we use the binutils that comes bundled within Chrome. # We should not use the binutils from the host system. gn_args['linux_use_bundled_binutils'] = True gyp_dict['host_clang'] = 1 # Need to reset these after the env vars have been fixed by # _SetupTCEnvironment. gn_args['cros_host_is_clang'] = True gn_args['cros_target_cc'] = env['CC'] gn_args['cros_target_cxx'] = env['CXX'] gn_args['cros_target_ld'] = env['LD'] # We need to reset extra C/CXX flags to remove references to # EBUILD_CFLAGS, EBUILD_CXXFLAGS gn_args['cros_target_extra_cflags'] = env['CFLAGS'] gn_args['cros_target_extra_cxxflags'] = env['CXXFLAGS'] gn_args['cros_host_cc'] = env['CC_host'] gn_args['cros_host_cxx'] = env['CXX_host'] gn_args['cros_host_ld'] = env['LD_host'] gn_args['cros_host_ar'] = env['AR_host'] gn_args['cros_v8_snapshot_cc'] = env['CC_host'] gn_args['cros_v8_snapshot_cxx'] = env['CXX_host'] gn_args['cros_v8_snapshot_ld'] = env['LD_host'] gn_args['cros_v8_snapshot_ar'] = env['AR_host'] # No need to adjust CFLAGS and CXXFLAGS for GN since the only # adjustment made in _SetupTCEnvironment is for split debug which # is done with 'use_debug_fission'. # Enable goma if requested. if goma_dir: gyp_dict['use_goma'] = 1 gn_args['use_goma'] = True gyp_dict['gomadir'] = goma_dir gn_args['goma_dir'] = goma_dir gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080 env['GYP_DEFINES'] = chrome_util.DictToGypDefines(gyp_dict) env['GN_ARGS'] = gn_helpers.ToGNString(gn_args) # PS1 sets the command line prompt and xterm window caption. full_version = sdk_ctx.version if full_version != CUSTOM_VERSION: full_version = self.sdk.GetFullVersion(sdk_ctx.version) env['PS1'] = self._CreatePS1(self.board, full_version, chroot=options.chroot) out_dir = 'out_%s' % self.board env['builddir_name'] = out_dir env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir env['GYP_CROSSCOMPILE'] = '1' # deploy_chrome relies on the 'gn' USE flag to locate .so (and potentially # other) files. Set this by default if GYP_CHROMIUM_NO_ACTION=1. # TODO(stevenjb): Maybe figure out a better way to set this by default. if os.environ.get('GYP_CHROMIUM_NO_ACTION', '') == '1': env['USE'] = 'gn' logging.notice( 'GYP_CHROMIUM_NO_ACTION=1, setting USE="gn" for deploy_chrome.' ) return env
def CreateAndUploadMergedAFDOProfile(gs_context, buildroot, unmerged_name, recent_to_merge=5, max_age_days=14): """Create a merged AFDO profile from recent AFDO profiles and upload it. If the upload would overwrite an existing merged file, this skips the upload. Args: gs_context: GS Context buildroot: The build root unmerged_name: name of the AFDO profile we've just uploaded. No profiles whose names are lexicographically ordered after this are candidates for selection. recent_to_merge: The maximum number of profiles to merge max_age_days: Don't merge profiles older than max_age_days days old. Returns: A (str, bool) of: - The name of a merged profile in GSURL_BASE_BENCH if the AFDO profile is a candidate for merging. Otherwise, None. - Whether we uploaded a merged profile. """ _, work_dir, chroot_work_dir = _BuildrootToWorkDirs(buildroot) profile_suffix = AFDO_SUFFIX + COMPRESSION_SUFFIX merged_suffix = '-merged' glob_url = os.path.join(GSURL_BASE_BENCH, '*' + profile_suffix) benchmark_listing = gs_context.List(glob_url, details=True) unmerged_version = _ParseBenchmarkProfileName(unmerged_name) def get_ordered_mergeable_profiles(benchmark_listing): """Returns a list of mergeable profiles ordered by increasing version.""" profile_versions = [ (_ParseBenchmarkProfileName(os.path.basename(x.url)), x) for x in benchmark_listing ] # Exclude merged profiles, because merging merged profiles into merged # profiles is likely bad. candidates = [(version, x) for version, x in profile_versions if unmerged_version >= version and not version.is_merged] candidates.sort() return [x for _, x in candidates] benchmark_profiles = get_ordered_mergeable_profiles(benchmark_listing) if not benchmark_profiles: logging.warning( 'Skipping merged profile creation: no merge candidates ' 'found') return None, False base_time = benchmark_profiles[-1].creation_time time_cutoff = base_time - datetime.timedelta(days=max_age_days) merge_candidates = [ p for p in benchmark_profiles if p.creation_time >= time_cutoff ] merge_candidates = merge_candidates[-recent_to_merge:] # This should never happen, but be sure we're not merging a profile into # itself anyway. It's really easy for that to silently slip through, and can # lead to overrepresentation of a single profile, which just causes more # noise. assert len(set(p.url for p in merge_candidates)) == len(merge_candidates) # Merging a profile into itself is pointless. if len(merge_candidates) == 1: logging.warning( 'Skipping merged profile creation: we only have a single ' 'merge candidate.') return None, False chroot_afdo_files = [] for candidate in merge_candidates: # It would be slightly less complex to just name these off as # profile-1.afdo, profile-2.afdo, ... but the logs are more readable if we # keep the basename from gs://. candidate_name = os.path.basename(candidate.url) candidate_uncompressed_name = candidate_name[:-len(COMPRESSION_SUFFIX)] copy_from = candidate.url copy_to = os.path.join(work_dir, candidate_name) copy_to_uncompressed = os.path.join(work_dir, candidate_uncompressed_name) chroot_file = os.path.join(chroot_work_dir, candidate_uncompressed_name) gs_context.Copy(copy_from, copy_to) cros_build_lib.UncompressFile(copy_to, copy_to_uncompressed) chroot_afdo_files.append(chroot_file) afdo_basename = os.path.basename(chroot_afdo_files[-1]) assert afdo_basename.endswith(AFDO_SUFFIX) afdo_basename = afdo_basename[:-len(AFDO_SUFFIX)] raw_merged_basename = 'raw-' + afdo_basename + merged_suffix + AFDO_SUFFIX chroot_raw_merged_output_path = os.path.join(chroot_work_dir, raw_merged_basename) # Weight all profiles equally. _MergeAFDOProfiles([(profile, 1) for profile in chroot_afdo_files], chroot_raw_merged_output_path) profile_to_upload_basename = afdo_basename + merged_suffix + AFDO_SUFFIX profile_to_upload_path = os.path.join(work_dir, profile_to_upload_basename) chroot_profile_to_upload_path = os.path.join(chroot_work_dir, profile_to_upload_basename) _RemoveIndirectCallTargetsFromProfile(chroot_raw_merged_output_path, chroot_profile_to_upload_path) result_basename = os.path.basename(profile_to_upload_path) return result_basename, _CompressAndUploadAFDOProfileIfNotPresent( gs_context, buildroot, GSURL_BASE_BENCH, profile_to_upload_path)
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None, goma_port=None): """Sets environment variables to export to the SDK shell.""" if options.chroot: sysroot = os.path.join(options.chroot, 'build', board) if not os.path.isdir(sysroot) and not options.cmd: logging.warning("Because --chroot is set, expected a sysroot to be at " "%s, but couldn't find one.", sysroot) else: sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path environment = os.path.join(sdk_ctx.key_map[constants.CHROME_ENV_TAR].path, 'environment') if options.chroot: # Override with the environment from the chroot if available (i.e. # build_packages or emerge chromeos-chrome has been run for |board|). env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base', 'chromeos-chrome-*') env_glob = glob.glob(env_path) if len(env_glob) != 1: logging.warning('Multiple Chrome versions in %s. This can be resolved' ' by running "eclean-$BOARD -d packages". Using' ' environment from: %s', env_path, environment) elif not os.path.isdir(env_glob[0]): logging.warning('Environment path not found: %s. Using enviroment from:' ' %s.', env_path, environment) else: chroot_env_file = os.path.join(env_glob[0], 'environment.bz2') if os.path.isfile(chroot_env_file): # Log a warning here since this is new behavior that is not obvious. logging.notice('Environment fetched from: %s', chroot_env_file) # Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment. chroot_cache = os.path.join(options.cache_dir, COMMAND_NAME, 'chroot') osutils.SafeMakedirs(chroot_cache) environment = os.path.join(chroot_cache, 'environment_%s' % board) cros_build_lib.UncompressFile(chroot_env_file, environment) env = osutils.SourceEnvironment(environment, self.EBUILD_ENV) gn_args = gn_helpers.FromGNArgs(env['GN_ARGS']) self._SetupTCEnvironment(sdk_ctx, options, env, gn_args['is_clang']) # Add managed components to the PATH. env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH']) env['PATH'] = '%s:%s' % (os.path.dirname(self.sdk.gs_ctx.gsutil_bin), env['PATH']) # Export internally referenced variables. os.environ[self.sdk.SDK_BOARD_ENV] = board if options.sdk_path: os.environ[self.sdk.SDK_PATH_ENV] = options.sdk_path os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version # Add board and sdk version as gn args so that tests can bind them in # test wrappers generated at compile time. gn_args['cros_board'] = board gn_args['cros_sdk_version'] = sdk_ctx.version # Export the board/version info in a more accessible way, so developers can # reference them in their chrome_sdk.bashrc files, as well as within the # chrome-sdk shell. for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]: env[var.lstrip('%')] = os.environ[var] # Export Goma information. if goma_dir: env[self.SDK_GOMA_DIR_ENV] = goma_dir if goma_port: env[self.SDK_GOMA_PORT_ENV] = goma_port # SYSROOT is necessary for Goma and the sysroot wrapper. env['SYSROOT'] = sysroot # Deprecated options warnings. TODO(stevenjb): Eliminate these entirely # once removed from any builders. if options.component: logging.warning('--component is deprecated, ignoring') if options.fastbuild: logging.warning('--fastbuild is deprecated, ignoring') gn_args['target_sysroot'] = sysroot gn_args.pop('pkg_config', None) # pkg_config only affects the target and comes from the sysroot. # host_pkg_config is used for programs compiled for use later in the build. gn_args['host_pkg_config'] = 'pkg-config' if options.clang: gn_args['is_clang'] = True if options.internal: gn_args['is_chrome_branded'] = True gn_args['is_official_build'] = True else: gn_args.pop('is_chrome_branded', None) gn_args.pop('is_official_build', None) gn_args.pop('internal_gles2_conform_tests', None) # For SimpleChrome, we use the binutils that comes bundled within Chrome. # We should not use the binutils from the host system. gn_args['linux_use_bundled_binutils'] = True # Need to reset these after the env vars have been fixed by # _SetupTCEnvironment. gn_args['cros_host_is_clang'] = True # v8 snapshot is built on the host, so we need to set this. # See crosbug/618346. gn_args['cros_v8_snapshot_is_clang'] = True # gn_args['cros_target_cc'] = env['CC'] gn_args['cros_target_cxx'] = env['CXX'] gn_args['cros_target_ld'] = env['LD'] gn_args['cros_target_extra_cflags'] = env.get('CFLAGS', '') gn_args['cros_target_extra_cxxflags'] = env.get('CXXFLAGS', '') gn_args['cros_host_cc'] = env['CC_host'] gn_args['cros_host_cxx'] = env['CXX_host'] gn_args['cros_host_ld'] = env['LD_host'] gn_args['cros_host_ar'] = env['AR_host'] gn_args['cros_v8_snapshot_cc'] = env['CC_host'] gn_args['cros_v8_snapshot_cxx'] = env['CXX_host'] gn_args['cros_v8_snapshot_ld'] = env['LD_host'] gn_args['cros_v8_snapshot_ar'] = env['AR_host'] # No need to adjust CFLAGS and CXXFLAGS for GN since the only # adjustment made in _SetupTCEnvironment is for split debug which # is done with 'use_debug_fission'. # Enable goma if requested. if goma_dir: gn_args['use_goma'] = True gn_args['goma_dir'] = goma_dir elif not options.goma: # If --nogoma option is explicitly set, disable goma, even if it is # used in the original GN_ARGS. gn_args['use_goma'] = False gn_args.pop('internal_khronos_glcts_tests', None) # crbug.com/588080 # Disable ThinLTO and CFI for simplechrome. Tryjob machines do not have # enough file descriptors to use. crbug.com/789607 if 'use_thin_lto' in gn_args: gn_args['use_thin_lto'] = False if 'is_cfi' in gn_args: gn_args['is_cfi'] = False if 'use_cfi_cast' in gn_args: gn_args['use_cfi_cast'] = False # We need to remove the flag below from cros_target_extra_ldflags. # The format of ld flags is something like # '-Wl,-O1 -Wl,-O2 -Wl,--as-needed -stdlib=libc++' extra_thinlto_flag = '-Wl,-plugin-opt,-import-instr-limit=30' extra_ldflags = gn_args.get('cros_target_extra_ldflags', '') if extra_thinlto_flag in extra_ldflags: gn_args['cros_target_extra_ldflags'] = extra_ldflags.replace( extra_thinlto_flag, '') # We removed webcore debug symbols on release builds on arm. # See crbug.com/792999. However, we want to keep the symbols # for simplechrome builds. # TODO: remove the 'remove_webcore_debug_symbols' once we # change the ebuild file. gn_args['remove_webcore_debug_symbols'] = False gn_args['blink_symbol_level'] = -1 if options.gn_extra_args: gn_args.update(gn_helpers.FromGNArgs(options.gn_extra_args)) gn_args_env = gn_helpers.ToGNString(gn_args) env['GN_ARGS'] = gn_args_env # PS1 sets the command line prompt and xterm window caption. full_version = sdk_ctx.version if full_version != CUSTOM_VERSION: full_version = self.sdk.GetFullVersion(sdk_ctx.version) env['PS1'] = self._CreatePS1(self.board, full_version, chroot=options.chroot) # Set the useful part of PS1 for users with a custom PROMPT_COMMAND. env['CROS_PS1_PREFIX'] = self._PS1Prefix(self.board, full_version, chroot=options.chroot) out_dir = 'out_%s' % self.board env['builddir_name'] = out_dir build_label = 'Release' # This is used by landmines.py to prevent collisions when building both # chromeos and android from shared source. # For context, see crbug.com/407417 env['CHROMIUM_OUT_DIR'] = os.path.join(options.chrome_src, out_dir) self._UpdateGnArgsIfStale(out_dir, build_label, gn_args, env['SDK_BOARD']) return env