def PerformStage(self):
buildroot = self._build_root
gs_context = gs.GSContext()
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=buildroot)
version_number = cpv.version
# We need the name of one board that has been setup in this
# builder to find the Chrome ebuild. The chrome ebuild should be
# the same for all the boards, so just use the first one.
# If we don't have any boards, leave the called function to guess.
board = self._boards[0] if self._boards else None
arch_profiles = {}
for arch in afdo.AFDO_ARCH_GENERATORS:
afdo_file = afdo.GetLatestAFDOFile(cpv, arch, buildroot,
gs_context)
if not afdo_file:
raise afdo.MissingAFDOData(
'Could not find appropriate AFDO profile')
state = 'current' if version_number in afdo_file else 'previous'
logging.info('Found %s %s AFDO profile %s', state, arch, afdo_file)
arch_profiles[arch] = afdo_file
# Now update the Chrome ebuild file with the AFDO profiles we found
# for each architecture.
afdo.UpdateChromeEbuildAFDOFile(board, arch_profiles)
def DetermineChromeVersion(self):
"""Determine the current Chrome version in buildroot now and return it.
This uses the typical portage logic to determine which version of Chrome
is active right now in the buildroot.
Returns:
The new value of attrs.chrome_version (e.g. "35.0.1863.0").
"""
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=self.buildroot)
return cpv.version_no_rev.partition('_')[0]
def DetermineAndroidVersion(self, boards=None):
"""Determine the current Android version in buildroot now and return it.
This uses the typical portage logic to determine which version of Android
is active right now in the buildroot.
Args:
boards: List of boards to check version of.
Returns:
The Android build ID of the container for the boards.
Raises:
NoAndroidVersionError: if no unique Android version can be determined.
"""
version = None
try:
if boards:
# Verify that all boards have the same version.
for board in boards:
cpv = portage_util.BestVisible(constants.ANDROID_CP,
buildroot=self.buildroot,
board=board)
if version is None:
version = cpv.version_no_rev
elif version != cpv.version_no_rev:
raise NoAndroidVersionError(
'Different Android versions (%s vs %s) for %s' %
(version, cpv.version_no_rev, boards))
else:
# This is expected to fail until ANDROID_CP is in the main overlays.
cpv = portage_util.BestVisible(constants.ANDROID_CP,
buildroot=self.buildroot)
version = cpv.version_no_rev
except cros_build_lib.RunCommandError as ex:
raise NoAndroidVersionError(
'Android version could not be determined for %s' % boards, ex)
return version
def PerformStage(self):
"""Collect a 'perf' profile and convert it into the AFDO format."""
super(AFDODataGenerateStage, self).PerformStage()
board = self._current_board
if not afdo.CanGenerateAFDOData(board):
logging.warning('Board %s cannot generate its own AFDO profile.',
board)
return
arch = self._GetCurrentArch()
buildroot = self._build_root
gs_context = gs.GSContext()
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=buildroot)
afdo_file = None
# Generation of AFDO could fail for different reasons.
# We will ignore the failures and let the master PFQ builder try
# to find an older AFDO profile.
try:
if afdo.WaitForAFDOPerfData(cpv, arch, buildroot, gs_context):
afdo_file = afdo.GenerateAFDOData(cpv, arch, board, buildroot,
gs_context)
assert afdo_file
logging.info('Generated %s AFDO profile %s', arch, afdo_file)
else:
raise afdo.MissingAFDOData(
'Could not find current "perf" profile. '
'Master PFQ builder will try to use stale '
'AFDO profile.')
# Will let system-exiting exceptions through.
except Exception:
logging.PrintBuildbotStepWarnings()
logging.warning('AFDO profile generation failed with exception ',
exc_info=True)
alert_msg = ('Please triage. This will become a fatal error.\n\n'
'arch=%s buildroot=%s\n\nURL=%s' %
(arch, buildroot, self._run.ConstructDashboardURL()))
subject_msg = (
'Failure in generation of AFDO Data for builder %s' %
self._run.config.name)
alerts.SendEmailLog(subject_msg,
afdo.AFDO_ALERT_RECIPIENTS,
server=alerts.SmtpServer(
constants.GOLO_SMTP_SERVER),
message=alert_msg)
# Re-raise whatever exception we got here. This stage will only
# generate a warning but we want to make sure the warning is
# generated.
raise
def PerformStage(self):
buildroot = self._build_root
gs_context = gs.GSContext()
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=buildroot)
# We need the name of one board that has been setup in this
# builder to find the Chrome ebuild. The chrome ebuild should be
# the same for all the boards, so just use the first one.
# If we don't have any boards, leave the called function to guess.
board = self._boards[0] if self._boards else None
profiles = {}
for source, getter in afdo.PROFILE_SOURCES.iteritems():
profile = getter(cpv, source, buildroot, gs_context)
if not profile:
raise afdo.MissingAFDOData(
'Could not find appropriate profile for %s' % source)
logging.info('Found AFDO profile %s for %s', profile, source)
profiles[source] = profile
# Now update the Chrome ebuild file with the AFDO profiles we found
# for each source.
afdo.UpdateChromeEbuildAFDOFile(board, profiles)
def PerformStage(self):
if self.suite_config.suite == constants.HWTEST_AFDO_SUITE:
arch = self._GetPortageEnvVar('ARCH', self._current_board)
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=self._build_root)
if afdo.CheckAFDOPerfData(cpv, arch, gs.GSContext()):
logging.info('AFDO profile already generated for arch %s '
'and Chrome %s. Not generating it again',
arch, cpv.version_no_rev.split('_')[0])
return
build = '/'.join([self._bot_id, self.version])
if (self._run.options.remote_trybot and (self._run.options.hwtest or
self._run.config.pre_cq)):
debug = self._run.options.debug_forced
else:
debug = self._run.options.debug
# Get the subsystems set for the board to test
per_board_dict = self._run.attrs.metadata.GetDict()['board-metadata']
current_board_dict = per_board_dict.get(self._current_board)
if current_board_dict:
subsystems = set(current_board_dict.get('subsystems_to_test', []))
# 'subsystem:all' indicates to skip the subsystem logic
if 'all' in subsystems:
subsystems = None
else:
subsystems = None
skip_duts_check = False
if config_lib.IsCanaryType(self._run.config.build_type):
skip_duts_check = True
build_id, db = self._run.GetCIDBHandle()
cmd_result = commands.RunHWTestSuite(
build, self.suite_config.suite, self._current_board,
pool=self.suite_config.pool, num=self.suite_config.num,
file_bugs=self.suite_config.file_bugs,
wait_for_results=self.wait_for_results,
priority=self.suite_config.priority,
timeout_mins=self.suite_config.timeout_mins,
retry=self.suite_config.retry,
max_retries=self.suite_config.max_retries,
minimum_duts=self.suite_config.minimum_duts,
suite_min_duts=self.suite_config.suite_min_duts,
offload_failures_only=self.suite_config.offload_failures_only,
debug=debug, subsystems=subsystems, skip_duts_check=skip_duts_check)
subsys_tuple = self.GenerateSubsysResult(cmd_result.json_dump_result,
subsystems)
if db:
if not subsys_tuple:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_UNUSED,
board=self._current_board)
else:
logging.info('pass_subsystems: %s, fail_subsystems: %s',
subsys_tuple[0], subsys_tuple[1])
for s in subsys_tuple[0]:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_PASS,
message_value=str(s), board=self._current_board)
for s in subsys_tuple[1]:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_FAIL,
message_value=str(s), board=self._current_board)
if cmd_result.to_raise:
raise cmd_result.to_raise
def PerformStage(self):
if self.suite_config.suite == constants.HWTEST_AFDO_SUITE:
arch = self._GetPortageEnvVar('ARCH', self._current_board)
cpv = portage_util.BestVisible(constants.CHROME_CP,
buildroot=self._build_root)
if afdo.CheckAFDOPerfData(cpv, arch, gs.GSContext()):
logging.info('AFDO profile already generated for arch %s '
'and Chrome %s. Not generating it again',
arch, cpv.version_no_rev.split('_')[0])
return
if self.suite_config.suite in [constants.HWTEST_CTS_FOLLOWER_SUITE,
constants.HWTEST_CTS_QUAL_SUITE,
constants.HWTEST_GTS_QUAL_SUITE]:
# Increase the priority for CTS/GTS qualification suite as we want stable
# build to have higher priority than beta build (again higher than dev).
try:
cros_vers = self._run.GetVersionInfo().VersionString().split('.')
if not isinstance(self.suite_config.priority, (int, long)):
# Convert CTS/GTS priority to corresponding integer value.
self.suite_config.priority = constants.HWTEST_PRIORITIES_MAP[
self.suite_config.priority]
# We add 1/10 of the branch version to the priority. This results in a
# modest priority bump the older the branch is. Typically beta priority
# would be dev + [1..4] and stable priority dev + [5..9].
self.suite_config.priority += int(math.ceil(float(cros_vers[1]) / 10.0))
except cbuildbot_run.VersionNotSetError:
logging.debug('Could not obtain version info. %s will use initial '
'priority value: %s', self.suite_config.suite,
self.suite_config.priority)
build = '/'.join([self._bot_id, self.version])
# Get the subsystems set for the board to test
if self.suite_config.suite == constants.HWTEST_PROVISION_SUITE:
subsystems = set()
else:
subsystems = self._GetSubsystems()
skip_duts_check = False
if config_lib.IsCanaryType(self._run.config.build_type):
skip_duts_check = True
build_id, db = self._run.GetCIDBHandle()
test_args = None
if config_lib.IsCQType(self._run.config.build_type):
test_args = {'fast': 'True'}
cmd_result = commands.RunHWTestSuite(
build, self.suite_config.suite, self._board_name,
model=self._model,
pool=self.suite_config.pool,
file_bugs=self.suite_config.file_bugs,
wait_for_results=self.wait_for_results,
priority=self.suite_config.priority,
timeout_mins=self.suite_config.timeout_mins,
retry=self.suite_config.retry,
max_retries=self.suite_config.max_retries,
minimum_duts=self.suite_config.minimum_duts,
suite_min_duts=self.suite_config.suite_min_duts,
suite_args=self.suite_config.suite_args,
offload_failures_only=self.suite_config.offload_failures_only,
debug=not self.TestsEnabled(self._run),
subsystems=subsystems,
skip_duts_check=skip_duts_check,
job_keyvals=self.GetJobKeyvals(),
test_args=test_args)
if config_lib.IsCQType(self._run.config.build_type):
self.ReportHWTestResults(cmd_result.json_dump_result, build_id, db)
subsys_tuple = self.GenerateSubsysResult(cmd_result.json_dump_result,
subsystems)
if db:
if not subsys_tuple:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_UNUSED,
board=self._current_board)
else:
logging.info('pass_subsystems: %s, fail_subsystems: %s',
subsys_tuple[0], subsys_tuple[1])
for s in subsys_tuple[0]:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_PASS,
message_value=str(s), board=self._current_board)
for s in subsys_tuple[1]:
db.InsertBuildMessage(build_id, message_type=constants.SUBSYSTEMS,
message_subtype=constants.SUBSYSTEM_FAIL,
message_value=str(s), board=self._current_board)
if cmd_result.to_raise:
raise cmd_result.to_raise
