def GetSubsysResultForSlaves(self):
"""Get the pass/fail HWTest subsystems results for each slave.
Returns:
A dictionary mapping a slave config name to a dictionary of the pass/fail
subsystems. E.g.
{'foo-paladin': {'pass_subsystems':{'A', 'B'},
'fail_subsystems':{'C'}}}
"""
# build_id is the master build id for the run
build_id, db = self._run.GetCIDBHandle()
assert db, 'No database connection to use.'
slave_msgs = db.GetSlaveBuildMessages(build_id)
slave_subsys_msgs = ([
m for m in slave_msgs if m['message_type'] == constants.SUBSYSTEMS
])
subsys_by_config = dict()
group_msg_by_config = cros_build_lib.GroupByKey(
slave_subsys_msgs, 'build_config')
for config, dict_list in group_msg_by_config.iteritems():
d = subsys_by_config.setdefault(config, {})
subsys_groups = cros_build_lib.GroupByKey(dict_list,
'message_subtype')
for k, v in subsys_groups.iteritems():
if k == constants.SUBSYSTEM_PASS:
d['pass_subsystems'] = set([x['message_value'] for x in v])
if k == constants.SUBSYSTEM_FAIL:
d['fail_subsystems'] = set([x['message_value'] for x in v])
# If message_subtype==subsystem_unused, keep d as an empty dict.
return subsys_by_config
def Gather(self,
start_date,
end_date,
master_config=constants.CQ_MASTER,
sort_by_build_number=True,
starting_build_number=None,
ending_build_number=None):
"""Fetches build data and failure reasons.
Args:
start_date: A datetime.date instance for the earliest build to
examine.
end_date: A datetime.date instance for the latest build to
examine.
master_config: Config name of master to gather data for.
Default to CQ_MASTER.
sort_by_build_number: Optional boolean. If True, builds will be
sorted by build number.
starting_build_number: (optional) The lowest build number to
include in the results.
ending_build_number: (optional) The highest build number to include
in the results.
"""
logging.info('Gathering data for %s from %s until %s', master_config,
start_date, end_date)
self.builds = self.db.GetBuildHistory(
master_config,
start_date=start_date,
end_date=end_date,
starting_build_number=starting_build_number,
num_results=self.db.NUM_RESULTS_NO_LIMIT)
if ending_build_number is not None:
self.builds = [
x for x in self.builds
if x['build_number'] <= ending_build_number
]
if self.builds:
logging.info('Fetched %d builds (build_id: %d to %d)',
len(self.builds), self.builds[0]['id'],
self.builds[-1]['id'])
else:
logging.info('Fetched no builds.')
if sort_by_build_number:
logging.info('Sorting by build number.')
self.builds.sort(key=lambda x: x['build_number'])
self.claction_history = self.db.GetActionHistory(start_date, end_date)
self.GatherBuildAnnotations()
self.builds_by_build_id.update({b['id']: b for b in self.builds})
# Gather slave statuses for each of the master builds. For now this is a
# separate query per CQ run, but this could be consolidated to a single
# query if necessary (requires adding a cidb.py API method).
for bid in self.builds_by_build_id:
self.slave_builds_by_master_id[bid] = self.db.GetSlaveStatuses(bid)
self.slave_builds_by_config = cros_build_lib.GroupByKey(
itertools.chain(*self.slave_builds_by_master_id.values()),
'build_config')
def PerformStage(self):
if not self._run.config.master:
logging.info('This stage is only meaningful for master builds. '
'Doing nothing.')
return
build_id, db = self._run.GetCIDBHandle()
if not db:
logging.info('No cidb connection for this build. '
'Doing nothing.')
return
slave_failures = db.GetSlaveFailures(build_id)
failures_by_build = cros_build_lib.GroupByKey(slave_failures,
'build_id')
for build_id, build_failures in sorted(failures_by_build.items()):
failures_by_stage = cros_build_lib.GroupByKey(
build_failures, 'build_stage_id')
# Surface a link to each slave stage that failed, in stage_id sorted
# order.
for stage_id in sorted(failures_by_stage):
failure = failures_by_stage[stage_id][0]
# Ignore failures that did not cause their enclosing stage to fail.
# Ignore slave builds that are still inflight, because some stage logs
# might not have been printed to buildbot yet.
# TODO(akeshet) revisit this approach, if we seem to be suppressing
# useful information as a result of it.
if (failure['stage_status'] != constants.BUILDER_STATUS_FAILED
or failure['build_status']
== constants.BUILDER_STATUS_INFLIGHT):
continue
waterfall_url = constants.WATERFALL_TO_DASHBOARD[
failure['waterfall']]
slave_stage_url = tree_status.ConstructDashboardURL(
waterfall_url, failure['builder_name'],
failure['build_number'], failure['stage_name'])
logging.PrintBuildbotLink(
'%s %s' % (failure['build_config'], failure['stage_name']),
slave_stage_url)
def FillInBuildStatusesWithStages(db, build_statuses):
"""Fill in a 'stages' value for a list of build_statuses.
Modifies the build_status objects in-place.
Args:
db: cidb.CIDBConnection object.
build_statuses: List of build_status dictionaries as returned by various
CIDB methods.
"""
ids = [status['id'] for status in build_statuses]
all_stages = db.GetBuildsStages(ids)
stages_by_build_id = cros_build_lib.GroupByKey(all_stages, 'build_id')
for status in build_statuses:
status['stages'] = stages_by_build_id.get(status['id'], [])
