def _ProcessMightSubmitChanges(self):
"""Process changes in might_submit set.
This method goes through all the changes in current might_submit set. For
each change, get a set of its relevant slaves. If all the relevant slaves
can either be verified by current CQ or passed in CQ history, move the
change to will_submit set.
"""
if not self.might_submit:
return
logging.info('Processing changes in might submit set.')
will_submit_changes = set()
for change in self.might_submit:
relevant_slaves = self.change_relevant_slaves_dict.get(
change, set())
if self._ChangeCanBeSubmitted(change, relevant_slaves,
self.buildbucket_info_dict,
self.build_ignorable_changes_dict,
self.change_passed_slaves_dict):
will_submit_changes.add(change)
if will_submit_changes:
self.will_submit.update(will_submit_changes)
self.might_submit.difference_update(will_submit_changes)
logging.info(
'Moving %s to will_submit set, because their relevant builds'
' completed successfully or all failures are ignorable or '
'passed in CQ history.',
cros_patch.GetChangesAsString(will_submit_changes))
def ShouldSelfDestruct(self):
"""Process builds and relevant changes, decide whether to self-destruct.
Returns:
A tuple of (boolean indicating if the master should self-destruct,
boolean indicating if the master should self-destruct with
with success)
"""
self._ProcessCompletedBuilds()
self._ProcessMightSubmitChanges()
logging.info(
'will_submit set contains %d changes: [%s]\n'
'might_submit set contains %d changes: [%s]\n'
'will_not_submit set contains %d changes: [%s]\n',
len(self.will_submit),
cros_patch.GetChangesAsString(self.will_submit),
len(self.might_submit),
cros_patch.GetChangesAsString(self.might_submit),
len(self.will_not_submit),
cros_patch.GetChangesAsString(self.will_not_submit))
# The master should wait for all the necessary slaves to pass the
# UploadPrebuiltsStage so the master can publish prebuilts after
# self-destruction with success. More context: crbug.com/703819
all_completed_slaves_passed = (
self._AllCompletedSlavesPassedUploadPrebuiltsStage())
all_uncompleted_slaves_passed = (
self._AllUncompletedSlavesPassedUploadPrebuiltsStage())
should_self_destruct = (bool(not self.might_submit)
and (not all_completed_slaves_passed
or all_uncompleted_slaves_passed))
should_self_destruct_with_success = (bool(not self.might_submit)
and bool(not self.will_not_submit)
and all_completed_slaves_passed
and all_uncompleted_slaves_passed)
return should_self_destruct, should_self_destruct_with_success
def ClassifyOverlayChanges(cls, changes, config, build_root, manifest,
packages_under_test):
"""Classifies overlay changes in |changes|.
Args:
changes: The list or set of GerritPatch instances.
config: The cbuildbot config.
build_root: Path to the build root.
manifest: A ManifestCheckout instance representing our build directory.
packages_under_test: A list of packages names included in the build
without version/revision (e.g. ['chromeos-base/chromite']). If None,
don't try to map overlay changes to packages.
Returns:
A (overlay_changes, irrelevant_overlay_changes) tuple; overlay_changes
is a subset of |changes| that have modified one or more overlays, and
irrelevant_overlay_changes is a subset of overlay_changes which are
irrelevant to |config|.
"""
visible_overlays = set(
portage_util.FindOverlays(config.overlays, None, build_root))
# The overlays relevant to this build.
relevant_overlays = GetRelevantOverlaysForConfig(config, build_root)
overlay_changes = set()
irrelevant_overlay_changes = set()
for change in changes:
affected_overlays = GetAffectedOverlays(change, manifest,
visible_overlays)
if affected_overlays is not None:
# The change modifies an overlay.
overlay_changes.add(change)
if not any(x in relevant_overlays for x in affected_overlays):
# The change touched an irrelevant overlay.
irrelevant_overlay_changes.add(change)
continue
if packages_under_test:
# If the change modifies packages that are not part of this
# build, they are considered irrelevant too.
packages = GetAffectedPackagesForOverlayChange(
change, manifest, visible_overlays)
if packages:
logging.info('%s affects packages %s',
cros_patch.GetChangesAsString([change]),
', '.join(packages))
if not any(x in packages_under_test for x in packages):
irrelevant_overlay_changes.add(change)
return overlay_changes, irrelevant_overlay_changes
def FindSuspects(cls,
changes,
messages,
infra_fail=False,
lab_fail=False,
sanity=True):
"""Find out what changes probably caused our failure.
In cases where there were no internal failures, we can assume that the
external failures are at fault. Otherwise, this function just defers to
_FindPackageBuildFailureSuspects and GetBlamedChanges as needed.
If the failures don't match either case, just fail everything.
Args:
changes: A list of cros_patch.GerritPatch instances to consider.
messages: A list of build failure messages, of type
BuildFailureMessage or of type NoneType.
infra_fail: The build failed purely due to infrastructure failures.
lab_fail: The build failed purely due to test lab infrastructure
failures.
sanity: The sanity checker builder passed and the tree was open when
the build started.
Returns:
A set of changes as suspects.
"""
bad_changes = cls.GetBlamedChanges(changes)
if bad_changes:
# If there are changes that have been set verified=-1 or
# code-review=-2, these changes are the ONLY suspects of the
# failed build.
logging.warning(
'Detected that some changes have been blamed for '
'the build failure. Only these CLs will be rejected: %s',
cros_patch.GetChangesAsString(bad_changes))
return set(bad_changes)
elif lab_fail:
logging.warning('Detected that the build failed purely due to HW '
'Test Lab failure(s). Will not reject any changes')
return set()
elif infra_fail:
# The non-lab infrastructure errors might have been caused
# by chromite changes.
logging.warning(
'Detected that the build failed due to non-lab infrastructure '
'issue(s). Will only reject chromite changes')
return set(cls.FilterChangesForInfraFail(changes))
return cls._FindPackageBuildFailureSuspects(changes, messages, sanity)
def _GetIgnorableChanges(self, build_config, builder_status,
relevant_changes):
"""Get changes that can ignore failures in BuilderStatus.
Some projects are configured with ignored-stages in COMMIT_QUEUE.ini. The CQ
can still submit changes from these projects if all failed statges are
listed in ignored-stages. Please refer to
cq_config.CQConfigParser.GetStagesToIgnore for more details.
1) if the builder_status is in 'pass' status, it means the build uploaded a
'pass' builder_status but failed other steps in or after the completion
stage. This is rare but still possible, and it should not blame any changes
as the build has finishes its testing. Returns all changes in
relevant_changes in this case.
2) else if the builder_status is in 'fail' with failure messages, it
calculates and returns all ignorable changes given the failure messages.
3) else, the builder_status is either in 'fail' status without failure
messages or in one of the 'inflight' and 'missing' statuses. It cannot
calculate ignorable changes without any failure message so should just
return an empty set.
Args:
build_config: The config name (string) of the build.
builder_status: An instance of build_status.BuilderStatus.
relevant_changes: A set of relevant changes for triage to get the
ignorable changes.
Returns:
A set of ignorable changes (GerritPatch instances).
"""
if builder_status.Passed():
return relevant_changes
elif builder_status.Failed() and builder_status.message:
ignoreable_changes = set()
for change in relevant_changes:
ignore_result = triage_lib.CalculateSuspects.CanIgnoreFailures(
[builder_status.message], change, self.build_root,
self.slave_subsys_dict)
if ignore_result[0]:
logging.debug(
'change %s is ignoreable for failures of %s.',
cros_patch.GetChangesAsString([change]), build_config)
ignoreable_changes.add(change)
return ignoreable_changes
else:
return set()
def FindHWTestFailureSuspects(cls, changes, build_root, failed_hwtests):
"""Find suspects for HWTest failures.
Args:
changes: A list of cros_patch.GerritPatch instances.
build_root: The path to the build root.
failed_hwtests: A list of names of failed hwtests got from CIDB (see the
return type of HWTestResultManager.GetFailedHWTestsFromCIDB), or None.
Returns:
A pair of suspects and no_assignee_hwtests. suspects is a set of
cros_patch.GerritPatch instances. no_assignee_hwtests is True when there
are failed hwtests without assigned suspects; else False.
"""
suspects = set()
no_assignee_hwtests = False
if not failed_hwtests:
logging.info(
'No failed HWTests, skip finding HWTest failure suspects')
return suspects, no_assignee_hwtests
manifest = git.ManifestCheckout.Cached(build_root)
hwtests_with_assignee = set()
for change in changes:
assigned = cls.GetFailedHwtestsAffectedByChange(
change, manifest, failed_hwtests)
if assigned:
hwtests_with_assignee.update(assigned)
suspects.add(change)
if suspects:
logging.info('Found suspects for HWTest failures: %s',
cros_patch.GetChangesAsString(suspects))
hwtests_without_assignee = failed_hwtests - hwtests_with_assignee
if hwtests_without_assignee:
logging.info(
'Didn\'t find changes to blame for failed HWtests: %s',
hwtests_without_assignee)
no_assignee_hwtests = True
return suspects, no_assignee_hwtests
def FindPackageBuildFailureSuspects(self, changes, failure):
"""Find suspects for a PackageBuild failure.
If a change touched a package and that package broke, this change is one of
the suspects; if multiple changes touched one failed package, all these
changes will be returned as suspects.
Args:
changes: A list of cros_patch.GerritPatch instances.
failure: An instance of StageFailureMessage(or its sub-class).
Returns:
A pair of suspects and no_assignee_packages. suspects is a set of
cros_patch.GerritPatch instances as suspects. no_assignee_packages is True
when there're failed packages without assigned suspects; else,
no_assignee_packages is False.
"""
suspects = set()
no_assignee_packages = False
packages_with_assignee = set()
failed_packages = failure.GetFailedPackages()
for package in failed_packages:
failed_projects = portage_util.FindWorkonProjects([package])
for change in changes:
if change.project in failed_projects:
suspects.add(change)
packages_with_assignee.add(package)
if suspects:
logging.info('Find suspects for BuildPackages failures: %s',
cros_patch.GetChangesAsString(suspects))
packages_without_assignee = set(failed_packages) - packages_with_assignee
if packages_without_assignee:
logging.info('Didn\'t find changes to blame for failed packages: %s',
list(packages_without_assignee))
no_assignee_packages = True
return suspects, no_assignee_packages
def GetFullyVerifiedChanges(cls, changes, changes_by_config,
subsys_by_config,
passed_in_history_slaves_by_change, failing,
inflight, no_stat, messages, build_root):
"""Examines build failures and returns a set of fully verified changes.
A change is fully verified if all the build configs relevant to
this change have either passed or failed in a manner that can be
safely ignored by the change.
Args:
changes: A list of GerritPatch instances to examine.
changes_by_config: A dictionary of relevant changes indexed by the
config names.
subsys_by_config: A dictionary of pass/fail HWTest subsystems indexed
by the config names.
passed_in_history_slaves_by_change: A dict mapping changes to their
relevant slaves (build config name strings) which passed in history.
failing: Names of the builders that failed.
inflight: Names of the builders that timed out.
no_stat: Set of builder names of slave builders that had status None.
messages: A list of build_failure_message.BuildFailureMessage or NoneType
objects from the failed slaves.
build_root: Build root directory.
Returns:
A dictionary mapping the fully verified changes to their string reasons
for submission. (Should be None or constant with name STRATEGY_* from
constants.py.)
"""
changes = set(changes)
no_stat = set(no_stat)
failing = set(failing)
inflight = set(inflight)
fully_verified = dict()
all_tested_changes = set()
for tested_changes in changes_by_config.itervalues():
all_tested_changes.update(tested_changes)
untested_changes = changes - all_tested_changes
if untested_changes:
# Some board overlay changes were not tested by CQ at all.
logging.info(
'These changes were not tested by any slaves, '
'so they will be submitted: %s',
cros_patch.GetChangesAsString(untested_changes))
fully_verified.update({
c: constants.STRATEGY_CQ_PARTIAL_NOT_TESTED
for c in untested_changes
})
not_completed = set.union(no_stat, inflight)
for change in all_tested_changes:
# If each of the relevant configs associated with a change satisifies one
# of the conditions:
# 1) passed successfully; OR
# 2) failed with failures which can be ignored by the change; OR
# 3) there are builds of the relevant build config passed in history.
# this change will be considered as fully verified.
verified = True
verified_reasons = set()
relevant_configs = [
k for k, v in changes_by_config.iteritems() if change in v
]
passed_in_history_slaves = passed_in_history_slaves_by_change.get(
change, set())
logging.info(
'Checking change %s; relevant configs %s; configs passed in '
'history %s.', change.PatchLink(), relevant_configs,
list(passed_in_history_slaves))
for build_config in relevant_configs:
if build_config in not_completed:
if build_config in passed_in_history_slaves:
verified_reasons.add(
constants.STRATEGY_CQ_PARTIAL_CQ_HISTORY)
else:
logging.info(
'Failed to verify change %s: relevant build %s isn\'t '
'completed in current run and didn\'t pass in history',
change.PatchLink(), build_config)
verified = False
break
elif build_config in failing:
failed_messages = [
x for x in messages if x.builder == build_config
]
ignore_result = cls.CanIgnoreFailures(
failed_messages, change, build_root, subsys_by_config)
if ignore_result[0]:
verified_reasons.add(ignore_result[1])
elif build_config in passed_in_history_slaves:
verified_reasons.add(
constants.STRATEGY_CQ_PARTIAL_CQ_HISTORY)
else:
logging.info(
'Failed to verify change %s: relevant build %s failed '
'with not ignorable failures in current run and '
'didn\'t pass in history', change.PatchLink(),
build_config)
verified = False
break
else:
verified_reasons.add(
constants.STRATEGY_CQ_PARTIAL_BUILDS_PASSED)
if verified:
reason = cls._GetVerifiedReason(verified_reasons)
fully_verified.update({change: reason})
logging.info(
'Change %s is verified with reasons %s, choose the final '
'reason %s.', change.PatchLink(), list(verified_reasons),
reason)
return fully_verified
def FindSuspects(cls,
changes,
messages,
infra_fail=False,
lab_fail=False,
build_root=None,
failed_hwtests=None,
sanity=True):
"""Find out what changes probably caused our failure.
1) if there're bad changes to blame, return the bad changes as the suspects;
2) else if there're only internal lab failures, return an empty suspects;
3) else if there're only internal infra failures, return infra changes as
the suspects;
4) else, find and return suspects by analyzing the failures.
Args:
changes: A list of cros_patch.GerritPatch instances to consider.
messages: A list of build failure messages, of type
build_failure_message.BuildFailureMessage or of type NoneType.
infra_fail: The build failed purely due to infrastructure failures.
lab_fail: The build failed purely due to test lab infrastructure
failures.
build_root: The path to the build root.
failed_hwtests: A list of names of failed hwtests got from CIDB (see the
return type of HWTestResultManager.GetFailedHWTestsFromCIDB) or a
NoneType instance.
sanity: The sanity checker builder passed and the tree was open when
the build started and ended.
Returns:
An instance of SuspectChanges.
"""
suspect_changes = SuspectChanges()
bad_changes = cls.GetBlamedChanges(changes)
if bad_changes:
# If there are changes that have been set verified=-1 or
# code-review=-2, these changes are the ONLY suspects of the
# failed build.
logging.warning(
'Detected that some changes have been blamed for '
'the build failure. Only these CLs will be rejected: %s',
cros_patch.GetChangesAsString(bad_changes))
suspect_changes.update(
{x: constants.SUSPECT_REASON_BAD_CHANGE
for x in bad_changes})
elif lab_fail:
logging.warning('Detected that the build failed purely due to HW '
'Test Lab failure(s). Will not reject any changes')
elif infra_fail:
# The non-lab infrastructure errors might have been caused
# by chromite changes.
logging.warning(
'Detected that the build failed due to non-lab infrastructure '
'issue(s). Will only reject chromite changes')
infra_changes = cls.FilterChangesForInfraFail(changes)
suspect_changes.update({
x: constants.SUSPECT_REASON_INFRA_FAIL
for x in infra_changes
})
else:
suspect_changes = cls.FindSuspectsForFailures(
changes, messages, build_root, failed_hwtests, sanity)
return suspect_changes
def _ProcessCompletedBuilds(self):
"""Process completed and not retriable builds.
This method goes through all the builds which completed without SUCCESS
result and will not be retried.
1) if the failed build didn't pass the sync stage, iterate all changes,
move the changes which didn't pass this build config in history to
will_not_submit (as well as their dependencies).
2) else, get BuilderStatus for the build (if there's no BuilderStatus
pickle file in GS, a BuilderStatus with 'missing' status will be returned).
Find not ignorable changes given the BuilderStatus, iterate the changes in
not ignorable changes, move the changes which didn't pass this build config
in history to will_not_submit (as well as their dependencies).
"""
# TODO(nxia): Improve SlaveBuilderStatus to take buildbucket_info_dict
# and cidb_status_dict as arguments to avoid extra queries.
slave_builder_statuses = builder_status_lib.SlaveBuilderStatus(
self.master_build_id, self.db, self.config, self.metadata,
self.buildbucket_client, self.builders_array, self.dry_run)
for build_config, bb_info in self.buildbucket_info_dict.iteritems():
if (build_config in self.completed_builds and bb_info.status
== constants.BUILDBUCKET_BUILDER_STATUS_COMPLETED
and bb_info.result !=
constants.BUILDBUCKET_BUILDER_RESULT_SUCCESS):
# This build didn't succeed and cannot be retried.
logging.info(
'Processing relevant changes of build %s status %s '
'result %s', build_config, bb_info.status, bb_info.result)
stages = self.slave_stages_dict[build_config]
if not self.PassedAnyOfStages(stages, self.STAGE_SYNC):
# The build_config didn't pass any of the sync stages. Find changes
# which don't have valid passed builds of this build_config in
# history. Move the changes and their dependencies to will_not_submit
# set.
will_not_submit_changes = self._GetWillNotSubmitChanges(
build_config, self.changes)
depend_changes = self.GetDependChanges(
will_not_submit_changes, self.dependency_map)
will_not_submit_changes |= depend_changes
if will_not_submit_changes:
self._UpdateWillNotSubmitChanges(
will_not_submit_changes)
logging.info(
'Build %s didn\'t pass any stage in %s. Will not'
' submit changes: %s', build_config,
self.STAGE_SYNC,
cros_patch.GetChangesAsString(
will_not_submit_changes))
else:
# The build passed the required sync stage. Get builder_status and
# get not ignorable changes based on the builder_status. Find changes
# in the not ignorable changes don't have valid passed builds of this
# build_config hi history. Move the changes and their dependencies to
# will_not_submit set.
relevant_changes = self.slave_changes_dict[build_config]
builder_status = slave_builder_statuses.GetBuilderStatusForBuild(
build_config)
ignorable_changes = self._GetIgnorableChanges(
build_config, builder_status, relevant_changes)
self.build_ignorable_changes_dict[
build_config] = ignorable_changes
not_ignorable_changes = relevant_changes - ignorable_changes
will_not_submit_changes = self._GetWillNotSubmitChanges(
build_config, not_ignorable_changes)
depend_changes = self.GetDependChanges(
will_not_submit_changes, self.dependency_map)
will_not_submit_changes = will_not_submit_changes | depend_changes
if will_not_submit_changes:
self._UpdateWillNotSubmitChanges(
will_not_submit_changes)
logging.info(
'Build %s failed with not ignorable failures, will not'
' submit changes: %s', build_config,
cros_patch.GetChangesAsString(
will_not_submit_changes))
if not self.might_submit:
# No need to process other completed builds, might_submit is empty.
return
def GetFullyVerifiedChanges(cls, changes, changes_by_config,
subsys_by_config, failing, inflight, no_stat,
messages, build_root):
"""Examines build failures and returns a set of fully verified changes.
A change is fully verified if all the build configs relevant to
this change have either passed or failed in a manner that can be
safely ignored by the change.
Args:
changes: A list of GerritPatch instances to examine.
changes_by_config: A dictionary of relevant changes indexed by the
config names.
subsys_by_config: A dictionary of pass/fail HWTest subsystems indexed
by the config names.
failing: Names of the builders that failed.
inflight: Names of the builders that timed out.
no_stat: Set of builder names of slave builders that had status None.
messages: A list of BuildFailureMessage or NoneType objects from
the failed slaves.
build_root: Build root directory.
Returns:
A dictionary mapping the fully verified changes to their string reasons
for submission. (Should be None or constant with name STRATEGY_* from
constants.py.)
"""
changes = set(changes)
no_stat = set(no_stat)
failing = set(failing)
inflight = set(inflight)
fully_verified = dict()
all_tested_changes = set()
for tested_changes in changes_by_config.itervalues():
all_tested_changes.update(tested_changes)
untested_changes = changes - all_tested_changes
if untested_changes:
# Some board overlay changes were not tested by CQ at all.
logging.info(
'These changes were not tested by any slaves, '
'so they will be submitted: %s',
cros_patch.GetChangesAsString(untested_changes))
fully_verified.update({c: None for c in untested_changes})
for change in all_tested_changes:
# If all relevant configs associated with a change passed, the
# change is fully verified.
relevant_configs = [
k for k, v in changes_by_config.iteritems() if change in v
]
if any(x in set.union(no_stat, inflight)
for x in relevant_configs):
continue
failed_configs = [x for x in relevant_configs if x in failing]
if not failed_configs:
logging.info(
'All the %s relevant config(s) for change %s passed, so '
'it will be submitted.', len(relevant_configs),
cros_patch.GetChangesAsString([change]))
fully_verified.update({change: None})
else:
# Examine the failures and see if we can safely ignore them
# for the change.
failed_messages = [
x for x in messages if x.builder in failed_configs
]
ignore_result = cls._CanIgnoreFailures(failed_messages, change,
build_root,
subsys_by_config)
if ignore_result[0]:
logging.info(
'All failures of relevant configs for change %s are '
'ignorable by this change, so it will be submitted.',
cros_patch.GetChangesAsString([change]))
fully_verified.update({change: ignore_result[1]})
return fully_verified
def CreateValidationFailureMessage(pre_cq_trybot, change, suspects, messages,
sanity=True, infra_fail=False,
lab_fail=False, no_stat=None,
retry=False, cl_status_url=None):
"""Create a message explaining why a validation failure occurred.
Args:
pre_cq_trybot: Whether the builder is a Pre-CQ trybot. (Note: The Pre-CQ
launcher is NOT considered a Pre-CQ trybot.)
change: The change we want to create a message for.
suspects: An instance of triage_lib.SuspectChanges.
messages: A list of build failure messages from supporting builders.
These must be BuildFailureMessage objects or NoneType objects.
sanity: A boolean indicating whether the build was considered sane. If
not sane, none of the changes will have their CommitReady bit modified.
infra_fail: The build failed purely due to infrastructure failures.
lab_fail: The build failed purely due to test lab infrastructure failures.
no_stat: A list of builders which failed prematurely without reporting
status.
retry: Whether we should retry automatically.
cl_status_url: URL of the CL status viewer for the change.
Returns:
A string that communicates what happened.
"""
msg = []
if no_stat:
msg.append('The following build(s) did not start or failed prematurely:')
msg.append(', '.join(no_stat))
if messages:
# Build a list of error messages. We don't want to build a ridiculously
# long comment, as Gerrit will reject it. See http://crbug.com/236831
max_error_len = 20000 / max(1, len(messages))
msg.append('The following build(s) failed:')
for message in map(str, messages):
if len(message) > max_error_len:
message = message[:max_error_len] + '... (truncated)'
msg.append(message)
# Create a list of changes other than this one that might be guilty.
# Limit the number of suspects to 20 so that the list of suspects isn't
# ridiculously long.
max_suspects = 20
other_suspects = set(suspects.keys()) - set([change])
if len(other_suspects) < max_suspects:
other_suspects_str = cros_patch.GetChangesAsString(other_suspects)
else:
other_suspects_str = ('%d other changes. See the blamelist for more '
'details.' % (len(other_suspects),))
if not sanity:
msg.append('The build was consider not sane because the sanity check '
'builder(s) failed. Your change will not be blamed for the '
'failure.')
assert retry
elif lab_fail:
msg.append('The build encountered Chrome OS Lab infrastructure issues. '
' Your change will not be blamed for the failure.')
assert retry
else:
if infra_fail:
msg.append('The build failure may have been caused by infrastructure '
'issues and/or bad %s changes.' % constants.INFRA_PROJECTS)
if change in suspects.keys():
if other_suspects_str:
msg.append('Your change may have caused this failure. There are '
'also other changes that may be at fault: %s'
% other_suspects_str)
else:
msg.append('This failure was probably caused by your change.')
msg.append('Please check whether the failure is your fault. If your '
'change is not at fault, you may mark it as ready again.')
else:
if len(suspects) == 1:
msg.append('This failure was probably caused by %s'
% other_suspects_str)
elif len(suspects) > 0:
msg.append('One of the following changes is probably at fault: %s'
% other_suspects_str)
assert retry
if pre_cq_trybot and cl_status_url:
msg.append(
'We notify the first failure only. Please find the full status at %s.'
% cl_status_url)
if retry:
bot = 'The Pre-Commit Queue' if pre_cq_trybot else 'The Commit Queue'
msg.insert(0, 'NOTE: %s will retry your change automatically.' % bot)
return '\n\n'.join(msg)
