def _ProcessAbort(self, request_id, pending_requests, manager):
"""Actually remove a given request that is being aborted."""
state = None
if self._RequestInState(request_id, self._PENDING):
pending_requests.remove(request_id)
state = self._PENDING
elif self._RequestInState(request_id, self._RUNNING):
manager.TerminateTask(request_id)
state = self._RUNNING
elif self._RequestInState(request_id, self._COMPLETE):
state = self._COMPLETE
# No check for "requested" state; our caller guarantees it's not
# needed.
#
# By design, we don't fail if the aborted request is already gone.
if state is not None:
logging.info('Abort is removing %s from state %s',
request_id, state)
try:
self._ClearRequest(request_id, state)
except OSError:
logging.exception('Request %s was not removed from %s.',
request_id, state)
else:
logging.info('Abort for non-existent request %s', request_id)
def _get_pubsub_service(self):
try:
return discovery.build(PUBSUB_SERVICE_NAME, PUBSUB_VERSION,
credentials=self.credential)
except errors.Error as e:
logging.exception('Failed to get pubsub resource object:%s', e)
raise PubSubException('Failed to get pubsub resource object')
def run_tidy(board: str, ebuild_list: List[portage_util.EBuild],
keep_dirs: bool,
parse_errors_are_nonfatal: bool) -> Set[TidyDiagnostic]:
"""Runs clang-tidy on the given ebuilds for the given board.
Returns the set of |TidyDiagnostic|s produced by doing so.
"""
# Since we rely on build actions _actually_ running, we can't live with a
# cache.
osutils.RmDir(
Path(cros_build_lib.GetSysroot(board)) / 'var' / 'cache' / 'portage',
ignore_missing=True,
sudo=True,
)
results = set()
# If clang-tidy dumps a lot of diags, it can take 1-10secs of CPU while
# holding the GIL to |yaml.load| on my otherwise-idle dev box. |yaml_pool|
# lets us do this in parallel.
with multiprocessing.pool.Pool() as yaml_pool:
for ebuild in ebuild_list:
lint_tmpdir = generate_lints(board, ebuild.ebuild_path)
try:
results |= collect_lints(lint_tmpdir, yaml_pool)
except ClangTidyParseError:
if not parse_errors_are_nonfatal:
raise
logging.exception('Working on %r', ebuild)
finally:
if keep_dirs:
logging.info('Lints for %r are in %r', ebuild.ebuild_path,
lint_tmpdir)
else:
osutils.RmDir(lint_tmpdir, ignore_missing=True, sudo=True)
return results
def SendPerfValues(self, test_results_dir):
"""Gather all perf values in |test_results_dir| and send them to chromeperf.
The uploading will be retried 3 times for each file.
Args:
test_results_dir: A path to the directory with perf files.
"""
# A dict of list of perf values, keyed by test name.
perf_entries = collections.defaultdict(list)
for root, _, filenames in os.walk(test_results_dir):
for relative_name in filenames:
if not image_test_lib.IsPerfFile(relative_name):
continue
full_name = os.path.join(root, relative_name)
entries = perf_uploader.LoadPerfValues(full_name)
test_name = image_test_lib.ImageTestCase.GetTestName(relative_name)
perf_entries[test_name].extend(entries)
platform_name = self._run.bot_id
cros_ver = self._run.GetVersionInfo().VersionString()
chrome_ver = self._run.DetermineChromeVersion()
for test_name, perf_values in perf_entries.iteritems():
try:
perf_uploader.UploadPerfValues(perf_values, platform_name, test_name,
cros_version=cros_ver,
chrome_version=chrome_ver)
except Exception:
logging.exception('Fail to upload perf result for test %s.', test_name)
def GetCLRiskReport(build_id):
"""Returns a string reporting the riskiest CLs.
Args:
build_id: The master build id in cidb.
Returns:
A dictionary mapping "CL=risk%" strings to the CLs' URLs.
"""
try:
risks = _GetCLRisks(build_id)
except requests.exceptions.HTTPError:
logging.exception('Encountered an error reaching CL-Scanner.')
return {'(encountered exception reaching CL-Scanner)': ''}
except timeout_util.TimeoutError:
logging.exception('Timed out reaching CL-Scanner.')
return {'(timeout reaching CL-Scanner)': ''}
logging.info('CL-Scanner risks: %s', _PrettyPrintCLRisks(risks))
if not risks:
return {}
# TODO(phobbs) this will need to be changed to handle internal CLs.
top_risky = _TopRisky(risks)
return {
_PrettyPrintCLRisk(cl, risk): _EXTERNAL_CL_LINK % cl
for cl, risk in top_risky.iteritems()
}
def PerformStage(self):
"""Performs the stage. Overridden from generic_stages.BuilderStage."""
# CreateTestRoot creates a results directory and returns its path relative
# to the chroot.
chroot_results_dir = commands.CreateTestRoot(self._build_root)
try:
got_exception = False
try:
self._RunAllSuites(self._run.config.tast_vm_tests,
chroot_results_dir)
except Exception:
# sys.exc_info() returns (None, None, None) in the finally block, so we
# need to record the fact that we already have an error here.
got_exception = True
raise
finally:
self._ProcessAndArchiveResults(
self._MakeChrootPathAbsolute(chroot_results_dir),
[t.suite_name for t in self._run.config.tast_vm_tests],
got_exception)
except Exception:
logging.exception('Tast VM tests failed')
raise
def publish_notifications(self, topic, messages=None):
"""Publishes a test result notification to a given pubsub topic.
@param topic: The Cloud pubsub topic.
@param messages: A list of notification messages.
@returns A list of pubsub message ids, and empty if fails.
@raises PubSubException if failed to publish the notification.
"""
if not messages:
return None
pubsub = self._get_pubsub_service()
try:
body = {'messages': messages}
resp = pubsub.projects().topics().publish(
topic=topic, body=body).execute(
num_retries=DEFAULT_PUBSUB_NUM_RETRIES)
msgIds = []
if resp:
msgIds = resp.get('messageIds')
if msgIds:
logging.debug('Published notification message')
else:
logging.error('Failed to published notification message')
return msgIds
except errors.Error as e:
logging.exception('Failed to publish test result notification:%s',
e)
raise PubSubException('Failed to publish the notification')
def _ConsumeMessages(message_q, setup_args, setup_kwargs):
"""Configures ts_mon and gets metrics from a message queue.
Args:
message_q: A multiprocessing.Queue to read metrics from.
setup_args: Arguments to pass to SetupTsMonGlobalState.
setup_kwargs: Keyword arguments to pass to SetupTsMonGlobalState.
"""
last_flush = 0
pending = False
# If our parent dies, finish flushing before exiting.
reset_after = []
parallel.ExitWithParent(signal.SIGHUP)
signal.signal(signal.SIGHUP,
lambda _sig, _stack: _WaitToFlush(last_flush,
reset_after=reset_after))
# Configure ts-mon, but don't start up a sending thread.
setup_kwargs['auto_flush'] = False
SetupTsMonGlobalState(*setup_args, **setup_kwargs)
message = message_q.get()
while message:
try:
cls = getattr(metrics, message.metric_name)
metric = cls(*message.metric_args, **message.metric_kwargs)
if message.reset_after:
reset_after.append(metric)
getattr(metric, message.method)(
*message.method_args,
**message.method_kwargs)
except Exception:
logging.exception('Caught an exception while running %s',
_MethodCallRepr(message.metric_name,
message.method,
message.method_args,
message.method_kwargs))
pending, last_flush, time_delta = _FlushIfReady(True, last_flush,
reset_after=reset_after)
try:
# Only wait until the next flush time if we have pending metrics.
timeout = FLUSH_INTERVAL - time_delta if pending else None
message = message_q.get(timeout=timeout)
except Queue.Empty:
# We had pending metrics, but we didn't get a new message. Flush and wait
# indefinitely.
pending, last_flush, _ = _FlushIfReady(pending, last_flush,
reset_after=reset_after)
# Wait as long as we need to for the next metric.
message = message_q.get()
if pending:
_WaitToFlush(last_flush, reset_after=reset_after)
def afe_rpc_call(hostname):
"""Perform one rpc call set on server
@param hostname: server's hostname to poll
"""
afe_monitor = AfeMonitor(hostname)
try:
afe_monitor.run()
except Exception as e:
metrics.Counter(METRIC_MONITOR_ERROR).increment(
fields={'target_hostname': hostname})
logging.exception(e)
def _MapIgnoringErrors(f, sequence, exception_type=Exception):
"""Maps a function over a stream ignoring exceptions.
Args:
f: A function to call.
sequence: An iterable to map over, forgiving exceptions
exception_type: The specific exception to forgive.
"""
for item in sequence:
try:
yield f(item)
except exception_type as e:
log.exception('Ignoring error while mapping: %s.', e)
def _CallMetric(self, message):
"""Calls the metric method from |message|, ignoring exceptions."""
try:
cls = getattr(metrics, message.metric_name)
metric = cls(*message.metric_args, **message.metric_kwargs)
if message.reset_after:
self.reset_after_flush.append(metric)
getattr(metric, message.method)(*message.method_args,
**message.method_kwargs)
self.pending = True
except Exception:
logging.exception('Caught an exception while running %s',
_MethodCallRepr(message))
def _UploadNinjaLog(self, compiler_proxy_path):
"""Uploads .ninja_log file and its related metadata.
This uploads the .ninja_log file generated by ninja to build Chrome.
Also, it appends some related metadata at the end of the file following
'# end of ninja log' marker.
Args:
compiler_proxy_path: Path to the compiler proxy, which will be contained
in the metadata.
Returns:
The name of the uploaded file.
"""
ninja_log_path = os.path.join(self._goma_log_dir, 'ninja_log')
if not os.path.exists(ninja_log_path):
logging.warning('ninja_log is not found: %s', ninja_log_path)
return None
ninja_log_content = osutils.ReadFile(ninja_log_path)
try:
st = os.stat(ninja_log_path)
ninja_log_mtime = datetime.datetime.fromtimestamp(st.st_mtime)
except OSError:
logging.exception('Failed to get timestamp: %s', ninja_log_path)
return None
ninja_log_info = self._BuildNinjaInfo(compiler_proxy_path)
# Append metadata at the end of the log content.
ninja_log_content += '# end of ninja log\n' + json.dumps(
ninja_log_info)
# Aligned with goma_utils in chromium bot.
pid = os.getpid()
upload_ninja_log_path = os.path.join(
self._goma_log_dir, 'ninja_log.%s.%s.%s.%d' %
(getpass.getuser(), cros_build_lib.GetHostName(),
ninja_log_mtime.strftime('%Y%m%d-%H%M%S'), pid))
osutils.WriteFile(upload_ninja_log_path, ninja_log_content)
uploaded_filename = os.path.basename(upload_ninja_log_path) + '.gz'
self._gs_context.CopyInto(upload_ninja_log_path,
self._remote_dir,
filename=uploaded_filename,
auto_compress=True,
headers=self._headers)
return uploaded_filename
def _CleanupBatch(files):
"""Remove each file in a list of files, warning if they don't exist.
Args:
files: A list of file paths to remove.
"""
for path in files:
try:
os.remove(path)
except OSError as error:
if error.errno == errno.ENOENT:
log.exception(
'warning: could not find %s while attempting to remove it.',
path)
else:
raise
def _UploadPerfValues(self, *args, **kwargs):
"""Helper for uploading perf values.
This currently handles common checks only. We could make perf values more
integrated in the overall stage running process in the future though if we
had more stages that cared about this.
"""
# Only upload perf data for buildbots as the data from local tryjobs
# probably isn't useful to us.
if not self._run.options.buildbot:
return
try:
retry_util.RetryException(perf_uploader.PerfUploadingError, 3,
perf_uploader.UploadPerfValues, *args,
**kwargs)
except perf_uploader.PerfUploadingError:
logging.exception('Uploading perf data failed')
def _get_credential(self):
"""Gets the pubsub service api handle."""
if not os.path.isfile(self.credential_file):
logging.error('No credential file found')
raise PubSubException('Credential file does not exist:' +
self.credential_file)
try:
credential = GoogleCredentials.from_stream(self.credential_file)
if credential.create_scoped_required():
credential = credential.create_scoped(PUBSUB_SCOPES)
return credential
except ApplicationDefaultCredentialsError as ex:
logging.exception('Failed to get credential:%s', ex)
except errors.Error as e:
logging.exception('Failed to get the pubsub service handle:%s', e)
raise PubSubException('Credential file %s does not exists:' %
self.credential_file)
def _ProcessAndArchiveResults(self, abs_results_dir, suite_names,
already_have_error):
"""Processes and archives test results.
Args:
abs_results_dir: Absolute path to directory containing test results.
suite_names: List of string test suite names.
already_have_error: Boolean for whether testing has already failed.
Raises:
failures_lib.TestFailure if one or more tests failed or results were
unavailable. Suppressed if already_have_error is True.
"""
if not os.path.isdir(abs_results_dir) or not os.listdir(
abs_results_dir):
raise failures_lib.TestFailure(FAILURE_NO_RESULTS %
abs_results_dir)
archive_base = constants.TAST_VM_TEST_RESULTS % {
'attempt': self._attempt
}
_CopyResultsDir(abs_results_dir,
os.path.join(self.archive_path, archive_base))
# TODO(crbug.com/770562): Collect stack traces once the tast executable is
# symbolizing and collecting them (see VMTestStage._ArchiveTestResults).
# Now archive the results to Cloud Storage.
logging.info('Uploading artifacts to Cloud Storage...')
self.UploadArtifact(archive_base, archive=False, strict=False)
self.PrintDownloadLink(archive_base, RESULTS_LINK_PREFIX)
try:
self._ProcessResultsFile(abs_results_dir, archive_base,
suite_names)
except Exception as e:
# Don't raise a new exception if testing already failed.
if already_have_error:
logging.exception(
'Got error while archiving or processing results')
else:
raise e
finally:
osutils.RmDir(abs_results_dir, ignore_missing=True, sudo=True)
def UpdateStateful(self):
"""Update the stateful partition of the device."""
try:
stateful_update_payload = os.path.join(
self.device.work_dir, auto_updater_transfer.STATEFUL_FILENAME)
updater = stateful_updater.StatefulUpdater(self.device)
updater.Update(
stateful_update_payload,
update_type=(
stateful_updater.StatefulUpdater.UPDATE_TYPE_CLOBBER
if self._clobber_stateful else None))
# Delete the stateful update file on success so it doesn't occupy extra
# disk space. On failure it will get cleaned up.
self.device.DeletePath(stateful_update_payload)
except stateful_updater.Error as e:
error_msg = 'Stateful update failed with error: %s' % str(e)
logging.exception(error_msg)
self.ResetStatefulPartition()
raise StatefulUpdateError(error_msg)
def CleanupChroot(buildroot):
"""Unmount/clean up an image-based chroot without deleting the backing image.
Args:
buildroot: Directory containing the chroot to be cleaned up.
"""
chroot_dir = os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR)
logging.info('Cleaning up chroot at %s', chroot_dir)
if os.path.exists(chroot_dir) or os.path.exists(chroot_dir + '.img'):
try:
cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=False)
except timeout_util.TimeoutError:
logging.exception('Cleaning up chroot timed out')
# Dump debug info to help https://crbug.com/1000034.
cros_build_lib.run(['mount'], error_code_ok=False)
cros_build_lib.run(['uname', '-a'], error_code_ok=False)
cros_build_lib.sudo_run(['losetup', '-a'], error_code_ok=False)
cros_build_lib.run(['dmesg'], error_code_ok=False)
logging.warning(
'Assuming the bot is going to reboot, so ignoring this '
'failure; see https://crbug.com/1000034')
def SendPerfValues(self, test_results_dir):
"""Gather all perf values in |test_results_dir| and send them to chromeperf.
The uploading will be retried 3 times for each file.
Args:
test_results_dir: A path to the directory with perf files.
"""
# A dict of list of perf values, keyed by test name.
perf_entries = collections.defaultdict(list)
for root, _, filenames in os.walk(test_results_dir):
for relative_name in filenames:
if not image_test_lib.IsPerfFile(relative_name):
continue
full_name = os.path.join(root, relative_name)
entries = perf_uploader.LoadPerfValues(full_name)
test_name = image_test_lib.ImageTestCase.GetTestName(
relative_name)
perf_entries[test_name].extend(entries)
platform_name = self._run.bot_id
try:
cros_ver = self._run.GetVersionInfo().VersionString()
except cbuildbot_run.VersionNotSetError:
logging.error('Could not obtain version info. '
'Failed to upload perf results.')
return
chrome_ver = self._run.DetermineChromeVersion()
for test_name, perf_values in perf_entries.iteritems():
try:
perf_uploader.UploadPerfValues(perf_values,
platform_name,
test_name,
cros_version=cros_ver,
chrome_version=chrome_ver)
except Exception:
logging.exception('Failed to upload perf result for test %s.',
test_name)
def get_puppet_summary(time_fn=time.time):
path = _LAST_RUN_FILE
try:
with open(path) as fh:
data = yaml.safe_load(fh)
except IOError:
# This is fine - the system probably isn't managed by puppet.
return
except yaml.YAMLError:
# This is less fine - the file exists but is invalid.
logging.exception('Failed to read puppet lastrunfile %s', path)
return
if not isinstance(data, dict):
return
try:
config_version.set(data['version']['config'])
except ts_mon.MonitoringInvalidValueTypeError:
# https://crbug.com/581749
logging.exception('lastrunfile contains invalid "config" value. '
'Please fix Puppet.')
except KeyError:
logging.warning('version/config not found in %s', path)
try:
puppet_version.set(data['version']['puppet'])
except ts_mon.MonitoringInvalidValueTypeError:
# https://crbug.com/581749
logging.exception('lastrunfile contains invalid puppet version. '
'Please fix Puppet.')
except KeyError:
logging.warning('version/puppet not found in %s', path)
try:
for key, value in data['events'].iteritems():
if key != 'total':
events.set(value, {'result': key})
except KeyError:
logging.warning('events not found in %s', path)
try:
for key, value in data['resources'].iteritems():
resources.set(value, {'action': key})
except KeyError:
logging.warning('resources not found in %s', path)
try:
for key, value in data['time'].iteritems():
if key == 'last_run':
age.set(time_fn() - value)
elif key != 'total':
times.set(value, {'step': key})
except KeyError:
logging.warning('time not found in %s', path)
def UploadPerfValues(perf_values, platform_name, test_name, revision=None,
cros_version=None, chrome_version=None,
dashboard=DASHBOARD_URL, master_name=None,
test_prefix=None, platform_prefix=None, dry_run=False):
"""Uploads any perf data associated with a test to the perf dashboard.
Note: If |revision| is used, then |cros_version| & |chrome_version| are not
necessary. Conversely, if |revision| is not used, then |cros_version| and
|chrome_version| must both be specified.
Args:
perf_values: List of PerformanceValue objects.
platform_name: A string identifying platform e.g. 'x86-release'. 'cros-'
will be prepended to |platform_name| internally, by _FormatForUpload.
test_name: A string identifying the test
revision: The raw X-axis value; normally it represents a VCS repo, but may
be any monotonic increasing value integer.
cros_version: A string identifying Chrome OS version e.g. '6052.0.0'.
chrome_version: A string identifying Chrome version e.g. '38.0.2091.2'.
dashboard: The dashboard to upload data to.
master_name: The "master" field to use; by default it is looked up in the
perf_dashboard_config.json database.
test_prefix: Arbitrary string to automatically prefix to the test name.
If None, then 'cbuildbot.' is used to guarantee namespacing.
platform_prefix: Arbitrary string to automatically prefix to
|platform_name|. If None, then 'cros-' is used to guarantee namespacing.
dry_run: Do everything but upload the data to the server.
"""
if not perf_values:
return
# Aggregate values from multiple iterations together.
perf_data = _AggregateIterations(perf_values)
# Compute averages and standard deviations as needed for measured perf
# values that exist in multiple iterations. Ultimately, we only upload a
# single measurement (with standard deviation) for every unique measured
# perf metric.
_ComputeAvgStddev(perf_data)
# Format the perf data for the upload, then upload it.
if revision is None:
# No "revision" field, calculate one. Chrome and CrOS fields must be given.
cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
revision = _ComputeRevisionFromVersions(chrome_version, cros_version)
try:
if master_name is None:
presentation_info = _GetPresentationInfo(test_name)
else:
presentation_info = PresentationInfo(master_name, test_name)
formatted_data = _FormatForUpload(perf_data, platform_name,
presentation_info,
revision=revision,
cros_version=cros_version,
chrome_version=chrome_version,
test_prefix=test_prefix,
platform_prefix=platform_prefix)
if dry_run:
logging.debug('UploadPerfValues: skipping upload due to dry-run')
else:
retry_util.RetryException(PerfUploadingError, 3, _SendToDashboard,
formatted_data, dashboard=dashboard)
except PerfUploadingError:
logging.exception('Error when uploading perf data to the perf '
'dashboard for test %s.', test_name)
raise
else:
logging.info('Successfully uploaded perf data to the perf '
'dashboard for test %s.', test_name)
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""For running module as script."""
from __future__ import print_function
import sys
from chromite.lib import cros_logging as logging
from chromite.scripts import sysmon
try:
sysmon.main(sys.argv[1:])
except Exception:
logging.exception('sysmon throws an error')
raise
def UploadPerfValues(perf_values, platform_name, test_name, revision=None,
cros_version=None, chrome_version=None,
dashboard=DASHBOARD_URL, master_name=None,
test_prefix=None, platform_prefix=None, dry_run=False):
"""Uploads any perf data associated with a test to the perf dashboard.
Note: If |revision| is used, then |cros_version| & |chrome_version| are not
necessary. Conversely, if |revision| is not used, then |cros_version| and
|chrome_version| must both be specified.
Args:
perf_values: List of PerformanceValue objects.
platform_name: A string identifying platform e.g. 'x86-release'. 'cros-'
will be prepended to |platform_name| internally, by _FormatForUpload.
test_name: A string identifying the test
revision: The raw X-axis value; normally it represents a VCS repo, but may
be any monotonic increasing value integer.
cros_version: A string identifying Chrome OS version e.g. '6052.0.0'.
chrome_version: A string identifying Chrome version e.g. '38.0.2091.2'.
dashboard: The dashboard to upload data to.
master_name: The "master" field to use; by default it is looked up in the
perf_dashboard_config.json database.
test_prefix: Arbitrary string to automatically prefix to the test name.
If None, then 'cbuildbot.' is used to guarantee namespacing.
platform_prefix: Arbitrary string to automatically prefix to
|platform_name|. If None, then 'cros-' is used to guarantee namespacing.
dry_run: Do everything but upload the data to the server.
"""
if not perf_values:
return
# Aggregate values from multiple iterations together.
perf_data = _AggregateIterations(perf_values)
# Compute averages and standard deviations as needed for measured perf
# values that exist in multiple iterations. Ultimately, we only upload a
# single measurement (with standard deviation) for every unique measured
# perf metric.
_ComputeAvgStddev(perf_data)
# Format the perf data for the upload, then upload it.
if revision is None:
# No "revision" field, calculate one. Chrome and CrOS fields must be given.
cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
revision = _ComputeRevisionFromVersions(chrome_version, cros_version)
try:
if master_name is None:
presentation_info = _GetPresentationInfo(test_name)
else:
presentation_info = PresentationInfo(master_name, test_name)
formatted_data = _FormatForUpload(perf_data, platform_name,
presentation_info,
revision=revision,
cros_version=cros_version,
chrome_version=chrome_version,
test_prefix=test_prefix,
platform_prefix=platform_prefix)
if dry_run:
logging.debug('UploadPerfValues: skipping upload due to dry-run')
else:
retry_util.GenericRetry(_RetryIfServerError, 3, _SendToDashboard,
formatted_data, dashboard=dashboard)
except PerfUploadingError:
logging.exception('Error when uploading perf data to the perf '
'dashboard for test %s.', test_name)
raise
else:
logging.info('Successfully uploaded perf data to the perf '
'dashboard for test %s.', test_name)
def GenerateAlertStage(build, stage, exceptions, aborted, buildinfo,
logdog_client):
"""Generate alert details for a single build stage.
Args:
build: Dictionary of build details from CIDB.
stage: Dictionary of stage details from CIDB.
exceptions: A list of instances of failure_message_lib.StageFailure.
aborted: Boolean indicated if the build was aborted.
buildinfo: BuildInfo build JSON file from MILO.
logdog_client: logdog.LogdogClient object.
Returns:
som.CrosStageFailure object if stage requires alert. None otherwise.
"""
STAGE_IGNORE_STATUSES = frozenset([
constants.BUILDER_STATUS_PASSED, constants.BUILDER_STATUS_PLANNED,
constants.BUILDER_STATUS_SKIPPED
])
ABORTED_IGNORE_STATUSES = frozenset([
constants.BUILDER_STATUS_INFLIGHT, constants.BUILDER_STATUS_FORGIVEN,
constants.BUILDER_STATUS_WAITING
])
NO_LOG_RETRY_STATUSES = frozenset(
[constants.BUILDER_STATUS_INFLIGHT, constants.BUILDER_STATUS_ABORTED])
# IGNORE_EXCEPTIONS should be ignored if they're the only exception.
IGNORE_EXCEPTIONS = frozenset(['ImportantBuilderFailedException'])
# ABORTED_DISREGARD_EXCEPTIONS should cause any failures of aborted stages
# to be entirely disregarded.
ABORTED_DISREGARD_EXCEPTIONS = frozenset(['_ShutDownException'])
if (stage['build_id'] != build['id']
or stage['status'] in STAGE_IGNORE_STATUSES):
return None
if aborted and stage['status'] in ABORTED_IGNORE_STATUSES:
return None
logging.info(' stage %s (id %d): %s', stage['name'], stage['id'],
stage['status'])
logs_links = []
notes = []
# Generate links to the logs of the stage and use them for classification.
if buildinfo and stage['name'] in buildinfo['steps']:
prefix = buildinfo['annotationStream']['prefix']
annotation = buildinfo['steps'][stage['name']]
AddLogsLink(logdog_client, 'stdout', buildinfo['project'], prefix,
annotation.get('stdoutStream'), logs_links)
AddLogsLink(logdog_client, 'stderr', buildinfo['project'], prefix,
annotation.get('stderrStream'), logs_links)
# Use the logs in an attempt to classify the failure.
if (annotation.get('stdoutStream')
and annotation['stdoutStream'].get('name')):
path = '%s/+/%s' % (prefix, annotation['stdoutStream']['name'])
try:
# If either the build or stage is reporting as being inflight,
# LogDog might still be waiting for logs so don't wait unnecesarily
# for them.
retry = (build['status'] not in NO_LOG_RETRY_STATUSES
and stage['status'] not in NO_LOG_RETRY_STATUSES)
logs = logdog_client.GetLines(buildinfo['project'],
path,
allow_retries=retry)
classification = classifier.ClassifyFailure(
stage['name'], logs)
for c in classification or []:
notes.append('Classification: %s' % (c))
except Exception as e:
logging.exception('Could not classify logs: %s', e)
notes.append('Warning: unable to classify logs: %s' % (e))
elif aborted:
# Aborted build with no stage logs is not worth reporting on.
return None
else:
notes.append('Warning: stage logs unavailable')
# Copy the links from the buildbot build JSON.
stage_links = []
if buildinfo:
if stage['status'] == constants.BUILDER_STATUS_FORGIVEN:
# TODO: Include these links but hide them by default in frontend.
pass
elif stage['name'] in buildinfo['steps']:
step = buildinfo['steps'][stage['name']]
stage_links = [
som.Link(l['label'], l['url'])
for l in step.get('otherLinks', [])
]
else:
steps = [
s for s in buildinfo['steps'].keys()
if s is not None and not isinstance(s, tuple)
]
logging.warn('Could not find stage %s in: %s', stage['name'],
', '.join(steps))
else:
notes.append('Warning: stage details unavailable')
# Limit the number of links that will be displayed for a single stage.
# Let there be one extra since it doesn't make sense to have a line
# saying there is one more.
# TODO: Move this to frontend so they can be unhidden by clicking.
if len(stage_links) > MAX_STAGE_LINKS + 1:
# Insert at the beginning of the notes which come right after the links.
notes.insert(
0, '... and %d more URLs' % (len(stage_links) - MAX_STAGE_LINKS))
del stage_links[MAX_STAGE_LINKS:]
# Add all exceptions recording in CIDB as notes.
has_other_exceptions = False
has_ignore_exception = False
for e in exceptions:
if e.build_stage_id == stage['id']:
notes.append('%s: %s' % (e.exception_type, e.exception_message))
if aborted and e.exception_type in ABORTED_DISREGARD_EXCEPTIONS:
# Don't generate alert if the exception indicates it should be
# entirely disregarded.
return None
elif e.exception_type in IGNORE_EXCEPTIONS:
# Ignore this exception (and stage if there aren't other exceptions).
has_ignore_exception = True
continue
has_other_exceptions = True
# If there is an ignored exception and no other exceptions, treat this
# stage as non-failed.
if has_ignore_exception and not has_other_exceptions:
return None
# Add the stage to the alert.
return som.CrosStageFailure(stage['name'],
MapCIDBToSOMStatus(stage['status']),
logs_links, stage_links, notes)