def testReportStageFailureOnPackageBuildFailure(self):
"""Test ReportStageFailure On PackageBuildFailure."""
fake_db = fake_cidb.FakeCIDBConnection()
msg = 'run command error'
short_name = 'short name'
failed_packages = ['chromeos-base/autotest', 'chromeos-base/telemetry']
error = cros_build_lib.RunCommandError(msg,
cros_build_lib.CommandResult())
build_failure = failures_lib.PackageBuildFailure(
error, short_name, failed_packages)
mock_build_stage_id = 1
failure_id = failures_lib.ReportStageFailure(fake_db,
mock_build_stage_id,
build_failure)
extra_info_json_string = json.dumps({
'shortname': short_name,
'failed_packages': failed_packages
})
self.assertEqual(len(fake_db.failureTable), 1)
values = fake_db.failureTable[failure_id]
self.assertEqual(values['exception_message'], str(build_failure))
self.assertEqual(values['outer_failure_id'], None)
self.assertEqual(values['extra_info'], extra_info_json_string)
self.assertEqual(
json.loads(values['extra_info'])['failed_packages'],
failed_packages)
def testReportStageFailureOnCompoundFailure(self):
"""Tests ReportStageFailure on CompoundFailure."""
fake_db = fake_cidb.FakeCIDBConnection()
inner_exception_1 = failures_lib.TestLabFailure()
inner_exception_2 = TypeError()
exc_infos = failures_lib.CreateExceptInfo(inner_exception_1, None)
exc_infos += failures_lib.CreateExceptInfo(inner_exception_2, None)
outer_exception = failures_lib.GoBFailure(exc_infos=exc_infos)
mock_build_stage_id = 9345
outer_failure_id = failures_lib.ReportStageFailure(
fake_db, mock_build_stage_id, outer_exception)
self.assertEqual(3, len(fake_db.failureTable))
for failure_id, failure in fake_db.failureTable.iteritems():
self.assertEqual(failure['build_stage_id'], mock_build_stage_id)
self.assertIsNone(failure['extra_info'])
if failure_id == outer_failure_id:
self.assertEqual(failure_id, outer_failure_id)
self.assertEqual(failure['exception_message'],
outer_exception.ToSummaryString())
elif failure[
'exception_type'] == failures_lib.TestLabFailure.__name__:
self.assertEqual(failure['outer_failure_id'], outer_failure_id)
self.assertEqual(failure['exception_category'],
constants.EXCEPTION_CATEGORY_LAB)
elif failure['exception_type'] == TypeError.__name__:
self.assertEqual(failure['outer_failure_id'], outer_failure_id)
self.assertEqual(failure['exception_category'],
constants.EXCEPTION_CATEGORY_UNKNOWN)
def testReportStageFailure(self):
"""Test ReportStageFailure."""
class FakeStepFailure(failures_lib.StepFailure):
"""A fake StepFailure subclass for unittest."""
EXCEPTION_CATEGORY = 'unittest'
fake_failure = FakeStepFailure('Toot! Toot!')
insert_failure_fn = self.PatchObject(failures_lib,
'_InsertFailureToMonarch')
failures_lib.ReportStageFailure(
fake_failure, {})
insert_failure_fn.assert_called_once_with(exception_category='unittest',
metrics_fields={})
def _RunParallelStages(stage_objs):
"""Run the specified stages in parallel.
Args:
stage_objs: BuilderStage objects.
"""
steps = [stage.Run for stage in stage_objs]
try:
parallel.RunParallelSteps(steps)
except BaseException as ex:
logging.error('BaseException in _RunParallelStages %s' % ex,
exc_info=True)
# If a stage threw an exception, it might not have correctly reported
# results (e.g. because it was killed before it could report the
# results.) In this case, attribute the exception to any stages that
# didn't report back correctly (if any).
for stage in stage_objs:
for name in stage.GetStageNames():
if not results_lib.Results.StageHasResults(name):
results_lib.Results.Record(
name, ex, str(ex), prefix=stage.StageNamePrefix())
if cidb.CIDBConnectionFactory.IsCIDBSetup():
db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(
)
for build_stage_id in stage.GetBuildStageIDs():
stage_status = db.GetBuildStage(build_stage_id)
# If no failures for this stage found in failureTable, and the stage
# has no status or has non-failure status in buildStageTable,
# report failures to failureTable for this stage.
if (not db.HasFailureMsgForStage(build_stage_id) and
(stage_status is None or stage_status['status']
not in constants.BUILDER_NON_FAILURE_STATUSES)):
failures_lib.ReportStageFailure(
db,
build_stage_id,
ex,
build_config=stage.build_config)
# If this stage has non_completed status in buildStageTable, mark
# the stage as 'fail' status in buildStageTable.
if (stage_status is not None and stage_status['status']
not in constants.BUILDER_COMPLETED_STATUSES):
db.FinishBuildStage(
build_stage_id,
constants.BUILDER_STATUS_FAILED)
raise
def testReportStageFailureOnBuildScriptFailure(self):
"""Test ReportStageFailure On BuildScriptFailure."""
fake_db = fake_cidb.FakeCIDBConnection()
msg = 'run command error'
short_name = 'short name'
error = cros_build_lib.RunCommandError(msg,
cros_build_lib.CommandResult())
build_failure = failures_lib.BuildScriptFailure(error, short_name)
mock_build_stage_id = 1
failure_id = failures_lib.ReportStageFailure(fake_db,
mock_build_stage_id,
build_failure)
extra_info_json_string = json.dumps({'shortname': short_name})
self.assertEqual(len(fake_db.failureTable), 1)
values = fake_db.failureTable[failure_id]
self.assertEqual(values['exception_message'], msg)
self.assertEqual(values['outer_failure_id'], None)
self.assertEqual(values['extra_info'], extra_info_json_string)
self.assertEqual(
json.loads(values['extra_info'])['shortname'], short_name)
def Run(self):
"""Have the builder execute the stage."""
skip_stage = self._ShouldSkipStage()
previous_record = results_lib.Results.PreviouslyCompletedRecord(
self.name)
if skip_stage:
self._BeginStepForBuildbot(' : [SKIPPED]')
elif previous_record is not None:
self._BeginStepForBuildbot(' : [PREVIOUSLY PROCESSED]')
else:
self._BeginStepForBuildbot()
try:
# Set default values
result = None
cidb_result = None
description = None
board = ''
elapsed_time = None
start_time = time.time()
if skip_stage:
self._StartBuildStageInCIDB()
self._PrintLoudly('Not running Stage %s' % self.name)
self.HandleSkip()
result = results_lib.Results.SKIPPED
return
if previous_record:
self._StartBuildStageInCIDB()
self._PrintLoudly('Stage %s processed previously' % self.name)
self.HandleSkip()
# Success is stored in the results log for a stage that completed
# successfully in a previous run. But, we report the truth to CIDB.
result = results_lib.Results.SUCCESS
cidb_result = constants.BUILDER_STATUS_SKIPPED
# Copy over metadata from the previous record. instead of returning
# metadata about the current run.
board = previous_record.board
elapsed_time = float(previous_record.time)
return
self._WaitBuildStageInCIDB()
ready = self.WaitUntilReady()
if not ready:
self._PrintLoudly(
'Stage %s precondition failed while waiting to start.' %
self.name)
# If WaitUntilReady is false, mark stage as skipped in Results and CIDB
result = results_lib.Results.SKIPPED
return
# Ready to start, mark buildStage as inflight in CIDB
self._Print('Preconditions for the stage successfully met. '
'Beginning to execute stage...')
self._StartBuildStageInCIDB()
start_time = time.time()
sys.stdout.flush()
sys.stderr.flush()
# TODO(davidjames): Verify that PerformStage always returns None. See
# crbug.com/264781
self.PerformStage()
result = results_lib.Results.SUCCESS
except SystemExit as e:
if e.code != 0:
result, description, _ = self._TopHandleStageException()
raise
except Exception as e:
if isinstance(e, failures_lib.ExitEarlyException):
# One stage finished and exited early, not a failure.
result = results_lib.Results.SUCCESS
raise
# Tell the build bot this step failed for the waterfall.
result, description, retrying = self._TopHandleStageException()
if result not in (results_lib.Results.FORGIVEN,
results_lib.Results.SUCCESS):
if isinstance(e, failures_lib.StepFailure):
raise
else:
raise failures_lib.StepFailure()
elif retrying:
raise failures_lib.RetriableStepFailure()
except BaseException:
result, description, _ = self._TopHandleStageException()
raise
finally:
# Some cases explicitly set a cidb status. For others, infer.
if cidb_result is None:
cidb_result = self._TranslateResultToCIDBStatus(result)
if elapsed_time is None:
elapsed_time = time.time() - start_time
self._RecordResult(self.name,
result,
description,
prefix=self._prefix,
board=board,
time=elapsed_time,
build_stage_id=self._build_stage_id)
self._FinishBuildStageInCIDBAndMonarch(cidb_result, elapsed_time)
if isinstance(result,
BaseException) and self._build_stage_id is not None:
_, db = self._run.GetCIDBHandle()
if db:
failures_lib.ReportStageFailure(
db,
self._build_stage_id,
result,
build_config=self.build_config)
try:
self.Finish()
except Exception as e:
# Failures here are OUTSIDE of the stage and not handled well. Log and
# continue with the assumption that the ReportStage will re-upload this
# data or report a failure correctly.
logging.warning('IGNORED: Finish failure: %s', e)
self._PrintLoudly('Finished Stage %s - %s' %
(self.name, cros_build_lib.UserDateTimeFormat()))
sys.stdout.flush()
sys.stderr.flush()