def testScheduleUnimportantSlaveBuildsFailure(self):
"""Test ScheduleSlaveBuilds with unimportant slave failures."""
stage = self.ConstructStage()
self.PatchObject(
scheduler_stages.ScheduleSlavesStage,
'PostSlaveBuildToBuildbucket',
side_effect=buildbucket_lib.BuildbucketResponseException)
slave_config_map = {
'slave_external': config_lib.BuildConfig(important=False),
}
self.PatchObject(generic_stages.BuilderStage,
'_GetSlaveConfigMap',
return_value=slave_config_map)
stage.ScheduleSlaveBuildsViaBuildbucket(important_only=False,
dryrun=True)
scheduled_slaves = self._run.attrs.metadata.GetValue(
constants.METADATA_SCHEDULED_IMPORTANT_SLAVES)
self.assertEqual(len(scheduled_slaves), 0)
unscheduled_slaves = self._run.attrs.metadata.GetValue(
constants.METADATA_UNSCHEDULED_SLAVES)
self.assertEqual(len(unscheduled_slaves), 1)
def _GetTestConfig(self):
test_config = config_lib.SiteConfig()
test_config.Add(
'master',
config_lib.BuildConfig(),
boards=[],
build_type=self.build_type,
master=True,
slave_configs=['test3', 'test5'],
manifest_version=True,
)
test_config.Add(
'test1',
config_lib.BuildConfig(),
boards=['amd64-generic'],
manifest_version=True,
build_type=constants.PFQ_TYPE,
overlays='public',
important=False,
chrome_rev=None,
branch=False,
internal=False,
master=False,
)
test_config.Add(
'test2',
config_lib.BuildConfig(),
boards=['amd64-generic'],
manifest_version=False,
build_type=constants.PFQ_TYPE,
overlays='public',
important=True,
chrome_rev=None,
branch=False,
internal=False,
master=False,
)
test_config.Add(
'test3',
config_lib.BuildConfig(),
boards=['amd64-generic'],
manifest_version=True,
build_type=constants.PFQ_TYPE,
overlays='both',
important=True,
chrome_rev=None,
branch=False,
internal=True,
master=False,
)
test_config.Add(
'test4',
config_lib.BuildConfig(),
boards=['amd64-generic'],
manifest_version=True,
build_type=constants.PFQ_TYPE,
overlays='both',
important=True,
chrome_rev=None,
branch=True,
internal=True,
master=False,
)
test_config.Add(
'test5',
config_lib.BuildConfig(),
boards=['amd64-generic'],
manifest_version=True,
build_type=constants.PFQ_TYPE,
overlays='public',
important=True,
chrome_rev=None,
branch=False,
internal=False,
master=False,
)
return test_config
def _ExtendDefaultConfig(**kwargs):
"""Extend DEFAULT_CONFIG with keys/values in kwargs."""
config_kwargs = DEFAULT_CONFIG.copy()
config_kwargs.update(kwargs)
return config_lib.BuildConfig(**config_kwargs)
DEFAULT_OPTIONS = cros_test_lib.EasyAttr(
archive_base=DEFAULT_ARCHIVE_BASE,
buildroot=DEFAULT_BUILDROOT,
buildnumber=DEFAULT_BUILDNUMBER,
buildbot=True,
branch=DEFAULT_BRANCH,
remote_trybot=False,
debug=False,
postsync_patch=True,
)
DEFAULT_CONFIG = config_lib.BuildConfig(
name=DEFAULT_BOT_NAME,
master=True,
boards=[DEFAULT_BOARD],
postsync_patch=True,
child_configs=[
config_lib.BuildConfig(name='foo', postsync_patch=False, boards=[]),
config_lib.BuildConfig(name='bar', postsync_patch=False, boards=[]),
],
)
DEFAULT_VERSION = '6543.2.1'
def _ExtendDefaultOptions(**kwargs):
"""Extend DEFAULT_OPTIONS with keys/values in kwargs."""
options_kwargs = DEFAULT_OPTIONS.copy()
options_kwargs.update(kwargs)
return cros_test_lib.EasyAttr(**options_kwargs)
# Access to protected member.
# pylint: disable=W0212
DEFAULT_OPTIONS = cros_test_lib.EasyAttr(
archive_base=DEFAULT_ARCHIVE_BASE,
buildroot=DEFAULT_BUILDROOT,
buildnumber=DEFAULT_BUILDNUMBER,
buildbot=True,
branch=DEFAULT_BRANCH,
remote_trybot=False,
debug=False,
)
DEFAULT_CONFIG = config_lib.BuildConfig(name=DEFAULT_BOT_NAME,
master=True,
boards=[DEFAULT_BOARD],
child_configs=[
config_lib.BuildConfig(name='foo'),
config_lib.BuildConfig(name='bar'),
],
gs_path=config_lib.GS_PATH_DEFAULT)
def _ExtendDefaultOptions(**kwargs):
"""Extend DEFAULT_OPTIONS with keys/values in kwargs."""
options_kwargs = DEFAULT_OPTIONS.copy()
options_kwargs.update(kwargs)
return cros_test_lib.EasyAttr(**options_kwargs)
def _ExtendDefaultConfig(**kwargs):
"""Extend DEFAULT_CONFIG with keys/values in kwargs."""
config_kwargs = DEFAULT_CONFIG.copy()
def VerifyStage(self,
failing,
inflight,
no_stat,
handle_failure=False,
handle_timeout=False,
sane_tot=True,
stage=None,
all_slaves=None,
slave_stages=None,
fatal=True,
self_destructed=False):
"""Runs and Verifies PerformStage.
Args:
failing: The names of the builders that failed.
inflight: The names of the buiders that timed out.
no_stat: The names of the builders that had no status.
handle_failure: If True, calls HandleValidationFailure.
handle_timeout: If True, calls HandleValidationTimeout.
sane_tot: If not true, assumes TOT is not sane.
stage: If set, use this constructed stage, otherwise create own.
all_slaves: Optional set of all slave configs.
slave_stages: Optional list of slave stages.
fatal: Optional boolean indicating whether the completion_stage failed
with fatal. Default to True.
self_destructed: Optional boolean indicating whether the completion_stage
self_destructed. Default to False.
"""
if not stage:
stage = self.ConstructStage()
stage._run.attrs.metadata.UpdateWithDict(
{constants.SELF_DESTRUCTED_BUILD: self_destructed})
# Setup the stage to look at the specified configs.
all_slaves = list(all_slaves or set(failing + inflight + no_stat))
all_started_slaves = list(all_slaves or set(failing + inflight))
configs = [config_lib.BuildConfig(name=x) for x in all_slaves]
self.PatchObject(stage, '_GetSlaveConfigs', return_value=configs)
statuses = {}
for x in failing:
statuses[x] = builder_status_lib.BuilderStatus(
constants.BUILDER_STATUS_FAILED, message=None)
for x in inflight:
statuses[x] = builder_status_lib.BuilderStatus(
constants.BUILDER_STATUS_INFLIGHT, message=None)
for x in no_stat:
statuses[x] = builder_status_lib.BuilderStatus(
constants.BUILDER_STATUS_MISSING, message=None)
self.completion_stage.GetSlaveStatuses.return_value = statuses
self.completion_stage.GetFatal.return_value = fatal
# Setup DB and provide list of slave stages.
mock_cidb = mock.MagicMock()
cidb.CIDBConnectionFactory.SetupMockCidb(mock_cidb)
if slave_stages is None:
slave_stages = []
critical_stages = (
relevant_changes.TriageRelevantChanges.STAGE_SYNC)
for stage_name, slave in itertools.product(critical_stages,
all_started_slaves):
slave_stages.append({
'name': stage_name,
'build_config': slave,
'status': constants.BUILDER_STATUS_PASSED
})
self.PatchObject(mock_cidb,
'GetSlaveStages',
return_value=slave_stages)
# Set up SubmitPartialPool to provide a list of changes to look at.
self.PatchObject(stage.sync_stage.pool,
'SubmitPartialPool',
return_value=self.other_changes)
# Actually run the stage.
stage.PerformStage()
if fatal:
stage.sync_stage.pool.submit_pool_mock.assert_not_called()
self.mock_record_metrics.assert_called_once_with(False)
else:
stage.sync_stage.pool.submit_pool_mock.assert_called_once_with(
reason=constants.STRATEGY_CQ_SUCCESS)
self.mock_record_metrics.assert_called_once_with(True)
if handle_failure:
stage.sync_stage.pool.handle_failure_mock.assert_called_once_with(
mock.ANY,
no_stat=set(no_stat),
sanity=sane_tot,
changes=self.other_changes,
failed_hwtests=mock.ANY)
if handle_timeout:
stage.sync_stage.pool.handle_timeout_mock.assert_called_once_with(
sanity=mock.ANY, changes=self.other_changes)
def testApplyCallable(self):
# Callable that adds a configurable amount.
append = lambda x: lambda base: base + ' ' + x
site_config = config_lib.SiteConfig()
site_config.AddTemplate('add1', foo=append('one'))
site_config.AddTemplate('add2', foo=append('two'))
site_config.AddTemplate('fixed', foo='fixed')
site_config.AddTemplate('derived', site_config.templates.add1)
site_config.AddTemplate('stacked', site_config.templates.add1,
site_config.templates.add2)
site_config.AddTemplate('stackedDeep',
site_config.templates.fixed,
site_config.templates.add1,
site_config.templates.add1,
site_config.templates.add1,
foo=append('deep'))
site_config.AddTemplate('stackedDeeper',
site_config.templates.stacked,
site_config.templates.stackedDeep,
foo=append('deeper'))
base = config_lib.BuildConfig(foo='base')
# Basic apply.
result = base.derive(site_config.templates.add1)
self.assertEqual(result.foo, 'base one')
# Callable template + local callable.
result = base.derive(site_config.templates.add1, foo=append('local'))
self.assertEqual(result.foo, 'base one local')
# Callable template + local fixed.
result = base.derive(site_config.templates.add1, foo='local')
self.assertEqual(result.foo, 'local')
# Derived template.
result = base.derive(site_config.templates.derived)
self.assertEqual(result.foo, 'base one')
# Template with fixed override after stacking template (all callable magic
# should disappear).
result = base.derive(site_config.templates.fixed)
self.assertEqual(result.foo, 'fixed')
# Template with stacked.
result = base.derive(site_config.templates.stacked)
self.assertEqual(result.foo, 'base one two')
# Callables on top of fixed from template.
result = base.derive(site_config.templates.stackedDeep)
self.assertEqual(result.foo, 'fixed one one one deep')
# Just get crazy with it.
result = base.derive(site_config.templates.stackedDeeper)
self.assertEqual(result.foo, 'fixed one one one deep deeper')
# Ensure objects derived from weren't modified.
self.assertEqual(base.foo, 'base')
self.assertEqual(site_config.templates.fixed.foo, 'fixed')
def testApplyMixed(self):
# Apply simple values..
config = config_lib.BuildConfig()
config.apply(self.fooConfig, self.barConfig, a=1, b=2, bar=3)
self.assertEqual(config, dict(name='bar', foo=1, bar=3, a=1, b=2))
def testAddForBoards(self):
per_board = {
'foo': config_lib.BuildConfig(value='foo'),
'bar': config_lib.BuildConfig(value='bar'),
'multiboard': config_lib.BuildConfig(boards=['foo', 'bar']),
}
# Test the minimal invocation.
self.site_config.AddForBoards(
'minimal',
['foo', 'bar'],
)
self.assertIn('foo-minimal', self.site_config)
self.assertEqual(self.site_config['foo-minimal'].boards, ['foo'])
self.assertEqual(self.site_config['bar-minimal'].boards, ['bar'])
# Test a partial set of per-board values specified.
self.site_config.AddForBoards(
'partial_per_board',
['foo', 'no_per'],
per_board,
)
self.assertIn('foo-partial_per_board', self.site_config)
self.assertIn('no_per-partial_per_board', self.site_config)
self.assertEqual(self.site_config['foo-partial_per_board'].value,
'foo')
# Test all boards with per_board values specified, and test we can
# override board listing in per_board values.
self.site_config.AddForBoards(
'per_board',
['foo', 'bar', 'multiboard'],
per_board,
)
self.assertEqual(self.site_config['foo-per_board'].value, 'foo')
self.assertEqual(self.site_config['bar-per_board'].value, 'bar')
self.assertEqual(self.site_config['multiboard-per_board'].boards,
['foo', 'bar'])
# Test using a template
self.site_config.AddForBoards(
'template',
['foo', 'bar'],
None,
self.site_config.templates.template,
)
self.assertEqual(self.site_config['foo-template'].value, 'template')
# Test a template, and a mixin.
self.site_config.AddForBoards(
'mixin',
['foo', 'bar'],
None,
self.site_config.templates.template,
self.site_config.templates.mixin,
)
self.assertEqual(self.site_config['foo-mixin'].value, 'mixin')
# Test a template, and a mixin, and a per-board.
self.site_config.AddForBoards(
'mixin_per_board',
['foo', 'bar'],
per_board,
self.site_config.templates.template,
self.site_config.templates.mixin,
)
self.assertEqual(self.site_config['foo-mixin_per_board'].value, 'foo')
# Test a template, and a mixin, and a per-board, and an override.
self.site_config.AddForBoards(
'override',
['foo', 'bar'],
per_board,
self.site_config.templates.template,
self.site_config.templates.mixin,
value='override',
)
self.assertEqual(self.site_config['foo-override'].value, 'override')
def ApplyCustomOverrides(site_config):
"""Method with to override specific flags for specific builders.
Generally try really hard to avoid putting anything here that isn't
a really special case for a single specific builder. This is performed
after every other bit of processing, so it always has the final say.
Args:
site_config: config_lib.SiteConfig containing builds to have their
waterfall values updated.
"""
overwritten_configs = {
'lakitu-release':
config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations, ),
# This is the full build of open-source overlay.
'lakitu-full':
config_lib.BuildConfig().apply(
# logging_CrashSender is expected to fail for lakitu-full.
# See b/111567339 for more details.
useflags=config_lib.append_useflags(['-tests_logging_CrashSender'
]), ),
'lakitu-gpu-release':
config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations, ),
'lakitu-nc-release':
config_lib.BuildConfig().apply(signer_tests=False, ),
'lakitu-st-release':
config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations, ),
'lakitu_next-release':
config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations,
signer_tests=False,
),
'guado_labstation-release': {
'hw_tests': [],
# 'hwqual':False,
'image_test': False,
# 'images':['test'],
'signer_tests': False,
'vm_tests': [],
},
'fizz-labstation-release': {
'hw_tests': [],
'image_test': False,
'signer_tests': False,
'vm_tests': [],
},
'betty-arc64-nyc-android-pfq':
site_config.templates.tast_vm_android_pfq_tests,
'betty-nyc-android-pfq':
site_config.templates.tast_vm_android_pfq_tests,
'betty-pi-arc-pi-android-pfq':
site_config.templates.tast_vm_android_pfq_tests,
# There's no amd64-generic-release builder, so we use amd64-generic-full
# to validate informational Tast tests on amd64-generic:
# https://crbug.com/946858
'amd64-generic-full':
site_config.templates.tast_vm_canary_tests,
'betty-arc64-release':
site_config.templates.tast_vm_canary_tests,
'betty-pi-arc-release':
site_config.templates.tast_vm_canary_tests,
'betty-release':
site_config.templates.tast_vm_canary_tests,
}
for config_name, overrides in overwritten_configs.items():
# TODO: Turn this assert into a unittest.
# config = site_config[config_name]
# for k, v in overrides.items():
# assert config[k] != v, ('Unnecessary override: %s: %s' %
# (config_name, k))
site_config[config_name].apply(**overrides)
def VerifyStage(self,
failing,
inflight,
handle_failure=True,
handle_timeout=False,
sane_tot=True,
submit_partial=False,
alert=False,
stage=None,
all_slaves=None,
slave_stages=None,
do_submit_partial=True,
build_passed=False):
"""Runs and Verifies PerformStage.
Args:
failing: The names of the builders that failed.
inflight: The names of the buiders that timed out.
handle_failure: If True, calls HandleValidationFailure.
handle_timeout: If True, calls HandleValidationTimeout.
sane_tot: If not true, assumes TOT is not sane.
submit_partial: If True, submit partial pool will submit some changes.
alert: If True, sends out an alert email for infra failures.
stage: If set, use this constructed stage, otherwise create own.
all_slaves: Optional set of all slave configs.
slave_stages: Optional list of slave stages.
do_submit_partial: If True, assert that there was no call to
SubmitPartialPool.
build_passed: Whether the build passed or failed.
"""
if not stage:
stage = self.ConstructStage()
# Setup the stage to look at the specified configs.
all_slaves = list(all_slaves or set(failing + inflight))
configs = [config_lib.BuildConfig(name=x) for x in all_slaves]
self.PatchObject(stage, '_GetSlaveConfigs', return_value=configs)
# Setup builder statuses.
stage._run.attrs.manifest_manager = mock.MagicMock()
statuses = {}
for x in failing:
statuses[x] = manifest_version.BuilderStatus(
constants.BUILDER_STATUS_FAILED, message=None)
for x in inflight:
statuses[x] = manifest_version.BuilderStatus(
constants.BUILDER_STATUS_INFLIGHT, message=None)
if self._run.config.master:
self.PatchObject(stage._run.attrs.manifest_manager,
'GetBuildersStatus',
return_value=statuses)
else:
self.PatchObject(stage,
'_GetLocalBuildStatus',
return_value=statuses)
# Setup DB and provide list of slave stages.
mock_cidb = mock.MagicMock()
cidb.CIDBConnectionFactory.SetupMockCidb(mock_cidb)
if slave_stages is None:
slave_stages = []
critical_stages = (
completion_stages.CommitQueueCompletionStage._CRITICAL_STAGES)
for stage_name, slave in itertools.product(critical_stages,
all_slaves):
slave_stages.append({
'name': stage_name,
'build_config': slave,
'status': constants.BUILDER_STATUS_PASSED
})
self.PatchObject(mock_cidb,
'GetSlaveStages',
return_value=slave_stages)
# Set up SubmitPartialPool to provide a list of changes to look at.
if submit_partial:
spmock = self.PatchObject(stage.sync_stage.pool,
'SubmitPartialPool',
return_value=self.other_changes)
handlefailure_changes = self.other_changes
else:
spmock = self.PatchObject(stage.sync_stage.pool,
'SubmitPartialPool',
return_value=self.changes)
handlefailure_changes = self.changes
# Track whether 'HandleSuccess' is called.
success_mock = self.PatchObject(stage, 'HandleSuccess')
# Actually run the stage.
if build_passed:
stage.PerformStage()
else:
with self.assertRaises(
completion_stages.ImportantBuilderFailedException):
stage.PerformStage()
# Verify the calls.
self.assertEqual(success_mock.called, build_passed)
if not build_passed and self._run.config.master:
self.tot_sanity_mock.assert_called_once_with(mock.ANY, mock.ANY)
if alert:
self.alert_email_mock.called_once_with(mock.ANY, mock.ANY,
mock.ANY, mock.ANY)
self.assertEqual(do_submit_partial, spmock.called)
if handle_failure:
stage.sync_stage.pool.handle_failure_mock.assert_called_once_with(
mock.ANY,
no_stat=set([]),
sanity=sane_tot,
changes=handlefailure_changes)
if handle_timeout:
stage.sync_stage.pool.handle_timeout_mock.assert_called_once_with(
sanity=mock.ANY, changes=self.changes)
