def setUp(self):
self.base_vmtests = [config_lib.VMTestConfig('base')]
self.override_vmtests = [config_lib.VMTestConfig('override')]
self.base_hwtests = [config_lib.HWTestConfig('base')]
self.override_hwtests = [config_lib.HWTestConfig('override')]
self.all_configs = MockSiteConfig()
self.all_configs.Add(
'no_tests_without_override',
vm_tests=[],
)
self.all_configs.Add(
'no_tests_with_override',
vm_tests=[],
vm_tests_override=self.override_vmtests,
hw_tests_override=self.override_hwtests,
)
self.all_configs.Add(
'tests_without_override',
vm_tests=self.base_vmtests,
hw_tests=self.base_hwtests,
)
self.all_configs.Add(
'tests_with_override',
vm_tests=self.base_vmtests,
vm_tests_override=self.override_vmtests,
hw_tests=self.base_hwtests,
hw_tests_override=self.override_hwtests,
)
def testForgivingVMTest(self):
"""Test if a test is warn-only, it actually warns."""
self._run.config['vm_tests'] = [
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
warn_only=True,
test_suite='bvt-perbuild'),
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
warn_only=False,
test_suite='bvt-arc')
]
# pylint: disable=unused-argument
def _MockRunTestSuite(buildroot, board, image_path, results_dir,
test_config, *args, **kwargs):
# Only raise exception in one test.
if test_config.test_suite == 'bvt-perbuild':
raise Exception()
# pylint: enable=unused-argument
self.PatchObject(vm_test_stages,
'RunTestSuite',
autospec=True,
side_effect=_MockRunTestSuite)
results_lib.Results.Clear()
self.RunStage()
result = results_lib.Results.Get()[0]
self.assertEqual(result.result, results_lib.Results.FORGIVEN)
# Make sure that all tests were actually run.
self.assertEqual(vm_test_stages.RunTestSuite.call_count,
len(self._run.config['vm_tests']))
def MockSiteConfig():
"""Create a SiteConfig object for convenient testing pleasure.
Shared amoung a number of unittest files, so be careful if changing it.
"""
result = config_lib.SiteConfig()
# Add a single, simple build config.
result.Add(
'x86-generic-paladin',
active_waterfall='chromiumos',
boards=['x86-generic'],
build_type='paladin',
chrome_sdk=True,
chrome_sdk_build_chrome=False,
description='Commit Queue',
doc='http://mock_url/',
image_test=True,
images=['base', 'test'],
important=True,
manifest_version=True,
prebuilts='public',
trybot_list=True,
upload_standalone_images=False,
vm_tests=[config_lib.VMTestConfig('smoke_suite')],
)
return result
def MockSiteConfig():
"""Create a SiteConfig object for convenient testing pleasure.
Shared amoung a number of unittest files, so be careful if changing it.
"""
result = config_lib.SiteConfig()
# Add a single, simple build config.
result.Add(
'amd64-generic-release',
boards=['amd64-generic'],
display_label='MockLabel',
build_type='canary',
chrome_sdk=True,
chrome_sdk_build_chrome=False,
description='LegacyRelease',
doc='http://mock_url/',
image_test=True,
images=['base', 'test'],
important=True,
manifest_version=True,
prebuilts='public',
upload_standalone_images=False,
vm_tests=[config_lib.VMTestConfig('smoke_suite')],
)
return result
def testReportTestResults(self):
"""Test trybot with reporting function."""
self._run.config['vm_tests'] = [
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE)
]
self._run.config['vm_test_report_to_dashboards'] = True
self.RunStage()
def testWhitelistChromeCrashes(self):
"""Test SMOKE config with allowing chrome crashes."""
config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)
self._RunTestSuite(config, allow_chrome_crashes=True)
self.assertCommandContains(['--autotest=suite:smoke'])
self.assertCommandContains(['--test_that-args=--allow-chrome-crashes'])
def testSmokeChromite(self):
"""Test SMOKE config using chromite VM code path."""
config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)
self._RunTestSuite(config)
self.assertCommandContains(['--autotest=suite:smoke'])
self.assertCommandContains(
['---test_that-args=-whitelist_chrome_crashes'], expected=False)
def getInfoVMTest():
suites = ['vmtest-informational4']
ret = []
for suite in suites:
ret.append(
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite=suite,
warn_only=True,
timeout=12 * 60 * 60))
return ret
def testVmTestOverride(self):
"""Verify that vm_tests override for trybots pay heed to original config."""
mock_options = mock.Mock()
old = self.site_config['x86-mario-paladin']
new = config_lib.OverrideConfigForTrybot(old, mock_options)
self.assertEquals(new['vm_tests'], [
config_lib.VMTestConfig(constants.SMOKE_SUITE_TEST_TYPE),
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)
])
# Don't override vm tests for arm boards.
old = self.site_config['daisy-paladin']
new = config_lib.OverrideConfigForTrybot(old, mock_options)
self.assertEquals(new['vm_tests'], old['vm_tests'])
# Don't override vm tests for brillo boards.
old = self.site_config['storm-paladin']
new = config_lib.OverrideConfigForTrybot(old, mock_options)
self.assertEquals(new['vm_tests'], old['vm_tests'])
def testGceTests(self):
"""Tests if GCE_VM_TEST_TYPE tests are run on GCE."""
self._run.config['vm_tests'] = [
config_lib.VMTestConfig(constants.GCE_VM_TEST_TYPE)
]
gce_tarball = constants.TEST_IMAGE_GCE_TAR
# pylint: disable=unused-argument
def _MockRunTestSuite(buildroot, board, image_path, results_dir,
test_type, *args, **kwargs):
self.assertEndsWith(image_path, gce_tarball)
self.assertEqual(test_type, constants.GCE_VM_TEST_TYPE)
# pylint: enable=unused-argument
commands.RunTestSuite.side_effect = _MockRunTestSuite
self.RunStage()
self.assertTrue(commands.RunTestSuite.called
and commands.RunTestSuite.call_count == 1)
def IncrementalBuilders(site_config):
"""Create all incremental test configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
"""
# incremental
site_config['amd64-generic-incremental'].apply(
site_config.templates.no_vmtest_builder, )
site_config['betty-incremental'].apply(
vm_tests=getInfoVMTest(),
vm_tests_override=getInfoVMTest(),
)
site_config['lakitu-incremental'].apply(
site_config.templates.lakitu_test_customizations, )
site_config['x32-generic-incremental'].apply(
site_config.templates.no_vmtest_builder, )
site_config['lakitu-gpu-incremental'].apply(
site_config.templates.lakitu_test_customizations, )
site_config['lakitu-st-incremental'].apply(
site_config.templates.lakitu_test_customizations, )
site_config['lakitu_next-incremental'].apply(
site_config.templates.lakitu_test_customizations, )
site_config['kumo-incremental'].apply(vm_tests=[
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')
], )
def testQuickTests(self):
"""Tests if quick unit and cros_au_test_harness tests are run correctly."""
self._run.config['vm_tests'] = [
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE)
]
self.RunStage()
def setUp(self):
self.complex_defaults = {
'value': 'default',
}
# Construct our test config.
site_config = config_lib.SiteConfig(defaults=self.complex_defaults)
site_config.AddTemplate('match', value='default')
site_config.AddTemplate('template', value='template')
site_config.AddTemplate('mixin', value='mixin')
site_config.AddTemplate('unused', value='unused')
site_config.AddTemplate('callable', value=self._callable)
default = site_config.Add('default')
default_with_override = site_config.Add('default_with_override',
value='override')
site_config.AddWithoutTemplate('default_with_mixin',
site_config.templates.mixin)
site_config.AddWithoutTemplate('mixin_with_override',
site_config.templates.mixin,
value='override')
site_config.Add('default_with_template',
site_config.templates.template)
site_config.Add('template_with_override',
site_config.templates.template,
value='override')
site_config.Add('template_with_mixin', site_config.templates.template,
site_config.templates.mixin)
site_config.Add('template_with_mixin_override',
site_config.templates.template,
site_config.templates.mixin,
value='override')
site_config.Add('match', value='default')
site_config.Add('template_back_to_default',
site_config.templates.template,
value='default')
site_config.Add('calling', site_config.templates.callable)
site_config.Add(
'vm_tests',
vm_tests=[config_lib.VMTestConfig('vm_suite')],
vm_tests_override=[config_lib.VMTestConfig('vm_override')])
site_config.Add(
'hw_tests',
hw_tests=[config_lib.HWTestConfig('hw_suite')],
hw_tests_override=[config_lib.HWTestConfig('hw_override')])
site_config.Add('tast_vm_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig(
'tast_vm_suite', ['(!disabled)'])
])
site_config.AddGroup('parent', default, default_with_override)
self.site_config = site_config
def testFullTests(self):
"""Tests if full unit and cros_au_test_harness tests are run correctly."""
self._run.config['vm_tests'] = [
config_lib.VMTestConfig(constants.FULL_AU_TEST_TYPE)
]
self.RunStage()
kwargs = kwargs.copy()
# Tast test suites run at most three jobs (for system, Chrome, and Android
# tests) and have short timeouts, so request at most 1 DUT (while retaining
# passed-in requests for 0 DUTs).
if kwargs.get('minimum_duts', 0):
kwargs['minimum_duts'] = 1
if kwargs.get('suite_min_duts', 0):
kwargs['suite_min_duts'] = 1
return config_lib.HWTestConfig(suite_name, **kwargs)
TRADITIONAL_VM_TESTS_SUPPORTED = [
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False),
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)
]
def InsertHwTestsOverrideDefaults(build):
"""Insert default hw_tests values for a given build.
Also updates child builds.
Args:
build: BuildConfig instance to modify in place.
"""
for child in build['child_configs']:
def setUp(self):
self.complex_defaults = {
'value': 'default',
}
self.complex_site_params = {
'site_foo': True,
'site_bar': False,
'nested': {
'sub1': 1,
'sub2': 2
},
}
# Construct our test config.
site_config = config_lib.SiteConfig(
defaults=self.complex_defaults,
site_params=self.complex_site_params)
site_config.AddTemplate('match', value='default')
site_config.AddTemplate('template', value='template')
site_config.AddTemplate('mixin', value='mixin')
site_config.AddTemplate('unused', value='unused')
site_config.AddTemplate('callable', value=self._callable)
default = site_config.Add('default')
default_with_override = site_config.Add('default_with_override',
value='override')
site_config.AddWithoutTemplate('default_with_mixin',
site_config.templates.mixin)
site_config.AddWithoutTemplate('mixin_with_override',
site_config.templates.mixin,
value='override')
site_config.Add('default_with_template',
site_config.templates.template)
site_config.Add('template_with_override',
site_config.templates.template,
value='override')
site_config.Add('template_with_mixin', site_config.templates.template,
site_config.templates.mixin)
site_config.Add('template_with_mixin_override',
site_config.templates.template,
site_config.templates.mixin,
value='override')
site_config.Add('match', value='default')
site_config.Add('template_back_to_default',
site_config.templates.template,
value='default')
site_config.Add('calling', site_config.templates.callable)
site_config.Add(
'vm_tests',
vm_tests=[config_lib.VMTestConfig('vm_suite')],
vm_tests_override=[config_lib.VMTestConfig('vm_override')])
site_config.Add(
'hw_tests',
hw_tests=[config_lib.HWTestConfig('hw_suite')],
hw_tests_override=[config_lib.HWTestConfig('hw_override')])
site_config.AddGroup('parent', default, default_with_override)
self.site_config = site_config
def testSmoke(self):
"""Test SMOKE config."""
config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')
self._RunTestSuite(config)
self.assertCommandContains(['--verify_suite_name=smoke'])
def testSimple(self):
"""Test SIMPLE config."""
config = config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE)
self._RunTestSuite(config)
self.assertCommandContains(['--test_prefix=SimpleTestVerify'])
def testAddedContents(self):
"""Verify that our complex config looks like we expect, before saving."""
expected = {
'default': {
'_template': None,
'name': 'default',
'value': 'default',
},
'default_with_override': {
'_template': None,
'name': 'default_with_override',
'value': 'override',
},
'default_with_mixin': {
'_template': None,
'name': 'default_with_mixin',
'value': 'mixin',
},
'mixin_with_override': {
'_template': None,
'name': 'mixin_with_override',
'value': 'override',
},
'default_with_template': {
'_template': 'template',
'name': 'default_with_template',
'value': 'template',
},
'template_with_override': {
'_template': 'template',
'name': 'template_with_override',
'value': 'override'
},
'template_with_mixin': {
'_template': 'template',
'name': 'template_with_mixin',
'value': 'mixin',
},
'template_with_mixin_override': {
'_template': 'template',
'name': 'template_with_mixin_override',
'value': 'override'
},
'calling': {
'_template': 'callable',
'name': 'calling',
'value': 'default extended',
},
'match': {
'_template': None,
'name': 'match',
'value': 'default',
},
'template_back_to_default': {
'_template': 'template',
'name': 'template_back_to_default',
'value': 'default',
},
'vm_tests': {
'_template': None,
'name': 'vm_tests',
'vm_tests': [config_lib.VMTestConfig('vm_suite')],
'vm_tests_override': [config_lib.VMTestConfig('vm_override')],
},
'hw_tests': {
'_template': None,
'name': 'hw_tests',
'hw_tests': [config_lib.HWTestConfig('hw_suite')],
'hw_tests_override': [config_lib.HWTestConfig('hw_override')],
},
'parent': {
'_template': None,
'name': 'parent',
'value': 'default',
},
}
# Make sure our expected build configs exist.
self.assertItemsEqual(self.site_config.keys(), expected.keys())
# Make sure each one contains
self.longMessage = True
for name in expected.keys():
self.assertDictContainsSubset(expected[name],
self.site_config[name], name)
# Special handling for child configs.
children = self.site_config['parent'].child_configs
self.assertEqual(len(children), 2)
self.assertDictContainsSubset(
{
'_template': None,
'name': 'default',
'value': 'default',
'grouped': True,
}, children[0])
self.assertDictContainsSubset(
{
'_template': None,
'name': 'default_with_override',
'value': 'override',
'grouped': True,
}, children[1])
