def setUp(self): self.base_vmtests = [config_lib.VMTestConfig('base')] self.override_vmtests = [config_lib.VMTestConfig('override')] self.base_hwtests = [config_lib.HWTestConfig('base')] self.override_hwtests = [config_lib.HWTestConfig('override')] self.all_configs = MockSiteConfig() self.all_configs.Add( 'no_tests_without_override', vm_tests=[], ) self.all_configs.Add( 'no_tests_with_override', vm_tests=[], vm_tests_override=self.override_vmtests, hw_tests_override=self.override_hwtests, ) self.all_configs.Add( 'tests_without_override', vm_tests=self.base_vmtests, hw_tests=self.base_hwtests, ) self.all_configs.Add( 'tests_with_override', vm_tests=self.base_vmtests, vm_tests_override=self.override_vmtests, hw_tests=self.base_hwtests, hw_tests_override=self.override_hwtests, )
def testForgivingVMTest(self): """Test if a test is warn-only, it actually warns.""" self._run.config['vm_tests'] = [ config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, warn_only=True, test_suite='bvt-perbuild'), config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, warn_only=False, test_suite='bvt-arc') ] # pylint: disable=unused-argument def _MockRunTestSuite(buildroot, board, image_path, results_dir, test_config, *args, **kwargs): # Only raise exception in one test. if test_config.test_suite == 'bvt-perbuild': raise Exception() # pylint: enable=unused-argument self.PatchObject(vm_test_stages, 'RunTestSuite', autospec=True, side_effect=_MockRunTestSuite) results_lib.Results.Clear() self.RunStage() result = results_lib.Results.Get()[0] self.assertEqual(result.result, results_lib.Results.FORGIVEN) # Make sure that all tests were actually run. self.assertEqual(vm_test_stages.RunTestSuite.call_count, len(self._run.config['vm_tests']))
def MockSiteConfig(): """Create a SiteConfig object for convenient testing pleasure. Shared amoung a number of unittest files, so be careful if changing it. """ result = config_lib.SiteConfig() # Add a single, simple build config. result.Add( 'x86-generic-paladin', active_waterfall='chromiumos', boards=['x86-generic'], build_type='paladin', chrome_sdk=True, chrome_sdk_build_chrome=False, description='Commit Queue', doc='http://mock_url/', image_test=True, images=['base', 'test'], important=True, manifest_version=True, prebuilts='public', trybot_list=True, upload_standalone_images=False, vm_tests=[config_lib.VMTestConfig('smoke_suite')], ) return result
def MockSiteConfig(): """Create a SiteConfig object for convenient testing pleasure. Shared amoung a number of unittest files, so be careful if changing it. """ result = config_lib.SiteConfig() # Add a single, simple build config. result.Add( 'amd64-generic-release', boards=['amd64-generic'], display_label='MockLabel', build_type='canary', chrome_sdk=True, chrome_sdk_build_chrome=False, description='LegacyRelease', doc='http://mock_url/', image_test=True, images=['base', 'test'], important=True, manifest_version=True, prebuilts='public', upload_standalone_images=False, vm_tests=[config_lib.VMTestConfig('smoke_suite')], ) return result
def testReportTestResults(self): """Test trybot with reporting function.""" self._run.config['vm_tests'] = [ config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE) ] self._run.config['vm_test_report_to_dashboards'] = True self.RunStage()
def testWhitelistChromeCrashes(self): """Test SMOKE config with allowing chrome crashes.""" config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite='smoke', use_ctest=False) self._RunTestSuite(config, allow_chrome_crashes=True) self.assertCommandContains(['--autotest=suite:smoke']) self.assertCommandContains(['--test_that-args=--allow-chrome-crashes'])
def testSmokeChromite(self): """Test SMOKE config using chromite VM code path.""" config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite='smoke', use_ctest=False) self._RunTestSuite(config) self.assertCommandContains(['--autotest=suite:smoke']) self.assertCommandContains( ['---test_that-args=-whitelist_chrome_crashes'], expected=False)
def getInfoVMTest(): suites = ['vmtest-informational4'] ret = [] for suite in suites: ret.append( config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite=suite, warn_only=True, timeout=12 * 60 * 60)) return ret
def testVmTestOverride(self): """Verify that vm_tests override for trybots pay heed to original config.""" mock_options = mock.Mock() old = self.site_config['x86-mario-paladin'] new = config_lib.OverrideConfigForTrybot(old, mock_options) self.assertEquals(new['vm_tests'], [ config_lib.VMTestConfig(constants.SMOKE_SUITE_TEST_TYPE), config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE), config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE) ]) # Don't override vm tests for arm boards. old = self.site_config['daisy-paladin'] new = config_lib.OverrideConfigForTrybot(old, mock_options) self.assertEquals(new['vm_tests'], old['vm_tests']) # Don't override vm tests for brillo boards. old = self.site_config['storm-paladin'] new = config_lib.OverrideConfigForTrybot(old, mock_options) self.assertEquals(new['vm_tests'], old['vm_tests'])
def testGceTests(self): """Tests if GCE_VM_TEST_TYPE tests are run on GCE.""" self._run.config['vm_tests'] = [ config_lib.VMTestConfig(constants.GCE_VM_TEST_TYPE) ] gce_tarball = constants.TEST_IMAGE_GCE_TAR # pylint: disable=unused-argument def _MockRunTestSuite(buildroot, board, image_path, results_dir, test_type, *args, **kwargs): self.assertEndsWith(image_path, gce_tarball) self.assertEqual(test_type, constants.GCE_VM_TEST_TYPE) # pylint: enable=unused-argument commands.RunTestSuite.side_effect = _MockRunTestSuite self.RunStage() self.assertTrue(commands.RunTestSuite.called and commands.RunTestSuite.call_count == 1)
def IncrementalBuilders(site_config): """Create all incremental test configs. Args: site_config: config_lib.SiteConfig to be modified by adding templates and configs. """ # incremental site_config['amd64-generic-incremental'].apply( site_config.templates.no_vmtest_builder, ) site_config['betty-incremental'].apply( vm_tests=getInfoVMTest(), vm_tests_override=getInfoVMTest(), ) site_config['lakitu-incremental'].apply( site_config.templates.lakitu_test_customizations, ) site_config['x32-generic-incremental'].apply( site_config.templates.no_vmtest_builder, ) site_config['lakitu-gpu-incremental'].apply( site_config.templates.lakitu_test_customizations, ) site_config['lakitu-st-incremental'].apply( site_config.templates.lakitu_test_customizations, ) site_config['lakitu_next-incremental'].apply( site_config.templates.lakitu_test_customizations, ) site_config['kumo-incremental'].apply(vm_tests=[ config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite='smoke') ], )
def testQuickTests(self): """Tests if quick unit and cros_au_test_harness tests are run correctly.""" self._run.config['vm_tests'] = [ config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE) ] self.RunStage()
def setUp(self): self.complex_defaults = { 'value': 'default', } # Construct our test config. site_config = config_lib.SiteConfig(defaults=self.complex_defaults) site_config.AddTemplate('match', value='default') site_config.AddTemplate('template', value='template') site_config.AddTemplate('mixin', value='mixin') site_config.AddTemplate('unused', value='unused') site_config.AddTemplate('callable', value=self._callable) default = site_config.Add('default') default_with_override = site_config.Add('default_with_override', value='override') site_config.AddWithoutTemplate('default_with_mixin', site_config.templates.mixin) site_config.AddWithoutTemplate('mixin_with_override', site_config.templates.mixin, value='override') site_config.Add('default_with_template', site_config.templates.template) site_config.Add('template_with_override', site_config.templates.template, value='override') site_config.Add('template_with_mixin', site_config.templates.template, site_config.templates.mixin) site_config.Add('template_with_mixin_override', site_config.templates.template, site_config.templates.mixin, value='override') site_config.Add('match', value='default') site_config.Add('template_back_to_default', site_config.templates.template, value='default') site_config.Add('calling', site_config.templates.callable) site_config.Add( 'vm_tests', vm_tests=[config_lib.VMTestConfig('vm_suite')], vm_tests_override=[config_lib.VMTestConfig('vm_override')]) site_config.Add( 'hw_tests', hw_tests=[config_lib.HWTestConfig('hw_suite')], hw_tests_override=[config_lib.HWTestConfig('hw_override')]) site_config.Add('tast_vm_tests', tast_vm_tests=[ config_lib.TastVMTestConfig( 'tast_vm_suite', ['(!disabled)']) ]) site_config.AddGroup('parent', default, default_with_override) self.site_config = site_config
def testFullTests(self): """Tests if full unit and cros_au_test_harness tests are run correctly.""" self._run.config['vm_tests'] = [ config_lib.VMTestConfig(constants.FULL_AU_TEST_TYPE) ] self.RunStage()
kwargs = kwargs.copy() # Tast test suites run at most three jobs (for system, Chrome, and Android # tests) and have short timeouts, so request at most 1 DUT (while retaining # passed-in requests for 0 DUTs). if kwargs.get('minimum_duts', 0): kwargs['minimum_duts'] = 1 if kwargs.get('suite_min_duts', 0): kwargs['suite_min_duts'] = 1 return config_lib.HWTestConfig(suite_name, **kwargs) TRADITIONAL_VM_TESTS_SUPPORTED = [ config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite='smoke', use_ctest=False), config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE), config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE) ] def InsertHwTestsOverrideDefaults(build): """Insert default hw_tests values for a given build. Also updates child builds. Args: build: BuildConfig instance to modify in place. """ for child in build['child_configs']:
def setUp(self): self.complex_defaults = { 'value': 'default', } self.complex_site_params = { 'site_foo': True, 'site_bar': False, 'nested': { 'sub1': 1, 'sub2': 2 }, } # Construct our test config. site_config = config_lib.SiteConfig( defaults=self.complex_defaults, site_params=self.complex_site_params) site_config.AddTemplate('match', value='default') site_config.AddTemplate('template', value='template') site_config.AddTemplate('mixin', value='mixin') site_config.AddTemplate('unused', value='unused') site_config.AddTemplate('callable', value=self._callable) default = site_config.Add('default') default_with_override = site_config.Add('default_with_override', value='override') site_config.AddWithoutTemplate('default_with_mixin', site_config.templates.mixin) site_config.AddWithoutTemplate('mixin_with_override', site_config.templates.mixin, value='override') site_config.Add('default_with_template', site_config.templates.template) site_config.Add('template_with_override', site_config.templates.template, value='override') site_config.Add('template_with_mixin', site_config.templates.template, site_config.templates.mixin) site_config.Add('template_with_mixin_override', site_config.templates.template, site_config.templates.mixin, value='override') site_config.Add('match', value='default') site_config.Add('template_back_to_default', site_config.templates.template, value='default') site_config.Add('calling', site_config.templates.callable) site_config.Add( 'vm_tests', vm_tests=[config_lib.VMTestConfig('vm_suite')], vm_tests_override=[config_lib.VMTestConfig('vm_override')]) site_config.Add( 'hw_tests', hw_tests=[config_lib.HWTestConfig('hw_suite')], hw_tests_override=[config_lib.HWTestConfig('hw_override')]) site_config.AddGroup('parent', default, default_with_override) self.site_config = site_config
def testSmoke(self): """Test SMOKE config.""" config = config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE, test_suite='smoke') self._RunTestSuite(config) self.assertCommandContains(['--verify_suite_name=smoke'])
def testSimple(self): """Test SIMPLE config.""" config = config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE) self._RunTestSuite(config) self.assertCommandContains(['--test_prefix=SimpleTestVerify'])
def testAddedContents(self): """Verify that our complex config looks like we expect, before saving.""" expected = { 'default': { '_template': None, 'name': 'default', 'value': 'default', }, 'default_with_override': { '_template': None, 'name': 'default_with_override', 'value': 'override', }, 'default_with_mixin': { '_template': None, 'name': 'default_with_mixin', 'value': 'mixin', }, 'mixin_with_override': { '_template': None, 'name': 'mixin_with_override', 'value': 'override', }, 'default_with_template': { '_template': 'template', 'name': 'default_with_template', 'value': 'template', }, 'template_with_override': { '_template': 'template', 'name': 'template_with_override', 'value': 'override' }, 'template_with_mixin': { '_template': 'template', 'name': 'template_with_mixin', 'value': 'mixin', }, 'template_with_mixin_override': { '_template': 'template', 'name': 'template_with_mixin_override', 'value': 'override' }, 'calling': { '_template': 'callable', 'name': 'calling', 'value': 'default extended', }, 'match': { '_template': None, 'name': 'match', 'value': 'default', }, 'template_back_to_default': { '_template': 'template', 'name': 'template_back_to_default', 'value': 'default', }, 'vm_tests': { '_template': None, 'name': 'vm_tests', 'vm_tests': [config_lib.VMTestConfig('vm_suite')], 'vm_tests_override': [config_lib.VMTestConfig('vm_override')], }, 'hw_tests': { '_template': None, 'name': 'hw_tests', 'hw_tests': [config_lib.HWTestConfig('hw_suite')], 'hw_tests_override': [config_lib.HWTestConfig('hw_override')], }, 'parent': { '_template': None, 'name': 'parent', 'value': 'default', }, } # Make sure our expected build configs exist. self.assertItemsEqual(self.site_config.keys(), expected.keys()) # Make sure each one contains self.longMessage = True for name in expected.keys(): self.assertDictContainsSubset(expected[name], self.site_config[name], name) # Special handling for child configs. children = self.site_config['parent'].child_configs self.assertEqual(len(children), 2) self.assertDictContainsSubset( { '_template': None, 'name': 'default', 'value': 'default', 'grouped': True, }, children[0]) self.assertDictContainsSubset( { '_template': None, 'name': 'default_with_override', 'value': 'override', 'grouped': True, }, children[1])