def testInheritedKeyvals(self): """Tests should inherit some whitelisted job keyvals.""" # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. job_keyvals = { constants.KEYVAL_CIDB_BUILD_ID: '111', constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 'your': 'name', } test_keyvals = { constants.KEYVAL_CIDB_BUILD_ID: '111', constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', } self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling( recorder, extra_keyvals=test_keyvals) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(None, job_keyvals) utils.write_keyval(None, mox.IgnoreArg()) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, job_keyvals=job_keyvals) suite.schedule(recorder.record_entry)
def testSuiteMaxRetries(self): """Test suite max retries.""" name_list = [ 'name-data_one', 'name-data_two', 'name-data_three', 'name-data_four', 'name-data_five', 'name-data_six', 'name-data_seven' ] keyval_dict = { constants.SCHEDULED_TEST_COUNT_KEY: 7, constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list) } self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(None, keyval_dict) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, job_retry=True, max_retries=1) suite.schedule(recorder.record_entry) self.assertEqual(suite._retry_handler._max_retries, 1) # Find the job_id of the test that allows retry job_id = suite._retry_handler._retry_map.iterkeys().next() suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) self.assertEqual(suite._retry_handler._max_retries, 0)
def testRetryMapAfterScheduling(self): """Test job-test and test-job mapping are correctly updated.""" name_list = ['name-data_one', 'name-data_two', 'name-data_three', 'name-data_four', 'name-data_five', 'name-data_six', 'name-data_seven'] keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(None, keyval_dict) all_files = self.files.items() # Sort tests in self.files so that they are in the same # order as they are scheduled. expected_retry_map = {} for n in range(len(all_files)): test = all_files[n][1] job_id = n + 1 job_retries = 1 if test.job_retries is None else test.job_retries if job_retries > 0: expected_retry_map[job_id] = { 'state': RetryHandler.States.NOT_ATTEMPTED, 'retry_max': job_retries} self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, job_retry=True) suite.schedule(recorder.record_entry) self.assertEqual(expected_retry_map, suite._retry_handler._retry_map)
def testSuiteDependencies(self): """Should add suite dependencies to tests scheduled.""" name_list = [ 'name-data_one', 'name-data_two', 'name-data_three', 'name-data_four', 'name-data_five', 'name-data_six', 'name-data_seven' ] keyval_dict = { constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list) } self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, suite_deps=['extra']) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(None, keyval_dict) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, extra_deps=['extra'], afe=self.afe, tko=self.tko) suite.schedule(recorder.record_entry)
def testRetryMapAfterScheduling(self): """Test job-test and test-job mapping are correctly updated.""" self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=True) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, job_retry=True) suite.schedule(recorder.record_entry, add_experimental=True) all_files = self.files.items() # Sort tests in self.files so that they are in the same # order as they are scheduled. all_files.sort(key=lambda record: record[1].experimental) expected_retry_map = {} for n in range(len(all_files)): test = all_files[n][1] job_id = n + 1 if test.job_retries > 0: expected_retry_map[job_id] = { 'state': RetryHandler.States.NOT_ATTEMPTED, 'retry_max': test.job_retries } self.assertEqual(expected_retry_map, suite._retry_handler._retry_map)
def testScheduleTestsAndRecord(self): """Should schedule stable and experimental tests with the AFE.""" name_list = [ 'name-data_one', 'name-data_two', 'name-data_three', 'name-data_four', 'name-data_five', 'name-data_six', 'name-data_seven' ] keyval_dict = { constants.SCHEDULED_TEST_COUNT_KEY: 7, constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list) } self.mock_control_file_parsing() self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, results_dir=self.tmpdir) self.mox.ResetAll() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, suite=suite) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(self.tmpdir, keyval_dict) self.mox.ReplayAll() suite.schedule(recorder.record_entry) for job in suite._jobs: self.assertTrue(hasattr(job, 'test_name'))
def testScheduleStableTests(self): """Should schedule only stable tests with the AFE.""" self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=False) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko) suite.schedule(recorder.record_entry, add_experimental=False)
def testScheduleUnrunnableTestsTESTNA(self): """Tests which fail to schedule should be TEST_NA.""" self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=True, raises=True) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko) suite.schedule(recorder.record_entry, add_experimental=True)
def testSuiteDependencies(self): """Should add suite dependencies to tests scheduled.""" self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=False, suite_deps=['extra']) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, extra_deps=['extra'], afe=self.afe, tko=self.tko) suite.schedule(recorder.record_entry, add_experimental=False)
def testScheduleUnrunnableTestsTESTNA(self): """Tests which fail to schedule should be TEST_NA.""" # Since all tests will be fail to schedule, the num of scheduled tests # will be zero. name_list = [] keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, raises=True) self.mox.StubOutWithMock(utils, 'write_keyval') utils.write_keyval(None, keyval_dict) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko) suite.schedule(recorder.record_entry)
def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): """Create a Suite, using mocked tests and control file contents. @return Suite object, after mocking out behavior needed to create it. """ self.expect_control_file_parsing() self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, self.getter, afe=self.afe, tko=self.tko, file_bugs=file_bugs, job_retry=True) self.mox.ResetAll() return suite
def testSuiteMaxRetries(self): self.mock_control_file_parsing() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=True) self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, job_retry=True, max_retries=1) suite.schedule(recorder.record_entry, add_experimental=True) self.assertEqual(suite._retry_handler._max_retries, 1) # Find the job_id of the test that allows retry job_id = suite._retry_handler._retry_map.iterkeys().next() suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) self.assertEqual(suite._retry_handler._max_retries, 0)
def testScheduleTestsAndRecord(self): """Should schedule stable and experimental tests with the AFE.""" self.mock_control_file_parsing() self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, self.devserver, afe=self.afe, tko=self.tko, results_dir=self.tmpdir) self.mox.ResetAll() recorder = self.mox.CreateMock(base_job.base_job) self.expect_job_scheduling(recorder, add_experimental=True, suite=suite) self.mox.ReplayAll() suite.schedule(recorder.record_entry, True) for job in suite._jobs: self.assertTrue(hasattr(job, 'test_name'))
def testStableUnstableFilter(self): """Should distinguish between experimental and stable tests.""" self.expect_control_file_parsing() self.mox.ReplayAll() suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, devserver=None, cf_getter=self.getter, afe=self.afe, tko=self.tko) self.assertTrue(self.files['one'] in suite.tests) self.assertTrue(self.files['two'] in suite.tests) self.assertTrue(self.files['one'] in suite.unstable_tests()) self.assertTrue(self.files['two'] in suite.stable_tests()) self.assertFalse(self.files['one'] in suite.stable_tests()) self.assertFalse(self.files['two'] in suite.unstable_tests()) # Sanity check. self.assertFalse(self.files['four'] in suite.tests)
def main(): """Entry point to run the suite enumerator command.""" parser, options, args = parse_options() if options.listall: if args: print 'Cannot use suite_name with --listall' parser.print_help() elif not args or len(args) != 1: parser.print_help() return fs_getter = Suite.create_fs_getter(options.autotest_dir) devserver = dev_server.ImageServer('') if options.listall: for suite in Suite.list_all_suites('', devserver, fs_getter): print suite return suite = Suite.create_from_name(args[0], {}, '', devserver, fs_getter) # If in test list, print firmware_FAFTSetup before other tests # NOTE: the test.name value can be *different* from the directory # name that appears in test.path PRETEST_LIST = [ 'firmware_FAFTSetup', ] for test in filter(lambda test: test.name in \ PRETEST_LIST, suite.stable_tests()): print test.path for test in filter(lambda test: test.name not in \ PRETEST_LIST, suite.stable_tests()): print test.path # Check if test_suites/control.suite_name exists. control_path = os.path.join(options.autotest_dir, 'test_suites', 'control.' + args[0]) if not os.path.exists(control_path): print >> sys.stderr, ('Warning! control file is missing: %s' % control_path)