def testInheritedKeyvals(self):
"""Tests should inherit some whitelisted job keyvals."""
# Only keyvals in constants.INHERITED_KEYVALS are inherited to tests.
job_keyvals = {
constants.KEYVAL_CIDB_BUILD_ID: '111',
constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222',
'your': 'name',
}
test_keyvals = {
constants.KEYVAL_CIDB_BUILD_ID: '111',
constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222',
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(
recorder,
extra_keyvals=test_keyvals)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, job_keyvals)
utils.write_keyval(None, mox.IgnoreArg())
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko,
job_keyvals=job_keyvals)
suite.schedule(recorder.record_entry)
def testSuiteMaxRetries(self):
"""Test suite max retries."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
job_retry=True,
max_retries=1)
suite.schedule(recorder.record_entry)
self.assertEqual(suite._retry_handler._max_retries, 1)
# Find the job_id of the test that allows retry
job_id = suite._retry_handler._retry_map.iterkeys().next()
suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10)
self.assertEqual(suite._retry_handler._max_retries, 0)
def testRetryMapAfterScheduling(self):
"""Test job-test and test-job mapping are correctly updated."""
name_list = ['name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven']
keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
all_files = self.files.items()
# Sort tests in self.files so that they are in the same
# order as they are scheduled.
expected_retry_map = {}
for n in range(len(all_files)):
test = all_files[n][1]
job_id = n + 1
job_retries = 1 if test.job_retries is None else test.job_retries
if job_retries > 0:
expected_retry_map[job_id] = {
'state': RetryHandler.States.NOT_ATTEMPTED,
'retry_max': job_retries}
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko,
job_retry=True)
suite.schedule(recorder.record_entry)
self.assertEqual(expected_retry_map, suite._retry_handler._retry_map)
def testSuiteDependencies(self):
"""Should add suite dependencies to tests scheduled."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: len(name_list),
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, suite_deps=['extra'])
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
extra_deps=['extra'],
afe=self.afe,
tko=self.tko)
suite.schedule(recorder.record_entry)
def testRetryMapAfterScheduling(self):
"""Test job-test and test-job mapping are correctly updated."""
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=True)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
job_retry=True)
suite.schedule(recorder.record_entry, add_experimental=True)
all_files = self.files.items()
# Sort tests in self.files so that they are in the same
# order as they are scheduled.
all_files.sort(key=lambda record: record[1].experimental)
expected_retry_map = {}
for n in range(len(all_files)):
test = all_files[n][1]
job_id = n + 1
if test.job_retries > 0:
expected_retry_map[job_id] = {
'state': RetryHandler.States.NOT_ATTEMPTED,
'retry_max': test.job_retries
}
self.assertEqual(expected_retry_map, suite._retry_handler._retry_map)
def testScheduleTestsAndRecord(self):
"""Should schedule stable and experimental tests with the AFE."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
results_dir=self.tmpdir)
self.mox.ResetAll()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, suite=suite)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(self.tmpdir, keyval_dict)
self.mox.ReplayAll()
suite.schedule(recorder.record_entry)
for job in suite._jobs:
self.assertTrue(hasattr(job, 'test_name'))
def testScheduleStableTests(self):
"""Should schedule only stable tests with the AFE."""
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=False)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko)
suite.schedule(recorder.record_entry, add_experimental=False)
def testScheduleUnrunnableTestsTESTNA(self):
"""Tests which fail to schedule should be TEST_NA."""
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder,
add_experimental=True,
raises=True)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko)
suite.schedule(recorder.record_entry, add_experimental=True)
def testSuiteDependencies(self):
"""Should add suite dependencies to tests scheduled."""
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder,
add_experimental=False,
suite_deps=['extra'])
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
extra_deps=['extra'],
afe=self.afe,
tko=self.tko)
suite.schedule(recorder.record_entry, add_experimental=False)
def testScheduleUnrunnableTestsTESTNA(self):
"""Tests which fail to schedule should be TEST_NA."""
# Since all tests will be fail to schedule, the num of scheduled tests
# will be zero.
name_list = []
keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, raises=True)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko)
suite.schedule(recorder.record_entry)
def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False):
"""Create a Suite, using mocked tests and control file contents.
@return Suite object, after mocking out behavior needed to create it.
"""
self.expect_control_file_parsing()
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
self.getter,
afe=self.afe,
tko=self.tko,
file_bugs=file_bugs,
job_retry=True)
self.mox.ResetAll()
return suite
def testSuiteMaxRetries(self):
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=True)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
job_retry=True,
max_retries=1)
suite.schedule(recorder.record_entry, add_experimental=True)
self.assertEqual(suite._retry_handler._max_retries, 1)
# Find the job_id of the test that allows retry
job_id = suite._retry_handler._retry_map.iterkeys().next()
suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10)
self.assertEqual(suite._retry_handler._max_retries, 0)
def testScheduleTestsAndRecord(self):
"""Should schedule stable and experimental tests with the AFE."""
self.mock_control_file_parsing()
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
results_dir=self.tmpdir)
self.mox.ResetAll()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder,
add_experimental=True,
suite=suite)
self.mox.ReplayAll()
suite.schedule(recorder.record_entry, True)
for job in suite._jobs:
self.assertTrue(hasattr(job, 'test_name'))
def testStableUnstableFilter(self):
"""Should distinguish between experimental and stable tests."""
self.expect_control_file_parsing()
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
devserver=None,
cf_getter=self.getter,
afe=self.afe,
tko=self.tko)
self.assertTrue(self.files['one'] in suite.tests)
self.assertTrue(self.files['two'] in suite.tests)
self.assertTrue(self.files['one'] in suite.unstable_tests())
self.assertTrue(self.files['two'] in suite.stable_tests())
self.assertFalse(self.files['one'] in suite.stable_tests())
self.assertFalse(self.files['two'] in suite.unstable_tests())
# Sanity check.
self.assertFalse(self.files['four'] in suite.tests)
def main():
"""Entry point to run the suite enumerator command."""
parser, options, args = parse_options()
if options.listall:
if args:
print 'Cannot use suite_name with --listall'
parser.print_help()
elif not args or len(args) != 1:
parser.print_help()
return
fs_getter = Suite.create_fs_getter(options.autotest_dir)
devserver = dev_server.ImageServer('')
if options.listall:
for suite in Suite.list_all_suites('', devserver, fs_getter):
print suite
return
suite = Suite.create_from_name(args[0], {}, '', devserver, fs_getter)
# If in test list, print firmware_FAFTSetup before other tests
# NOTE: the test.name value can be *different* from the directory
# name that appears in test.path
PRETEST_LIST = [
'firmware_FAFTSetup',
]
for test in filter(lambda test: test.name in \
PRETEST_LIST, suite.stable_tests()):
print test.path
for test in filter(lambda test: test.name not in \
PRETEST_LIST, suite.stable_tests()):
print test.path
# Check if test_suites/control.suite_name exists.
control_path = os.path.join(options.autotest_dir, 'test_suites',
'control.' + args[0])
if not os.path.exists(control_path):
print >> sys.stderr, ('Warning! control file is missing: %s' %
control_path)