def testFetchSuiteTestView(self):
"""Test that it fetches the correct suite test views."""
suite_job_id = 100
suite_name = 'dummy'
build = 'R23-1.1.1.1'
server_job_view = self._build_view(
10, 'SERVER_JOB', '----', 'GOOD', suite_job_id)
test_to_ignore = self._build_view(
11, 'dummy_Pass', '101-user/host/dummy_Pass',
'GOOD', suite_job_id)
test_to_include = self._build_view(
12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id)
self._mock_afe_get_jobs(suite_job_id, [])
self._mock_tko_get_detailed_test_views(
[server_job_view, test_to_ignore, test_to_include])
collector = run_suite.ResultCollector(
'fake_server', self.afe, self.tko,
build='fake/build', board='fake', suite_name='dummy',
suite_job_id=suite_job_id)
suite_views = collector._fetch_relevant_test_views_of_suite()
suite_views = sorted(suite_views, key=lambda view: view['test_idx'])
# Verify that SERVER_JOB is renamed to 'Suite Prep'
self.assertEqual(suite_views[0].get_testname(),
run_suite.TestView.SUITE_PREP)
# Verify that the test with a subidr is not included.
self.assertEqual(suite_views[0]['test_idx'], 10)
self.assertEqual(suite_views[1]['test_idx'], 12)
def testGenerateLinks(self):
"""Test that it generates correct web and buildbot links."""
suite_job_id = 100
suite_name = 'my_suite'
build = 'lumpy-release/R36-5788.0.0'
board = 'lumpy'
fake_job = mock.MagicMock()
fake_job.parent = suite_job_id
suite_job_view = run_suite.TestView(
self._build_view(
20, 'Suite job', '----', 'GOOD', suite_job_id),
fake_job, suite_name, build, 'chromeos-test')
good_test = run_suite.TestView(
self._build_view(
21, 'test_Pass', 'fake/subdir', 'GOOD', 101),
fake_job, suite_name, build, 'chromeos-test')
bad_test = run_suite.TestView(
self._build_view(
23, 'test_Fail', 'fake/subdir', 'FAIL', 102),
fake_job, suite_name, build, 'chromeos-test')
collector = run_suite.ResultCollector(
'fake_server', self.afe, self.tko,
build, board, suite_name, suite_job_id, user='******')
collector._suite_views = [suite_job_view]
collector._test_views = [suite_job_view, good_test, bad_test]
collector._max_testname_width = max(
[len(v.get_testname()) for v in collector._test_views]) + 3
collector._generate_web_and_buildbot_links()
URL_PATTERN = run_suite._URL_PATTERN
# expected_web_links is list of (anchor, url) tuples we
# are expecting.
expected_web_links = [
(v.get_testname(),
URL_PATTERN % ('fake_server',
'%s-%s' % (v['afe_job_id'], 'chromeos-test')))
for v in collector._test_views]
# Verify web links are generated correctly.
for i in range(len(collector._web_links)):
expect = expected_web_links[i]
self.assertEqual(collector._web_links[i].anchor, expect[0])
self.assertEqual(collector._web_links[i].url, expect[1])
expected_buildbot_links = [
(v.get_testname(),
URL_PATTERN % ('fake_server',
'%s-%s' % (v['afe_job_id'], 'chromeos-test')))
for v in collector._test_views if v['status'] != 'GOOD']
# Verify buildbot links are generated correctly.
for i in range(len(collector._buildbot_links)):
expect = expected_buildbot_links[i]
self.assertEqual(collector._buildbot_links[i].anchor, expect[0])
self.assertEqual(collector._buildbot_links[i].url, expect[1])
self.assertEqual(collector._buildbot_links[i].retry_count, 0)
# Assert that a wmatrix retry dashboard link is created.
self.assertNotEqual(
collector._buildbot_links[i].GenerateWmatrixRetryLink(),'')
def testFetchTestViewOfChildJobs(self):
"""Test that it fetches the correct child test views."""
build = 'lumpy-release/R36-5788.0.0'
board = 'lumpy'
suite_name = 'my_suite'
suite_job_id = 100
invalid_job_id = 101
invalid_job_name = '%s/%s/test_Pass' % (build, suite_name)
good_job_id = 102
good_job_name = '%s/%s/test_Pass' % (build, suite_name)
bad_job_id = 103
bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name)
missing_job_id = 104
invalid_test = self._build_view(
19, 'test_Pass_Old', 'fake/subdir',
'FAIL', invalid_job_id, invalid_job_name)
good_job_server_job = self._build_view(
20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name)
good_job_test = self._build_view(
21, 'test_Pass', 'fake/subdir', 'GOOD',
good_job_id, good_job_name,
job_keyvals={'retry_original_job_id': invalid_job_id})
bad_job_server_job = self._build_view(
22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name)
bad_job_test = self._build_view(
23, 'test_ServerJobFail', 'fake/subdir', 'GOOD',
bad_job_id, bad_job_name)
self._mock_tko_get_detailed_test_views(
[good_job_server_job, good_job_test,
bad_job_server_job, bad_job_test, invalid_test],
[missing_job_id])
self._mock_afe_get_jobs(suite_job_id,
[good_job_id, bad_job_id, missing_job_id])
collector = run_suite.ResultCollector(
'fake_server', self.afe, self.tko,
build, suite_name, suite_job_id,
return_code_function=run_suite._ReturnCodeComputer())
child_views, retry_counts, missing_results = (
collector._fetch_test_views_of_child_jobs())
# child_views should contain tests 21, 22, 23
child_views = sorted(child_views, key=lambda view: view['test_idx'])
# Verify that the SERVER_JOB has been renamed properly
self.assertEqual(child_views[1].get_testname(),
'test_ServerJobFail_SERVER_JOB')
self.assertEqual(missing_results, {'test.104': [104]})
# Verify that failed SERVER_JOB and actual invalid tests are included,
expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'],
bad_job_test['test_idx']]
child_view_ids = [v['test_idx'] for v in child_views]
self.assertEqual(child_view_ids, expected)
self.afe.get_jobs.assert_called_once_with(
parent_job_id=suite_job_id)
# Verify the retry_counts is calculated correctly
self.assertEqual(len(retry_counts), 1)
self.assertEqual(retry_counts[21], 1)
def output_results(moblab, parent_job):
"""Output the Brillo PTS and it's subjobs results.
@param moblab: MoblabHost representing the MobLab being used for testing.
@param parent_job: autotest_lib.server.frontend.Job object representing the
test job.
"""
solo_test_run = len(moblab.afe.get_jobs(parent_job=parent_job.id)) == 0
rc = run_suite.ResultCollector(moblab.web_address,
moblab.afe,
moblab.tko,
None,
None,
parent_job.name,
parent_job.id,
user='******',
solo_test_run=solo_test_run)
rc.run()
rc.output_results()
def testFetchSuiteTestView(self):
"""Test that it fetches the correct suite test views."""
suite_job_id = 100
suite_name = 'dummy'
build = 'R23-1.1.1.1'
server_job_view = self._build_view(10, 'SERVER_JOB', '----', 'GOOD',
suite_job_id)
test_to_ignore = self._build_view(11, 'dummy_Pass',
'101-user/host/dummy_Pass', 'GOOD',
suite_job_id)
test_to_include = self._build_view(12, 'dummy_Pass.bluetooth', None,
'TEST_NA', suite_job_id)
test_missing = self._build_view(13, 'dummy_Missing', None, 'ABORT',
suite_job_id)
self._mock_afe_get_jobs(suite_job_id, [])
self._mock_tko_get_detailed_test_views(
[server_job_view, test_to_ignore, test_to_include, test_missing])
collector = run_suite.ResultCollector(
'fake_server',
self.afe,
self.tko,
build='fake/build',
board='fake',
suite_name='dummy',
suite_job_id=suite_job_id,
return_code_function=run_suite._ReturnCodeComputer())
collector._missing_results = {
test_missing['test_name']: [14, 15],
}
suite_views = collector._fetch_relevant_test_views_of_suite()
suite_views = sorted(suite_views, key=lambda view: view['test_idx'])
# Verify that SERVER_JOB is renamed to 'Suite job'
self.assertEqual(suite_views[0].get_testname(),
run_suite.TestView.SUITE_JOB)
# Verify that the test with a subidr is not included.
self.assertEqual(suite_views[0]['test_idx'], 10)
self.assertEqual(suite_views[1]['test_idx'], 12)
self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id)
# Verify that the test with missing results had it's AFE job id
# replaced.
self.assertEqual(suite_views[2]['test_idx'], 13)
self.assertEqual(suite_views[2]['afe_job_id'], 14)
def _end_to_end_test_helper(
self, include_bad_test=False, include_warn_test=False,
include_experimental_bad_test=False, include_timeout_test=False,
include_self_aborted_test=False,
include_aborted_by_suite_test=False,
include_good_retry=False, include_bad_retry=False,
suite_job_timed_out=False, suite_job_status='GOOD'):
"""A helper method for testing ResultCollector end-to-end.
This method mocks the retrieving of required test views,
and call ResultCollector.run() to collect the results.
@param include_bad_test:
If True, include a view of a test which has status 'FAIL'.
@param include_warn_test:
If True, include a view of a test which has status 'WARN'
@param include_experimental_bad_test:
If True, include a view of an experimental test
which has status 'FAIL'.
@param include_timeout_test:
If True, include a view of a test which was aborted before
started.
@param include_self_aborted_test:
If True, include a view of test which was aborted after
started and hit hits own timeout.
@param include_self_aborted_by_suite_test:
If True, include a view of test which was aborted after
started but has not hit its own timeout.
@param include_good_retry:
If True, include a test that passed after retry.
@param include_bad_retry:
If True, include a test that failed after retry.
@param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING'
@returns: A ResultCollector instance.
"""
suite_job_id = 100
good_job_id = 101
bad_job_id = 102
warn_job_id = 102
experimental_bad_job_id = 102
timeout_job_id = 100
self_aborted_job_id = 104
aborted_by_suite_job_id = 105
good_retry_job_id = 106
bad_retry_job_id = 107
invalid_job_id_1 = 90
invalid_job_id_2 = 91
suite_name = 'dummy'
build = 'lumpy-release/R27-3888.0.0'
suite_job_keyvals = {
constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20',
constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25',
constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'}
suite_job_started_time = '2014-04-29 13:14:37'
if suite_job_timed_out:
suite_job_keyvals['aborted_by'] = 'test_user'
suite_job_finished_time = '2014-04-29 13:25:37'
suite_job_status = 'ABORT'
else:
suite_job_finished_time = '2014-04-29 13:23:37'
server_job_view = self._build_view(
10, 'SERVER_JOB', '----', suite_job_status, suite_job_id,
'lumpy-release/R27-3888.0.0-test_suites/control.dummy',
'', suite_job_keyvals, '2014-04-29 13:14:37',
'2014-04-29 13:20:27', job_started_time=suite_job_started_time,
job_finished_time=suite_job_finished_time)
good_test = self._build_view(
11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD',
good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass',
'', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36')
bad_test = self._build_view(
12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL',
bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail',
'always fail', {}, '2014-04-29 13:16:00',
'2014-04-29 13:16:02')
warn_test = self._build_view(
13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN',
warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn',
'always warn', {}, '2014-04-29 13:16:00',
'2014-04-29 13:16:02')
experimental_bad_test = self._build_view(
14, 'experimental_dummy_Fail.Fail',
'102-user/host/dummy_Fail.Fail', 'FAIL',
experimental_bad_job_id,
'lumpy-release/R27-3888.0.0/dummy/experimental_dummy_Fail.Fail',
'always fail', {'experimental': 'True'}, '2014-04-29 13:16:06',
'2014-04-29 13:16:07')
timeout_test = self._build_view(
15, 'dummy_Timeout', '', 'ABORT',
timeout_job_id,
'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout',
'child job did not run', {}, '2014-04-29 13:15:37',
'2014-04-29 13:15:38')
self_aborted_test = self._build_view(
16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT',
self_aborted_job_id,
'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
'child job aborted', {'aborted_by': 'test_user'},
'2014-04-29 13:15:39', '2014-04-29 13:15:40',
job_started_time='2014-04-29 13:15:39',
job_finished_time='2014-04-29 13:25:40')
aborted_by_suite = self._build_view(
17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite',
'RUNNING', aborted_by_suite_job_id,
'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
'aborted by suite', {'aborted_by': 'test_user'},
'2014-04-29 13:15:39', '2014-04-29 13:15:40',
job_started_time='2014-04-29 13:15:39',
job_finished_time='2014-04-29 13:15:40')
good_retry = self._build_view(
18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD',
good_retry_job_id,
'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
'', {'retry_original_job_id': invalid_job_id_1},
'2014-04-29 13:15:37',
'2014-04-29 13:15:38', invalidates_test_idx=1)
bad_retry = self._build_view(
19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL',
bad_retry_job_id,
'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
'retry failed', {'retry_original_job_id': invalid_job_id_2},
'2014-04-29 13:15:39', '2014-04-29 13:15:40',
invalidates_test_idx=2)
invalid_test_1 = self._build_view(
1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD',
invalid_job_id_1,
'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
'original test failed', {}, '2014-04-29 13:10:00',
'2014-04-29 13:10:01')
invalid_test_2 = self._build_view(
2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL',
invalid_job_id_2,
'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
'original test failed', {},
'2014-04-29 13:10:03', '2014-04-29 13:10:04')
test_views = [server_job_view, good_test]
child_jobs = set([good_job_id])
if include_bad_test:
test_views.append(bad_test)
child_jobs.add(bad_job_id)
if include_warn_test:
test_views.append(warn_test)
child_jobs.add(warn_job_id)
if include_experimental_bad_test:
test_views.append(experimental_bad_test)
child_jobs.add(experimental_bad_job_id)
if include_timeout_test:
test_views.append(timeout_test)
if include_self_aborted_test:
test_views.append(self_aborted_test)
child_jobs.add(self_aborted_job_id)
if include_good_retry:
test_views.extend([good_retry, invalid_test_1])
child_jobs.add(good_retry_job_id)
if include_bad_retry:
test_views.extend([bad_retry, invalid_test_2])
child_jobs.add(bad_retry_job_id)
if include_aborted_by_suite_test:
test_views.append(aborted_by_suite)
child_jobs.add(aborted_by_suite_job_id)
self._mock_tko_get_detailed_test_views(test_views)
self._mock_afe_get_jobs(suite_job_id, child_jobs)
collector = run_suite.ResultCollector(
'fake_server', self.afe, self.tko,
'lumpy-release/R36-5788.0.0', 'lumpy', 'dummy', suite_job_id)
collector.run()
return collector