def launch(self):
"""Entrance to launch tests in this runner."""
success, log = self._launch_variations_smoke_test()
test_status = TestStatus.PASS if success else TestStatus.FAIL
# Report a single test named |VariationsSmokeTest| as part of runner output.
overall_result = ResultCollection(test_results=[
TestResult('VariationsSmokeTest', test_status, test_log=log)
])
overall_result.report_to_result_sink()
self.test_results = overall_result.standard_json_output(
path_delimiter='/')
self.logs.update(overall_result.test_runner_logs())
self.tear_down()
return success
class ResultCollectionTest(test_runner_test.TestCase):
"""Tests ResultCollection class APIs."""
def setUp(self):
super(ResultCollectionTest, self).setUp()
self.full_collection = ResultCollection(test_results=[
PASSED_RESULT, FAILED_RESULT, FAILED_RESULT_DUPLICATE,
DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT, CRASHED_RESULT,
FLAKY_PASS_RESULT, FLAKY_FAIL_RESULT, ABORTED_RESULT
])
def test_init(self):
"""Tests class initialization."""
collection = ResultCollection(test_results=[
PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT
],
crashed=True)
self.assertTrue(collection.crashed)
self.assertEqual(collection.crash_message, '')
self.assertEqual(
collection.test_results,
[PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT])
def test_add_result(self):
"""Tests add_test_result."""
collection = ResultCollection(test_results=[FAILED_RESULT])
collection.add_test_result(DISABLED_RESULT)
self.assertEqual(collection.test_results,
[FAILED_RESULT, DISABLED_RESULT])
def test_add_result_collection_default(self):
"""Tests add_result_collection default (merge crash info)."""
collection = ResultCollection(test_results=[FAILED_RESULT])
self.assertFalse(collection.crashed)
collection.append_crash_message('Crash1')
crashed_collection = ResultCollection(test_results=[PASSED_RESULT],
crashed=True)
crashed_collection.append_crash_message('Crash2')
collection.add_result_collection(crashed_collection)
self.assertTrue(collection.crashed)
self.assertEqual(collection.crash_message, 'Crash1\nCrash2')
self.assertEqual(collection.test_results,
[FAILED_RESULT, PASSED_RESULT])
def test_add_result_collection_overwrite(self):
"""Tests add_result_collection overwrite."""
collection = ResultCollection(test_results=[FAILED_RESULT],
crashed=True)
self.assertTrue(collection.crashed)
collection.append_crash_message('Crash1')
crashed_collection = ResultCollection(test_results=[PASSED_RESULT])
collection.add_result_collection(crashed_collection,
overwrite_crash=True)
self.assertFalse(collection.crashed)
self.assertEqual(collection.crash_message, '')
self.assertEqual(collection.test_results,
[FAILED_RESULT, PASSED_RESULT])
def test_add_result_collection_ignore(self):
"""Tests add_result_collection overwrite."""
collection = ResultCollection(test_results=[FAILED_RESULT])
self.assertFalse(collection.crashed)
crashed_collection = ResultCollection(test_results=[PASSED_RESULT],
crashed=True)
crashed_collection.append_crash_message('Crash2')
collection.add_result_collection(crashed_collection, ignore_crash=True)
self.assertFalse(collection.crashed)
self.assertEqual(collection.crash_message, '')
self.assertEqual(collection.test_results,
[FAILED_RESULT, PASSED_RESULT])
def test_add_results(self):
"""Tests add_results."""
collection = ResultCollection(test_results=[PASSED_RESULT])
collection.add_results([FAILED_RESULT, DISABLED_RESULT])
self.assertEqual(collection.test_results,
[PASSED_RESULT, FAILED_RESULT, DISABLED_RESULT])
def test_add_name_prefix_to_tests(self):
"""Tests add_name_prefix_to_tests."""
passed = copy.copy(PASSED_RESULT)
disabeld = copy.copy(DISABLED_RESULT)
collection = ResultCollection(test_results=[passed, disabeld])
some_prefix = 'Some/prefix'
collection.add_name_prefix_to_tests(some_prefix)
for test_result in collection.test_results:
self.assertTrue(test_result.name.startswith(some_prefix))
def test_add_test_names_status(self):
"""Tests add_test_names_status."""
test_names = ['test1', 'test2', 'test3']
collection = ResultCollection(test_results=[PASSED_RESULT])
collection.add_test_names_status(test_names, TestStatus.SKIP)
disabled_test_names = ['test4', 'test5', 'test6']
collection.add_test_names_status(disabled_test_names,
TestStatus.SKIP,
expected_status=TestStatus.SKIP)
self.assertEqual(collection.test_results[0], PASSED_RESULT)
unexpected_skipped = collection.tests_by_expression(
lambda t: not t.expected() and t.status == TestStatus.SKIP)
self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
self.assertEqual(collection.disabled_tests(),
set(['test4', 'test5', 'test6']))
@mock.patch('test_result_util.TestResult.report_to_result_sink')
@mock.patch('result_sink_util.ResultSinkClient.close')
@mock.patch('result_sink_util.ResultSinkClient.__init__',
return_value=None)
def test_add_and_report_test_names_status(self, mock_sink_init,
mock_sink_close, mock_report):
"""Tests add_test_names_status."""
test_names = ['test1', 'test2', 'test3']
collection = ResultCollection(test_results=[PASSED_RESULT])
collection.add_and_report_test_names_status(test_names,
TestStatus.SKIP)
self.assertEqual(collection.test_results[0], PASSED_RESULT)
unexpected_skipped = collection.tests_by_expression(
lambda t: not t.expected() and t.status == TestStatus.SKIP)
self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
self.assertEqual(1, len(mock_sink_init.mock_calls))
self.assertEqual(3, len(mock_report.mock_calls))
self.assertEqual(1, len(mock_sink_close.mock_calls))
def testappend_crash_message(self):
"""Tests append_crash_message."""
collection = ResultCollection(test_results=[PASSED_RESULT])
collection.append_crash_message('Crash message 1.')
self.assertEqual(collection.crash_message, 'Crash message 1.')
collection.append_crash_message('Crash message 2.')
self.assertEqual(collection.crash_message,
'Crash message 1.\nCrash message 2.')
def test_tests_by_expression(self):
"""Tests tests_by_expression."""
collection = self.full_collection
exp = lambda result: result.status == TestStatus.SKIP
skipped_tests = collection.tests_by_expression(exp)
self.assertEqual(skipped_tests,
set(['unexpected/skipped_test', 'disabled/test']))
def test_get_spcific_tests(self):
"""Tests getting sets of tests of specific status."""
collection = self.full_collection
self.assertEqual(
collection.all_test_names(),
set([
'passed/test', 'disabled/test', 'failed/test',
'unexpected/skipped_test', 'crashed/test', 'flaky/test',
'aborted/test'
]))
self.assertEqual(collection.crashed_tests(), set(['crashed/test']))
self.assertEqual(collection.disabled_tests(), set(['disabled/test']))
self.assertEqual(collection.expected_tests(),
set(['passed/test', 'disabled/test', 'flaky/test']))
self.assertEqual(
collection.unexpected_tests(),
set([
'failed/test', 'unexpected/skipped_test', 'crashed/test',
'flaky/test', 'aborted/test'
]))
self.assertEqual(collection.passed_tests(),
set(['passed/test', 'flaky/test']))
self.assertEqual(collection.failed_tests(),
set(['failed/test', 'flaky/test']))
self.assertEqual(collection.flaky_tests(), set(['flaky/test']))
self.assertEqual(
collection.never_expected_tests(),
set([
'failed/test', 'unexpected/skipped_test', 'crashed/test',
'aborted/test'
]))
self.assertEqual(collection.pure_expected_tests(),
set(['passed/test', 'disabled/test']))
@mock.patch('test_result_util.TestResult.report_to_result_sink')
@mock.patch('result_sink_util.ResultSinkClient.close')
@mock.patch('result_sink_util.ResultSinkClient.__init__',
return_value=None)
def test_add_and_report_crash(self, mock_sink_init, mock_sink_close,
mock_report):
"""Tests add_and_report_crash."""
collection = copy.copy(self.full_collection)
self.assertFalse('BUILD_INTERRUPTED' in collection.crashed_tests())
collection.add_and_report_crash('Prefix Line')
self.assertEqual(collection.crash_message, 'Prefix Line\n')
self.assertTrue('BUILD_INTERRUPTED' in collection.crashed_tests())
mock_sink_init.assert_called_once()
mock_report.assert_called_once()
mock_sink_close.assert_called_once()
@mock.patch('test_result_util.TestResult.report_to_result_sink')
@mock.patch('result_sink_util.ResultSinkClient.close')
@mock.patch('result_sink_util.ResultSinkClient.__init__',
return_value=None)
def test_report_to_result_sink(self, mock_sink_init, mock_sink_close,
mock_report):
"""Tests report_to_result_sink."""
collection = copy.copy(self.full_collection)
collection.report_to_result_sink()
mock_sink_init.assert_called_once()
self.assertEqual(len(collection.test_results),
len(mock_report.mock_calls))
mock_sink_close.assert_called()
@mock.patch('shard_util.shard_index', return_value=0)
@mock.patch('time.time', return_value=10000)
def test_standard_json_output(self, *args):
"""Tests standard_json_output."""
passed_test_value = {
'expected': 'PASS',
'actual': 'PASS',
'shard': 0,
'is_unexpected': False
}
failed_test_value = {
'expected': 'PASS',
'actual': 'FAIL FAIL',
'shard': 0,
'is_unexpected': True
}
disabled_test_value = {
'expected': 'SKIP',
'actual': 'SKIP',
'shard': 0,
'is_unexpected': False
}
unexpected_skip_test_value = {
'expected': 'PASS',
'actual': 'SKIP',
'shard': 0,
'is_unexpected': True
}
crashed_test_value = {
'expected': 'PASS',
'actual': 'CRASH',
'shard': 0,
'is_unexpected': True
}
flaky_test_value = {
'expected': 'PASS',
'actual': 'PASS FAIL',
'shard': 0,
'is_unexpected': False,
'is_flaky': True
}
aborted_test_value = {
'expected': 'PASS',
'actual': 'TIMEOUT',
'shard': 0,
'is_unexpected': True
}
expected_tests = collections.OrderedDict()
expected_tests['passed/test'] = passed_test_value
expected_tests['failed/test'] = failed_test_value
expected_tests['disabled/test'] = disabled_test_value
expected_tests['unexpected/skipped_test'] = unexpected_skip_test_value
expected_tests['crashed/test'] = crashed_test_value
expected_tests['flaky/test'] = flaky_test_value
expected_tests['aborted/test'] = aborted_test_value
expected_num_failures_by_type = {
'PASS': 2,
'FAIL': 1,
'CRASH': 1,
'SKIP': 2,
'TIMEOUT': 1
}
expected_json = {
'version': 3,
'path_delimiter': '/',
'seconds_since_epoch': 10000,
'interrupted': False,
'num_failures_by_type': expected_num_failures_by_type,
'tests': expected_tests
}
self.assertEqual(
self.full_collection.standard_json_output(path_delimiter='/'),
expected_json)
def test_test_runner_logs(self):
"""Test test_runner_logs."""
expected_logs = collections.OrderedDict()
expected_logs['passed tests'] = ['passed/test']
expected_logs['disabled tests'] = ['disabled/test']
flaky_logs = ['Failure log of attempt 1:', 'line1', 'line2']
failed_logs = [
'Failure log of attempt 1:', 'line1', 'line2',
'Failure log of attempt 2:', 'line3', 'line4'
]
no_logs = ['Failure log of attempt 1:', '']
expected_logs['flaked tests'] = {'flaky/test': flaky_logs}
expected_logs['failed tests'] = {
'failed/test': failed_logs,
'crashed/test': no_logs,
'unexpected/skipped_test': no_logs,
'aborted/test': no_logs
}
expected_logs['failed/test'] = failed_logs
expected_logs['unexpected/skipped_test'] = no_logs
expected_logs['flaky/test'] = flaky_logs
expected_logs['crashed/test'] = no_logs
expected_logs['aborted/test'] = no_logs
generated_logs = self.full_collection.test_runner_logs()
keys = [
'passed tests', 'disabled tests', 'flaked tests', 'failed tests',
'failed/test', 'unexpected/skipped_test', 'flaky/test',
'crashed/test', 'aborted/test'
]
for key in keys:
self.assertEqual(generated_logs[key], expected_logs[key])
def launch(self):
"""Launches the test app."""
self.set_up()
# The overall ResultCorrection object holding all runs of all tests in the
# runner run. It will be updated with each test application launch.
overall_result = ResultCollection()
destination = 'id=%s' % self.udid
test_app = self.get_launch_test_app()
out_dir = os.path.join(self.out_dir, 'TestResults')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
try:
result = self._run(cmd=cmd, shards=self.shards or 1)
if result.crashed and not result.crashed_tests():
# If the app crashed but not during any particular test case, assume
# it crashed on startup. Try one more time.
self.shutdown_and_restart()
LOGGER.warning('Crashed on startup, retrying...\n')
out_dir = os.path.join(self.out_dir,
'retry_after_crash_on_startup')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
result = self._run(cmd)
result.report_to_result_sink()
if result.crashed and not result.crashed_tests():
raise AppLaunchError
overall_result.add_result_collection(result)
try:
while result.crashed and result.crashed_tests():
# If the app crashes during a specific test case, then resume at the
# next test case. This is achieved by filtering out every test case
# which has already run.
LOGGER.warning('Crashed during %s, resuming...\n',
list(result.crashed_tests()))
test_app.excluded_tests = list(
overall_result.all_test_names())
retry_out_dir = os.path.join(
self.out_dir,
'retry_after_crash_%d' % int(time.time()))
result = self._run(
self.get_launch_command(
test_app,
os.path.join(retry_out_dir, str(int(time.time()))),
destination))
result.report_to_result_sink()
# Only keep the last crash status in crash retries in overall crash
# status.
overall_result.add_result_collection(result,
overwrite_crash=True)
except OSError as e:
if e.errno == errno.E2BIG:
LOGGER.error('Too many test cases to resume.')
else:
raise
# Retry failed test cases.
test_app.excluded_tests = []
never_expected_tests = overall_result.never_expected_tests()
if self.retries and never_expected_tests:
LOGGER.warning('%s tests failed and will be retried.\n',
len(never_expected_tests))
for i in xrange(self.retries):
tests_to_retry = list(
overall_result.never_expected_tests())
for test in tests_to_retry:
LOGGER.info('Retry #%s for %s.\n', i + 1, test)
test_app.included_tests = [test]
retry_out_dir = os.path.join(self.out_dir,
test + '_failed',
'retry_%d' % i)
retry_result = self._run(
self.get_launch_command(test_app, retry_out_dir,
destination))
if not retry_result.all_test_names():
retry_result.add_test_result(
TestResult(
test,
TestStatus.SKIP,
test_log=
'In single test retry, result of this test '
'didn\'t appear in log.'))
retry_result.report_to_result_sink()
# No unknown tests might be skipped so do not change
# |overall_result|'s crash status.
overall_result.add_result_collection(retry_result,
ignore_crash=True)
interrupted = overall_result.crashed
if interrupted:
overall_result.add_and_report_crash(
crash_message_prefix_line=
'Test application crashed when running '
'tests which might have caused some tests never ran or finished.'
)
self.test_results = overall_result.standard_json_output()
self.logs.update(overall_result.test_runner_logs())
return not overall_result.never_expected_tests(
) and not interrupted
finally:
self.tear_down()