Ejemplo n.º 1
0
    def launch(self):
        """Entrance to launch tests in this runner."""
        success, log = self._launch_variations_smoke_test()

        test_status = TestStatus.PASS if success else TestStatus.FAIL
        # Report a single test named |VariationsSmokeTest| as part of runner output.
        overall_result = ResultCollection(test_results=[
            TestResult('VariationsSmokeTest', test_status, test_log=log)
        ])
        overall_result.report_to_result_sink()
        self.test_results = overall_result.standard_json_output(
            path_delimiter='/')
        self.logs.update(overall_result.test_runner_logs())
        self.tear_down()

        return success
Ejemplo n.º 2
0
class ResultCollectionTest(test_runner_test.TestCase):
    """Tests ResultCollection class APIs."""
    def setUp(self):
        super(ResultCollectionTest, self).setUp()
        self.full_collection = ResultCollection(test_results=[
            PASSED_RESULT, FAILED_RESULT, FAILED_RESULT_DUPLICATE,
            DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT, CRASHED_RESULT,
            FLAKY_PASS_RESULT, FLAKY_FAIL_RESULT, ABORTED_RESULT
        ])

    def test_init(self):
        """Tests class initialization."""
        collection = ResultCollection(test_results=[
            PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT
        ],
                                      crashed=True)
        self.assertTrue(collection.crashed)
        self.assertEqual(collection.crash_message, '')
        self.assertEqual(
            collection.test_results,
            [PASSED_RESULT, DISABLED_RESULT, UNEXPECTED_SKIPPED_RESULT])

    def test_add_result(self):
        """Tests add_test_result."""
        collection = ResultCollection(test_results=[FAILED_RESULT])
        collection.add_test_result(DISABLED_RESULT)
        self.assertEqual(collection.test_results,
                         [FAILED_RESULT, DISABLED_RESULT])

    def test_add_result_collection_default(self):
        """Tests add_result_collection default (merge crash info)."""
        collection = ResultCollection(test_results=[FAILED_RESULT])
        self.assertFalse(collection.crashed)
        collection.append_crash_message('Crash1')

        crashed_collection = ResultCollection(test_results=[PASSED_RESULT],
                                              crashed=True)
        crashed_collection.append_crash_message('Crash2')

        collection.add_result_collection(crashed_collection)
        self.assertTrue(collection.crashed)
        self.assertEqual(collection.crash_message, 'Crash1\nCrash2')
        self.assertEqual(collection.test_results,
                         [FAILED_RESULT, PASSED_RESULT])

    def test_add_result_collection_overwrite(self):
        """Tests add_result_collection overwrite."""
        collection = ResultCollection(test_results=[FAILED_RESULT],
                                      crashed=True)
        self.assertTrue(collection.crashed)
        collection.append_crash_message('Crash1')

        crashed_collection = ResultCollection(test_results=[PASSED_RESULT])

        collection.add_result_collection(crashed_collection,
                                         overwrite_crash=True)
        self.assertFalse(collection.crashed)
        self.assertEqual(collection.crash_message, '')
        self.assertEqual(collection.test_results,
                         [FAILED_RESULT, PASSED_RESULT])

    def test_add_result_collection_ignore(self):
        """Tests add_result_collection overwrite."""
        collection = ResultCollection(test_results=[FAILED_RESULT])
        self.assertFalse(collection.crashed)

        crashed_collection = ResultCollection(test_results=[PASSED_RESULT],
                                              crashed=True)
        crashed_collection.append_crash_message('Crash2')

        collection.add_result_collection(crashed_collection, ignore_crash=True)
        self.assertFalse(collection.crashed)
        self.assertEqual(collection.crash_message, '')
        self.assertEqual(collection.test_results,
                         [FAILED_RESULT, PASSED_RESULT])

    def test_add_results(self):
        """Tests add_results."""
        collection = ResultCollection(test_results=[PASSED_RESULT])
        collection.add_results([FAILED_RESULT, DISABLED_RESULT])
        self.assertEqual(collection.test_results,
                         [PASSED_RESULT, FAILED_RESULT, DISABLED_RESULT])

    def test_add_name_prefix_to_tests(self):
        """Tests add_name_prefix_to_tests."""
        passed = copy.copy(PASSED_RESULT)
        disabeld = copy.copy(DISABLED_RESULT)
        collection = ResultCollection(test_results=[passed, disabeld])
        some_prefix = 'Some/prefix'
        collection.add_name_prefix_to_tests(some_prefix)
        for test_result in collection.test_results:
            self.assertTrue(test_result.name.startswith(some_prefix))

    def test_add_test_names_status(self):
        """Tests add_test_names_status."""
        test_names = ['test1', 'test2', 'test3']
        collection = ResultCollection(test_results=[PASSED_RESULT])
        collection.add_test_names_status(test_names, TestStatus.SKIP)
        disabled_test_names = ['test4', 'test5', 'test6']
        collection.add_test_names_status(disabled_test_names,
                                         TestStatus.SKIP,
                                         expected_status=TestStatus.SKIP)
        self.assertEqual(collection.test_results[0], PASSED_RESULT)
        unexpected_skipped = collection.tests_by_expression(
            lambda t: not t.expected() and t.status == TestStatus.SKIP)
        self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
        self.assertEqual(collection.disabled_tests(),
                         set(['test4', 'test5', 'test6']))

    @mock.patch('test_result_util.TestResult.report_to_result_sink')
    @mock.patch('result_sink_util.ResultSinkClient.close')
    @mock.patch('result_sink_util.ResultSinkClient.__init__',
                return_value=None)
    def test_add_and_report_test_names_status(self, mock_sink_init,
                                              mock_sink_close, mock_report):
        """Tests add_test_names_status."""
        test_names = ['test1', 'test2', 'test3']
        collection = ResultCollection(test_results=[PASSED_RESULT])
        collection.add_and_report_test_names_status(test_names,
                                                    TestStatus.SKIP)
        self.assertEqual(collection.test_results[0], PASSED_RESULT)
        unexpected_skipped = collection.tests_by_expression(
            lambda t: not t.expected() and t.status == TestStatus.SKIP)
        self.assertEqual(unexpected_skipped, set(['test1', 'test2', 'test3']))
        self.assertEqual(1, len(mock_sink_init.mock_calls))
        self.assertEqual(3, len(mock_report.mock_calls))
        self.assertEqual(1, len(mock_sink_close.mock_calls))

    def testappend_crash_message(self):
        """Tests append_crash_message."""
        collection = ResultCollection(test_results=[PASSED_RESULT])
        collection.append_crash_message('Crash message 1.')
        self.assertEqual(collection.crash_message, 'Crash message 1.')
        collection.append_crash_message('Crash message 2.')
        self.assertEqual(collection.crash_message,
                         'Crash message 1.\nCrash message 2.')

    def test_tests_by_expression(self):
        """Tests tests_by_expression."""
        collection = self.full_collection
        exp = lambda result: result.status == TestStatus.SKIP
        skipped_tests = collection.tests_by_expression(exp)
        self.assertEqual(skipped_tests,
                         set(['unexpected/skipped_test', 'disabled/test']))

    def test_get_spcific_tests(self):
        """Tests getting sets of tests of specific status."""
        collection = self.full_collection
        self.assertEqual(
            collection.all_test_names(),
            set([
                'passed/test', 'disabled/test', 'failed/test',
                'unexpected/skipped_test', 'crashed/test', 'flaky/test',
                'aborted/test'
            ]))
        self.assertEqual(collection.crashed_tests(), set(['crashed/test']))
        self.assertEqual(collection.disabled_tests(), set(['disabled/test']))
        self.assertEqual(collection.expected_tests(),
                         set(['passed/test', 'disabled/test', 'flaky/test']))
        self.assertEqual(
            collection.unexpected_tests(),
            set([
                'failed/test', 'unexpected/skipped_test', 'crashed/test',
                'flaky/test', 'aborted/test'
            ]))
        self.assertEqual(collection.passed_tests(),
                         set(['passed/test', 'flaky/test']))
        self.assertEqual(collection.failed_tests(),
                         set(['failed/test', 'flaky/test']))
        self.assertEqual(collection.flaky_tests(), set(['flaky/test']))
        self.assertEqual(
            collection.never_expected_tests(),
            set([
                'failed/test', 'unexpected/skipped_test', 'crashed/test',
                'aborted/test'
            ]))
        self.assertEqual(collection.pure_expected_tests(),
                         set(['passed/test', 'disabled/test']))

    @mock.patch('test_result_util.TestResult.report_to_result_sink')
    @mock.patch('result_sink_util.ResultSinkClient.close')
    @mock.patch('result_sink_util.ResultSinkClient.__init__',
                return_value=None)
    def test_add_and_report_crash(self, mock_sink_init, mock_sink_close,
                                  mock_report):
        """Tests add_and_report_crash."""
        collection = copy.copy(self.full_collection)
        self.assertFalse('BUILD_INTERRUPTED' in collection.crashed_tests())

        collection.add_and_report_crash('Prefix Line')
        self.assertEqual(collection.crash_message, 'Prefix Line\n')
        self.assertTrue('BUILD_INTERRUPTED' in collection.crashed_tests())

        mock_sink_init.assert_called_once()
        mock_report.assert_called_once()
        mock_sink_close.assert_called_once()

    @mock.patch('test_result_util.TestResult.report_to_result_sink')
    @mock.patch('result_sink_util.ResultSinkClient.close')
    @mock.patch('result_sink_util.ResultSinkClient.__init__',
                return_value=None)
    def test_report_to_result_sink(self, mock_sink_init, mock_sink_close,
                                   mock_report):
        """Tests report_to_result_sink."""
        collection = copy.copy(self.full_collection)
        collection.report_to_result_sink()

        mock_sink_init.assert_called_once()
        self.assertEqual(len(collection.test_results),
                         len(mock_report.mock_calls))
        mock_sink_close.assert_called()

    @mock.patch('shard_util.shard_index', return_value=0)
    @mock.patch('time.time', return_value=10000)
    def test_standard_json_output(self, *args):
        """Tests standard_json_output."""
        passed_test_value = {
            'expected': 'PASS',
            'actual': 'PASS',
            'shard': 0,
            'is_unexpected': False
        }
        failed_test_value = {
            'expected': 'PASS',
            'actual': 'FAIL FAIL',
            'shard': 0,
            'is_unexpected': True
        }
        disabled_test_value = {
            'expected': 'SKIP',
            'actual': 'SKIP',
            'shard': 0,
            'is_unexpected': False
        }
        unexpected_skip_test_value = {
            'expected': 'PASS',
            'actual': 'SKIP',
            'shard': 0,
            'is_unexpected': True
        }
        crashed_test_value = {
            'expected': 'PASS',
            'actual': 'CRASH',
            'shard': 0,
            'is_unexpected': True
        }
        flaky_test_value = {
            'expected': 'PASS',
            'actual': 'PASS FAIL',
            'shard': 0,
            'is_unexpected': False,
            'is_flaky': True
        }
        aborted_test_value = {
            'expected': 'PASS',
            'actual': 'TIMEOUT',
            'shard': 0,
            'is_unexpected': True
        }
        expected_tests = collections.OrderedDict()
        expected_tests['passed/test'] = passed_test_value
        expected_tests['failed/test'] = failed_test_value
        expected_tests['disabled/test'] = disabled_test_value
        expected_tests['unexpected/skipped_test'] = unexpected_skip_test_value
        expected_tests['crashed/test'] = crashed_test_value
        expected_tests['flaky/test'] = flaky_test_value
        expected_tests['aborted/test'] = aborted_test_value
        expected_num_failures_by_type = {
            'PASS': 2,
            'FAIL': 1,
            'CRASH': 1,
            'SKIP': 2,
            'TIMEOUT': 1
        }
        expected_json = {
            'version': 3,
            'path_delimiter': '/',
            'seconds_since_epoch': 10000,
            'interrupted': False,
            'num_failures_by_type': expected_num_failures_by_type,
            'tests': expected_tests
        }
        self.assertEqual(
            self.full_collection.standard_json_output(path_delimiter='/'),
            expected_json)

    def test_test_runner_logs(self):
        """Test test_runner_logs."""
        expected_logs = collections.OrderedDict()
        expected_logs['passed tests'] = ['passed/test']
        expected_logs['disabled tests'] = ['disabled/test']
        flaky_logs = ['Failure log of attempt 1:', 'line1', 'line2']
        failed_logs = [
            'Failure log of attempt 1:', 'line1', 'line2',
            'Failure log of attempt 2:', 'line3', 'line4'
        ]
        no_logs = ['Failure log of attempt 1:', '']
        expected_logs['flaked tests'] = {'flaky/test': flaky_logs}
        expected_logs['failed tests'] = {
            'failed/test': failed_logs,
            'crashed/test': no_logs,
            'unexpected/skipped_test': no_logs,
            'aborted/test': no_logs
        }
        expected_logs['failed/test'] = failed_logs
        expected_logs['unexpected/skipped_test'] = no_logs
        expected_logs['flaky/test'] = flaky_logs
        expected_logs['crashed/test'] = no_logs
        expected_logs['aborted/test'] = no_logs
        generated_logs = self.full_collection.test_runner_logs()
        keys = [
            'passed tests', 'disabled tests', 'flaked tests', 'failed tests',
            'failed/test', 'unexpected/skipped_test', 'flaky/test',
            'crashed/test', 'aborted/test'
        ]
        for key in keys:
            self.assertEqual(generated_logs[key], expected_logs[key])
Ejemplo n.º 3
0
    def launch(self):
        """Launches the test app."""
        self.set_up()
        # The overall ResultCorrection object holding all runs of all tests in the
        # runner run. It will be updated with each test application launch.
        overall_result = ResultCollection()
        destination = 'id=%s' % self.udid
        test_app = self.get_launch_test_app()
        out_dir = os.path.join(self.out_dir, 'TestResults')
        cmd = self.get_launch_command(test_app, out_dir, destination,
                                      self.shards)
        try:
            result = self._run(cmd=cmd, shards=self.shards or 1)
            if result.crashed and not result.crashed_tests():
                # If the app crashed but not during any particular test case, assume
                # it crashed on startup. Try one more time.
                self.shutdown_and_restart()
                LOGGER.warning('Crashed on startup, retrying...\n')
                out_dir = os.path.join(self.out_dir,
                                       'retry_after_crash_on_startup')
                cmd = self.get_launch_command(test_app, out_dir, destination,
                                              self.shards)
                result = self._run(cmd)

            result.report_to_result_sink()

            if result.crashed and not result.crashed_tests():
                raise AppLaunchError

            overall_result.add_result_collection(result)

            try:
                while result.crashed and result.crashed_tests():
                    # If the app crashes during a specific test case, then resume at the
                    # next test case. This is achieved by filtering out every test case
                    # which has already run.
                    LOGGER.warning('Crashed during %s, resuming...\n',
                                   list(result.crashed_tests()))
                    test_app.excluded_tests = list(
                        overall_result.all_test_names())
                    retry_out_dir = os.path.join(
                        self.out_dir,
                        'retry_after_crash_%d' % int(time.time()))
                    result = self._run(
                        self.get_launch_command(
                            test_app,
                            os.path.join(retry_out_dir, str(int(time.time()))),
                            destination))
                    result.report_to_result_sink()
                    # Only keep the last crash status in crash retries in overall crash
                    # status.
                    overall_result.add_result_collection(result,
                                                         overwrite_crash=True)

            except OSError as e:
                if e.errno == errno.E2BIG:
                    LOGGER.error('Too many test cases to resume.')
                else:
                    raise

            # Retry failed test cases.
            test_app.excluded_tests = []
            never_expected_tests = overall_result.never_expected_tests()
            if self.retries and never_expected_tests:
                LOGGER.warning('%s tests failed and will be retried.\n',
                               len(never_expected_tests))
                for i in xrange(self.retries):
                    tests_to_retry = list(
                        overall_result.never_expected_tests())
                    for test in tests_to_retry:
                        LOGGER.info('Retry #%s for %s.\n', i + 1, test)
                        test_app.included_tests = [test]
                        retry_out_dir = os.path.join(self.out_dir,
                                                     test + '_failed',
                                                     'retry_%d' % i)
                        retry_result = self._run(
                            self.get_launch_command(test_app, retry_out_dir,
                                                    destination))

                        if not retry_result.all_test_names():
                            retry_result.add_test_result(
                                TestResult(
                                    test,
                                    TestStatus.SKIP,
                                    test_log=
                                    'In single test retry, result of this test '
                                    'didn\'t appear in log.'))
                        retry_result.report_to_result_sink()
                        # No unknown tests might be skipped so do not change
                        # |overall_result|'s crash status.
                        overall_result.add_result_collection(retry_result,
                                                             ignore_crash=True)

            interrupted = overall_result.crashed

            if interrupted:
                overall_result.add_and_report_crash(
                    crash_message_prefix_line=
                    'Test application crashed when running '
                    'tests which might have caused some tests never ran or finished.'
                )

            self.test_results = overall_result.standard_json_output()
            self.logs.update(overall_result.test_runner_logs())

            return not overall_result.never_expected_tests(
            ) and not interrupted
        finally:
            self.tear_down()