Ejemplo n.º 1
0
    def test_overall_stats(self):

        data = ResultData()
        data.add_results('chrome', [
            self._build_result('error'),
            self._build_result('error'),
            self._build_result('fail'),
            self._build_result('fail'),
            self._build_result('fail'),
            self._build_result('pass'),
            self._build_result('pass'),
            self._build_result('pass'),
            self._build_result('pass'),
            self._build_result('skip'),
            self._build_result('skip'),
            self._build_result('skip'),
            self._build_result('skip'),
            self._build_result('skip'),
        ])

        data.add_results('firefox', [
            self._build_result('error'),
            self._build_result('fail'),
            self._build_result('pass'),
            self._build_result('skip'),
        ])

        # Expect that the overall stats (no browser specified)
        # is the sum of all browser-specific stats
        expected_stats = self._build_stats(
            num_error=3, num_failed=4,
            num_passed=5, num_skipped=6
        )

        self.assertEqual(data.stats(None), expected_stats)
Ejemplo n.º 2
0
    def test_stats(self):

        # Tuples relating input results to stats
        cases = [
            ([], self._build_stats()),
            ([self._build_result('error')], self._build_stats(num_error=1)),
            ([self._build_result('fail')], self._build_stats(num_failed=1)),
            ([self._build_result('skip')], self._build_stats(num_skipped=1)),
            ([self._build_result('pass')], self._build_stats(num_passed=1)),
            ([
                self._build_result('error'),
                self._build_result('error'),
                self._build_result('fail'),
                self._build_result('fail'),
                self._build_result('fail'),
                self._build_result('pass'),
                self._build_result('pass'),
                self._build_result('pass'),
                self._build_result('pass'),
                self._build_result('skip'),
                self._build_result('skip'),
                self._build_result('skip'),
                self._build_result('skip'),
                self._build_result('skip'),
            ], self._build_stats(
                num_error=2, num_failed=3,
                num_passed=4, num_skipped=5
            ))
        ]

        for input_results, expected_stats in cases:
            data = ResultData()
            data.add_results('chrome', input_results)
            self.assertEqual(data.stats('chrome'), expected_stats)
Ejemplo n.º 3
0
    def testadd_results(self):

        data = ResultData()
        input_results = [self._build_result('pass'), self._build_result('fail')]

        data.add_results('chrome', input_results)
        self.assertEqual(data.browsers(), ['chrome'])
        self.assertEqual(data.test_results('chrome'), input_results)
        self.assertEqual(data.all_passed(), False)
Ejemplo n.º 4
0
    def run(self):
        """
        Execute each available test suite page and record whether
        each test passed/failed.

        If configured, will write test results using reporters.

        Returns a `ResultData` object containing the test results.
        """

        # Start the suite page server running on a local port
        self._suite_page_server.start()

        # Create an object to hold results data for all browsers
        results_data = ResultData()

        try:

            for browser in self._browser_list:

                # Run the test suite with one of our browsers
                results_data.add_results(
                    browser.name(),
                    self._run_with_browser(browser)
                )

            # After all browsers have loaded their pages,
            # Block until all coverage data received
            # (if coverage is not configured, this will
            # not do anything).
            try:
                self._report_coverage_data = self._suite_page_server.all_coverage_data()

            # If we timed out, log it, but don't exit with failure
            except TimeoutError:
                msg = dedent("""
                Did not receive all coverage data.  No coverage reports will be written.
                (This sometimes occurs when JSCover does not have enough memory to run.)
                """).strip()
                LOGGER.warning(msg)

        # Re-raise any exceptions that occur
        except:
            raise

        # Stop the suite page server, freeing up the port
        finally:
            self._suite_page_server.stop()

        # Generate test result reports
        for reporter in self._result_reporters:
            reporter.write_report(results_data)

        return results_data
Ejemplo n.º 5
0
class ResultReporterTestCase(TestCase):
    """
    Base class for reporter tests.
    """

    # Subclasses override this to specify the class under test
    REPORTER_CLASS = None

    def setUp(self):
        self.result_data = ResultData()

    def add_result(self, browser_name, group_name, test_name, status, detail):
        """
        Add test results for the browser with `browser_name`.

        `group_name`: name of the test group (e.g. 'Adder tests')
        `test_name`: name of the specific test case
                     (e.g. 'it should start at zero')
        `status`: pass | fail | skip
        `detail`: details of the test case (e.g. a stack trace)
        """
        self.result_data.add_results(
            browser_name, [{
                'test_group': group_name,
                'test_name': test_name,
                'status': status,
                'detail': detail
            }]
        )

    def assert_report(self, expected_report):
        """
        Assert that the console report matches `expected_report`.
        """
        assert_long_str_equal(self.build_report(), expected_report, strip=True)

    def build_report(self):
        """
        Build a report.
        """
        output = StringIO()
        self.REPORTER_CLASS(output).write_report(self.result_data)
        return output.getvalue()
Ejemplo n.º 6
0
    def test_all_passed(self):

        # Tuples relating all_passed to input results from each browser.
        cases = [
            (True, [], []),
            (True, [self._build_result('pass')], []),
            (True, [self._build_result('pass'), self._build_result('pass')], []),
            (True, [self._build_result('pass')], [self._build_result('pass')]),
            (False, [self._build_result('fail')], [self._build_result('pass')]),
            (False, [self._build_result('error')], [self._build_result('pass')]),
            (False, [self._build_result('fail')], [self._build_result('fail')]),
            (False, [self._build_result('error')], [self._build_result('fail')]),
            (False, [self._build_result('error')], [self._build_result('error')]),
        ]

        for (expect_passed, chrome_results, firefox_results) in cases:
            data = ResultData()
            data.add_results('chrome', chrome_results)
            data.add_results('firefox', firefox_results)
            self.assertEqual(data.all_passed(), expect_passed)