Beispiel #1
0
    def test_harness_remove_all(self):
        """Tests that removing all expectations doesn't delete the file.

        Make sure we're prepared for the day when we exterminated flakes.
        """

        self._define_builders({
            "WebKit Linux Trusty": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Release']
            },
            "WebKit Linux Trusty (dbg)": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Debug']
            },
        })

        # Set up the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(
            host,
            all_build_types=('release', 'debug'),
            all_systems=(('trusty', 'x86_64'),))

        # Write out a fake TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        test_expectations = """
            # Remove since passing on both bots.
            Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""

        files = {
            test_expectation_path: test_expectations
        }
        host.filesystem = MockFileSystem(files)
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {
            'WebKit Linux Trusty': {
                "test/a.html": ["PASS", "PASS", "PASS"],
            },
            'WebKit Linux Trusty (dbg)': {
                "test/a.html": ["PASS", "PASS", "PASS"],
            },
        }

        main(host, expectation_factory, [])

        self.assertTrue(host.filesystem.isfile(test_expectation_path))
        self.assertEqual(host.filesystem.files[test_expectation_path], '')
    def test_harness_remove_all(self):
        """Tests that removing all expectations doesn't delete the file.

        Make sure we're prepared for the day when we exterminated flakes.
        """

        self._define_builders({
            "WebKit Linux Trusty": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Release']
            },
            "WebKit Linux Trusty (dbg)": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Debug']
            },
        })

        # Setup the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(
            host,
            all_build_types=('release', 'debug'),
            all_systems=(('trusty', 'x86_64'),))

        # Write out a fake TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        test_expectations = """
            # Remove since passing on both bots.
            Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""

        files = {
            test_expectation_path: test_expectations
        }
        host.filesystem = MockFileSystem(files)
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {
            'WebKit Linux Trusty': {
                "test/a.html": ["PASS", "PASS", "PASS"],
            },
            'WebKit Linux Trusty (dbg)': {
                "test/a.html": ["PASS", "PASS", "PASS"],
            },
        }

        main(host, expectation_factory, [])

        self.assertTrue(host.filesystem.isfile(test_expectation_path))
        self.assertEqual(host.filesystem.files[test_expectation_path], '')
Beispiel #3
0
    def test_harness_no_expectations(self):
        """Tests behavior when TestExpectations file doesn't exist.

        Tests that a warning is outputted if the TestExpectations file
        doesn't exist.
        """

        # Setup the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(host)

        # Write the test file but not the TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        host.filesystem = MockFileSystem()
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {}

        self.assertFalse(host.filesystem.isfile(test_expectation_path))

        return_code = main(host, expectation_factory, [])

        self.assertEqual(return_code, 1)

        self.assertLog([
            "WARNING: Didn't find generic expectations file at: %s\n" %
            test_expectation_path
        ])
        self.assertFalse(host.filesystem.isfile(test_expectation_path))
Beispiel #4
0
    def test_harness_no_expectations(self):
        """Tests behavior when TestExpectations file doesn't exist.

        Tests that a warning is outputted if the TestExpectations file
        doesn't exist.
        """

        # Set up the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(host)

        # Write the test file but not the TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        host.filesystem = MockFileSystem()
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {}

        self.assertFalse(host.filesystem.isfile(test_expectation_path))

        return_code = main(host, expectation_factory, [])

        self.assertEqual(return_code, 1)

        self.assertLog([
            "WARNING: Didn't find generic expectations file at: %s\n" % test_expectation_path
        ])
        self.assertFalse(host.filesystem.isfile(test_expectation_path))
Beispiel #5
0
    def test_harness_updates_file(self):
        """Tests that the call harness updates the TestExpectations file."""

        self._define_builders({
            "WebKit Linux Precise": {
                "port_name": "linux-precise",
                "specifiers": ['Precise', 'Release']
            },
            "WebKit Linux Precise (dbg)": {
                "port_name": "linux-precise",
                "specifiers": ['Precise', 'Debug']
            },
        })

        # Setup the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(host)
        host.port_factory._all_build_types = ('release', 'debug')
        host.port_factory._all_systems = (('precise', 'x86_64'), )

        # Write out a fake TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        test_expectations = """
            # Remove since passing on both bots.
            Bug(test) [ Linux ] test/a.html [ Failure Pass ]
            # Keep since there's a failure on release bot.
            Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
            # Remove since it's passing on both builders.
            Bug(test) test/c.html [ Failure Pass ]
            # Keep since there's a failure on debug bot.
            Bug(test) [ Linux ] test/d.html [ Failure ]"""
        files = {test_expectation_path: test_expectations}
        host.filesystem = MockFileSystem(files)
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {
            'WebKit Linux Precise': {
                "test/a.html": ["PASS", "PASS", "PASS"],
                "test/b.html": ["PASS", "IMAGE", "PASS"],
                "test/c.html": ["PASS", "PASS", "PASS"],
                "test/d.html": ["PASS", "PASS", "PASS"],
            },
            'WebKit Linux Precise (dbg)': {
                "test/a.html": ["PASS", "PASS", "PASS"],
                "test/b.html": ["PASS", "PASS", "PASS"],
                "test/c.html": ["PASS", "PASS", "PASS"],
                "test/d.html": ["IMAGE", "PASS", "PASS"],
            },
        }

        main(host, expectation_factory, [])

        self.assertEqual(
            host.filesystem.files[test_expectation_path],
            ("""            # Keep since there's a failure on release bot.
            Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
            # Keep since there's a failure on debug bot.
            Bug(test) [ Linux ] test/d.html [ Failure ]"""))
Beispiel #6
0
    def test_harness_updates_file(self):
        """Tests that the call harness updates the TestExpectations file."""

        self._define_builders({
            "WebKit Linux Trusty": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Release']
            },
            "WebKit Linux Trusty (dbg)": {
                "port_name": "linux-trusty",
                "specifiers": ['Trusty', 'Debug']
            },
        })

        # Setup the mock host and port.
        host = MockHost()
        host.port_factory = FakePortFactory(
            host,
            all_build_types=('release', 'debug'),
            all_systems=(('trusty', 'x86_64'),))

        # Write out a fake TestExpectations file.
        test_expectation_path = (
            host.port_factory.get().path_to_generic_test_expectations_file())
        test_expectations = """
            # Remove since passing on both bots.
            Bug(test) [ Linux ] test/a.html [ Failure Pass ]
            # Keep since there's a failure on release bot.
            Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
            # Remove since it's passing on both builders.
            Bug(test) test/c.html [ Failure Pass ]
            # Keep since there's a failure on debug bot.
            Bug(test) [ Linux ] test/d.html [ Failure ]"""
        files = {
            test_expectation_path: test_expectations
        }
        host.filesystem = MockFileSystem(files)
        self._write_tests_into_filesystem(host.filesystem)

        # Write out the fake builder bot results.
        expectation_factory = FakeBotTestExpectationsFactory()
        expectation_factory._all_results_by_builder = {
            'WebKit Linux Trusty': {
                "test/a.html": ["PASS", "PASS", "PASS"],
                "test/b.html": ["PASS", "IMAGE", "PASS"],
                "test/c.html": ["PASS", "PASS", "PASS"],
                "test/d.html": ["PASS", "PASS", "PASS"],
            },
            'WebKit Linux Trusty (dbg)': {
                "test/a.html": ["PASS", "PASS", "PASS"],
                "test/b.html": ["PASS", "PASS", "PASS"],
                "test/c.html": ["PASS", "PASS", "PASS"],
                "test/d.html": ["IMAGE", "PASS", "PASS"],
            },
        }

        main(host, expectation_factory, [])

        self.assertEqual(host.filesystem.files[test_expectation_path], (
            """            # Keep since there's a failure on release bot.
            Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
            # Keep since there's a failure on debug bot.
            Bug(test) [ Linux ] test/d.html [ Failure ]"""))