Exemplo n.º 1
0
    def test_serial_parallel_stop_on_failure(self):
        """Run a sequence of serial and parallel tests and stop on failure.

        """

        sequencer = Sequencer("serial_parallel_stop_on_failure")

        # Run a bunch of tests.
        passed, failed, skipped = sequencer.run(
            NamedTest("a"),
            [NamedTest("b")],
            # Tests in a tuple are executed in parallel.
            (NamedTest("c", work_time=0.2), NamedTest("d"), NamedTest("e"),
             (NamedTest("f"), NamedTest("g")), NamedTest("h"),
             (NamedTest("i", work_time=0.1), NamedTest("j"), FailTest("a")),
             [FailTest("b"), [NotExecutedTest("a")]], FailTest("c")),
            [NotExecutedTest("b")],
            NamedTest("k"),
            NamedTest("l"),
            continue_on_failure=False)

        sequencer.report()

        self.assertTrue(passed >= 2)
        self.assertTrue(failed >= 1)
        self.assertTrue(skipped >= 1)
        self.assertEqual(passed + failed + skipped, 17)
        self.assertTrue(NamedTest.count >= 2)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertTrue(FailTest.count >= 1)
Exemplo n.º 2
0
    def test_testcase_filter(self):
        """Use the test execution filter to run a specific testcase in a
        sequence.

        """

        # A testcase in `testcase_skip_filter` will be skipped even if it is
        # in `testcase_filter`.
        sequencer = Sequencer("filter",
                              testcase_filter=["fail_1", "test_b"],
                              testcase_skip_filter=["fail_1"])

        sequencer.run(
            FailTest("1"),
            NamedTest("a"),
            [
                FailTest("2"),
                NamedTest("b")
            ]
        )

        sequencer.report()

        self.assertEqual(NamedTest.count, 1)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 0)
Exemplo n.º 3
0
    def test_skipped_and_failed_summary_messages(self):
        """Verify the skipped and failed messages in the summary.

        """

        sequencer = Sequencer(
            "execute_test_skipped_and_failed_summary_messages")

        result = sequencer.run(
            [SkipTest("1", 'My skip message.'),
             FailTest("1")])

        self.assert_result(result, Result(0, 1, 1))

        sequencer.report()

        json_report = sequencer.summary_json()
        self.assertEqual(json_report["testcases"][0]['name'], 'skip_1')
        self.assertEqual(json_report["testcases"][0]['result'], 'SKIPPED')
        self.assertEqual(json_report["testcases"][0]['message'],
                         'My skip message.')
        self.assertEqual(json_report["testcases"][1]['name'], 'fail_1')
        self.assertEqual(json_report["testcases"][1]['result'], 'FAILED')
        self.assertTrue(json_report["testcases"][1]['message'].endswith(
            '1 is not equal to 0'))
Exemplo n.º 4
0
    def test_passed_asserts(self):
        """Test the various asserts.

        """

        sequencer = Sequencer("passed_asserts")

        result = sequencer.run(AssertsEqualTest(1,
                                                1), AssertsNotEqualTest(2, 1),
                               AssertsTrueTest(True), AssertsTrueTest(1),
                               AssertsTrueTest([1]), AssertsFalseTest(False),
                               AssertsFalseTest(0), AssertsFalseTest([]),
                               AssertsInTest(1, [0, 1, 2]),
                               AssertsNotInTest(1, [0, 2]),
                               AssertsIsNoneTest(None),
                               AssertsIsNotNoneTest(0),
                               AssertsGreaterTest(2, 1),
                               AssertsGreaterEqualTest(2, 2),
                               AssertsGreaterEqualTest(2, 1),
                               AssertsLessTest(1,
                                               2), AssertsLessEqualTest(1, 2),
                               AssertsLessEqualTest(2, 2),
                               AssertsRaisesSubclassExceptionTest(),
                               AssertsRaisesExceptionTest(),
                               AssertsRaisesExceptionTupleTest(),
                               AssertsNoneTest(None))

        sequencer.report()

        self.assert_result(result, Result(22, 0, 0))
Exemplo n.º 5
0
    def test_serial_parallel(self):
        """Run a sequence of serial and parallel tests.

        """

        sequencer = Sequencer("serial_parallel")

        # Run a bunch of tests.
        passed, failed, skipped = sequencer.run(
            NamedTest("a"),
            [NamedTest("b")],
            # Tests in a tuple are executed in parallel.
            (NamedTest("c", work_time=0.2), NamedTest("d"), NamedTest("e"),
             (NamedTest("f"), NamedTest("g")), NamedTest("h"),
             (NamedTest("i", work_time=0.1), NamedTest("j"), FailTest("a")),
             [FailTest("b"), [NotExecutedTest("a")]], FailTest("c")),
            [NotExecutedTest("b")],
            NamedTest("k"),
            NamedTest("l"))

        sequencer.report()

        self.assertEqual(passed, 12)
        self.assertEqual(failed, 3)
        self.assertEqual(skipped, 2)
        self.assertEqual(NamedTest.count, 12)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 3)

        json_report = sequencer.summary_json()
        print(json.dumps(json_report, indent=4))
        self.assertEqual(json_report["name"], "serial_parallel")
        self.assertEqual(len(json_report["testcases"]), 17)
Exemplo n.º 6
0
    def test_serial_parallel(self):
        """Run a sequence of serial and parallel tests.

        """

        sequencer = Sequencer("serial_parallel")

        # Run test a and b.
        sequencer.run(
            NamedTest("a"),
            NamedTest("b")
        )
        
        # Run a bunch of tests.
        try:
            sequencer.run([
                # Tests in a tuple are executed in parallel.
                (
                    NamedTest("c", work_time=0.2),
                    [
                        NamedTest("d"),
                        NamedTest("e"),
                        (
                            NamedTest("f"),
                            NamedTest("g")
                        ),
                        NamedTest("h"),
                        (
                            NamedTest("i", work_time=0.1),
                            NamedTest("j"),
                            FailTest("a")
                        ),
                        NotExecutedTest("a")
                    ],
                    FailTest("b"),
                    FailTest("c")
                ),
                NotExecutedTest("b")
            ])
            
            # sequencer.run() should throw the
            # SequencerTestFailedError exception since the test
            # FailTest() fails
            raise
            
        except SequencerTestFailedError:
            sequencer.run(
                NamedTest("k"),
                NamedTest("l")
            )

        sequencer.report()

        self.assertEqual(NamedTest.count, 12)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 3)
Exemplo n.º 7
0
    def test_expected_failure(self):
        """Expected failute test.

        """

        sequencer = Sequencer("expected_failure")

        result = sequencer.run(XFailPassedTest(), XFailFailedTest(),
                               XFailSkippedTest())

        sequencer.report()

        self.assert_result(result, Result(0, 0, 1, 1, 1))
Exemplo n.º 8
0
    def test_setup_teardown(self):
        """Test the setup and teardown methods.

        """

        sequencer = Sequencer("setup_teardown")

        result = sequencer.run(FailSetupTest("1"), FailTearDownTest("1"))

        sequencer.report()

        self.assert_result(result, Result(0, 2, 0))
        self.assertEqual(FailSetupTest.count, 1)
        self.assertEqual(FailTearDownTest.count, 1)
Exemplo n.º 9
0
    def test_force_serial_execution(self):
        """Force serial test case execution.

        """

        sequencer = Sequencer("forced_serial_execution",
                              force_serial_execution=True)

        result = sequencer.run((FailTest("1"), NamedTest("a")))

        sequencer.report()

        self.assert_result(result, Result(1, 1, 0))
        self.assertEqual(NamedTest.count, 1)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 1)
Exemplo n.º 10
0
    def test_continue_on_failure(self):
        """Run all tests even if a test fails.

        """

        sequencer = Sequencer("continue_on_failure")

        result = sequencer.run(FailTest("1"), NamedTest("a"),
                               (FailTest("2"), NamedTest("b")))

        sequencer.report()

        self.assert_result(result, Result(2, 2, 0))
        self.assertEqual(NamedTest.count, 2)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 2)
Exemplo n.º 11
0
    def test_cleanup(self):
        """Test calling the run function twice, second time as cleanup. The
        run_failed state is reset for each call to run.

        """

        sequencer = Sequencer("cleanup")

        sequencer.run(FailTest("1"))
        passed, failed, skipped = sequencer.run(NamedTest("cleanup"))

        sequencer.report()

        self.assertEqual(passed, 1)
        self.assertEqual(failed, 1)
        self.assertEqual(skipped, 0)

        self.assertEqual(NamedTest.count, 1)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 1)
Exemplo n.º 12
0
    def test_testcase_description(self):
        """Test the testcase descriptions.

        """

        sequencer = Sequencer("testcase descriptions")

        result = sequencer.run(DescriptionNoneTest(), DescriptionEmptyTest(),
                               DescriptionBlankTest(),
                               DescriptionMultiLineTest())

        sequencer.report()

        self.assert_result(result, Result(4, 0, 0))

        json_report = sequencer.summary_json()
        self.assertEqual(json_report["testcases"][0]['description'], [])
        self.assertEqual(json_report["testcases"][1]['description'], [])
        self.assertEqual(json_report["testcases"][2]['description'], [])
        self.assertEqual(json_report["testcases"][3]['description'],
                         ['Line 1.', 'Line 2.', 'Line 3.'])
Exemplo n.º 13
0
    def test_force_serial_execution(self):
        """Force serial test case execution.

        """

        sequencer = Sequencer("forced_serial_execution",
                              force_serial_execution=True)

        try:
            sequencer.run((
                FailTest("1"),
                NamedTest("a")
            ))
        except SequencerTestFailedError:
            pass
        
        sequencer.report()

        self.assertEqual(NamedTest.count, 0)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 1)
Exemplo n.º 14
0
    def test_dot_digraph(self):
        """Create a dot digraph of given sequence.

        """

        sequencer = Sequencer("dot_digraph", dry_run=True)

        result = sequencer.run(
            NamedTest("a"), NamedTest("b"), (NamedTest("c", work_time=0.2), [
                NamedTest("d"),
                NamedTest("e"),
                (NamedTest("f", work_time=1.4), NamedTest("g")),
                NamedTest("h"),
                (NamedTest("i", work_time=0.1), NamedTest(
                    "j", work_time=3.2), FailTest("a")),
                NotExecutedTest("a")
            ], FailTest("b"), FailTest("c")), NotExecutedTest("b"),
            NamedTest("k"), NamedTest("l"))

        sequencer.report()

        self.assert_result(result, Result(17, 0, 0))
Exemplo n.º 15
0
    def test_execute_test_twice_two_run(self):
        """Execute the same test twice in two separate calls to run. This
        tests that a DOT graph can be created if multiple tests have
        the same name.

        """

        sequencer = Sequencer("execute_test_twice")

        result = sequencer.run([NamedTest("1"), NamedTest("1")])

        self.assert_result(result, Result(2, 0, 0))

        result = sequencer.run([NamedTest("1"), NamedTest("1")])

        self.assert_result(result, Result(4, 0, 0))

        sequencer.report()

        self.assertEqual(NamedTest.count, 4)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 0)
Exemplo n.º 16
0
    def test_testcase_filter(self):
        """Use the test execution filter to run a specific testcase in a
        sequence.

        """

        # A testcase in `testcase_skip_filter` will be skipped even if it is
        # in `testcase_filter`.
        sequencer = Sequencer("filter",
                              testcase_filter=["fail_1", "test_b", "test_d"],
                              testcase_skip_filter=["fail_1"])

        result = sequencer.run(FailTest("1"),
                               NamedTest("a"), (FailTest("2"), NamedTest("b")),
                               NamedTest("c"), [NamedTest("d")])

        sequencer.report()

        self.assert_result(result, Result(2, 0, 4))
        self.assertEqual(NamedTest.count, 2)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 0)
Exemplo n.º 17
0
    def test_continue_on_failure(self):
        """Run all tests even if a test fails.

        """

        sequencer = Sequencer("continue_on_failure",
                              stop_on_failure=False)

        sequencer.run(
            FailTest("1"),
            NamedTest("a"),
            [
                FailTest("2"),
                NamedTest("b")
            ]
        )

        sequencer.report()

        self.assertEqual(NamedTest.count, 2)
        self.assertEqual(NotExecutedTest.count, 0)
        self.assertEqual(FailTest.count, 2)
Exemplo n.º 18
0
    def test_dot_digraph(self):
        """Create a dot digraph of given sequence.

        """

        sequencer = Sequencer("dot_digraph",
                              dry_run=True,
                              stop_on_failure=False)

        sequencer.run(
            NamedTest("a"),
            NamedTest("b"),
            (
                NamedTest("c", work_time=0.2),
                [
                    NamedTest("d"),
                    NamedTest("e"),
                    (
                        NamedTest("f", work_time=1.4),
                        NamedTest("g")
                    ),
                    NamedTest("h"),
                    (
                        NamedTest("i", work_time=0.1),
                        NamedTest("j", work_time=3.2),
                        FailTest("a")
                    ),
                    NotExecutedTest("a")
                ],
                FailTest("b"),
                FailTest("c")
            ),
            NotExecutedTest("b"),
            NamedTest("k"),
            NamedTest("l")
        )

        sequencer.report()
Exemplo n.º 19
0
    def test_failed_asserts(self):
        """Test the various asserts.

        """

        sequencer = Sequencer("failed_asserts")

        result = sequencer.run(AssertsEqualTest(1,
                                                2), AssertsNotEqualTest(2, 2),
                               AssertsTrueTest(False), AssertsFalseTest(True),
                               AssertsInTest(1, [0, 2]),
                               AssertsNotInTest(1, [0, 1, 2]),
                               AssertsIsNoneTest(0),
                               AssertsIsNotNoneTest(None),
                               AssertsGreaterTest(2, 2),
                               AssertsGreaterEqualTest(1, 2),
                               AssertsLessTest(2,
                                               2), AssertsLessEqualTest(2, 1),
                               AssertsRaisesNoExceptionTest(),
                               AssertsRaisesNoExceptionTupleTest(),
                               AssertsRaisesWrongExceptionTest(),
                               AssertsRaisesWrongExceptionTupleTest(),
                               AssertsNoneTest(0))

        sequencer.report()

        self.assert_result(result, Result(0, 17, 0))

        # Failure messages.
        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsEqualTest(1, 2).run()

        self.assertTrue(str(cm.exception).endswith(': 1 is not equal to 2'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsNotEqualTest(2, 2).run()

        self.assertTrue(str(cm.exception).endswith(': 2 is equal to 2'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsTrueTest(False).run()

        self.assertTrue(str(cm.exception).endswith(': False is not true'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsFalseTest(True).run()

        self.assertTrue(str(cm.exception).endswith(': True is not false'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsInTest(1, [0, 2]).run()

        self.assertTrue(str(cm.exception).endswith(': 1 not found in [0, 2]'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsNotInTest(1, [0, 1, 2]).run()

        self.assertTrue(str(cm.exception).endswith(': 1 found in [0, 1, 2]'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsIsNoneTest(0).run()

        self.assertTrue(str(cm.exception).endswith(': 0 is not None'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsIsNotNoneTest(None).run()

        self.assertTrue(str(cm.exception).endswith(': None is None'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsGreaterTest(2, 2).run()

        self.assertTrue(
            str(cm.exception).endswith(': 2 is not greater than 2'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsGreaterEqualTest(1, 2).run()

        self.assertTrue(
            str(cm.exception).endswith(
                ': 1 is not greater than or equal to 2'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsLessTest(2, 2).run()

        self.assertTrue(str(cm.exception).endswith(': 2 is not less than 2'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsLessEqualTest(2, 1).run()

        self.assertTrue(
            str(cm.exception).endswith(': 2 is not less than or equal to 1'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsRaisesNoExceptionTest().run()

        self.assertTrue(str(cm.exception).endswith(': ValueError not raised'))

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsRaisesNoExceptionTupleTest().run()

        self.assertTrue(
            str(cm.exception).endswith(': ValueError or TypeError not raised'))

        with self.assertRaises(TypeError) as cm:
            AssertsRaisesWrongExceptionTest().run()

        self.assertEqual(str(cm.exception), 'This is not a value error.')

        with self.assertRaises(IndexError) as cm:
            AssertsRaisesWrongExceptionTupleTest().run()

        self.assertEqual(str(cm.exception),
                         'This is not a value error or type error.')

        with self.assertRaises(TestCaseFailedError) as cm:
            AssertsNoneTest(0).run()

        self.assertTrue(str(cm.exception).endswith(': 0 is not None'))