def test_exclude_tags(self):
runner = DiscoverRunner(tags=['fast'], exclude_tags=['core'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1)
runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 0)
runner = DiscoverRunner(exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 4)
def test_exclude_tags(self):
runner = DiscoverRunner(tags=['fast'], exclude_tags=['core'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 1)
runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 0)
runner = DiscoverRunner(exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 0)
def test_reverse(self):
"""
Reverse should reorder tests while maintaining the grouping specified
by ``DiscoverRunner.reorder_by``.
"""
runner = DiscoverRunner(reverse=True, verbosity=0)
suite = runner.build_suite(
test_labels=('test_runner_apps.sample', 'test_runner_apps.simple'))
self.assertIn('test_runner_apps.simple', next(iter(suite)).id(),
msg="Test labels should be reversed.")
suite = runner.build_suite(test_labels=('test_runner_apps.simple',))
suite = tuple(suite)
self.assertIn('DjangoCase', suite[0].id(),
msg="Test groups should not be reversed.")
self.assertIn('SimpleCase', suite[4].id(),
msg="Test groups order should be preserved.")
self.assertIn('DjangoCase2', suite[0].id(),
msg="Django test cases should be reversed.")
self.assertIn('SimpleCase2', suite[4].id(),
msg="Simple test cases should be reversed.")
self.assertIn('UnittestCase2', suite[8].id(),
msg="Unittest test cases should be reversed.")
self.assertIn('test_2', suite[0].id(),
msg="Methods of Django cases should be reversed.")
self.assertIn('test_2', suite[4].id(),
msg="Methods of simple cases should be reversed.")
self.assertIn('test_2', suite[9].id(),
msg="Methods of unittest cases should be reversed.")
def test_reverse(self):
"""
Reverse should reorder tests while maintaining the grouping specified
by ``DiscoverRunner.reorder_by``.
"""
runner = DiscoverRunner(reverse=True)
suite = runner.build_suite(
test_labels=('test_runner_apps.sample', 'test_runner_apps.simple'))
self.assertIn('test_runner_apps.simple', next(iter(suite)).id(),
msg="Test labels should be reversed.")
suite = runner.build_suite(test_labels=('test_runner_apps.simple',))
suite = tuple(suite)
self.assertIn('DjangoCase', suite[0].id(),
msg="Test groups should not be reversed.")
self.assertIn('SimpleCase', suite[4].id(),
msg="Test groups order should be preserved.")
self.assertIn('DjangoCase2', suite[0].id(),
msg="Django test cases should be reversed.")
self.assertIn('SimpleCase2', suite[4].id(),
msg="Simple test cases should be reversed.")
self.assertIn('UnittestCase2', suite[8].id(),
msg="Unittest test cases should be reversed.")
self.assertIn('test_2', suite[0].id(),
msg="Methods of Django cases should be reversed.")
self.assertIn('test_2', suite[4].id(),
msg="Methods of simple cases should be reversed.")
self.assertIn('test_2', suite[8].id(),
msg="Methods of unittest cases should be reversed.")
def test_exclude_tags(self):
runner = DiscoverRunner(tags=['fast'], exclude_tags=['core'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1)
runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 0)
runner = DiscoverRunner(exclude_tags=['slow'])
self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 4)
def test_tags(self):
runner = DiscoverRunner(tags=['core'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 1)
runner = DiscoverRunner(tags=['fast'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 2)
runner = DiscoverRunner(tags=['slow'])
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 2)
def test_exclude_tags(self):
runner = DiscoverRunner(tags=['fast'], exclude_tags=['core'], verbosity=0)
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 1)
runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow'], verbosity=0)
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 0)
runner = DiscoverRunner(exclude_tags=['slow'], verbosity=0)
self.assertEqual(runner.build_suite(['test_runner_apps.tagged.tests']).countTestCases(), 0)
def test_loader_patterns_not_mutated_when_test_label_is_file_path(self):
runner = DiscoverRunner(test_name_patterns=["test_sample"],
verbosity=0)
with change_cwd("."), change_loader_patterns(["UnittestCase1"]):
with self.assertRaises(RuntimeError):
runner.build_suite(["test_discover_runner.py"])
self.assertEqual(runner.test_loader.testNamePatterns,
["UnittestCase1"])
def test_number_of_tests_found_displayed(self):
runner = DiscoverRunner()
with captured_stdout() as stdout:
runner.build_suite([
'test_runner_apps.sample.tests_sample.TestDjangoTestCase',
'test_runner_apps.simple',
])
self.assertIn('Found 14 test(s).\n', stdout.getvalue())
def test_duplicates_ignored(self):
"""
Tests shouldn't be discovered twice when discovering on overlapping paths.
"""
base_app = "forms_tests"
sub_app = "forms_tests.field_tests"
runner = DiscoverRunner(verbosity=0)
with self.modify_settings(INSTALLED_APPS={"append": sub_app}):
single = runner.build_suite([base_app]).countTestCases()
dups = runner.build_suite([base_app, sub_app]).countTestCases()
self.assertEqual(single, dups)
def test_exclude_tags(self):
runner = DiscoverRunner(tags=["fast"], exclude_tags=["core"], verbosity=0)
self.assertEqual(
runner.build_suite(["test_runner_apps.tagged.tests"]).countTestCases(), 1
)
runner = DiscoverRunner(tags=["fast"], exclude_tags=["slow"], verbosity=0)
self.assertEqual(
runner.build_suite(["test_runner_apps.tagged.tests"]).countTestCases(), 0
)
runner = DiscoverRunner(exclude_tags=["slow"], verbosity=0)
self.assertEqual(
runner.build_suite(["test_runner_apps.tagged.tests"]).countTestCases(), 0
)
def test_build_suite_shuffling(self):
# These will result in unittest.loader._FailedTest instances rather
# than TestCase objects, but they are sufficient for testing.
labels = ["label1", "label2", "label3", "label4"]
cases = [
({}, ["label1", "label2", "label3", "label4"]),
({
"reverse": True
}, ["label4", "label3", "label2", "label1"]),
({
"shuffle": 8
}, ["label4", "label1", "label3", "label2"]),
({
"shuffle": 8,
"reverse": True
}, ["label2", "label3", "label1", "label4"]),
]
for kwargs, expected in cases:
with self.subTest(kwargs=kwargs):
# Prevent writing the seed to stdout.
runner = DiscoverRunner(**kwargs, verbosity=0)
tests = runner.build_suite(test_labels=labels)
# The ids have the form "unittest.loader._FailedTest.label1".
names = [test.id().split(".")[-1] for test in tests]
self.assertEqual(names, expected)
class DjangoAutoTestSuite(unittest.TestSuite):
"""
This test suite configures django settings (which should be in test_settings.py), and starts a test runner.
It allows us to run the django tests with setup.py test.
"""
def __init__(self, *args, **kwargs):
self._configure()
self.test_runner = DiscoverRunner()
tests = self.test_runner.build_suite()
super(DjangoAutoTestSuite, self).__init__(tests=tests, *args, **kwargs)
self.test_runner.setup_test_environment()
self.test_dbs = self.test_runner.setup_databases()
def _configure(self):
test_settings = importlib.import_module("test_settings")
setting_attrs = {
attr: getattr(test_settings, attr)
for attr in dir(test_settings)
if "__" not in attr
}
if not django.conf.settings.configured:
django.conf.settings.configure(**setting_attrs)
django.setup()
def run(self, result_obj, *args, **kwargs):
result = super(DjangoAutoTestSuite, self).run(result_obj, *args, **kwargs)
self.test_runner.teardown_databases(self.test_dbs)
self.test_runner.teardown_test_environment()
return result
def test_transaction_test_case_before_simple_test_case(self):
runner = DiscoverRunner()
suite = runner.build_suite(
['test_discovery_sample3.tests_transaction_test_case_ordering'])
suite = tuple(suite)
# TransactionTestCase is second after TestCase.
self.assertIn('TestTransactionTestCase', suite[1].id())
def test_number_of_databases_parallel_test_suite(self):
"""
Number of databases doesn't exceed the number of TestCases with
parallel tests.
"""
runner = DiscoverRunner(parallel=8, verbosity=0)
suite = runner.build_suite(["test_runner_apps.tagged"])
self.assertEqual(suite.processes, len(suite.subsuites))
self.assertEqual(runner.parallel, suite.processes)
def test_buffer_mode_test_fail(self):
runner = DiscoverRunner(buffer=True, verbose=0)
with captured_stdout() as stdout, captured_stderr() as stderr:
suite = runner.build_suite([
'test_runner_apps.buffer.tests_buffer.WriteToStdoutStderrTestCase.test_fail',
])
runner.run_suite(suite)
self.assertIn('Write to stderr.', stderr.getvalue())
self.assertIn('Write to stdout.', stdout.getvalue())
def test_number_of_databases_no_parallel_test_suite(self):
"""
Number of databases doesn't exceed the number of TestCases with
non-parallel tests.
"""
runner = DiscoverRunner(parallel=8, verbosity=0)
suite = runner.build_suite(["test_runner_apps.simple.tests.DjangoCase1"])
self.assertEqual(runner.parallel, 1)
self.assertIsInstance(suite, TestSuite)
def test_buffer_mode_test_pass(self):
runner = DiscoverRunner(buffer=True, verbosity=0)
with captured_stdout() as stdout, captured_stderr() as stderr:
suite = runner.build_suite([
"test_runner_apps.buffer.tests_buffer.WriteToStdoutStderrTestCase."
"test_pass",
])
runner.run_suite(suite)
self.assertNotIn("Write to stderr.", stderr.getvalue())
self.assertNotIn("Write to stdout.", stdout.getvalue())
def test_tag_fail_to_load(self):
with self.assertRaises(SyntaxError):
import_module('test_runner_apps.tagged.tests_syntax_error')
runner = DiscoverRunner(tags=['syntax_error'], verbosity=0)
# A label that doesn't exist or cannot be loaded due to syntax errors
# is always considered matching.
suite = runner.build_suite(['doesnotexist', 'test_runner_apps.tagged'])
self.assertEqual([test.id() for test in suite], [
'unittest.loader._FailedTest.doesnotexist',
'unittest.loader._FailedTest.test_runner_apps.tagged.tests_syntax_error',
])
def test_loader_patterns_not_mutated(self):
runner = DiscoverRunner(test_name_patterns=['test_sample'], verbosity=0)
tests = [
('test_runner_apps.sample.tests', 1),
('test_runner_apps.sample.tests.Test.test_sample', 1),
('test_runner_apps.sample.empty', 0),
('test_runner_apps.sample.tests_sample.EmptyTestCase', 0),
]
for test_labels, tests_count in tests:
with self.subTest(test_labels=test_labels):
with change_loader_patterns(['UnittestCase1']):
count = runner.build_suite([test_labels]).countTestCases()
self.assertEqual(count, tests_count)
self.assertEqual(runner.test_loader.testNamePatterns, ['UnittestCase1'])
def test_suite_result_with_failure(self):
cases = [
(1, 'FailureTestCase'),
(1, 'ErrorTestCase'),
(0, 'ExpectedFailureTestCase'),
(1, 'UnexpectedSuccessTestCase'),
]
runner = DiscoverRunner(verbosity=0)
for expected_failures, testcase in cases:
with self.subTest(testcase=testcase):
suite = runner.build_suite([
f'test_runner_apps.failures.tests_failures.{testcase}',
])
with captured_stderr():
result = runner.run_suite(suite)
failures = runner.suite_result(suite, result)
self.assertEqual(failures, expected_failures)
def test_transaction_test_case_next_serialized_rollback_option(self):
runner = DiscoverRunner()
suite = runner.build_suite(
['test_discovery_sample3.tests_transaction_test_case_mixed'])
django_test_case, first_transaction_test_case, middle_transaction_test_case, \
last_transaction_test_case, vanilla_test_case = suite
# TransactionTestCase1._next_serialized_rollback is
# TransactionTestCase2.serialize_rollback.
self.assertEqual(first_transaction_test_case._next_serialized_rollback,
middle_transaction_test_case.serialized_rollback)
# TransactionTestCase2._next_serialized_rollback is
# TransactionTestCase3.serialize_rollback.
self.assertEqual(
middle_transaction_test_case._next_serialized_rollback,
last_transaction_test_case.serialized_rollback)
# The last TransactionTestCase of the suite has
# _next_serialized_rollback to = True.
self.assertIs(last_transaction_test_case._next_serialized_rollback,
True)
def test_transaction_test_case_next_serialized_rollback_option(self):
runner = DiscoverRunner()
suite = runner.build_suite(['test_discovery_sample3.tests_transaction_test_case_mixed'])
django_test_case, first_transaction_test_case, middle_transaction_test_case, \
last_transaction_test_case, vanilla_test_case = suite
# TransactionTestCase1._next_serialized_rollback is
# TransactionTestCase2.serialize_rollback.
self.assertEqual(
first_transaction_test_case._next_serialized_rollback,
middle_transaction_test_case.serialized_rollback
)
# TransactionTestCase2._next_serialized_rollback is
# TransactionTestCase3.serialize_rollback.
self.assertEqual(
middle_transaction_test_case._next_serialized_rollback,
last_transaction_test_case.serialized_rollback
)
# The last TransactionTestCase of the suite has
# _next_serialized_rollback to = True.
self.assertIs(last_transaction_test_case._next_serialized_rollback, True)
def test_transaction_test_case_next_serialized_rollback_option(self):
runner = DiscoverRunner()
suite = runner.build_suite(
['test_discovery_sample3.tests_transaction_test_case_mixed'])
django_test_case, first_transaction_test_case, second_transaction_test_case, \
third_transaction_test_case, fourth_transaction_test_case, vanilla_test_case = suite
# TransactionTestCase1._next_serialized_rollback is
# TransactionTestCase2.serialize_rollback.
self.assertEqual(first_transaction_test_case._next_serialized_rollback,
second_transaction_test_case.serialized_rollback)
# TransactionTestCase2._next_serialized_rollback is
# TransactionTestCase4.serialize_rollback because TransactionTestCase3
# is skipped.
self.assertEqual(
second_transaction_test_case._next_serialized_rollback,
fourth_transaction_test_case.serialized_rollback)
# The last TransactionTestCase of the suite has
# _next_serialized_rollback = True.
self.assertIs(fourth_transaction_test_case._next_serialized_rollback,
True)
def test_excluded_tags_displayed(self):
runner = DiscoverRunner(exclude_tags=['foo', 'bar'], verbosity=3)
with captured_stdout() as stdout:
runner.build_suite(['test_runner_apps.tagged.tests'])
self.assertIn('Excluding test tag(s): bar, foo.\n',
stdout.getvalue())
def build_suite(self, test_labels=None, extra_tests=None, **kwargs):
os.environ["DUMMY"] = "VALUE"
suite = DiscoverRunner.build_suite(self, test_labels, extra_tests,
**kwargs)
return filter_tests_by_explicit_tags(suite, self.tags)
def test_transaction_test_case_before_simple_test_case(self):
runner = DiscoverRunner()
suite = runner.build_suite(['test_discovery_sample3.tests_transaction_test_case_ordering'])
suite = tuple(suite)
# TransactionTestCase is second after TestCase.
self.assertIn('TestTransactionTestCase', suite[1].id())
def test_excluded_tags_displayed(self):
runner = DiscoverRunner(exclude_tags=['foo', 'bar'], verbosity=3)
with captured_stdout() as stdout:
runner.build_suite(['test_runner_apps.tagged.tests'])
self.assertIn('Excluding test tag(s): bar, foo.\n', stdout.getvalue())
def test_excluded_tags_displayed(self):
runner = DiscoverRunner(exclude_tags=["foo", "bar"], verbosity=3)
with captured_stdout() as stdout:
runner.build_suite(["test_runner_apps.tagged.tests"])
self.assertIn("Excluding test tag(s): bar, foo.\n", stdout.getvalue())
def test_number_of_parallel_workers(self):
"""Number of processes doesn't exceed the number of TestCases."""
runner = DiscoverRunner(parallel=5, verbosity=0)
suite = runner.build_suite(["test_runner_apps.tagged"])
self.assertEqual(suite.processes, len(suite.subsuites))
