class TestCase(six.with_metaclass(MetaTestCase, object)):
"""The TestCase class defines test methods and fixture methods; it is the meat and potatoes of testing.
QuickStart:
define a test method, instantiate an instance and call test_case.run()
Extended information:
TestCases can contain any number of test methods, as well as class-level
setup/teardown methods and setup/teardowns to be wrapped around each test
method. These are defined by decorators.
The phases of execution are thus:
class_setup
setup
test_method_1
teardown
setup
test_method_2
teardown
class_teardown
The results of test methods are stored in TestResult objects.
Additional behavior beyond running tests, such as logging results, is achieved
by registered callbacks. For more information see the docstrings for:
register_on_complete_test_method_callback
register_on_run_test_method_callback
"""
__test__ = False
STAGE_UNSTARTED = 0
STAGE_CLASS_SETUP = 1
STAGE_SETUP = 2
STAGE_TEST_METHOD = 3
STAGE_TEARDOWN = 4
STAGE_CLASS_TEARDOWN = 5
EVENT_ON_RUN_TEST_METHOD = 1
EVENT_ON_COMPLETE_TEST_METHOD = 2
EVENT_ON_RUN_CLASS_SETUP_METHOD = 3
EVENT_ON_COMPLETE_CLASS_SETUP_METHOD = 4
EVENT_ON_RUN_CLASS_TEARDOWN_METHOD = 5
EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD = 6
EVENT_ON_RUN_TEST_CASE = 7
EVENT_ON_COMPLETE_TEST_CASE = 8
log = class_logger.ClassLogger()
# For now, we still support the use of unittest-style assertions defined on
# the TestCase instance
for _name in dir(deprecated_assertions):
if _name.startswith(('assert', 'fail')):
locals()[_name] = classmethod(
getattr(deprecated_assertions, _name))
del _name
def __init__(self, *args, **kwargs):
super(TestCase, self).__init__()
self.__test_fixtures = TestFixtures.discover_from(self)
self.__suites_exclude = kwargs.get('suites_exclude', set())
self.__suites_require = kwargs.get('suites_require', set())
self.__name_overrides = kwargs.get('name_overrides', None)
TestResult.debug = kwargs.get('debugger') # sorry :(
# callbacks for various stages of execution, used for stuff like logging
self.__callbacks = defaultdict(list)
self.__all_test_results = []
self._stage = self.STAGE_UNSTARTED
self.failure_limit = kwargs.pop('failure_limit', None)
self.failure_count = 0
@property
def test_result(self):
return self.__all_test_results[-1] if self.__all_test_results else None
def _generate_test_method(self, method_name, function):
"""Allow tests to define new test methods in their __init__'s and have appropriate suites applied."""
suite(*getattr(self, '_suites', set()))(function)
setattr(
self,
method_name,
# http://stackoverflow.com/q/4364565
function.__get__(self, type(self)),
)
def runnable_test_methods(self):
"""Generator method to yield runnable test methods.
This will pick out the test methods from this TestCase, and then exclude any in
any of our exclude_suites. If there are any require_suites, it will then further
limit itself to test methods in those suites.
"""
for member_name in dir(self):
if not member_name.startswith("test"):
continue
member = getattr(self, member_name)
if not inspect.ismethod(member):
continue
member_suites = self.suites(member)
# if there are any exclude suites, exclude methods under them
if self.__suites_exclude and self.__suites_exclude & member_suites:
continue
# if there are any require suites, only run methods in *all* of those suites
if self.__suites_require and not ((self.__suites_require & member_suites) == self.__suites_require):
continue
# if there are any name overrides, only run the named methods
if self.__name_overrides is None or member.__name__ in self.__name_overrides:
yield member
def run(self):
"""Delegator method encapsulating the flow for executing a TestCase instance.
"""
# The TestResult constructor wants an actual method, which it inspects
# to determine the method name (and class name, so it must be a method
# and not a function!). self.run is as good a method as any.
test_case_result = TestResult(self.run)
test_case_result.start()
self.fire_event(self.EVENT_ON_RUN_TEST_CASE, test_case_result)
self._stage = self.STAGE_CLASS_SETUP
with self.__test_fixtures.class_context(
setup_callbacks=[
functools.partial(self.fire_event, self.EVENT_ON_RUN_CLASS_SETUP_METHOD),
functools.partial(self.fire_event, self.EVENT_ON_COMPLETE_CLASS_SETUP_METHOD),
],
teardown_callbacks=[
functools.partial(self.fire_event, self.EVENT_ON_RUN_CLASS_TEARDOWN_METHOD),
functools.partial(self.fire_event, self.EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD),
],
) as class_fixture_failures:
# if we have class fixture failures, we're not going to bother
# running tests, but we need to generate bogus results for them all
# and mark them as failed.
self.__run_test_methods(class_fixture_failures)
self._stage = self.STAGE_CLASS_TEARDOWN
# class fixture failures count towards our total
self.failure_count += len(class_fixture_failures)
# Once a test case completes we should trigger
# EVENT_ON_COMPLETE_TEST_CASE event so that we can log/report test case
# results.
if not test_case_result.complete:
test_case_result.end_in_success()
self.fire_event(self.EVENT_ON_COMPLETE_TEST_CASE, test_case_result)
@classmethod
def in_suite(cls, method, suite_name):
"""Return a bool denoting whether the given method is in the given suite."""
return suite_name in getattr(method, '_suites', set())
def suites(self, method=None):
"""Returns the suites associated with this test case and, optionally, the given method."""
suites = set(getattr(self, '_suites', []))
if method is not None:
suites |= getattr(method, '_suites', set())
return suites
def results(self):
"""Available after calling `self.run()`."""
if self._stage != self.STAGE_CLASS_TEARDOWN:
raise RuntimeError('results() called before tests have executed')
return list(self.__all_test_results)
def method_excluded(self, method):
"""Given this TestCase's included/excluded suites, is this test method excluded?
Returns a set of the excluded suites that the argument method is in, or an empty
suite if none.
"""
method_suites = set(getattr(method, '_suites', set()))
return (self.__suites_exclude & method_suites)
def __run_test_methods(self, class_fixture_failures):
"""Run this class's setup fixtures / test methods / teardown fixtures.
These are run in the obvious order - setup and teardown go before and after,
respectively, every test method. If there was a failure in the class_setup
phase, no method-level fixtures or test methods will be run, and we'll eventually
skip all the way to the class_teardown phase. If a given test method is marked
as disabled, neither it nor its fixtures will be run. If there is an exception
during the setup phase, the test method will not be run and execution
will continue with the teardown phase.
"""
for test_method in self.runnable_test_methods():
result = TestResult(test_method)
# Sometimes, test cases want to take further action based on
# results, e.g. further clean-up or reporting if a test method
# fails. (Yelp's Selenium test cases do this.) If you need to
# programatically inspect test results, you should use
# self.results().
# NOTE: THIS IS INCORRECT -- im_self is shared among all test
# methods on the TestCase instance. This is preserved for backwards
# compatibility and should be removed eventually.
try:
# run "on-run" callbacks. e.g. print out the test method name
self.fire_event(self.EVENT_ON_RUN_TEST_METHOD, result)
result.start()
self.__all_test_results.append(result)
# if class setup failed, this test has already failed.
self._stage = self.STAGE_CLASS_SETUP
for exc_info in class_fixture_failures:
result.end_in_failure(exc_info)
if result.complete:
continue
# first, run setup fixtures
self._stage = self.STAGE_SETUP
with self.__test_fixtures.instance_context() as fixture_failures:
# we haven't had any problems in class/instance setup, onward!
if not fixture_failures:
self._stage = self.STAGE_TEST_METHOD
result.record(test_method)
self._stage = self.STAGE_TEARDOWN
# maybe something broke during teardown -- record it
for exc_info in fixture_failures:
result.end_in_failure(exc_info)
if result.interrupted:
raise Interruption
# if nothing's gone wrong, it's not about to start
if not result.complete:
result.end_in_success()
finally:
self.fire_event(self.EVENT_ON_COMPLETE_TEST_METHOD, result)
if not result.success:
self.failure_count += 1
if self.failure_limit and self.failure_count >= self.failure_limit:
break
def addfinalizer(self, teardown_func):
if self._stage in (self.STAGE_SETUP, self.STAGE_TEST_METHOD, self.STAGE_TEARDOWN):
self.__extra_test_teardowns.append(teardown_func)
elif self._stage in (self.STAGE_CLASS_SETUP, self.STAGE_CLASS_TEARDOWN):
self.__extra_class_teardowns.append(teardown_func)
else:
raise RuntimeError('Tried to add a teardown while the test was not being executed.')
@test_fixtures.class_setup_teardown
def __setup_extra_class_teardowns(self):
self.__extra_class_teardowns = []
yield
for teardown in reversed(self.__extra_class_teardowns):
teardown()
@test_fixtures.setup_teardown
def __setup_extra_test_teardowns(self):
self.__extra_test_teardowns = []
yield
for teardown in reversed(self.__extra_test_teardowns):
teardown()
def register_callback(self, event, callback):
"""Register a callback for an internal event, usually used for logging.
The argument to the callback will be the test method object itself.
Fixture objects can be distinguished by the running them through
inspection.is_fixture_method().
"""
self.__callbacks[event].append(callback)
def fire_event(self, event, result):
for callback in self.__callbacks[event]:
callback(result.to_dict())
def classSetUp(self):
pass
def setUp(self):
pass
def tearDown(self):
pass
def classTearDown(self):
pass
def runTest(self):
pass
class TestCase(object):
"""The TestCase class defines test methods and fixture methods; it is the meat and potatoes of testing.
QuickStart:
define a test method, instantiate an instance and call test_case.run()
Extended information:
TestCases can contain any number of test methods, as well as class-level
setup/teardown methods and setup/teardowns to be wrapped around each test
method. These are defined by decorators.
The phases of execution are thus:
class_setup
setup
test_method_1
teardown
setup
test_method_2
teardown
class_teardown
The results of test methods are stored in TestResult objects.
Additional behavior beyond running tests, such as logging results, is achieved
by registered callbacks. For more information see the docstrings for:
register_on_complete_test_method_callback
register_on_run_test_method_callback
"""
__metaclass__ = MetaTestCase
__test__ = False
STAGE_CLASS_SETUP = 1
STAGE_SETUP = 2
STAGE_TEST_METHOD = 3
STAGE_TEARDOWN = 4
STAGE_CLASS_TEARDOWN = 5
EVENT_ON_RUN_TEST_METHOD = 1
EVENT_ON_COMPLETE_TEST_METHOD = 2
EVENT_ON_RUN_CLASS_SETUP_METHOD = 3
EVENT_ON_COMPLETE_CLASS_SETUP_METHOD = 4
EVENT_ON_RUN_CLASS_TEARDOWN_METHOD = 5
EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD = 6
EVENT_ON_RUN_FIXTURE_METHOD = 7
EVENT_ON_COMPLETE_FIXTURE_METHOD = 8
log = class_logger.ClassLogger()
def __init__(self, *args, **kwargs):
super(TestCase, self).__init__()
self._method_level = False
# ascend the class hierarchy and discover fixture methods
self.__init_fixture_methods()
self.__suites_include = kwargs.get('suites_include', set())
self.__suites_exclude = kwargs.get('suites_exclude', set())
self.__suites_require = kwargs.get('suites_require', set())
self.__name_overrides = kwargs.get('name_overrides', None)
self.__debugger = kwargs.get('debugger')
# callbacks for various stages of execution, used for stuff like logging
self.__callbacks = defaultdict(list)
# one of these will later be populated with exception info if there's an
# exception in the class_setup/class_teardown stage
self.__class_level_failure = None
self.__class_level_error = None
# for now, we still support the use of unittest-style assertions defined on the TestCase instance
for name in dir(deprecated_assertions):
if name.startswith(('assert', 'fail')):
setattr(
self, name,
instancemethod(getattr(deprecated_assertions, name), self,
self.__class__))
self.failure_limit = kwargs.pop('failure_limit', None)
self.failure_count = 0
def __init_fixture_methods(self):
"""Initialize and populate the lists of fixture methods for this TestCase.
Fixture methods are identified by the fixture_decorator_factory when the
methods are created. This means in order to figure out all the fixtures
this particular TestCase will need, we have to test all of its attributes
for 'fixture-ness'.
See __fixture_decorator_factory for more info.
"""
# init our self.(class_setup|setup|teardown|class_teardown)_fixtures lists
for fixture_type in FIXTURE_TYPES:
setattr(self, "%s_fixtures" % fixture_type, [])
# the list of classes in our heirarchy, starting with the highest class
# (object), and ending with our class
reverse_mro_list = [x for x in reversed(type(self).mro())]
# discover which fixures are on this class, including mixed-in ones
self._fixture_methods = defaultdict(list)
# we want to know everything on this class (including stuff inherited
# from bases), but we don't want to trigger any lazily loaded
# attributes, so dir() isn't an option; this traverses __bases__/__dict__
# correctly for us.
for classified_attr in inspect.classify_class_attrs(type(self)):
# have to index here for Python 2.5 compatibility
attr_name = classified_attr[0]
unbound_method = classified_attr[3]
defining_class = classified_attr[2]
# skip everything that's not a function/method
if not inspect.isroutine(unbound_method):
continue
# if this is an old setUp/tearDown/etc, tag it as a fixture
if attr_name in DEPRECATED_FIXTURE_TYPE_MAP:
fixture_type = DEPRECATED_FIXTURE_TYPE_MAP[attr_name]
fixture_decorator = globals()[fixture_type]
unbound_method = fixture_decorator(unbound_method)
# collect all of our fixtures in appropriate buckets
if inspection.is_fixture_method(unbound_method):
# where in our MRO this fixture was defined
defining_class_depth = reverse_mro_list.index(defining_class)
inspection.callable_setattr(
unbound_method,
'_defining_class_depth',
defining_class_depth,
)
# we grabbed this from the class and need to bind it to us
instance_method = instancemethod(unbound_method, self,
self.__class__)
self._fixture_methods[instance_method._fixture_type].append(
instance_method)
# arrange our fixture buckets appropriately
for fixture_type, fixture_methods in self._fixture_methods.iteritems():
# sort our fixtures in order of oldest (smaller id) to newest, but
# also grouped by class to correctly place deprecated fixtures
fixture_methods.sort(
key=lambda x: (x._defining_class_depth, x._fixture_id))
# for setup methods, we want methods defined further back in the
# class hierarchy to execute first. for teardown methods though,
# we want the opposite while still maintaining the class-level
# definition order, so we reverse only on class depth.
if fixture_type in REVERSED_FIXTURE_TYPES:
fixture_methods.sort(key=lambda x: x._defining_class_depth,
reverse=True)
fixture_list_name = "%s_fixtures" % fixture_type
setattr(self, fixture_list_name, fixture_methods)
def _generate_test_method(self, method_name, function):
"""Allow tests to define new test methods in their __init__'s and have appropriate suites applied."""
suite(*getattr(self, '_suites', set()))(function)
setattr(self, method_name,
instancemethod(function, self, self.__class__))
def runnable_test_methods(self):
"""Generator method to yield runnable test methods.
This will pick out the test methods from this TestCase, and then exclude any in
any of our exclude_suites. If there are any include_suites, it will then further
limit itself to test methods in those suites.
"""
for member_name in dir(self):
if not member_name.startswith("test"):
continue
member = getattr(self, member_name)
if not inspect.ismethod(member):
continue
member_suites = self.suites(member)
# if there are any exclude suites, exclude methods under them
if self.__suites_exclude and self.__suites_exclude & member_suites:
continue
# if there are any include suites, only run methods in them
if self.__suites_include and not (self.__suites_include
& member_suites):
continue
# if there are any require suites, only run methods in *all* of those suites
if self.__suites_require and not (
(self.__suites_require & member_suites)
== self.__suites_require):
continue
# if there are any name overrides, only run the named methods
if self.__name_overrides is None or member.__name__ in self.__name_overrides:
yield member
def run(self):
"""Delegator method encapsulating the flow for executing a TestCase instance"""
self.__run_class_setup_fixtures()
self.__enter_class_context_managers(self.class_setup_teardown_fixtures,
self.__run_test_methods)
self.__run_class_teardown_fixtures()
def __run_class_setup_fixtures(self):
"""Running the class's class_setup method chain."""
self.__run_class_fixtures(
self.STAGE_CLASS_SETUP,
self.class_setup_fixtures,
self.EVENT_ON_RUN_CLASS_SETUP_METHOD,
self.EVENT_ON_COMPLETE_CLASS_SETUP_METHOD,
)
def __run_class_teardown_fixtures(self):
"""End the process of running tests. Run the class's class_teardown methods"""
self.__run_class_fixtures(
self.STAGE_CLASS_TEARDOWN,
self.class_teardown_fixtures,
self.EVENT_ON_RUN_CLASS_TEARDOWN_METHOD,
self.EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD,
)
def __run_class_fixtures(self, stage, fixtures, callback_on_run_event,
callback_on_complete_event):
"""Set the current _stage, run a set of fixtures, calling callbacks before and after each."""
self._stage = stage
for fixture_method in fixtures:
result = TestResult(fixture_method)
try:
self.fire_event(callback_on_run_event, result)
result.start()
if self.__execute_block_recording_exceptions(
fixture_method, result, is_class_level=True):
result.end_in_success()
else:
if self.__class_level_failure:
result.end_in_failure(self.__class_level_failure)
### Bump failure count?
### Something about failure_limit?
elif self.__class_level_error:
result.end_in_error(self.__class_level_error)
### Bump failure count?
### Something about failure_limit?
else:
raise Exception(
"Couldn't find a class-level failure or error even"
" though we failed while executing a class-level fixture."
" This should not be possible. Aborting.")
except (KeyboardInterrupt, SystemExit):
result.end_in_interruption(sys.exc_info())
raise
finally:
self.fire_event(callback_on_complete_event, result)
@classmethod
def in_suite(cls, method, suite_name):
"""Return a bool denoting whether the given method is in the given suite."""
return suite_name in getattr(method, '_suites', set())
def suites(self, method=None):
"""Returns the suites associated with this test case and, optionally, the given method."""
suites = set(getattr(self, '_suites', []))
if method is not None:
suites |= getattr(method, '_suites', set())
return suites
def method_excluded(self, method):
"""Given this TestCase's included/excluded suites, is this test method excluded?
Returns a set of the excluded suites that the argument method is in, or an empty
suite if none.
"""
method_suites = set(getattr(method, '_suites', set()))
return (self.__suites_exclude & method_suites)
def __enter_class_context_managers(self, fixture_methods, callback):
"""Transform each fixture_method into a context manager with contextlib.contextmanager, enter them recursively, and call callback"""
if fixture_methods:
fixture_method = fixture_methods[0]
ctm = contextmanager(fixture_method)()
enter_result = TestResult(fixture_method)
enter_result.start()
self.fire_event(self.EVENT_ON_RUN_CLASS_SETUP_METHOD, enter_result)
if self.__execute_block_recording_exceptions(
ctm.__enter__, enter_result):
enter_result.end_in_success()
self.fire_event(self.EVENT_ON_COMPLETE_CLASS_SETUP_METHOD,
enter_result)
self.__enter_context_managers(fixture_methods[1:], callback)
exit_result = TestResult(fixture_method)
exit_result.start()
self.fire_event(self.EVENT_ON_RUN_CLASS_TEARDOWN_METHOD,
exit_result)
if self.__execute_block_recording_exceptions(
lambda: ctm.__exit__(None, None, None), exit_result):
exit_result.end_in_success()
self.fire_event(self.EVENT_ON_COMPLETE_CLASS_TEARDOWN_METHOD,
exit_result)
else:
callback()
def __enter_context_managers(self, fixture_methods, callback):
"""Transform each fixture_method into a context manager with contextlib.contextmanager, enter them recursively, and call callback"""
if fixture_methods:
with contextmanager(fixture_methods[0])():
self.__enter_context_managers(fixture_methods[1:], callback)
else:
callback()
def __run_test_methods(self):
"""Run this class's setup fixtures / test methods / teardown fixtures.
These are run in the obvious order - setup and teardown go before and after,
respectively, every test method. If there was a failure in the class_setup
phase, no method-level fixtures or test methods will be run, and we'll eventually
skip all the way to the class_teardown phase. If a given test method is marked
as disabled, neither it nor its fixtures will be run. If there is an exception
during the setup phase, the test method will not be run and execution
will continue with the teardown phase.
"""
for test_method in self.runnable_test_methods():
result = TestResult(test_method)
try:
self._method_level = True # Flag that we're currently running method-level stuff (rather than class-level)
# run "on-run" callbacks. e.g. print out the test method name
self.fire_event(self.EVENT_ON_RUN_TEST_METHOD, result)
result.start()
if self.__class_level_failure:
result.end_in_failure(self.__class_level_failure)
elif self.__class_level_error:
result.end_in_error(self.__class_level_error)
else:
# first, run setup fixtures
self._stage = self.STAGE_SETUP
def _setup_block():
for fixture_method in self.setup_fixtures:
fixture_method()
self.__execute_block_recording_exceptions(
_setup_block, result)
def _run_test_block():
# then run the test method itself, assuming setup was successful
self._stage = self.STAGE_TEST_METHOD
if not result.complete:
self.__execute_block_recording_exceptions(
test_method, result)
def _setup_teardown_block():
self.__enter_context_managers(
self.setup_teardown_fixtures, _run_test_block)
# then run any setup_teardown fixtures, assuming setup was successful.
if not result.complete:
self.__execute_block_recording_exceptions(
_setup_teardown_block, result)
# finally, run the teardown phase
self._stage = self.STAGE_TEARDOWN
def _teardown_block():
for fixture_method in self.teardown_fixtures:
fixture_method()
self.__execute_block_recording_exceptions(
_teardown_block, result)
# if nothing's gone wrong, it's not about to start
if not result.complete:
result.end_in_success()
except (KeyboardInterrupt, SystemExit):
result.end_in_interruption(sys.exc_info())
raise
finally:
self.fire_event(self.EVENT_ON_COMPLETE_TEST_METHOD, result)
self._method_level = False
if not result.success:
self.failure_count += 1
if self.failure_limit and self.failure_count >= self.failure_limit:
return
def register_callback(self, event, callback):
"""Register a callback for an internal event, usually used for logging.
The argument to the callback will be the test method object itself.
Fixture objects can be distinguished by the running them through
inspection.is_fixture_method().
"""
self.__callbacks[event].append(callback)
def fire_event(self, event, result):
for callback in self.__callbacks[event]:
callback(result.to_dict())
def __execute_block_recording_exceptions(self,
block_fxn,
result,
is_class_level=False):
"""Excerpted code for executing a block of code that might raise an
exception, requiring us to update a result object.
Return value is a boolean describing whether the block was successfully
executed without exceptions.
"""
try:
block_fxn()
except (KeyboardInterrupt, SystemExit):
raise
except TwistedFailureError, exception:
# We provide special support for handling the failures that are generated from twisted.
# Due to complexities in error handling and cleanup, it's difficult to get the raw exception
# data from an asynchcronous failure, so we really get a pseudo traceback object.
failure = exception.args[0]
exc_info = (failure.type, failure.value,
failure.getTracebackObject())
result.end_in_error(exc_info)
if is_class_level:
self.__class_level_failure = exc_info
except Exception, exception:
# some code may want to use an alternative exc_info for an exception
# (for instance, in an event loop). You can signal an alternative
# stack to use by adding a _testify_exc_tb attribute to the
# exception object
if hasattr(exception, '_testify_exc_tb'):
exc_info = (type(exception), exception,
exception._testify_exc_tb)
else:
exc_info = sys.exc_info()
if isinstance(exception, AssertionError):
result.end_in_failure(exc_info)
if is_class_level:
self.__class_level_failure = exc_info
else:
result.end_in_error(exc_info)
if is_class_level:
self.__class_level_error = exc_info
if self.__debugger:
exc, val, tb = exc_info
print "\nDEBUGGER"
print "\n".join(result.format_exception_info())
import ipdb
ipdb.post_mortem(tb)
return False