def _generate_func(function, name_func, tag_func, docstring_func, tag_dict,
kwargs):
"""
Generates a new function using the original function, name generation
function and parametrized kwargs.
Also attaches parametrized and explicit tags and apply custom wrappers.
"""
def _generated(self, env, result):
return function(self, env, result, **kwargs)
_generated.__doc__ = docstring_func(function.__doc__, kwargs)\
if docstring_func else None
_generated.__name__ = _name_func_wrapper(name_func=name_func,
func_name=function.__name__,
kwargs=kwargs)
# Tags generated via `tag_func` will be assigned as native tags
_generated.__tags__ = tagging.validate_tag_value(tag_func(kwargs)) \
if tag_func else {}
# Tags index will be merged tag ctx of tag_dict & generated tags
_generated.__tags_index__ = tagging.merge_tag_dicts(
_generated.__tags__, tag_dict)
_generated._parametrization_template = function.__name__
return _generated
def _generate_func(function, name_func, tag_func, docstring_func, tags,
kwargs):
"""
Generates a new function using the original function, name generation
function and parametrized kwargs.
Also attaches parametrized and explicit tags and apply custom wrappers.
"""
def _generated(self, env, result):
return function(self, env, result, **kwargs)
_generated.__doc__ = docstring_func(function.__doc__, kwargs)\
if docstring_func else None
_generated.__name__ = _name_func_wrapper(name_func=name_func,
func_name=function.__name__,
kwargs=kwargs)
# Tagging
parametrized_tags = tagging.validate_tag_value(tag_func(kwargs))\
if tag_func else {}
tagging.attach_testcase_tags(
_generated, tagging.merge_tag_dicts(tags, parametrized_tags))
_generated._parametrization_template = function.__name__
return _generated
def propagate_tag_indices(self, parent_tags=None):
"""
Distribute native tag data onto `tags_index` attributes on the nodes
of the test report. This distribution happens 2 ways.
"""
tags_index = tagging.merge_tag_dicts(self.tags, parent_tags or {})
for child in self:
if isinstance(child, TestGroupReport):
child.propagate_tag_indices(parent_tags=tags_index)
elif isinstance(child, TestCaseReport):
child.tags_index = tagging.merge_tag_dicts(
child.tags, tags_index)
self.tags_index = tagging.merge_tag_dicts(tags_index,
self._collect_tag_indices())
def update_tag_index(obj, tag_dict):
"""
Utility for updating ``__tags_index__`` attribute of an object.
"""
if isinstance(obj, types.MethodType):
obj = obj.__func__
obj.__tags_index__ = tagging.merge_tag_dicts(
tag_dict, getattr(obj, '__tags_index__', {}))
def _collect_tag_indices(self):
"""Collect tag indices from the current report and its children."""
tag_dicts = [self.tags_index]
for child in self:
if isinstance(child, TestCaseReport):
tag_dicts.append(child.tags_index)
elif isinstance(child, TestGroupReport):
tag_dicts.append(child._collect_tag_indices())
return tagging.merge_tag_dicts(*tag_dicts)
def get_tags_index(self):
"""
Tags index for a multitest is its native tags merged with tag indices
from all of its suites. (Suite tag indices will also contain tag
indices from their testcases as well).
"""
if self._tags_index is None:
self._tags_index = tagging.merge_tag_dicts(
self.cfg.tags or {}, *[s.__tags_index__ for s in self.suites])
return self._tags_index
def _testsuite(klass):
"""
Actual decorator that transforms a class into a suite and registers
testcases.
"""
# nasty, but smallest possible evil that has to be perpetrated in order
# to preserve the order of definition of the testcases and make sure
# they get executed in the same order
# pylint: disable=global-statement
global __GENERATED_TESTCASES__
global __TESTCASES__
global __SKIP__
klass.__testcases__ = __TESTCASES__
klass.__skip__ = __SKIP__
if not hasattr(klass, "__tags__"):
klass.__tags__ = {} # used for UI
klass.__tags_index__ = {} # used for actual filtering
# Attributes defined in test suite will be saved in test report,
# they should be normal objects which can be serialized with json
klass.__extra_attributes__ = {
attrib: getattr(klass, attrib)
for attrib in dir(klass)
if not (attrib.startswith("__") or callable(getattr(klass, attrib))
or isinstance(getattr(klass, attrib), property) or attrib in
klass.__testcases__ or getattr(getattr(
klass, attrib), "__parametrization_template__", False))
}
klass.get_testcases = get_testcase_methods
for func in __GENERATED_TESTCASES__:
setattr(klass, func.__name__, func)
testcase_methods = get_testcase_methods(klass)
# propagate suite's native tags onto itself, which
# will propagate them further to the suite's testcases
propagate_tag_indices(klass, klass.__tags__)
# Collect tag indices from testcase methods and update suite's tag index.
update_tag_index(
obj=klass,
tag_dict=tagging.merge_tag_dicts(
*[tc.__tags_index__ for tc in testcase_methods]),
)
__GENERATED_TESTCASES__ = []
__TESTCASES__ = []
__SKIP__ = defaultdict(tuple)
return klass
def tags_index(self):
"""
Root report only has tag indexes, which is only useful when
we run searches against multiple test reports.
(e.g Give me all test runs from all projects that have these tags)
"""
from testplan.testing.tagging import merge_tag_dicts
if self._tags_index is None:
self._tags_index = merge_tag_dicts(
*[child.tags_index for child in self])
return self._tags_index
def __call__(self, parser, namespace, values, option_string=None):
from testplan.testing import tagging
items = getattr(namespace, self.dest) or []
tag_arg = [tagging.parse_tag_arguments(v) for v in values]
tag_arg = tagging.merge_tag_dicts(*tag_arg)
items.append(tag_arg)
setattr(namespace, self.dest, items)
def _collect_tag_indices(self):
"""
Recursively collect tag indices from children (and their children etc)
"""
tag_dicts = [self.tags]
for child in self:
if isinstance(child, TestGroupReport):
tag_dicts.append(child._collect_tag_indices())
elif isinstance(child, TestCaseReport):
tag_dicts.append(child.tags)
return tagging.merge_tag_dicts(*tag_dicts)
def wrapper(function):
"""Meta logic for test case goes here"""
if tags:
tagging.attach_testcase_tags(function, tags)
if parameters is not None: # Empty tuple / dict checks happen later
functions = parametrization.generate_functions(
function=function,
parameters=parameters,
name_func=name_func,
docstring_func=docstring_func,
tag_func=tag_func,
tags=tags,
summarize=summarize,
num_passing=defaults.SUMMARY_NUM_PASSING,
num_failing=defaults.SUMMARY_NUM_FAILING)
# Register generated functions as test_cases
for func in functions:
_validate_testcase(func)
# this has to be called before wrappers otherwise wrappers can
# fail if they rely on __testcase__
_mark_function_as_testcase(func)
wrappers = custom_wrappers or []
if not isinstance(wrappers, (list, tuple)):
wrappers = [wrappers]
for wrapper_func in wrappers:
func = wrapper_func(func)
# so that CodeDetails gets the correct line number
func.wrapper_of = function
__TESTCASES__.append(func.__name__)
__GENERATED_TESTCASES__.append(func)
# Assign tags (native & tags collected from generated functions)
function.generated_tags = tagging.merge_tag_dicts(
*
[tagging.get_native_testcase_tags(func) for func in functions])
return function
else:
function.summarize = summarize
function.summarize_num_passing = num_passing
function.summarize_num_failing = num_failing
return _testcase(function)
def propagate_tag_indices(self):
"""
When a test is run and test instance report is populated with children
we may need to tag indices of the report tree.
This is more likely to happen for tests that are
run via 3rd party testing libraries.
"""
for child in self:
if isinstance(child, (TestGroupReport, TestCaseReport)):
child.tags_index = tagging.merge_tag_dicts(
self.tags_index, child.tags_index)
if isinstance(child, TestGroupReport):
child.propagate_tag_indices()
self.tags_index = self._collect_tag_indices()
def _generate_func(
function, name, name_func, tag_func, docstring_func, tag_dict, kwargs
):
"""
Generates a new function using the original function, name generation
function and parametrized kwargs.
Also attaches parametrized and explicit tags and apply custom wrappers.
"""
def _generated(self, env, result):
return function(self, env, result, **kwargs)
# If we were given a docstring function, we use it to generate the
# docstring for each testcase. Otherwise we just copy the docstring from
# the template method.
if docstring_func:
_generated.__doc__ = docstring_func(function.__doc__, kwargs)
else:
_generated.__doc__ = function.__doc__
_generated.__name__ = _parametrization_name_func_wrapper(
func_name=function.__name__, kwargs=kwargs
)
_generated.name = name_func(name, kwargs) if name_func else name
if hasattr(function, "__xfail__"):
_generated.__xfail__ = function.__xfail__
# Tags generated via `tag_func` will be assigned as native tags
_generated.__tags__ = (
tagging.validate_tag_value(tag_func(kwargs)) if tag_func else {}
)
# Tags index will be merged tag ctx of tag_dict & generated tags
_generated.__tags_index__ = tagging.merge_tag_dicts(
_generated.__tags__, tag_dict
)
_generated._parametrization_template = function.__name__
_generated._parametrization_kwargs = kwargs
return _generated
def _testsuite(klass):
"""
Actual decorator that transforms a class into a suite and registers
testcases.
"""
# nasty, but smallest possible evil that has to be perpetrated in order
# to preserve the order of definition of the testcases and make sure
# they get executed in the same order
# pylint: disable=global-statement
global __GENERATED_TESTCASES__
global __TESTCASES__
global __SKIP__
klass.__testcases__ = __TESTCASES__
klass.__skip__ = __SKIP__
if not hasattr(klass, '__tags__'):
klass.__tags__ = {} # used for UI
klass.__tags_index__ = {} # used for actual filtering
klass.get_testcases = get_testcase_methods
for func in __GENERATED_TESTCASES__:
setattr(klass, func.__name__, func)
testcase_methods = get_testcase_methods(klass)
# propagate suite's native tags onto itself, which
# will propagate them further to the suite's testcases
propagate_tag_indices(klass, klass.__tags__)
# Collect tag indices from testcase methods and update suite's tag index.
update_tag_index(
obj=klass,
tag_dict=tagging.merge_tag_dicts(
*[tc.__tags_index__ for tc in testcase_methods]))
__GENERATED_TESTCASES__ = []
__TESTCASES__ = []
__SKIP__ = defaultdict(tuple)
return klass
def test_basic_suite_tags():
mysuite = MySuite2()
assert mysuite.__tags__ == {'simple': {'A'}}
case_dict = {
'case1': {
'simple': {'B'}
},
'case2': {
'c': {'C'}
},
'case3': {
'd': {'D2', 'D1'}
}
}
for method in mysuite.get_testcases():
assert method.__tags__ == case_dict[method.__name__]
assert method.__tags_index__ == tagging.merge_tag_dicts(
case_dict[method.__name__], mysuite.__tags__)
def test_basic_suite_tags():
mysuite = MySuite2()
assert mysuite.__tags__ == {"simple": {"A"}}
case_dict = {
"case1": {
"simple": {"B"}
},
"case2": {
"c": {"C"}
},
"case3": {
"d": {"D2", "D1"}
},
}
for method in mysuite.get_testcases():
assert method.__tags__ == case_dict[method.__name__]
assert method.__tags_index__ == tagging.merge_tag_dicts(
case_dict[method.__name__], mysuite.__tags__)
def _run_suite(self, testsuite, testcases, testsuite_report):
"""Runs a testsuite object and populates its report object."""
post_testcase = getattr(testsuite, 'post_testcase', None)
pre_testcase = getattr(testsuite, 'pre_testcase', None)
with testsuite_report.logged_exceptions():
self._run_suite_related(testsuite, 'setup', testsuite_report)
if not testsuite_report.passed:
with testsuite_report.logged_exceptions():
self._run_suite_related(testsuite, 'teardown',
testsuite_report)
return
param_rep_lookup = {}
while self.active:
if self.status.tag == Runnable.STATUS.RUNNING:
try:
testcase = testcases.pop(0)
except IndexError:
with testsuite_report.logged_exceptions():
self._run_suite_related(testsuite, 'teardown',
testsuite_report)
break
else:
param_template = getattr(testcase,
'_parametrization_template', None)
if param_template:
if param_template not in param_rep_lookup:
param_method = getattr(testsuite, param_template)
param_report = TestGroupReport(
name=param_template,
description=param_method.__doc__,
category=Categories.PARAMETRIZATION,
tags=tagging.get_native_testcase_tags(
param_method),
tags_index=tagging.merge_tag_dicts(
param_method.generated_tags,
tagging.get_native_suite_tags(testsuite)))
param_rep_lookup[param_template] = param_report
testsuite_report.append(param_report)
parent_report = param_rep_lookup[param_template]
else:
parent_report = testsuite_report
testcase_report = self._run_testcase(
testcase=testcase,
pre_testcase=pre_testcase,
post_testcase=post_testcase)
parent_report.append(testcase_report)
# Break the suite execution if a testcase raised.
if testcase_report.status == Status.ERROR:
with testsuite_report.logged_exceptions():
self._run_suite_related(testsuite, 'teardown',
testsuite_report)
break
time.sleep(self.cfg.active_loop_sleep)
if self.get_stdout_style(testsuite_report.passed).display_suite:
log_suite_status(testsuite_report)
def _testsuite(klass):
"""
Actual decorator that transforms a class into a suite and registers
testcases.
"""
# nasty, but smallest possible evil that has to be perpetrated in order
# to preserve the order of definition of the testcases and make sure
# they get executed in the same order
_ensure_unique_generated_testcase_names(
__TESTCASES__ + __PARAMETRIZATION_TEMPLATE__, __GENERATED_TESTCASES__)
klass.__testcases__ = [None] * _number_of_testcases()
klass.__skip__ = __SKIP__
for testcase_name in __TESTCASES__:
klass.__testcases__[getattr(
klass, testcase_name).__seq_number__] = testcase_name
for func in __GENERATED_TESTCASES__:
klass.__testcases__[func.__seq_number__] = func.__name__
setattr(klass, func.__name__, func)
assert all(testcase for testcase in klass.__testcases__)
# Attributes `name` and `__tags__` are added only when class is
# decorated by @testsuite(...) which has the following parentheses.
if not hasattr(klass, "name"):
klass.name = None
if callable(klass.name):
try:
interface.check_signature(klass.name, ["cls_name", "suite"])
except interface.MethodSignatureMismatch as err:
_reset_globals()
raise err
elif not (klass.name is None or isinstance(klass.name, six.string_types)):
_reset_globals()
raise TypeError('"name" should be a string or a callable or `None`')
if not hasattr(klass, "__tags__"):
klass.__tags__ = {} # used for UI
klass.__tags_index__ = {} # used for actual filtering
klass.get_testcases = get_testcase_methods
testcase_methods = get_testcase_methods(klass)
# propagate suite's native tags onto itself, which
# will propagate them further to the suite's testcases
propagate_tag_indices(klass, klass.__tags__)
# Collect tag indices from testcase methods and update suite's tag index.
update_tag_index(
obj=klass,
tag_dict=tagging.merge_tag_dicts(
*[tc.__tags_index__ for tc in testcase_methods]),
)
# Suite resolved, clear global variables for resolving the next suite.
_reset_globals()
return klass
def collect_testcase_tags(testcases):
return tagging.merge_tag_dicts(*[tc.__tags_index__ for tc in testcases])
def _testsuite(klass):
"""
Actual decorator that transforms a class into a suite and registers
testcases.
"""
# nasty, but smallest possible evil that has to be perpetrated in order
# to preserve the order of definition of the testcases and make sure
# they get executed in the same order
_ensure_unique_generated_testcase_names(
__TESTCASES__ + __PARAMETRIZATION_TEMPLATE__, __GENERATED_TESTCASES__
)
klass.__testcases__ = [None] * _number_of_testcases()
klass.__skip__ = __SKIP__
for testcase_name in __TESTCASES__:
klass.__testcases__[
getattr(klass, testcase_name).__seq_number__
] = testcase_name
for func in __GENERATED_TESTCASES__:
klass.__testcases__[func.__seq_number__] = func.__name__
setattr(klass, func.__name__, func)
assert all(testcases for testcases in klass.__testcases__)
# Attributes `custom_name` and `__tags__` are added only when class is
# decorated by @testsuite(...) which has the following parentheses.
if not hasattr(klass, "custom_name"):
klass.custom_name = None
if not hasattr(klass, "__tags__"):
klass.__tags__ = {} # used for UI
klass.__tags_index__ = {} # used for actual filtering
# Attributes defined in test suite will be saved in test report,
# they should be normal objects which can be serialized with json
# TODO: Attribute `__extra_attributes__` will be removed later
klass.__extra_attributes__ = {
attrib: getattr(klass, attrib)
for attrib in dir(klass)
if not (
attrib.startswith("__")
or attrib == "custom_name"
or callable(getattr(klass, attrib))
or isinstance(getattr(klass, attrib), property)
or attrib in klass.__testcases__
or getattr(
getattr(klass, attrib), "__parametrization_template__", False
)
)
}
klass.get_testcases = get_testcase_methods
testcase_methods = get_testcase_methods(klass)
# propagate suite's native tags onto itself, which
# will propagate them further to the suite's testcases
propagate_tag_indices(klass, klass.__tags__)
# Collect tag indices from testcase methods and update suite's tag index.
update_tag_index(
obj=klass,
tag_dict=tagging.merge_tag_dicts(
*[tc.__tags_index__ for tc in testcase_methods]
),
)
# Suite resolved, clear global variables for resolving the next suite.
_reset_globals()
return klass