def irunner():
"""Set up an irunner instance for testing."""
target = runnable.TestRunner(name="TestRunner")
local_runner = runners.LocalRunner()
test_uids = ["test_1", "test_2", "test_3"]
test_objs = [
multitest.MultiTest(
name=uid,
suites=[Suite()],
test_filter=filtering.Filter(),
test_sorter=ordering.NoopSorter(),
stdout_style=defaults.STDOUT_STYLE,
environment=[driver.Driver(name="mock_driver")],
) for uid in test_uids
]
for test in test_objs:
local_runner.add(test, test.uid())
target.resources.add(local_runner)
with mock.patch("cheroot.wsgi.Server"):
irunner = base.TestRunnerIHandler(target)
irunner.setup()
yield irunner
irunner.teardown()
def get_options(cls):
return {
'name':
str,
ConfigOption('logger_level', default=TEST_INFO):
int,
ConfigOption('runpath',
default=default_runpath,
block_propagation=False):
Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True):
bool,
ConfigOption('all_tasks_local', default=False):
bool,
ConfigOption('shuffle', default=[]):
list, # list of string choices
ConfigOption('shuffle_seed',
default=float(random.randint(1, 9999))):
float,
ConfigOption('exporters', default=None):
Use(get_exporters),
ConfigOption('stdout_style',
default=defaults.STDOUT_STYLE,
block_propagation=False):
Style,
ConfigOption('report_dir',
default=defaults.REPORT_DIR,
block_propagation=False):
str,
ConfigOption('xml_dir', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('json_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_style',
default=defaults.PDF_STYLE,
block_propagation=False):
Style,
ConfigOption('report_tags', default=[], block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all',
default=[],
block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('browse', default=False):
bool,
ConfigOption('test_filter',
default=filtering.Filter(),
block_propagation=False):
filtering.BaseFilter,
ConfigOption('test_sorter',
default=ordering.NoopSorter(),
block_propagation=False):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None, block_propagation=False):
Or(None, listing.BaseLister)
}
def get_options(cls):
return {
'name': str,
ConfigOption('description', default=None): Or(str, None),
ConfigOption('logger_level', default=logger.TEST_INFO): int,
ConfigOption('file_log_level', default=logger.DEBUG): Or(int, None),
ConfigOption(
'runpath',
default=default_runpath,
): Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True): bool,
ConfigOption('all_tasks_local', default=False): bool,
ConfigOption('shuffle', default=[]): list, # list of string choices
ConfigOption(
'shuffle_seed', default=float(random.randint(1, 9999))): float,
ConfigOption(
'exporters', default=None): Use(get_exporters),
ConfigOption(
'stdout_style', default=defaults.STDOUT_STYLE): Style,
ConfigOption(
'report_dir', default=defaults.REPORT_DIR): Or(str, None),
ConfigOption('xml_dir', default=None,): Or(str, None),
ConfigOption('pdf_path', default=None,): Or(str, None),
ConfigOption('json_path', default=None,): Or(str, None),
ConfigOption('http_url', default=None): Or(str, None),
ConfigOption('pdf_style', default=defaults.PDF_STYLE,): Style,
ConfigOption('report_tags', default=[]
): [Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all', default=[]
): [Use(tagging.validate_tag_value)],
ConfigOption('merge_scheduled_parts', default=False): bool,
ConfigOption('browse', default=False): bool,
ConfigOption('ui_port', default=None): Or(None, int),
ConfigOption('web_server_startup_timeout',
default=defaults.WEB_SERVER_TIMEOUT): int,
ConfigOption('test_filter', default=filtering.Filter()
): filtering.BaseFilter,
ConfigOption('test_sorter', default=ordering.NoopSorter()
): ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None
):Or(None, listing.BaseLister),
ConfigOption('verbose', default=False): bool,
ConfigOption('debug', default=False): bool,
ConfigOption(
'timeout', default=None): Or(
None, And(Or(int, float), lambda t: t >= 0)),
ConfigOption('interactive_handler', default=TestRunnerIHandler):
object,
ConfigOption('extra_deps', default=[]): list
}
def configuration_schema(self):
"""
Schema for options validation and assignment of default values.
"""
overrides = Schema(
{
'name':
str,
ConfigOption('logger_level', default=TEST_INFO):
int,
ConfigOption('runpath', default=default_runpath):
Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True):
bool,
ConfigOption('all_tasks_local', default=False):
bool,
ConfigOption('shuffle', default=[]):
list, # list of string choices
ConfigOption('shuffle_seed',
default=float(random.randint(1, 9999))):
float,
ConfigOption('exporters', default=None):
Use(get_exporters),
ConfigOption('stdout_style', default=defaults.STDOUT_STYLE):
Style,
ConfigOption('report_dir', default=defaults.REPORT_DIR):
str,
ConfigOption('xml_dir', default=None):
Or(str, None),
ConfigOption('pdf_path', default=None):
Or(str, None),
ConfigOption('json_path', default=None):
Or(str, None),
ConfigOption('pdf_style', default=defaults.PDF_STYLE):
Style,
ConfigOption('report_tags', default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all', default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption('browse', default=False):
bool,
ConfigOption('test_filter', default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption('test_sorter', default=ordering.NoopSorter()):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None):
Or(None, listing.BaseLister)
},
ignore_extra_keys=True)
return self.inherit_schema(overrides, super(TestRunnerConfig, self))
def get_options(cls):
return {
'name':
str,
ConfigOption('description', default=None):
str,
ConfigOption('test_filter', default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption('test_sorter', default=ordering.NoopSorter()):
ordering.BaseSorter,
ConfigOption('stdout_style', default=defaults.STDOUT_STYLE):
test_styles.Style,
ConfigOption('tags', default=None):
Or(None, Use(tagging.validate_tag_value))
}
def configuration_schema(self):
"""
Schema for options validation and assignment of default values.
"""
overrides = {
'name':
str,
ConfigOption('description', default=None):
str,
ConfigOption('test_filter', default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption('test_sorter', default=ordering.NoopSorter()):
ordering.BaseSorter,
ConfigOption('stdout_style', default=defaults.STDOUT_STYLE):
test_styles.Style
}
return self.inherit_schema(overrides, super(TestConfig, self))
def get_options(cls):
return {
# 'name': And(str, lambda s: s.count(' ') == 0),
'name':
str,
ConfigOption('description', default=None):
str,
ConfigOption('test_filter',
default=filtering.Filter(),
block_propagation=False):
filtering.BaseFilter,
ConfigOption('test_sorter',
default=ordering.NoopSorter(),
block_propagation=False):
ordering.BaseSorter,
ConfigOption('stdout_style',
default=defaults.STDOUT_STYLE,
block_propagation=False):
test_styles.Style,
ConfigOption('tags', default=None):
Or(None, Use(tagging.validate_tag_value))
}
def get_options(cls):
start_stop_signature = Or(
None,
validate_func('env'),
validate_func('env', 'result'),
)
return {
# 'name': And(str, lambda s: s.count(' ') == 0),
'name':
str,
ConfigOption('description', default=None):
str,
ConfigOption('environment', default=[]): [Resource],
ConfigOption('before_start', default=None):
start_stop_signature,
ConfigOption('after_start', default=None):
start_stop_signature,
ConfigOption('before_stop', default=None):
start_stop_signature,
ConfigOption('after_stop', default=None):
start_stop_signature,
ConfigOption('test_filter',
default=filtering.Filter(),
block_propagation=False):
filtering.BaseFilter,
ConfigOption('test_sorter',
default=ordering.NoopSorter(),
block_propagation=False):
ordering.BaseSorter,
ConfigOption('stdout_style',
default=defaults.STDOUT_STYLE,
block_propagation=False):
test_styles.Style,
ConfigOption('tags', default=None):
Or(None, Use(tagging.validate_tag_value))
}
" test_a",
"Secondary",
" Gamma",
" test_c",
" test_b",
" test_a",
)
@pytest.mark.parametrize(
"listing_obj,filter_obj,sorter_obj,expected_output",
[
# Basic name listing
(
listing.ExpandedNameLister(),
filtering.Filter(),
ordering.NoopSorter(),
DEFAULT_NAME_OUTPUT,
),
# Basic pattern listing
(
listing.ExpandedPatternLister(),
filtering.Filter(),
ordering.NoopSorter(),
DEFAULT_PATTERN_OUTPUT,
),
# Basic count listing
(
listing.CountLister(),
filtering.Filter(),
ordering.NoopSorter(),
def get_options(cls):
return {
"name":
str,
ConfigOption("description", default=None):
Or(str, None),
ConfigOption("logger_level", default=logger.TEST_INFO):
int,
ConfigOption("file_log_level", default=logger.DEBUG):
int,
ConfigOption("runpath", default=default_runpath):
Or(None, str, lambda x: callable(x)),
ConfigOption("path_cleanup", default=True):
bool,
ConfigOption("all_tasks_local", default=False):
bool,
ConfigOption("shuffle", default=[]):
list, # list of string choices
ConfigOption("shuffle_seed",
default=float(random.randint(1, 9999))):
float,
ConfigOption("exporters", default=None):
Use(get_exporters),
ConfigOption("stdout_style", default=defaults.STDOUT_STYLE):
Style,
ConfigOption("report_dir", default=defaults.REPORT_DIR):
Or(str, None),
ConfigOption("xml_dir", default=None):
Or(str, None),
ConfigOption("pdf_path", default=None):
Or(str, None),
ConfigOption("json_path", default=None):
Or(str, None),
ConfigOption("http_url", default=None):
Or(str, None),
ConfigOption("pdf_style", default=defaults.PDF_STYLE):
Style,
ConfigOption("report_tags", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("report_tags_all", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("merge_scheduled_parts", default=False):
bool,
ConfigOption("browse", default=False):
bool,
ConfigOption("ui_port", default=None):
Or(None, int),
ConfigOption(
"web_server_startup_timeout",
default=defaults.WEB_SERVER_TIMEOUT,
):
int,
ConfigOption("test_filter", default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption("test_sorter", default=ordering.NoopSorter()):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption("test_lister", default=None):
Or(None, listing.BaseLister),
ConfigOption("verbose", default=False):
bool,
ConfigOption("debug", default=False):
bool,
ConfigOption("timeout", default=defaults.TESTPLAN_TIMEOUT):
Or(None, And(int, lambda t: t >= 0)),
ConfigOption("abort_wait_timeout", default=60):
int,
ConfigOption("interactive_handler", default=TestRunnerIHandler):
object,
ConfigOption("extra_deps", default=[]):
[Or(str, lambda x: inspect.ismodule(x))],
ConfigOption("label", default=None):
Or(None, str),
}
"""Unit tests for MultiTest base functionality."""
import os
from testplan.common.utils import path
from testplan.testing import multitest
from testplan.testing.multitest import base
from testplan.testing import filtering
from testplan.testing import ordering
from testplan import defaults
from testplan import report
from testplan.common import entity
# TODO: shouldn't need to specify these...
MTEST_DEFAULT_PARAMS = {
"test_filter": filtering.Filter(),
"test_sorter": ordering.NoopSorter(),
"stdout_style": defaults.STDOUT_STYLE,
}
def test_multitest_runpath():
"""Test setting of runpath."""
class Parent(object):
def __init__(self, runpath):
self.runpath = runpath
global_runpath = os.path.join("", "var", "tmp", "global_level")
local_runpath = os.path.join("", "var", "tmp", "local_runpath")
par = Parent(global_runpath)
def main_wrapper(cls,
name,
description=None,
parse_cmdline=True,
interactive=False,
port=None,
abort_signals=None,
logger_level=logger.TEST_INFO,
file_log_level=logger.DEBUG,
runpath=path.default_runpath,
path_cleanup=True,
all_tasks_local=False,
shuffle=None,
shuffle_seed=float(random.randint(1, 9999)),
exporters=None,
stdout_style=defaults.STDOUT_STYLE,
report_dir=defaults.REPORT_DIR,
xml_dir=None,
json_path=None,
http_url=None,
pdf_path=None,
pdf_style=defaults.PDF_STYLE,
report_tags=None,
report_tags_all=None,
merge_scheduled_parts=False,
browse=False,
ui_port=None,
web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT,
test_filter=filtering.Filter(),
test_sorter=ordering.NoopSorter(),
test_lister=None,
verbose=False,
debug=False,
timeout=None,
interactive_handler=TestRunnerIHandler,
extra_deps=None,
**options):
"""
Decorator that will be used for wrapping `main` methods in test scripts.
It accepts all arguments of a
:py:class:`~testplan.base.Testplan` entity.
"""
def test_plan_inner(definition):
"""
This is being passed the user-defined testplan entry point.
"""
def test_plan_inner_inner():
"""
This is the callable returned in the end, it executes the plan
and the associated reporting
"""
plan = cls(
name=name,
description=description,
parse_cmdline=parse_cmdline,
interactive=interactive,
port=port,
abort_signals=abort_signals,
logger_level=logger_level,
file_log_level=file_log_level,
runpath=runpath,
path_cleanup=path_cleanup,
all_tasks_local=all_tasks_local,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
exporters=exporters,
stdout_style=stdout_style,
report_dir=report_dir,
xml_dir=xml_dir,
json_path=json_path,
http_url=http_url,
pdf_path=pdf_path,
pdf_style=pdf_style,
report_tags=report_tags,
report_tags_all=report_tags_all,
merge_scheduled_parts=merge_scheduled_parts,
browse=browse,
ui_port=ui_port,
web_server_startup_timeout=web_server_startup_timeout,
test_filter=test_filter,
test_sorter=test_sorter,
test_lister=test_lister,
verbose=verbose,
debug=debug,
timeout=timeout,
interactive_handler=interactive_handler,
extra_deps=extra_deps,
**options)
try:
if arity(definition) == 2:
returned = definition(plan, plan.parser)
else:
returned = definition(plan)
except Exception:
print('Exception in test_plan definition, aborting plan..')
plan.abort()
raise
plan_result = plan.run()
plan_result.decorated_value = returned
return plan_result
return test_plan_inner_inner
return test_plan_inner
def __init__(self,
name,
description=None,
parse_cmdline=True,
interactive_port=None,
abort_signals=None,
logger_level=logger.TEST_INFO,
file_log_level=logger.DEBUG,
runpath=path.default_runpath,
path_cleanup=True,
all_tasks_local=False,
shuffle=None,
shuffle_seed=float(random.randint(1, 9999)),
exporters=None,
stdout_style=defaults.STDOUT_STYLE,
report_dir=defaults.REPORT_DIR,
xml_dir=None,
json_path=None,
http_url=None,
pdf_path=None,
pdf_style=defaults.PDF_STYLE,
report_tags=None,
report_tags_all=None,
merge_scheduled_parts=False,
browse=False,
ui_port=None,
web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT,
test_filter=filtering.Filter(),
test_sorter=ordering.NoopSorter(),
test_lister=None,
verbose=False,
debug=False,
timeout=None,
interactive_handler=TestRunnerIHandler,
extra_deps=None,
**options):
# Set mutable defaults.
if abort_signals is None:
abort_signals = entity.DEFAULT_RUNNABLE_ABORT_SIGNALS[:]
if shuffle is None:
shuffle = []
if extra_deps is None:
extra_deps = []
if report_tags is None:
report_tags = []
if report_tags_all is None:
report_tags_all = []
super(Testplan, self).__init__(
name=name,
description=description,
parse_cmdline=parse_cmdline,
interactive_port=interactive_port,
abort_signals=abort_signals,
logger_level=logger_level,
file_log_level=file_log_level,
runpath=runpath,
path_cleanup=path_cleanup,
all_tasks_local=all_tasks_local,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
exporters=exporters,
stdout_style=stdout_style,
report_dir=report_dir,
xml_dir=xml_dir,
json_path=json_path,
http_url=http_url,
pdf_path=pdf_path,
pdf_style=pdf_style,
report_tags=report_tags,
report_tags_all=report_tags_all,
merge_scheduled_parts=merge_scheduled_parts,
browse=browse,
ui_port=ui_port,
web_server_startup_timeout=web_server_startup_timeout,
test_filter=test_filter,
test_sorter=test_sorter,
test_lister=test_lister,
verbose=verbose,
debug=debug,
timeout=timeout,
interactive_handler=interactive_handler,
extra_deps=extra_deps,
**options)
for resource in self._cfg.resources:
self._runnable.add_resource(resource)
# Stores local tests.
self._runnable.add_resource(LocalRunner(), uid='local_runner')
# Stores independent environments.
self._runnable.add_resource(Environments(), uid='environments')
pass
@testcase(tags={'color': ('blue', 'yellow')})
def test_two(self, env, result):
pass
@testcase(tags={'color': ('red', 'green')})
def test_three(self, env, result):
pass
@pytest.mark.parametrize(
'filter_obj, report_ctx',
(
# Case 1
(filtering.Filter(), [('XXX', [
('Alpha', ['test_one', 'test_two', 'test_three']),
('Beta', ['test_one', 'test_two', 'test_three']),
]), ('YYY', (('Gamma', ['test_one', 'test_two', 'test_three']), ))]),
# Case 2
(filtering.Pattern('*:*:test_two'), [('XXX', [
('Alpha', ['test_two']),
('Beta', ['test_two']),
]), ('YYY', [
('Gamma', ['test_two']),
])]),
# Case 3
(filtering.Pattern('XXX:Beta:test_two'), [
('XXX', [('Beta', ['test_two'])]),
]),
# Case 4 - testcase name match AND tag match
' test_a',
'Secondary',
' Gamma',
' test_c',
' test_b',
' test_a',
)
@pytest.mark.parametrize(
'listing_obj,filter_obj,sorter_obj,expected_output',
[
# Basic name listing
(
listing.ExpandedNameLister(),
filtering.Filter(),
ordering.NoopSorter(),
DEFAULT_NAME_OUTPUT,
),
# Basic pattern listing
(listing.ExpandedPatternLister(), filtering.Filter(),
ordering.NoopSorter(), DEFAULT_PATTERN_OUTPUT),
# Basic count listing
(listing.CountLister(), filtering.Filter(), ordering.NoopSorter(),
to_stdout(
'Primary: (2 suites, 6 testcases)',
'Secondary: (1 suite, 3 testcases)',
)),
# Custom sort & name listing
(listing.ExpandedNameLister(), filtering.Filter(),
ordering.AlphanumericSorter(),
