from testplan.common import entity
from testplan.common.utils import path
from testplan.common.utils import testing
from testplan.common.utils.thread import Barrier
from testplan.testing import multitest
from testplan.testing import filtering
from testplan.testing import ordering
from testplan import defaults
from testplan import report
# TODO: shouldn't need to specify these...
MTEST_DEFAULT_PARAMS = {
"test_filter": filtering.Filter(),
"test_sorter": ordering.NoopSorter(),
"stdout_style": defaults.STDOUT_STYLE,
}
def test_multitest_runpath():
"""Test setting of runpath."""
class Parent(object):
def __init__(self, runpath):
self.runpath = runpath
global_runpath = os.path.join("", "var", "tmp", "global_level")
local_runpath = os.path.join("", "var", "tmp", "local_runpath")
par = Parent(global_runpath)
def __init__(self,
name,
description=None,
parse_cmdline=True,
parser=TestplanParser,
interactive_port=None,
abort_signals=None,
logger_level=logger.TEST_INFO,
file_log_level=logger.DEBUG,
runpath=path.default_runpath,
path_cleanup=True,
all_tasks_local=False,
shuffle=None,
shuffle_seed=float(random.randint(1, 9999)),
exporters=None,
stdout_style=defaults.STDOUT_STYLE,
report_dir=defaults.REPORT_DIR,
xml_dir=None,
json_path=None,
http_url=None,
pdf_path=None,
pdf_style=defaults.PDF_STYLE,
report_tags=None,
report_tags_all=None,
merge_scheduled_parts=False,
browse=False,
ui_port=None,
web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT,
test_filter=filtering.Filter(),
test_sorter=ordering.NoopSorter(),
test_lister=None,
verbose=False,
debug=False,
timeout=defaults.TESTPLAN_TIMEOUT,
interactive_handler=TestRunnerIHandler,
extra_deps=None,
label=None,
**options):
# Set mutable defaults.
if abort_signals is None:
abort_signals = entity.DEFAULT_RUNNABLE_ABORT_SIGNALS[:]
if shuffle is None:
shuffle = []
if extra_deps is None:
extra_deps = []
if report_tags is None:
report_tags = []
if report_tags_all is None:
report_tags_all = []
# Define instance attributes
self._parsed_args = argparse.Namespace()
self._processed_args = {}
self._default_options = {}
super(Testplan, self).__init__(
name=name,
description=description,
parse_cmdline=parse_cmdline,
parser=parser,
interactive_port=interactive_port,
abort_signals=abort_signals,
logger_level=logger_level,
file_log_level=file_log_level,
runpath=runpath,
path_cleanup=path_cleanup,
all_tasks_local=all_tasks_local,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
exporters=exporters,
stdout_style=stdout_style,
report_dir=report_dir,
xml_dir=xml_dir,
json_path=json_path,
http_url=http_url,
pdf_path=pdf_path,
pdf_style=pdf_style,
report_tags=report_tags,
report_tags_all=report_tags_all,
merge_scheduled_parts=merge_scheduled_parts,
browse=browse,
ui_port=ui_port,
web_server_startup_timeout=web_server_startup_timeout,
test_filter=test_filter,
test_sorter=test_sorter,
test_lister=test_lister,
verbose=verbose,
debug=debug,
timeout=timeout,
interactive_handler=interactive_handler,
extra_deps=extra_deps,
label=label,
**options)
# By default, a LocalRunner is added to store and execute the tests.
self._runnable.add_resource(LocalRunner(), uid="local_runner")
# Stores independent environments.
self._runnable.add_resource(Environments(), uid="environments")
def main_wrapper(cls,
name,
description=None,
parse_cmdline=True,
parser=TestplanParser,
interactive_port=None,
abort_signals=None,
logger_level=logger.TEST_INFO,
file_log_level=logger.DEBUG,
runpath=path.default_runpath,
path_cleanup=True,
all_tasks_local=False,
shuffle=None,
shuffle_seed=float(random.randint(1, 9999)),
exporters=None,
stdout_style=defaults.STDOUT_STYLE,
report_dir=defaults.REPORT_DIR,
xml_dir=None,
json_path=None,
http_url=None,
pdf_path=None,
pdf_style=defaults.PDF_STYLE,
report_tags=None,
report_tags_all=None,
merge_scheduled_parts=False,
browse=False,
ui_port=None,
web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT,
test_filter=filtering.Filter(),
test_sorter=ordering.NoopSorter(),
test_lister=None,
verbose=False,
debug=False,
timeout=defaults.TESTPLAN_TIMEOUT,
interactive_handler=TestRunnerIHandler,
extra_deps=None,
label=None,
**options):
"""
Decorator that will be used for wrapping `main` methods in test scripts.
It accepts all arguments of a
:py:class:`~testplan.base.Testplan` entity.
"""
def test_plan_inner(definition):
"""
This is being passed the user-defined testplan entry point.
"""
def test_plan_inner_inner():
"""
This is the callable returned in the end, it executes the plan
and the associated reporting
"""
plan = cls(
name=name,
description=description,
parse_cmdline=parse_cmdline,
parser=parser,
interactive_port=interactive_port,
abort_signals=abort_signals,
logger_level=logger_level,
file_log_level=file_log_level,
runpath=runpath,
path_cleanup=path_cleanup,
all_tasks_local=all_tasks_local,
shuffle=shuffle,
shuffle_seed=shuffle_seed,
exporters=exporters,
stdout_style=stdout_style,
report_dir=report_dir,
xml_dir=xml_dir,
json_path=json_path,
http_url=http_url,
pdf_path=pdf_path,
pdf_style=pdf_style,
report_tags=report_tags,
report_tags_all=report_tags_all,
merge_scheduled_parts=merge_scheduled_parts,
browse=browse,
ui_port=ui_port,
web_server_startup_timeout=web_server_startup_timeout,
test_filter=test_filter,
test_sorter=test_sorter,
test_lister=test_lister,
verbose=verbose,
debug=debug,
timeout=timeout,
interactive_handler=interactive_handler,
extra_deps=extra_deps,
label=label,
**options)
try:
if arity(definition) == 2:
returned = definition(plan, plan.parser)
else:
returned = definition(plan)
except Exception:
print("Exception in test_plan definition, aborting plan..")
plan.abort()
raise
plan_result = plan.run()
plan_result.decorated_value = returned
return plan_result
return test_plan_inner_inner
return test_plan_inner
@testcase
def test_aaa(self, env, result):
pass
@testcase(parameters=[3, 2, 1])
def test_yyy(self, env, result, val):
pass
@pytest.mark.parametrize(
"sorter, report_ctx",
(
# Case 1, noop, original declaration & insertion order
(
ordering.NoopSorter(),
[(
"Multitest",
[
(
"Beta",
[
"test_ccc",
"test_xxx",
(
"test_bbb",
["test_bbb <val=0>", "test_bbb <val=1>"],
),
"test_aaa",
(
"test_yyy",
def get_options(cls):
return {
'name':
str,
ConfigOption('description', default=None):
Or(str, None),
ConfigOption('logger_level', default=logger.TEST_INFO):
int,
ConfigOption('file_log_level', default=logger.DEBUG):
Or(int, None),
ConfigOption(
'runpath',
default=default_runpath,
):
Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True):
bool,
ConfigOption('all_tasks_local', default=False):
bool,
ConfigOption('shuffle', default=[]):
list, # list of string choices
ConfigOption('shuffle_seed',
default=float(random.randint(1, 9999))):
float,
ConfigOption('exporters', default=None):
Use(get_exporters),
ConfigOption('stdout_style', default=defaults.STDOUT_STYLE):
Style,
ConfigOption('report_dir', default=defaults.REPORT_DIR):
Or(str, None),
ConfigOption(
'xml_dir',
default=None,
):
Or(str, None),
ConfigOption(
'pdf_path',
default=None,
):
Or(str, None),
ConfigOption(
'json_path',
default=None,
):
Or(str, None),
ConfigOption('http_url', default=None):
Or(str, None),
ConfigOption(
'pdf_style',
default=defaults.PDF_STYLE,
):
Style,
ConfigOption('report_tags', default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all', default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption('merge_scheduled_parts', default=False):
bool,
ConfigOption('browse', default=None):
Or(None, bool),
ConfigOption('ui_port', default=None):
Or(None, int),
ConfigOption('web_server_startup_timeout',
default=defaults.WEB_SERVER_TIMEOUT):
int,
ConfigOption('test_filter', default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption('test_sorter', default=ordering.NoopSorter()):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None):
Or(None, listing.BaseLister),
ConfigOption('verbose', default=False):
bool,
ConfigOption('debug', default=False):
bool,
ConfigOption('timeout', default=None):
Or(None, And(Or(int, float), lambda t: t >= 0)),
ConfigOption('interactive_handler', default=TestRunnerIHandler):
object,
ConfigOption('extra_deps', default=[]):
list
}
def get_options(cls):
return {
"name":
str,
ConfigOption("description", default=None):
Or(str, None),
ConfigOption("logger_level", default=logger.TEST_INFO):
int,
ConfigOption("file_log_level", default=logger.DEBUG):
int,
ConfigOption("runpath", default=default_runpath):
Or(None, str, lambda x: callable(x)),
ConfigOption("path_cleanup", default=True):
bool,
ConfigOption("all_tasks_local", default=False):
bool,
ConfigOption("shuffle", default=[]):
list, # list of string choices
ConfigOption("shuffle_seed",
default=float(random.randint(1, 9999))):
float,
ConfigOption("exporters", default=None):
Use(get_exporters),
ConfigOption("stdout_style", default=defaults.STDOUT_STYLE):
Style,
ConfigOption("report_dir", default=defaults.REPORT_DIR):
Or(str, None),
ConfigOption("xml_dir", default=None):
Or(str, None),
ConfigOption("pdf_path", default=None):
Or(str, None),
ConfigOption("json_path", default=None):
Or(str, None),
ConfigOption("http_url", default=None):
Or(str, None),
ConfigOption("pdf_style", default=defaults.PDF_STYLE):
Style,
ConfigOption("report_tags", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("report_tags_all", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("merge_scheduled_parts", default=False):
bool,
ConfigOption("browse", default=False):
bool,
ConfigOption("ui_port", default=None):
Or(None, int),
ConfigOption(
"web_server_startup_timeout",
default=defaults.WEB_SERVER_TIMEOUT,
):
int,
ConfigOption("test_filter", default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption("test_sorter", default=ordering.NoopSorter()):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption("test_lister", default=None):
Or(None, listing.BaseLister),
ConfigOption("verbose", default=False):
bool,
ConfigOption("debug", default=False):
bool,
ConfigOption("timeout", default=defaults.TESTPLAN_TIMEOUT):
Or(None, And(int, lambda t: t >= 0)),
# active_loop_sleep impacts cpu usage in interactive mode
ConfigOption("active_loop_sleep", default=0.05):
float,
ConfigOption("interactive_handler", default=TestRunnerIHandler):
object,
ConfigOption("extra_deps", default=[]):
[Or(str, lambda x: inspect.ismodule(x))],
ConfigOption("label", default=None):
Or(None, str),
}
'Secondary',
' Gamma',
' test_c',
' test_b',
' test_a',
)
@pytest.mark.parametrize(
'listing_obj,filter_obj,sorter_obj,expected_output',
[
# Basic name listing
(
listing.ExpandedNameLister(),
filtering.Filter(),
ordering.NoopSorter(),
DEFAULT_NAME_OUTPUT,
),
# Basic pattern listing
(listing.ExpandedPatternLister(), filtering.Filter(),
ordering.NoopSorter(), DEFAULT_PATTERN_OUTPUT),
# Basic count listing
(listing.CountLister(), filtering.Filter(), ordering.NoopSorter(),
to_stdout(
'Primary: (2 suites, 6 testcases)',
'Secondary: (1 suite, 3 testcases)',
)),
# Custom sort & name listing
(listing.ExpandedNameLister(), filtering.Filter(),
ordering.AlphanumericSorter(),
to_stdout(
def get_options(cls):
return {
'name':
str,
ConfigOption('logger_level', default=TEST_INFO):
int,
ConfigOption('runpath',
default=default_runpath,
block_propagation=False):
Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True):
bool,
ConfigOption('all_tasks_local', default=False):
bool,
ConfigOption('shuffle', default=[]):
list, # list of string choices
ConfigOption('shuffle_seed',
default=float(random.randint(1, 9999))):
float,
ConfigOption('exporters', default=None):
Use(get_exporters),
ConfigOption('stdout_style',
default=defaults.STDOUT_STYLE,
block_propagation=False):
Style,
ConfigOption('report_dir',
default=defaults.REPORT_DIR,
block_propagation=False):
str,
ConfigOption('xml_dir', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('json_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_style',
default=defaults.PDF_STYLE,
block_propagation=False):
Style,
ConfigOption('report_tags', default=[], block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all',
default=[],
block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('browse', default=False):
bool,
ConfigOption('test_filter',
default=filtering.Filter(),
block_propagation=False):
filtering.BaseFilter,
ConfigOption('test_sorter',
default=ordering.NoopSorter(),
block_propagation=False):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None, block_propagation=False):
Or(None, listing.BaseLister),
ConfigOption('verbose', default=False, block_propagation=False):
bool,
ConfigOption('debug', default=False, block_propagation=False):
bool
}
