def irunner(): """Set up an irunner instance for testing.""" target = runnable.TestRunner(name="TestRunner") local_runner = runners.LocalRunner() test_uids = ["test_1", "test_2", "test_3"] test_objs = [ multitest.MultiTest( name=uid, suites=[Suite()], test_filter=filtering.Filter(), test_sorter=ordering.NoopSorter(), stdout_style=defaults.STDOUT_STYLE, environment=[driver.Driver(name="mock_driver")], ) for uid in test_uids ] for test in test_objs: local_runner.add(test, test.uid()) target.resources.add(local_runner) with mock.patch("cheroot.wsgi.Server"): irunner = base.TestRunnerIHandler(target) irunner.setup() yield irunner irunner.teardown()
def get_options(cls): return { 'name': str, ConfigOption('logger_level', default=TEST_INFO): int, ConfigOption('runpath', default=default_runpath, block_propagation=False): Or(None, str, lambda x: callable(x)), ConfigOption('path_cleanup', default=True): bool, ConfigOption('all_tasks_local', default=False): bool, ConfigOption('shuffle', default=[]): list, # list of string choices ConfigOption('shuffle_seed', default=float(random.randint(1, 9999))): float, ConfigOption('exporters', default=None): Use(get_exporters), ConfigOption('stdout_style', default=defaults.STDOUT_STYLE, block_propagation=False): Style, ConfigOption('report_dir', default=defaults.REPORT_DIR, block_propagation=False): str, ConfigOption('xml_dir', default=None, block_propagation=False): Or(str, None), ConfigOption('pdf_path', default=None, block_propagation=False): Or(str, None), ConfigOption('json_path', default=None, block_propagation=False): Or(str, None), ConfigOption('pdf_style', default=defaults.PDF_STYLE, block_propagation=False): Style, ConfigOption('report_tags', default=[], block_propagation=False): [Use(tagging.validate_tag_value)], ConfigOption('report_tags_all', default=[], block_propagation=False): [Use(tagging.validate_tag_value)], ConfigOption('browse', default=False): bool, ConfigOption('test_filter', default=filtering.Filter(), block_propagation=False): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter(), block_propagation=False): ordering.BaseSorter, # Test lister is None by default, otherwise Testplan would # list tests, not run them ConfigOption('test_lister', default=None, block_propagation=False): Or(None, listing.BaseLister) }
def get_options(cls): return { 'name': str, ConfigOption('description', default=None): Or(str, None), ConfigOption('logger_level', default=logger.TEST_INFO): int, ConfigOption('file_log_level', default=logger.DEBUG): Or(int, None), ConfigOption( 'runpath', default=default_runpath, ): Or(None, str, lambda x: callable(x)), ConfigOption('path_cleanup', default=True): bool, ConfigOption('all_tasks_local', default=False): bool, ConfigOption('shuffle', default=[]): list, # list of string choices ConfigOption( 'shuffle_seed', default=float(random.randint(1, 9999))): float, ConfigOption( 'exporters', default=None): Use(get_exporters), ConfigOption( 'stdout_style', default=defaults.STDOUT_STYLE): Style, ConfigOption( 'report_dir', default=defaults.REPORT_DIR): Or(str, None), ConfigOption('xml_dir', default=None,): Or(str, None), ConfigOption('pdf_path', default=None,): Or(str, None), ConfigOption('json_path', default=None,): Or(str, None), ConfigOption('http_url', default=None): Or(str, None), ConfigOption('pdf_style', default=defaults.PDF_STYLE,): Style, ConfigOption('report_tags', default=[] ): [Use(tagging.validate_tag_value)], ConfigOption('report_tags_all', default=[] ): [Use(tagging.validate_tag_value)], ConfigOption('merge_scheduled_parts', default=False): bool, ConfigOption('browse', default=False): bool, ConfigOption('ui_port', default=None): Or(None, int), ConfigOption('web_server_startup_timeout', default=defaults.WEB_SERVER_TIMEOUT): int, ConfigOption('test_filter', default=filtering.Filter() ): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter() ): ordering.BaseSorter, # Test lister is None by default, otherwise Testplan would # list tests, not run them ConfigOption('test_lister', default=None ):Or(None, listing.BaseLister), ConfigOption('verbose', default=False): bool, ConfigOption('debug', default=False): bool, ConfigOption( 'timeout', default=None): Or( None, And(Or(int, float), lambda t: t >= 0)), ConfigOption('interactive_handler', default=TestRunnerIHandler): object, ConfigOption('extra_deps', default=[]): list }
def configuration_schema(self): """ Schema for options validation and assignment of default values. """ overrides = Schema( { 'name': str, ConfigOption('logger_level', default=TEST_INFO): int, ConfigOption('runpath', default=default_runpath): Or(None, str, lambda x: callable(x)), ConfigOption('path_cleanup', default=True): bool, ConfigOption('all_tasks_local', default=False): bool, ConfigOption('shuffle', default=[]): list, # list of string choices ConfigOption('shuffle_seed', default=float(random.randint(1, 9999))): float, ConfigOption('exporters', default=None): Use(get_exporters), ConfigOption('stdout_style', default=defaults.STDOUT_STYLE): Style, ConfigOption('report_dir', default=defaults.REPORT_DIR): str, ConfigOption('xml_dir', default=None): Or(str, None), ConfigOption('pdf_path', default=None): Or(str, None), ConfigOption('json_path', default=None): Or(str, None), ConfigOption('pdf_style', default=defaults.PDF_STYLE): Style, ConfigOption('report_tags', default=[]): [Use(tagging.validate_tag_value)], ConfigOption('report_tags_all', default=[]): [Use(tagging.validate_tag_value)], ConfigOption('browse', default=False): bool, ConfigOption('test_filter', default=filtering.Filter()): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter()): ordering.BaseSorter, # Test lister is None by default, otherwise Testplan would # list tests, not run them ConfigOption('test_lister', default=None): Or(None, listing.BaseLister) }, ignore_extra_keys=True) return self.inherit_schema(overrides, super(TestRunnerConfig, self))
def get_options(cls): return { 'name': str, ConfigOption('description', default=None): str, ConfigOption('test_filter', default=filtering.Filter()): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter()): ordering.BaseSorter, ConfigOption('stdout_style', default=defaults.STDOUT_STYLE): test_styles.Style, ConfigOption('tags', default=None): Or(None, Use(tagging.validate_tag_value)) }
def configuration_schema(self): """ Schema for options validation and assignment of default values. """ overrides = { 'name': str, ConfigOption('description', default=None): str, ConfigOption('test_filter', default=filtering.Filter()): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter()): ordering.BaseSorter, ConfigOption('stdout_style', default=defaults.STDOUT_STYLE): test_styles.Style } return self.inherit_schema(overrides, super(TestConfig, self))
def get_options(cls): return { # 'name': And(str, lambda s: s.count(' ') == 0), 'name': str, ConfigOption('description', default=None): str, ConfigOption('test_filter', default=filtering.Filter(), block_propagation=False): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter(), block_propagation=False): ordering.BaseSorter, ConfigOption('stdout_style', default=defaults.STDOUT_STYLE, block_propagation=False): test_styles.Style, ConfigOption('tags', default=None): Or(None, Use(tagging.validate_tag_value)) }
def get_options(cls): start_stop_signature = Or( None, validate_func('env'), validate_func('env', 'result'), ) return { # 'name': And(str, lambda s: s.count(' ') == 0), 'name': str, ConfigOption('description', default=None): str, ConfigOption('environment', default=[]): [Resource], ConfigOption('before_start', default=None): start_stop_signature, ConfigOption('after_start', default=None): start_stop_signature, ConfigOption('before_stop', default=None): start_stop_signature, ConfigOption('after_stop', default=None): start_stop_signature, ConfigOption('test_filter', default=filtering.Filter(), block_propagation=False): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter(), block_propagation=False): ordering.BaseSorter, ConfigOption('stdout_style', default=defaults.STDOUT_STYLE, block_propagation=False): test_styles.Style, ConfigOption('tags', default=None): Or(None, Use(tagging.validate_tag_value)) }
" test_a", "Secondary", " Gamma", " test_c", " test_b", " test_a", ) @pytest.mark.parametrize( "listing_obj,filter_obj,sorter_obj,expected_output", [ # Basic name listing ( listing.ExpandedNameLister(), filtering.Filter(), ordering.NoopSorter(), DEFAULT_NAME_OUTPUT, ), # Basic pattern listing ( listing.ExpandedPatternLister(), filtering.Filter(), ordering.NoopSorter(), DEFAULT_PATTERN_OUTPUT, ), # Basic count listing ( listing.CountLister(), filtering.Filter(), ordering.NoopSorter(),
def get_options(cls): return { "name": str, ConfigOption("description", default=None): Or(str, None), ConfigOption("logger_level", default=logger.TEST_INFO): int, ConfigOption("file_log_level", default=logger.DEBUG): int, ConfigOption("runpath", default=default_runpath): Or(None, str, lambda x: callable(x)), ConfigOption("path_cleanup", default=True): bool, ConfigOption("all_tasks_local", default=False): bool, ConfigOption("shuffle", default=[]): list, # list of string choices ConfigOption("shuffle_seed", default=float(random.randint(1, 9999))): float, ConfigOption("exporters", default=None): Use(get_exporters), ConfigOption("stdout_style", default=defaults.STDOUT_STYLE): Style, ConfigOption("report_dir", default=defaults.REPORT_DIR): Or(str, None), ConfigOption("xml_dir", default=None): Or(str, None), ConfigOption("pdf_path", default=None): Or(str, None), ConfigOption("json_path", default=None): Or(str, None), ConfigOption("http_url", default=None): Or(str, None), ConfigOption("pdf_style", default=defaults.PDF_STYLE): Style, ConfigOption("report_tags", default=[]): [Use(tagging.validate_tag_value)], ConfigOption("report_tags_all", default=[]): [Use(tagging.validate_tag_value)], ConfigOption("merge_scheduled_parts", default=False): bool, ConfigOption("browse", default=False): bool, ConfigOption("ui_port", default=None): Or(None, int), ConfigOption( "web_server_startup_timeout", default=defaults.WEB_SERVER_TIMEOUT, ): int, ConfigOption("test_filter", default=filtering.Filter()): filtering.BaseFilter, ConfigOption("test_sorter", default=ordering.NoopSorter()): ordering.BaseSorter, # Test lister is None by default, otherwise Testplan would # list tests, not run them ConfigOption("test_lister", default=None): Or(None, listing.BaseLister), ConfigOption("verbose", default=False): bool, ConfigOption("debug", default=False): bool, ConfigOption("timeout", default=defaults.TESTPLAN_TIMEOUT): Or(None, And(int, lambda t: t >= 0)), ConfigOption("abort_wait_timeout", default=60): int, ConfigOption("interactive_handler", default=TestRunnerIHandler): object, ConfigOption("extra_deps", default=[]): [Or(str, lambda x: inspect.ismodule(x))], ConfigOption("label", default=None): Or(None, str), }
"""Unit tests for MultiTest base functionality.""" import os from testplan.common.utils import path from testplan.testing import multitest from testplan.testing.multitest import base from testplan.testing import filtering from testplan.testing import ordering from testplan import defaults from testplan import report from testplan.common import entity # TODO: shouldn't need to specify these... MTEST_DEFAULT_PARAMS = { "test_filter": filtering.Filter(), "test_sorter": ordering.NoopSorter(), "stdout_style": defaults.STDOUT_STYLE, } def test_multitest_runpath(): """Test setting of runpath.""" class Parent(object): def __init__(self, runpath): self.runpath = runpath global_runpath = os.path.join("", "var", "tmp", "global_level") local_runpath = os.path.join("", "var", "tmp", "local_runpath") par = Parent(global_runpath)
def main_wrapper(cls, name, description=None, parse_cmdline=True, interactive=False, port=None, abort_signals=None, logger_level=logger.TEST_INFO, file_log_level=logger.DEBUG, runpath=path.default_runpath, path_cleanup=True, all_tasks_local=False, shuffle=None, shuffle_seed=float(random.randint(1, 9999)), exporters=None, stdout_style=defaults.STDOUT_STYLE, report_dir=defaults.REPORT_DIR, xml_dir=None, json_path=None, http_url=None, pdf_path=None, pdf_style=defaults.PDF_STYLE, report_tags=None, report_tags_all=None, merge_scheduled_parts=False, browse=False, ui_port=None, web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT, test_filter=filtering.Filter(), test_sorter=ordering.NoopSorter(), test_lister=None, verbose=False, debug=False, timeout=None, interactive_handler=TestRunnerIHandler, extra_deps=None, **options): """ Decorator that will be used for wrapping `main` methods in test scripts. It accepts all arguments of a :py:class:`~testplan.base.Testplan` entity. """ def test_plan_inner(definition): """ This is being passed the user-defined testplan entry point. """ def test_plan_inner_inner(): """ This is the callable returned in the end, it executes the plan and the associated reporting """ plan = cls( name=name, description=description, parse_cmdline=parse_cmdline, interactive=interactive, port=port, abort_signals=abort_signals, logger_level=logger_level, file_log_level=file_log_level, runpath=runpath, path_cleanup=path_cleanup, all_tasks_local=all_tasks_local, shuffle=shuffle, shuffle_seed=shuffle_seed, exporters=exporters, stdout_style=stdout_style, report_dir=report_dir, xml_dir=xml_dir, json_path=json_path, http_url=http_url, pdf_path=pdf_path, pdf_style=pdf_style, report_tags=report_tags, report_tags_all=report_tags_all, merge_scheduled_parts=merge_scheduled_parts, browse=browse, ui_port=ui_port, web_server_startup_timeout=web_server_startup_timeout, test_filter=test_filter, test_sorter=test_sorter, test_lister=test_lister, verbose=verbose, debug=debug, timeout=timeout, interactive_handler=interactive_handler, extra_deps=extra_deps, **options) try: if arity(definition) == 2: returned = definition(plan, plan.parser) else: returned = definition(plan) except Exception: print('Exception in test_plan definition, aborting plan..') plan.abort() raise plan_result = plan.run() plan_result.decorated_value = returned return plan_result return test_plan_inner_inner return test_plan_inner
def __init__(self, name, description=None, parse_cmdline=True, interactive_port=None, abort_signals=None, logger_level=logger.TEST_INFO, file_log_level=logger.DEBUG, runpath=path.default_runpath, path_cleanup=True, all_tasks_local=False, shuffle=None, shuffle_seed=float(random.randint(1, 9999)), exporters=None, stdout_style=defaults.STDOUT_STYLE, report_dir=defaults.REPORT_DIR, xml_dir=None, json_path=None, http_url=None, pdf_path=None, pdf_style=defaults.PDF_STYLE, report_tags=None, report_tags_all=None, merge_scheduled_parts=False, browse=False, ui_port=None, web_server_startup_timeout=defaults.WEB_SERVER_TIMEOUT, test_filter=filtering.Filter(), test_sorter=ordering.NoopSorter(), test_lister=None, verbose=False, debug=False, timeout=None, interactive_handler=TestRunnerIHandler, extra_deps=None, **options): # Set mutable defaults. if abort_signals is None: abort_signals = entity.DEFAULT_RUNNABLE_ABORT_SIGNALS[:] if shuffle is None: shuffle = [] if extra_deps is None: extra_deps = [] if report_tags is None: report_tags = [] if report_tags_all is None: report_tags_all = [] super(Testplan, self).__init__( name=name, description=description, parse_cmdline=parse_cmdline, interactive_port=interactive_port, abort_signals=abort_signals, logger_level=logger_level, file_log_level=file_log_level, runpath=runpath, path_cleanup=path_cleanup, all_tasks_local=all_tasks_local, shuffle=shuffle, shuffle_seed=shuffle_seed, exporters=exporters, stdout_style=stdout_style, report_dir=report_dir, xml_dir=xml_dir, json_path=json_path, http_url=http_url, pdf_path=pdf_path, pdf_style=pdf_style, report_tags=report_tags, report_tags_all=report_tags_all, merge_scheduled_parts=merge_scheduled_parts, browse=browse, ui_port=ui_port, web_server_startup_timeout=web_server_startup_timeout, test_filter=test_filter, test_sorter=test_sorter, test_lister=test_lister, verbose=verbose, debug=debug, timeout=timeout, interactive_handler=interactive_handler, extra_deps=extra_deps, **options) for resource in self._cfg.resources: self._runnable.add_resource(resource) # Stores local tests. self._runnable.add_resource(LocalRunner(), uid='local_runner') # Stores independent environments. self._runnable.add_resource(Environments(), uid='environments')
pass @testcase(tags={'color': ('blue', 'yellow')}) def test_two(self, env, result): pass @testcase(tags={'color': ('red', 'green')}) def test_three(self, env, result): pass @pytest.mark.parametrize( 'filter_obj, report_ctx', ( # Case 1 (filtering.Filter(), [('XXX', [ ('Alpha', ['test_one', 'test_two', 'test_three']), ('Beta', ['test_one', 'test_two', 'test_three']), ]), ('YYY', (('Gamma', ['test_one', 'test_two', 'test_three']), ))]), # Case 2 (filtering.Pattern('*:*:test_two'), [('XXX', [ ('Alpha', ['test_two']), ('Beta', ['test_two']), ]), ('YYY', [ ('Gamma', ['test_two']), ])]), # Case 3 (filtering.Pattern('XXX:Beta:test_two'), [ ('XXX', [('Beta', ['test_two'])]), ]), # Case 4 - testcase name match AND tag match
' test_a', 'Secondary', ' Gamma', ' test_c', ' test_b', ' test_a', ) @pytest.mark.parametrize( 'listing_obj,filter_obj,sorter_obj,expected_output', [ # Basic name listing ( listing.ExpandedNameLister(), filtering.Filter(), ordering.NoopSorter(), DEFAULT_NAME_OUTPUT, ), # Basic pattern listing (listing.ExpandedPatternLister(), filtering.Filter(), ordering.NoopSorter(), DEFAULT_PATTERN_OUTPUT), # Basic count listing (listing.CountLister(), filtering.Filter(), ordering.NoopSorter(), to_stdout( 'Primary: (2 suites, 6 testcases)', 'Secondary: (1 suite, 3 testcases)', )), # Custom sort & name listing (listing.ExpandedNameLister(), filtering.Filter(), ordering.AlphanumericSorter(),