def print_summary(self):
print(_('\n{0} Test(s) Executed!'.format(self.total)))
if len(self.failed_tests) > 0:
print(_('\nFailed test information:'))
for wrapper in self.failed_tests:
self.print_error(wrapper)
def print_summary(self):
print(_('\n{0} Test(s) Executed!'.format(self.total)))
if len(self.failed_tests) > 0:
print(_('\nFailed test information:'))
for wrapper in self.failed_tests:
self.print_error(wrapper)
def add_arguments(self, argparser):
argparser.add_argument(
'--show-all-expects', dest='show_all_expects', action='store_true',
help=_('Displays all expectations for test cases'))
argparser.add_argument(
'--ascii-only', dest='ascii_only', action='store_true',
help=_('Disables color and uses only ascii characters '
'(useful for CI systems)'))
def setup_argparse(self):
self.arg_parser.add_argument(
'--coverage', dest='coverage', action='store_true',
help=_('Activates coverage.py integration'))
self.arg_parser.add_argument(
'--search', type=str, dest='search', metavar='',
help=_('The spec suite folder path.'))
self.arg_parser.add_argument(
'--no-art', dest='no_art', action='store_true',
help=_('Disables ASCII art'))
self.arg_parser.add_argument(
'--select-module', dest='select_module', metavar='',
help=_('Selects a module path to run. '
'Ex: spec.sample.TestClass'),
default=None)
self.arg_parser.add_argument(
'--select-by-metadata', dest='select_meta', metavar='',
help=_('Selects tests to run by specifying a list of '
'key=value pairs you wish to run'),
default=[], nargs='*')
self.arg_parser.add_argument(
'--no-color', dest='no_color', action='store_true',
help=_('Disables all ASCII color codes.'))
self.arg_parser.add_argument(
'--xunit-results', dest='xunit_results', metavar='',
help=_('Saves out xUnit compatible results to a specifed file'))
self.arg_parser.add_argument(
'--parallel', dest='parallel', action='store_true',
help=_('Activate parallel testing mode'))
self.arg_parser.add_argument(
'--num-processes', dest='num_processes', default=6, metavar='',
help=_('Specifies the number of processes to use under '
'parallel mode (default: 6)'))
def print_param(value, param, indent, prefix=None):
if not expect.success and not hardcoded(param):
msg_list = str(value).splitlines() or ['']
prefix = _('{0}: {1}').format(param or prefix, msg_list[0])
print_indent_msg(prefix, indent)
if len(msg_list) > 1:
print_msg_list(msg_list[1:], indent)
def raise_a(self, exception):
self.expected = exception
self.actions.extend(['raise', exception])
condition = False
try:
self.target(*self.caller_args)
except Exception as e:
condition = True and type(e) is exception
self.success = condition if not self.used_negative else not condition
if not self.success:
was = 'wasn\'t' if self.used_negative else 'was'
# Make sure we have a name to use
if hasattr(self.expected, '__name__'):
name = self.expected.__name__
else:
name = type(self.expected).__name__
msg = _(
'function {func_name} {was} expected to raise "{exc}"'.format(
func_name=self.target_src_param,
exc=name,
was=was
)
)
self.custom_msg = msg
def print_param(value, param, indent, prefix):
if not expect.success and not hardcoded(param):
msg_list = str(value).splitlines() or ['']
prefix = _('{0}: {1}').format(param or prefix, msg_list[0])
print_indent_msg(prefix, indent)
if len(msg_list) > 1:
print_msg_list(msg_list[1:], indent)
def setup_argparse(self):
self.arg_parser.add_argument(
'--coverage',
dest='coverage',
action='store_true',
help=_('Activates coverage.py integration'))
self.arg_parser.add_argument('--search',
type=str,
dest='search',
metavar='',
help=_('The spec suite folder path.'))
self.arg_parser.add_argument('--no-art',
dest='no_art',
action='store_true',
help=_('Disables ASCII art'))
self.arg_parser.add_argument(
'--select-module',
dest='select_module',
metavar='',
help=_('Selects a module path to run. Ex: sample.TestClass'),
default=None)
self.arg_parser.add_argument(
'--select-tests',
dest='select_tests',
metavar='',
help=_('Selects tests by name (comma delimited list).'),
type=lambda s: [item.strip() for item in s.split(',')],
default=None)
self.arg_parser.add_argument(
'--select-by-metadata',
dest='select_meta',
metavar='',
help=_('Selects tests to run by specifying a list of '
'key=value pairs you wish to run'),
default=[],
nargs='*')
self.arg_parser.add_argument('--no-color',
dest='no_color',
action='store_true',
help=_('Disables all ASCII color codes.'))
self.arg_parser.add_argument('--parallel',
dest='parallel',
action='store_true',
help=_('Activate parallel testing mode'))
self.arg_parser.add_argument(
'--num-processes',
dest='num_processes',
type=int,
default=6,
metavar='',
help=_('Specifies the number of processes to use under '
'parallel mode (default: 6)'))
def run(self, args):
select_meta = None
self.arguments = self.arg_parser.parse_args(args)
self.reporter_manager = ReporterPluginManager()
# Let each reporter parse cli arguments
self.reporter_manager.process_arguments(self.arguments)
if self.arguments.parallel:
self.parallel_manager = ParallelManager(
num_processes=self.arguments.num_processes,
track_coverage=self.arguments.coverage,
coverage_omit=self.get_coverage_omit_list())
if self.arguments.select_meta:
metas = [meta.split('=') for meta in self.arguments.select_meta]
select_meta = {meta[0]: meta[1].strip('"\'') for meta in metas}
if not self.arguments.no_art:
print(self.generate_ascii_art())
if self.arguments.coverage:
print(_(' - Running with coverage enabled - '))
self.coverage = coverage(omit=self.get_coverage_omit_list(),
data_suffix=self.arguments.parallel)
self.coverage._warn_no_data = False
self.coverage.start()
self.suite_types = self.suite_scanner.scan(
search_path=self.arguments.search,
module_name=self.arguments.select_module)
# Serial: Add and Execute | Parallel: Collect all with the add process
for suite_type in self.suite_types:
suite = suite_type()
self.suites.append(suite)
self.reporter_manager.subscribe_all_to_describe(suite)
suite.execute(select_metadata=select_meta,
parallel_manager=self.parallel_manager)
# Actually execute the tests for parallel now
if self.arguments.parallel:
self.parallel_manager.execute_all()
# Save coverage data if enabled
if self.coverage:
self.coverage.stop()
self.coverage.save()
if self.arguments.parallel:
self.coverage.combine()
# Print all console summaries
for reporter in self.reporter_manager.get_console_reporters():
reporter.print_summary()
self.reporter_manager.finish_all()
self.suite_scanner.destroy()
def almost_equal(self, expected, places=7):
if not isinstance(places, int):
raise TypeError('Places must be an integer')
self._compare(action_name=_('almost equal'),
expected=expected,
condition=round(abs(self.target - expected),
places) == 0)
def almost_equal(self, expected, places=7):
if not isinstance(places, int):
raise TypeError('Places must be an integer')
self._compare(
action_name=_('almost equal'),
expected=expected,
condition=round(abs(self.target - expected), places) == 0
)
def _add_expect_to_wrapper(obj_to_add):
try:
for frame in inspect.stack():
if frame[3] == 'execute':
wrapper = frame[0].f_locals.get('self')
if type(wrapper) is CaseWrapper:
wrapper.expects.append(obj_to_add)
except Exception as error:
raise Exception(_('Error attempting to add expect to parent '
'wrapper: {err}').format(err=error))
def __init__(self, target, required=False, src_params=None):
super(ExpectAssert, self).__init__()
self.prefix = _('expect')
self.target = target
self.target_src_param = src_params[0] if src_params else None
self.expected_src_param = src_params[1] if src_params else None
self.actions = [target]
self.success = False
self.used_negative = False
self.required = required
def _add_expect_to_wrapper(obj_to_add):
try:
for frame in inspect.stack():
if frame[3] == 'execute':
wrapper = frame[0].f_locals.get('self')
if type(wrapper) is CaseWrapper:
wrapper.expects.append(obj_to_add)
except Exception as error:
raise Exception(
_('Error attempting to add expect to parent '
'wrapper: {err}').format(err=error))
def run(self, args):
select_meta = None
self.arguments = self.arg_parser.parse_args(args)
# Let each reporter parse cli arguments
self.reporter_manager.process_arguments(self.arguments)
if self.arguments.select_meta:
metas = [meta.split('=') for meta in self.arguments.select_meta]
select_meta = {meta[0]: meta[1].strip('"\'') for meta in metas}
if not self.arguments.no_art:
print(self.generate_ascii_art())
if self.arguments.coverage:
print(_(' - Running with coverage enabled - '))
self.coverage = coverage(omit=['*/pyevents/event.py',
'*/pyevents/manager.py',
'*/specter/spec.py',
'*/specter/expect.py',
'*/specter/reporting/__init__.py',
'*/specter/reporting/console.py',
'*/specter/__init__.py'])
self.coverage._warn_no_data = False
self.suite_types = self.suite_scanner.scan(
search_path=self.arguments.search,
module_name=self.arguments.select_module)
for suite_type in self.suite_types:
# Start Coverage Capture
if self.coverage:
self.coverage.start()
suite = suite_type()
self.suites.append(suite)
self.reporter_manager.subscribe_all_to_describe(suite)
suite.execute(select_metadata=select_meta)
# Start Coverage Capture
if self.coverage:
self.coverage.stop()
# Save coverage data if enabled
if self.coverage:
self.coverage.save()
# Print all console summaries
for reporter in self.reporter_manager.get_console_reporters():
reporter.print_summary()
self.reporter_manager.finish_all()
self.suite_scanner.destroy()
def generate_ascii_art(self):
tag_line = _('Keeping the Bogeyman away from your code!')
ascii_art = """
___
_/ @@\\
~- ( \\ O/__ Specter
~- \\ \\__) ~~~~~~~~~~
~- / \\ {tag}
~- / _\\
~~~~~~~~~
""".format(tag=tag_line)
return ascii_art
def generate_ascii_art(self):
tag_line = _('Keeping the boogy man away from your code!')
ascii_art = """
___
_/ @@\\
~- ( \\ O/__ Specter
~- \\ \\__) ~~~~~~~~~~
~- / \\ {tag}
~- / _\\
~~~~~~~~~
""".format(tag=tag_line)
return ascii_art
def __init__(self, target, required=False, src_params=None,
caller_args=[]):
super(ExpectAssert, self).__init__()
self.prefix = _('expect')
self.target = target
self.src_params = src_params
self.actions = [target]
self.success = False
self.used_negative = False
self.required = required
self.caller_args = caller_args
self.custom_msg = None
self.custom_report_vars = {}
def __init__(self,
target,
required=False,
src_params=None,
caller_args=[]):
super(ExpectAssert, self).__init__()
self.prefix = _('expect')
self.target = target
self.src_params = src_params
self.actions = [target]
self.success = False
self.used_negative = False
self.required = required
self.caller_args = caller_args
self.custom_msg = None
self.custom_report_vars = {}
def raise_a(self, exception):
self.expected = exception
self.actions.extend(['raise', exception])
condition = False
raised_exc = 'nothing'
try:
self.target(*self.caller_args)
except Exception as e:
condition = type(e) == exception
raised_exc = e
# We didn't raise anything
if self.used_negative and not isinstance(raised_exc, Exception):
self.success = True
# Raised, but it didn't match
elif self.used_negative and type(raised_exc) != exception:
self.success = False
elif self.used_negative:
self.success = not condition
else:
self.success = condition
if not self.success:
was = 'wasn\'t' if self.used_negative else 'was'
# Make sure we have a name to use
if hasattr(self.expected, '__name__'):
name = self.expected.__name__
else:
name = type(self.expected).__name__
msg = _('function {func_name} {was} expected to raise "{exc}".')
self.custom_msg = msg.format(
func_name=self.target_src_param,
exc=name,
was=was
)
self.custom_report_vars['Raised Exception'] = (
type(raised_exc).__name__
)
def raise_a(self, exception):
self.expected = exception
self.actions.extend(['raise', exception])
condition = False
try:
self.target(*self.caller_args)
except Exception as e:
condition = True and type(e) is exception
self.success = condition if not self.used_negative else not condition
if not self.success:
was = 'wasn\'t' if self.used_negative else 'was'
msg = _('Function {func_name} {was} expected to raise "{excpt}"'
''.format(func_name=self.target_src_param,
excpt=self.expected.__name__,
was=was))
self.custom_msg = msg
def raise_a(self, exception):
self.expected = exception
self.actions.extend(['raise', exception])
condition = False
raised_exc = 'nothing'
try:
self.target(*self.caller_args)
except Exception as e:
condition = type(e) == exception
raised_exc = e
# We didn't raise anything
if self.used_negative and not isinstance(raised_exc, Exception):
self.success = True
# Raised, but it didn't match
elif self.used_negative and type(raised_exc) != exception:
self.success = False
elif self.used_negative:
self.success = not condition
else:
self.success = condition
if not self.success:
was = 'wasn\'t' if self.used_negative else 'was'
# Make sure we have a name to use
if hasattr(self.expected, '__name__'):
name = self.expected.__name__
else:
name = type(self.expected).__name__
msg = _('function {func_name} {was} expected to raise "{exc}".')
self.custom_msg = msg.format(func_name=self.target_src_param,
exc=name,
was=was)
self.custom_report_vars['Raised Exception'] = (
type(raised_exc).__name__)
def setup_argparse(self):
self.arg_parser.add_argument(
'--coverage', dest='coverage', action='store_true',
help=_('Activates coverage.py integration'))
self.arg_parser.add_argument(
'--search', type=str, dest='search',
help=_('The spec suite folder path.'))
self.arg_parser.add_argument(
'--no-art', dest='no_art', action='store_true',
help=_('Disables ASCII art'))
self.arg_parser.add_argument(
'--select-module', dest='select_module',
help=_('Selects a module path to run. '
'Ex: spec.sample.TestClass'),
default=None)
self.arg_parser.add_argument(
'--select-by-metadata', dest='select_meta',
help=_('Selects tests to run by specifying a list of '
'key=value pairs you wish to run'),
default=[], nargs='*')
self.arg_parser.add_argument(
'--no-color', dest='no_color', action='store_true',
help=_('Disables all ASCII color codes.'))
def be_in(self, expected):
self._compare(action_name=_('be in'), expected=expected,
condition=expected in self.target)
def be_true(self):
self._compare(action_name=_('be'), expected=True,
condition=self.target == True)
def be_less_than(self, expected):
self._compare(action_name=_('be less than'), expected=expected,
condition=self.target < expected)
def equal(self, expected):
self._compare(action_name=_('equal'), expected=expected,
condition=self.target == expected)
def not_to(self):
self.actions.append(_('not'))
self.used_negative = not self.used_negative
return self.to
class SpecterRunner(object):
DESCRIPTION = _('Specter is a spec-based testing library to help '
'facilitate BDD in Python.')
def __init__(self):
super(SpecterRunner, self).__init__()
self.coverage = None
self.suite_scanner = None
self.arg_parser = ArgumentParser(description=self.DESCRIPTION)
self.setup_argparse()
self.suites = []
self.reporter_manager = None
self.parallel_manager = None
def setup_argparse(self):
self.arg_parser.add_argument(
'--coverage',
dest='coverage',
action='store_true',
help=_('Activates coverage.py integration'))
self.arg_parser.add_argument('--search',
type=str,
dest='search',
metavar='',
help=_('The spec suite folder path.'))
self.arg_parser.add_argument('--no-art',
dest='no_art',
action='store_true',
help=_('Disables ASCII art'))
self.arg_parser.add_argument(
'--select-module',
dest='select_module',
metavar='',
help=_('Selects a module path to run. Ex: sample.TestClass'),
default=None)
self.arg_parser.add_argument(
'--select-tests',
dest='select_tests',
metavar='',
help=_('Selects tests by name (comma delimited list).'),
type=lambda s: [item.strip() for item in s.split(',')],
default=None)
self.arg_parser.add_argument(
'--select-by-metadata',
dest='select_meta',
metavar='',
help=_('Selects tests to run by specifying a list of '
'key=value pairs you wish to run'),
default=[],
nargs='*')
self.arg_parser.add_argument('--no-color',
dest='no_color',
action='store_true',
help=_('Disables all ASCII color codes.'))
self.arg_parser.add_argument('--parallel',
dest='parallel',
action='store_true',
help=_('Activate parallel testing mode'))
self.arg_parser.add_argument(
'--num-processes',
dest='num_processes',
type=int,
default=6,
metavar='',
help=_('Specifies the number of processes to use under '
'parallel mode (default: 6)'))
def generate_ascii_art(self):
tag_line = _('Keeping the Bogeyman away from your code!')
ascii_art = """
___
_/ @@\\
~- ( \\ O/__ Specter
~- \\ \\__) ~~~~~~~~~~
~- / \\ {tag}
~- / _\\
~~~~~~~~~
""".format(tag=tag_line)
return ascii_art
def get_coverage_omit_list(self):
omit_list = [
'*/pyevents/event.py', '*/pyevents/manager.py',
'*/specter/spec.py', '*/specter/expect.py',
'*/specter/parallel.py', '*/specter/scanner.py',
'*/specter/runner.py', '*/specter/util.py',
'*/specter/reporting/__init__.py',
'*/specter/reporting/console.py', '*/specter/reporting/dots.py',
'*/specter/reporting/xunit.py', '*/specter/__init__.py'
]
return omit_list
def combine_coverage_reports(self, omit, parallel):
""" Method to force the combination of parallel coverage reports."""
tmp_cov = coverage.coverage(omit=omit, data_suffix=parallel)
tmp_cov.load()
tmp_cov.combine()
tmp_cov.save()
def run(self, args):
select_meta = None
self.reporter_manager = ReporterPluginManager()
self.reporter_manager.add_to_arguments(self.arg_parser)
self.arguments = self.arg_parser.parse_args(args)
# Let each reporter parse cli arguments
self.reporter_manager.process_arguments(self.arguments)
if self.arguments.parallel:
coverage.process_startup()
self.parallel_manager = ParallelManager(
num_processes=self.arguments.num_processes,
track_coverage=self.arguments.coverage,
coverage_omit=self.get_coverage_omit_list())
if self.arguments.select_meta:
metas = [meta.split('=') for meta in self.arguments.select_meta]
select_meta = {meta[0]: meta[1].strip('"\'') for meta in metas}
if not self.arguments.no_art:
print(self.generate_ascii_art())
if self.arguments.coverage:
print(_(' - Running with coverage enabled - '))
self.coverage = coverage.coverage(
omit=self.get_coverage_omit_list(),
data_suffix=self.arguments.parallel)
self.coverage._warn_no_data = False
self.coverage.start()
self.suite_scanner = SuiteScanner(self.arguments.search or 'spec')
self.suite_types = self.suite_scanner.scan(
self.arguments.select_module)
# Serial: Add and Execute | Parallel: Collect all with the add process
for suite_type in self.suite_types:
suite = suite_type()
self.suites.append(suite)
self.reporter_manager.subscribe_all_to_spec(suite)
suite.execute(select_metadata=select_meta,
parallel_manager=self.parallel_manager,
select_tests=self.arguments.select_tests)
# Actually execute the tests for parallel now
if self.arguments.parallel:
self.parallel_manager.execute_all()
# Save coverage data if enabled
if self.coverage:
self.coverage.stop()
self.coverage.save()
if self.arguments.parallel:
self.combine_coverage_reports(self.get_coverage_omit_list(),
self.arguments.parallel)
# Print all console summaries
for reporter in self.reporter_manager.get_console_reporters():
reporter.print_summary()
self.reporter_manager.finish_all()
self.suite_scanner.destroy()
def be_false(self):
self._compare(action_name=_('be'),
expected=False,
condition=not self.target)
def be_a(self, expected):
self._compare(action_name=_('be a'), expected=expected,
condition=type(self.target) is expected)
def be_none(self):
self._compare(action_name=_('be'),
expected=None,
condition=self.target is None)
def be_an_instance_of(self, expected):
self._compare(action_name=_('be an instance of'),
expected=expected,
condition=isinstance(self.target, expected))
def be_a(self, expected):
self._compare(action_name=_('be a'),
expected=expected,
condition=type(self.target) is expected)
def be_in(self, expected):
self._compare(action_name=_('be in'),
expected=expected,
condition=self.target in expected)
def contain(self, expected):
self._compare(action_name=_('contain'),
expected=expected,
condition=expected in self.target)
def __init__(self, target, src_params=None, caller_args=[]):
super(RequireAssert, self).__init__(target=target,
required=True,
src_params=src_params,
caller_args=caller_args)
self.prefix = _('require')
def add_arguments(self, argparser):
argparser.add_argument(
'--json-results',
dest='json_results',
metavar='',
help=_('Saves Specter JSON results into a specifed file'))
def be_an_instance_of(self, expected):
self._compare(action_name=_('be an instance of'), expected=expected,
condition=isinstance(self.target, expected))
def run(self, args):
select_meta = None
self.reporter_manager = ReporterPluginManager()
self.reporter_manager.add_to_arguments(self.arg_parser)
self.arguments = self.arg_parser.parse_args(args)
# Let each reporter parse cli arguments
self.reporter_manager.process_arguments(self.arguments)
if self.arguments.parallel:
coverage.process_startup()
self.parallel_manager = ParallelManager(
num_processes=self.arguments.num_processes,
track_coverage=self.arguments.coverage,
coverage_omit=self.get_coverage_omit_list())
if self.arguments.select_meta:
metas = [meta.split('=') for meta in self.arguments.select_meta]
select_meta = {meta[0]: meta[1].strip('"\'') for meta in metas}
if not self.arguments.no_art:
print(self.generate_ascii_art())
if self.arguments.coverage:
print(_(' - Running with coverage enabled - '))
self.coverage = coverage.coverage(
omit=self.get_coverage_omit_list(),
data_suffix=self.arguments.parallel)
self.coverage._warn_no_data = False
self.coverage.start()
self.suite_scanner = SuiteScanner(self.arguments.search or 'spec')
self.suite_types = self.suite_scanner.scan(
self.arguments.select_module)
# Serial: Add and Execute | Parallel: Collect all with the add process
for suite_type in self.suite_types:
suite = suite_type()
self.suites.append(suite)
self.reporter_manager.subscribe_all_to_spec(suite)
suite.execute(select_metadata=select_meta,
parallel_manager=self.parallel_manager,
select_tests=self.arguments.select_tests)
# Actually execute the tests for parallel now
if self.arguments.parallel:
self.parallel_manager.execute_all()
# Save coverage data if enabled
if self.coverage:
self.coverage.stop()
self.coverage.save()
if self.arguments.parallel:
self.combine_coverage_reports(self.get_coverage_omit_list(),
self.arguments.parallel)
# Print all console summaries
for reporter in self.reporter_manager.get_console_reporters():
reporter.print_summary()
self.reporter_manager.finish_all()
self.suite_scanner.destroy()
def skip_wrapper(*args, **kwargs):
raise TestIncompleteException(test_func, _('Test is incomplete'))
def skip_wrapper(*args, **kwargs):
raise TestIncompleteException(test_func, _('Test is incomplete'))
def not_to(self):
self.actions.append(_('not'))
self.used_negative = not self.used_negative
return self.to
def to(self):
self.actions.append(_('to'))
return self
def to(self):
self.actions.append(_('to'))
return self
def be_greater_than(self, expected):
self._compare(action_name=_('be greater than'), expected=expected,
condition=self.target > expected)
def equal(self, expected):
self._compare(action_name=_('equal'),
expected=expected,
condition=self.target == expected)
def be_none(self):
self._compare(action_name=_('be'), expected=None,
condition=self.target is None)
def get_name(self):
return _('Dots Reporter')
def be_false(self):
self._compare(action_name=_('be'), expected=False,
condition=self.target == False)
def be_greater_than(self, expected):
self._compare(action_name=_('be greater than'),
expected=expected,
condition=self.target > expected)
def __init__(self, target, src_params=None):
super(RequireAssert, self).__init__(target=target, required=True,
src_params=src_params)
self.prefix = _('require')
def be_less_than(self, expected):
self._compare(action_name=_('be less than'),
expected=expected,
condition=self.target < expected)
def add_arguments(self, argparser):
argparser.add_argument(
"--json-results", dest="json_results", metavar="", help=_("Saves Specter JSON results into a specifed file")
)
def get_name(self):
return _('Dots Reporter')
def add_arguments(self, argparser):
argparser.add_argument(
'--xunit-results', dest='xunit_results', metavar='',
help=_('Saves out xUnit compatible results to a specifed file'))
def be_true(self):
self._compare(action_name=_('be'),
expected=True,
condition=self.target)
