class BaseTestSuite(_TestAndSuiteHelper):
"""Base class for TestSuite used in runtime and by rebot."""
def __init__(self, name, source=None, parent=None):
_TestAndSuiteHelper.__init__(self, name, parent)
self.source = source is not None and utils.normpath(source) or None
self.metadata = utils.NormalizedDict()
self.suites = []
self.tests = []
self.critical = _Critical()
self.critical_stats = Stat()
self.all_stats = Stat()
if parent:
parent.suites.append(self)
def set_name(self, name):
if name:
self.name = name
elif not self.parent and self.name == '': # MultiSourceSuite
self.name = ' & '.join([suite.name for suite in self.suites])
def set_critical_tags(self, critical, non_critical):
if critical is not None or non_critical is not None:
self.critical.set(critical, non_critical)
self._set_critical_tags(self.critical)
def _set_critical_tags(self, critical):
self.critical = critical
for suite in self.suites:
suite._set_critical_tags(critical)
for test in self.tests:
test.set_criticality(critical)
def set_doc(self, doc):
if doc is not None:
self.doc = doc
def set_metadata(self, metalist):
for metastr in metalist:
try:
name, value = metastr.split(':', 1)
except ValueError:
name, value = metastr, ''
self.metadata[name] = value
def get_metadata(self, html=False):
names = sorted(self.metadata.keys())
values = [ self.metadata[n] for n in names ]
if html:
values = [ utils.html_escape(v, formatting=True) for v in values ]
return zip(names, values)
def get_test_count(self):
count = len(self.tests)
for suite in self.suites:
count += suite.get_test_count()
return count
def get_full_message(self, html=False):
"""Returns suite's message including statistics message"""
stat_msg = self.get_stat_message(html)
if self.message == '':
return stat_msg
if not html:
return '%s\n\n%s' % (self.message, stat_msg)
return '%s<br /><br />%s' % (utils.html_escape(self.message), stat_msg)
def get_stat_message(self, html=False):
ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats)
atotal, aend, apass, afail = self._get_counts(self.all_stats)
msg = ('%%d critical test%%s, %%d passed, %(cfail)s%%d failed%(end)s\n'
'%%d test%%s total, %%d passed, %(afail)s%%d failed%(end)s')
if html:
msg = msg.replace(' ', ' ').replace('\n', '<br />')
msg = msg % {'cfail': '<span%s>' % (cfail and ' class="fail"' or ''),
'afail': '<span%s>' % (afail and ' class="fail"' or ''),
'end': '</span>'}
else:
msg = msg % {'cfail': '', 'afail': '', 'end': ''}
return msg % (ctotal, cend, cpass, cfail, atotal, aend, apass, afail)
def _get_counts(self, stat):
total = stat.passed + stat.failed
ending = utils.plural_or_not(total)
return total, ending, stat.passed, stat.failed
def set_status(self):
"""Sets status and statistics based on subsuite and test statuses.
Can/should be used when statuses have been changed somehow.
"""
self._set_stats()
self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL'
def _set_stats(self):
self.critical_stats = Stat()
self.all_stats = Stat()
for suite in self.suites:
suite.set_status()
self._add_suite_to_stats(suite)
for test in self.tests:
self._add_test_to_stats(test)
def _add_test_to_stats(self, test):
self.all_stats.add_test(test)
if test.critical == 'yes':
self.critical_stats.add_test(test)
def _add_suite_to_stats(self, suite):
self.critical_stats.add_stat(suite.critical_stats)
self.all_stats.add_stat(suite.all_stats)
def suite_teardown_failed(self, message=None):
if message is not None:
self._set_teardown_fail_msg(message)
self.critical_stats.fail_all()
self.all_stats.fail_all()
self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL'
sub_message = 'Teardown of the parent suite failed.'
for suite in self.suites:
suite.suite_teardown_failed(sub_message)
for test in self.tests:
test.suite_teardown_failed(sub_message)
def set_tags(self, tags):
if tags:
for test in self.tests:
test.tags = utils.normalize_tags(test.tags + tags)
for suite in self.suites:
suite.set_tags(tags)
def filter(self, suites=None, tests=None, includes=None, excludes=None):
self.filter_by_names(suites, tests)
self.filter_by_tags(includes, excludes)
def filter_by_names(self, suites=None, tests=None):
suites = [ ([], name.split('.')) for name in suites or [] ]
tests = tests or []
if (suites or tests) and not self._filter_by_names(suites, tests):
self._raise_no_tests_filtered_by_names(suites, tests)
def _filter_by_names(self, suites, tests):
suites = self._filter_suite_names(suites)
self.suites = [ suite for suite in self.suites
if suite._filter_by_names(suites, tests) ]
if not suites:
self.tests = [ test for test in self.tests if tests == [] or
utils.matches_any(test.name, tests, ignore=['_']) ]
else:
self.tests = []
return self.suites or self.tests
def _filter_suite_names(self, suites):
try:
return [ self._filter_suite_name(p, s) for p, s in suites ]
except StopIteration:
return []
def _filter_suite_name(self, parent, suite):
if utils.matches(self.name, suite[0], ignore=['_']):
if len(suite) == 1:
raise StopIteration('Match found')
return (parent + [suite[0]], suite[1:])
return ([], parent + suite)
def _raise_no_tests_filtered_by_names(self, suites, tests):
tests = utils.seq2str(tests, lastsep=' or ')
suites = utils.seq2str([ '.'.join(p + s) for p, s in suites ],
lastsep=' or ')
if not suites:
msg = 'test cases named %s.' % tests
elif not tests:
msg = 'test suites named %s.' % suites
else:
msg = 'test cases %s in suites %s.' % (tests, suites)
raise DataError("Suite '%s' contains no %s" % (self.name, msg))
def filter_by_tags(self, includes=None, excludes=None):
if not (includes or excludes):
return
if not includes: includes = []
if not excludes: excludes = []
if not self._filter_by_tags(includes, excludes):
self._raise_no_tests_filtered_by_tags(includes, excludes)
def _filter_by_tags(self, incls, excls):
self.suites = [ suite for suite in self.suites
if suite._filter_by_tags(incls, excls) ]
self.tests = [ test for test in self.tests
if test.is_included(incls, excls) ]
return len(self.suites) + len(self.tests) > 0
def _raise_no_tests_filtered_by_tags(self, incls, excls):
incl = utils.seq2str(incls)
excl = utils.seq2str(excls)
msg = "Suite '%s' with " % self.name
if incl:
msg += 'includes %s ' % incl
if excl:
msg += 'and '
if excl:
msg += 'excludes %s ' % excl
raise DataError(msg + 'contains no test cases.')
def set_runmode(self, runmode):
runmode = runmode.upper()
if runmode == 'EXITONFAILURE':
self._run_mode_exit_on_failure = True
elif runmode == 'SKIPTEARDOWNONEXIT':
self._run_mode_skip_teardowns_on_exit = True
elif runmode == 'DRYRUN':
self._run_mode_dry_run = True
elif runmode == 'RANDOM:TEST':
random.shuffle(self.tests)
elif runmode == 'RANDOM:SUITE':
random.shuffle(self.suites)
elif runmode == 'RANDOM:ALL':
random.shuffle(self.suites)
random.shuffle(self.tests)
else:
return
for suite in self.suites:
suite.set_runmode(runmode)
def set_options(self, settings):
self.set_tags(settings['SetTag'])
self.filter(settings['SuiteNames'], settings['TestNames'],
settings['Include'], settings['Exclude'])
self.set_name(settings['Name'])
self.set_doc(settings['Doc'])
self.set_metadata(settings['Metadata'])
self.set_critical_tags(settings['Critical'], settings['NonCritical'])
try:
for runmode in settings['RunMode']:
self.set_runmode(runmode)
except (KeyError, AttributeError) : # Only applicable when running tcs
pass
if not self.suites:
settings['SplitOutputs'] = -2
try:
self.remove_keywords(settings['RemoveKeywords'])
except (KeyError, AttributeError): # Only applicable with Rebot
pass
def serialize(self, serializer):
serializer.start_suite(self)
if self.setup is not None:
self.setup.serialize(serializer)
if self.teardown is not None:
self.teardown.serialize(serializer)
for suite in self.suites:
suite.serialize(serializer)
for test in self.tests:
test.serialize(serializer)
serializer.end_suite(self)
class BaseTestSuite(_TestAndSuiteHelper):
"""Base class for TestSuite used in runtime and by rebot."""
def __init__(self, name, source=None, parent=None):
_TestAndSuiteHelper.__init__(self, name, parent)
self.source = utils.abspath(source) if source else None
self._id = None
self.metadata = utils.NormalizedDict()
self.suites = []
self.tests = []
self.critical = _Critical()
self.critical_stats = Stat()
self.all_stats = Stat()
if parent:
parent.suites.append(self)
def set_name(self, name):
if name:
self.name = name
elif self._is_multi_source_suite():
self.name = ' & '.join(suite.name for suite in self.suites)
def _is_multi_source_suite(self):
return self.parent is None and self.name == ''
@property
def id(self):
if not self._id:
self._find_root()._set_id()
return self._id
def _find_root(self):
if self.parent:
return self.parent._find_root()
return self
def _set_id(self):
if not self._id:
self._id = 's1'
for index, suite in enumerate(self.suites):
suite._id = '%s-s%s' % (self._id, index+1)
suite._set_id()
def set_critical_tags(self, critical, non_critical):
if critical is not None or non_critical is not None:
self.critical.set(critical, non_critical)
self._set_critical_tags(self.critical)
def _set_critical_tags(self, critical):
self.critical = critical
for suite in self.suites:
suite._set_critical_tags(critical)
for test in self.tests:
test.set_criticality(critical)
def set_doc(self, doc):
if doc:
self.doc = doc
def set_metadata(self, metalist):
for name, value in metalist:
self.metadata[name] = value
def get_metadata(self):
return self.metadata.items()
def get_test_count(self):
count = len(self.tests)
for suite in self.suites:
count += suite.get_test_count()
return count
def get_full_message(self):
"""Returns suite's message including statistics message"""
stat_msg = self.get_stat_message()
if not self.message:
return stat_msg
return '%s\n\n%s' % (self.message, stat_msg)
def get_stat_message(self):
ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats)
atotal, aend, apass, afail = self._get_counts(self.all_stats)
return ('%d critical test%s, %d passed, %d failed\n'
'%d test%s total, %d passed, %d failed'
% (ctotal, cend, cpass, cfail, atotal, aend, apass, afail))
def _get_counts(self, stat):
ending = utils.plural_or_not(stat.total)
return stat.total, ending, stat.passed, stat.failed
def set_status(self):
"""Sets status and statistics based on subsuite and test statuses.
Can/should be used when statuses have been changed somehow.
"""
self.status = self._set_stats()
def _set_stats(self):
self.critical_stats = Stat()
self.all_stats = Stat()
for suite in self.suites:
suite.set_status()
self._add_suite_to_stats(suite)
for test in self.tests:
self._add_test_to_stats(test)
return self._get_status()
def _get_status(self):
return 'PASS' if not self.critical_stats.failed else 'FAIL'
def _add_test_to_stats(self, test):
self.all_stats.add_test(test)
if test.critical == 'yes':
self.critical_stats.add_test(test)
def _add_suite_to_stats(self, suite):
self.critical_stats.add_stat(suite.critical_stats)
self.all_stats.add_stat(suite.all_stats)
def suite_teardown_failed(self, message=None):
if message:
self._set_teardown_fail_msg(message)
self.critical_stats.fail_all()
self.all_stats.fail_all()
self.status = self._get_status()
sub_message = 'Teardown of the parent suite failed.'
for suite in self.suites:
suite.suite_teardown_failed(sub_message)
for test in self.tests:
test.suite_teardown_failed(sub_message)
def set_tags(self, tags):
if tags:
for test in self.tests:
test.tags = utils.normalize_tags(test.tags + tags)
for suite in self.suites:
suite.set_tags(tags)
def filter(self, suites=None, tests=None, includes=None, excludes=None,
zero_tests_ok=False):
if suites or tests:
self.filter_by_names(suites, tests, zero_tests_ok)
if includes or excludes:
self.filter_by_tags(includes, excludes, zero_tests_ok)
def filter_by_names(self, suites=None, tests=None, zero_tests_ok=False):
suites = [([], name.split('.')) for name in suites or []]
tests = tests or []
if not self._filter_by_names(suites, tests) and not zero_tests_ok:
self._raise_no_tests_filtered_by_names(suites, tests)
def _filter_by_names(self, suites, tests):
suites = self._filter_suite_names(suites)
self.suites = [suite for suite in self.suites
if suite._filter_by_names(suites, tests)]
if not suites:
self.tests = [test for test in self.tests if tests == [] or
any(utils.matches_any(name, tests, ignore=['_'])
for name in [test.name, test.longname])]
else:
self.tests = []
return bool(self.suites or self.tests)
def _filter_suite_names(self, suites):
try:
return [self._filter_suite_name(p, s) for p, s in suites]
except StopIteration:
return []
def _filter_suite_name(self, parent, suite):
if utils.matches(self.name, suite[0], ignore=['_']):
if len(suite) == 1:
raise StopIteration('Match found')
return (parent + [suite[0]], suite[1:])
return ([], parent + suite)
def _raise_no_tests_filtered_by_names(self, suites, tests):
tests = utils.seq2str(tests, lastsep=' or ')
suites = utils.seq2str(['.'.join(p + s) for p, s in suites],
lastsep=' or ')
if not suites:
msg = 'test cases named %s.' % tests
elif not tests:
msg = 'test suites named %s.' % suites
else:
msg = 'test cases %s in suites %s.' % (tests, suites)
raise DataError("Suite '%s' contains no %s" % (self.name, msg))
def filter_by_tags(self, includes=None, excludes=None, zero_tests_ok=False):
includes = includes or []
excludes = excludes or []
if not self._filter_by_tags(includes, excludes) and not zero_tests_ok:
self._raise_no_tests_filtered_by_tags(includes, excludes)
def _filter_by_tags(self, incls, excls):
self.suites = [suite for suite in self.suites
if suite._filter_by_tags(incls, excls)]
self.tests = [test for test in self.tests
if test.is_included(incls, excls)]
return bool(self.suites or self.tests)
def _raise_no_tests_filtered_by_tags(self, incls, excls):
incl = utils.seq2str(incls)
excl = utils.seq2str(excls)
msg = "Suite '%s' with " % self.name
if incl:
msg += 'includes %s ' % incl
if excl:
msg += 'and '
if excl:
msg += 'excludes %s ' % excl
raise DataError(msg + 'contains no test cases.')
def set_runmode(self, runmode):
runmode = runmode.upper()
if runmode == 'EXITONFAILURE':
self._run_mode_exit_on_failure = True
elif runmode == 'SKIPTEARDOWNONEXIT':
self._run_mode_skip_teardowns_on_exit = True
elif runmode == 'DRYRUN':
self._run_mode_dry_run = True
elif runmode == 'RANDOM:TEST':
random.shuffle(self.tests)
elif runmode == 'RANDOM:SUITE':
random.shuffle(self.suites)
elif runmode == 'RANDOM:ALL':
random.shuffle(self.suites)
random.shuffle(self.tests)
else:
return
for suite in self.suites:
suite.set_runmode(runmode)
def set_options(self, settings):
self.set_tags(settings['SetTag'])
self.filter(settings['SuiteNames'], settings['TestNames'],
settings['Include'], settings['Exclude'],
settings['RunEmptySuite'])
self.set_name(settings['Name'])
self.set_doc(settings['Doc'])
self.set_metadata(settings['Metadata'])
self.set_critical_tags(settings['Critical'], settings['NonCritical'])
self._return_status_rc = not settings['NoStatusRC']
if 'RunMode' in settings:
map(self.set_runmode, settings['RunMode'])
if 'RemoveKeywords' in settings:
self.remove_keywords(settings['RemoveKeywords'])
def serialize(self, serializer):
serializer.start_suite(self)
if self.setup is not None:
self.setup.serialize(serializer)
if self.teardown is not None:
self.teardown.serialize(serializer)
for suite in self.suites:
suite.serialize(serializer)
for test in self.tests:
test.serialize(serializer)
serializer.end_suite(self)
@property
def return_code(self):
rc = min(self.critical_stats.failed, 250)
return rc if self._return_status_rc else 0
class BaseTestSuite(_TestAndSuiteHelper):
"""Base class for TestSuite used in runtime and by rebot."""
def __init__(self, name, source=None, parent=None):
_TestAndSuiteHelper.__init__(self, name, parent)
self.source = source is not None and utils.normpath(source) or None
self.metadata = utils.NormalizedDict()
self.suites = []
self.tests = []
self.critical = _Critical()
self.critical_stats = Stat()
self.all_stats = Stat()
if parent:
parent.suites.append(self)
def set_name(self, name):
if name:
self.name = name
elif not self.parent and self.name == '': # MultiSourceSuite
self.name = ' & '.join([suite.name for suite in self.suites])
def set_critical_tags(self, critical, non_critical):
if critical is not None or non_critical is not None:
self.critical.set(critical, non_critical)
self._set_critical_tags(self.critical)
def _set_critical_tags(self, critical):
self.critical = critical
for suite in self.suites:
suite._set_critical_tags(critical)
for test in self.tests:
test.set_criticality(critical)
def set_doc(self, doc):
if doc is not None:
self.doc = doc
def set_metadata(self, metalist):
for metastr in metalist:
try:
name, value = metastr.split(':', 1)
except ValueError:
name, value = metastr, ''
self.metadata[name] = value
def get_metadata(self, html=False):
names = sorted(self.metadata.keys())
values = [self.metadata[n] for n in names]
if html:
values = [utils.html_escape(v, formatting=True) for v in values]
return zip(names, values)
def get_test_count(self):
count = len(self.tests)
for suite in self.suites:
count += suite.get_test_count()
return count
def get_full_message(self, html=False):
"""Returns suite's message including statistics message"""
stat_msg = self.get_stat_message(html)
if self.message == '':
return stat_msg
if not html:
return '%s\n\n%s' % (self.message, stat_msg)
return '%s<br /><br />%s' % (utils.html_escape(self.message), stat_msg)
def get_stat_message(self, html=False):
ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats)
atotal, aend, apass, afail = self._get_counts(self.all_stats)
msg = ('%%d critical test%%s, %%d passed, %(cfail)s%%d failed%(end)s\n'
'%%d test%%s total, %%d passed, %(afail)s%%d failed%(end)s')
if html:
msg = msg.replace(' ', ' ').replace('\n', '<br />')
msg = msg % {
'cfail': '<span%s>' % (cfail and ' class="fail"' or ''),
'afail': '<span%s>' % (afail and ' class="fail"' or ''),
'end': '</span>'
}
else:
msg = msg % {'cfail': '', 'afail': '', 'end': ''}
return msg % (ctotal, cend, cpass, cfail, atotal, aend, apass, afail)
def _get_counts(self, stat):
total = stat.passed + stat.failed
ending = utils.plural_or_not(total)
return total, ending, stat.passed, stat.failed
def set_status(self):
"""Sets status and statistics based on subsuite and test statuses.
Can/should be used when statuses have been changed somehow.
"""
self._set_stats()
self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL'
def _set_stats(self):
self.critical_stats = Stat()
self.all_stats = Stat()
for suite in self.suites:
suite.set_status()
self._add_suite_to_stats(suite)
for test in self.tests:
self._add_test_to_stats(test)
def _add_test_to_stats(self, test):
self.all_stats.add_test(test)
if test.critical == 'yes':
self.critical_stats.add_test(test)
def _add_suite_to_stats(self, suite):
self.critical_stats.add_stat(suite.critical_stats)
self.all_stats.add_stat(suite.all_stats)
def suite_teardown_failed(self, message=None):
if message is not None:
self._set_teardown_fail_msg(message)
self.critical_stats.fail_all()
self.all_stats.fail_all()
self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL'
sub_message = 'Teardown of the parent suite failed.'
for suite in self.suites:
suite.suite_teardown_failed(sub_message)
for test in self.tests:
test.suite_teardown_failed(sub_message)
def set_tags(self, tags):
if tags:
for test in self.tests:
test.tags = utils.normalize_tags(test.tags + tags)
for suite in self.suites:
suite.set_tags(tags)
def filter(self, suites=None, tests=None, includes=None, excludes=None):
self.filter_by_names(suites, tests)
self.filter_by_tags(includes, excludes)
def filter_by_names(self, suites=None, tests=None):
suites = [([], name.split('.')) for name in suites or []]
tests = tests or []
if (suites or tests) and not self._filter_by_names(suites, tests):
self._raise_no_tests_filtered_by_names(suites, tests)
def _filter_by_names(self, suites, tests):
suites = self._filter_suite_names(suites)
self.suites = [
suite for suite in self.suites
if suite._filter_by_names(suites, tests)
]
if not suites:
self.tests = [
test for test in self.tests if tests == []
or utils.matches_any(test.name, tests, ignore=['_'])
]
else:
self.tests = []
return self.suites or self.tests
def _filter_suite_names(self, suites):
try:
return [self._filter_suite_name(p, s) for p, s in suites]
except StopIteration:
return []
def _filter_suite_name(self, parent, suite):
if utils.matches(self.name, suite[0], ignore=['_']):
if len(suite) == 1:
raise StopIteration('Match found')
return (parent + [suite[0]], suite[1:])
return ([], parent + suite)
def _raise_no_tests_filtered_by_names(self, suites, tests):
tests = utils.seq2str(tests, lastsep=' or ')
suites = utils.seq2str(['.'.join(p + s) for p, s in suites],
lastsep=' or ')
if not suites:
msg = 'test cases named %s.' % tests
elif not tests:
msg = 'test suites named %s.' % suites
else:
msg = 'test cases %s in suites %s.' % (tests, suites)
raise DataError("Suite '%s' contains no %s" % (self.name, msg))
def filter_by_tags(self, includes=None, excludes=None):
if not (includes or excludes):
return
if not includes: includes = []
if not excludes: excludes = []
if not self._filter_by_tags(includes, excludes):
self._raise_no_tests_filtered_by_tags(includes, excludes)
def _filter_by_tags(self, incls, excls):
self.suites = [
suite for suite in self.suites
if suite._filter_by_tags(incls, excls)
]
self.tests = [
test for test in self.tests if test.is_included(incls, excls)
]
return len(self.suites) + len(self.tests) > 0
def _raise_no_tests_filtered_by_tags(self, incls, excls):
incl = utils.seq2str(incls)
excl = utils.seq2str(excls)
msg = "Suite '%s' with " % self.name
if incl:
msg += 'includes %s ' % incl
if excl:
msg += 'and '
if excl:
msg += 'excludes %s ' % excl
raise DataError(msg + 'contains no test cases.')
def set_runmode(self, runmode):
runmode = runmode.upper()
if runmode == 'EXITONFAILURE':
self._run_mode_exit_on_failure = True
elif runmode == 'SKIPTEARDOWNONEXIT':
self._run_mode_skip_teardowns_on_exit = True
elif runmode == 'DRYRUN':
self._run_mode_dry_run = True
elif runmode == 'RANDOM:TEST':
random.shuffle(self.tests)
elif runmode == 'RANDOM:SUITE':
random.shuffle(self.suites)
elif runmode == 'RANDOM:ALL':
random.shuffle(self.suites)
random.shuffle(self.tests)
else:
return
for suite in self.suites:
suite.set_runmode(runmode)
def set_options(self, settings):
self.set_tags(settings['SetTag'])
self.filter(settings['SuiteNames'], settings['TestNames'],
settings['Include'], settings['Exclude'])
self.set_name(settings['Name'])
self.set_doc(settings['Doc'])
self.set_metadata(settings['Metadata'])
self.set_critical_tags(settings['Critical'], settings['NonCritical'])
try:
for runmode in settings['RunMode']:
self.set_runmode(runmode)
except (KeyError, AttributeError): # Only applicable when running tcs
pass
if not self.suites:
settings['SplitOutputs'] = -2
try:
self.remove_keywords(settings['RemoveKeywords'])
except (KeyError, AttributeError): # Only applicable with Rebot
pass
def serialize(self, serializer):
serializer.start_suite(self)
if self.setup is not None:
self.setup.serialize(serializer)
if self.teardown is not None:
self.teardown.serialize(serializer)
for suite in self.suites:
suite.serialize(serializer)
for test in self.tests:
test.serialize(serializer)
serializer.end_suite(self)
