class BaseTestSuite(_TestAndSuiteHelper): """Base class for TestSuite used in runtime and by rebot.""" def __init__(self, name, source=None, parent=None): _TestAndSuiteHelper.__init__(self, name, parent) self.source = source is not None and utils.normpath(source) or None self.metadata = utils.NormalizedDict() self.suites = [] self.tests = [] self.critical = _Critical() self.critical_stats = Stat() self.all_stats = Stat() if parent: parent.suites.append(self) def set_name(self, name): if name: self.name = name elif not self.parent and self.name == '': # MultiSourceSuite self.name = ' & '.join([suite.name for suite in self.suites]) def set_critical_tags(self, critical, non_critical): if critical is not None or non_critical is not None: self.critical.set(critical, non_critical) self._set_critical_tags(self.critical) def _set_critical_tags(self, critical): self.critical = critical for suite in self.suites: suite._set_critical_tags(critical) for test in self.tests: test.set_criticality(critical) def set_doc(self, doc): if doc is not None: self.doc = doc def set_metadata(self, metalist): for metastr in metalist: try: name, value = metastr.split(':', 1) except ValueError: name, value = metastr, '' self.metadata[name] = value def get_metadata(self, html=False): names = sorted(self.metadata.keys()) values = [ self.metadata[n] for n in names ] if html: values = [ utils.html_escape(v, formatting=True) for v in values ] return zip(names, values) def get_test_count(self): count = len(self.tests) for suite in self.suites: count += suite.get_test_count() return count def get_full_message(self, html=False): """Returns suite's message including statistics message""" stat_msg = self.get_stat_message(html) if self.message == '': return stat_msg if not html: return '%s\n\n%s' % (self.message, stat_msg) return '%s<br /><br />%s' % (utils.html_escape(self.message), stat_msg) def get_stat_message(self, html=False): ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats) atotal, aend, apass, afail = self._get_counts(self.all_stats) msg = ('%%d critical test%%s, %%d passed, %(cfail)s%%d failed%(end)s\n' '%%d test%%s total, %%d passed, %(afail)s%%d failed%(end)s') if html: msg = msg.replace(' ', ' ').replace('\n', '<br />') msg = msg % {'cfail': '<span%s>' % (cfail and ' class="fail"' or ''), 'afail': '<span%s>' % (afail and ' class="fail"' or ''), 'end': '</span>'} else: msg = msg % {'cfail': '', 'afail': '', 'end': ''} return msg % (ctotal, cend, cpass, cfail, atotal, aend, apass, afail) def _get_counts(self, stat): total = stat.passed + stat.failed ending = utils.plural_or_not(total) return total, ending, stat.passed, stat.failed def set_status(self): """Sets status and statistics based on subsuite and test statuses. Can/should be used when statuses have been changed somehow. """ self._set_stats() self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL' def _set_stats(self): self.critical_stats = Stat() self.all_stats = Stat() for suite in self.suites: suite.set_status() self._add_suite_to_stats(suite) for test in self.tests: self._add_test_to_stats(test) def _add_test_to_stats(self, test): self.all_stats.add_test(test) if test.critical == 'yes': self.critical_stats.add_test(test) def _add_suite_to_stats(self, suite): self.critical_stats.add_stat(suite.critical_stats) self.all_stats.add_stat(suite.all_stats) def suite_teardown_failed(self, message=None): if message is not None: self._set_teardown_fail_msg(message) self.critical_stats.fail_all() self.all_stats.fail_all() self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL' sub_message = 'Teardown of the parent suite failed.' for suite in self.suites: suite.suite_teardown_failed(sub_message) for test in self.tests: test.suite_teardown_failed(sub_message) def set_tags(self, tags): if tags: for test in self.tests: test.tags = utils.normalize_tags(test.tags + tags) for suite in self.suites: suite.set_tags(tags) def filter(self, suites=None, tests=None, includes=None, excludes=None): self.filter_by_names(suites, tests) self.filter_by_tags(includes, excludes) def filter_by_names(self, suites=None, tests=None): suites = [ ([], name.split('.')) for name in suites or [] ] tests = tests or [] if (suites or tests) and not self._filter_by_names(suites, tests): self._raise_no_tests_filtered_by_names(suites, tests) def _filter_by_names(self, suites, tests): suites = self._filter_suite_names(suites) self.suites = [ suite for suite in self.suites if suite._filter_by_names(suites, tests) ] if not suites: self.tests = [ test for test in self.tests if tests == [] or utils.matches_any(test.name, tests, ignore=['_']) ] else: self.tests = [] return self.suites or self.tests def _filter_suite_names(self, suites): try: return [ self._filter_suite_name(p, s) for p, s in suites ] except StopIteration: return [] def _filter_suite_name(self, parent, suite): if utils.matches(self.name, suite[0], ignore=['_']): if len(suite) == 1: raise StopIteration('Match found') return (parent + [suite[0]], suite[1:]) return ([], parent + suite) def _raise_no_tests_filtered_by_names(self, suites, tests): tests = utils.seq2str(tests, lastsep=' or ') suites = utils.seq2str([ '.'.join(p + s) for p, s in suites ], lastsep=' or ') if not suites: msg = 'test cases named %s.' % tests elif not tests: msg = 'test suites named %s.' % suites else: msg = 'test cases %s in suites %s.' % (tests, suites) raise DataError("Suite '%s' contains no %s" % (self.name, msg)) def filter_by_tags(self, includes=None, excludes=None): if not (includes or excludes): return if not includes: includes = [] if not excludes: excludes = [] if not self._filter_by_tags(includes, excludes): self._raise_no_tests_filtered_by_tags(includes, excludes) def _filter_by_tags(self, incls, excls): self.suites = [ suite for suite in self.suites if suite._filter_by_tags(incls, excls) ] self.tests = [ test for test in self.tests if test.is_included(incls, excls) ] return len(self.suites) + len(self.tests) > 0 def _raise_no_tests_filtered_by_tags(self, incls, excls): incl = utils.seq2str(incls) excl = utils.seq2str(excls) msg = "Suite '%s' with " % self.name if incl: msg += 'includes %s ' % incl if excl: msg += 'and ' if excl: msg += 'excludes %s ' % excl raise DataError(msg + 'contains no test cases.') def set_runmode(self, runmode): runmode = runmode.upper() if runmode == 'EXITONFAILURE': self._run_mode_exit_on_failure = True elif runmode == 'SKIPTEARDOWNONEXIT': self._run_mode_skip_teardowns_on_exit = True elif runmode == 'DRYRUN': self._run_mode_dry_run = True elif runmode == 'RANDOM:TEST': random.shuffle(self.tests) elif runmode == 'RANDOM:SUITE': random.shuffle(self.suites) elif runmode == 'RANDOM:ALL': random.shuffle(self.suites) random.shuffle(self.tests) else: return for suite in self.suites: suite.set_runmode(runmode) def set_options(self, settings): self.set_tags(settings['SetTag']) self.filter(settings['SuiteNames'], settings['TestNames'], settings['Include'], settings['Exclude']) self.set_name(settings['Name']) self.set_doc(settings['Doc']) self.set_metadata(settings['Metadata']) self.set_critical_tags(settings['Critical'], settings['NonCritical']) try: for runmode in settings['RunMode']: self.set_runmode(runmode) except (KeyError, AttributeError) : # Only applicable when running tcs pass if not self.suites: settings['SplitOutputs'] = -2 try: self.remove_keywords(settings['RemoveKeywords']) except (KeyError, AttributeError): # Only applicable with Rebot pass def serialize(self, serializer): serializer.start_suite(self) if self.setup is not None: self.setup.serialize(serializer) if self.teardown is not None: self.teardown.serialize(serializer) for suite in self.suites: suite.serialize(serializer) for test in self.tests: test.serialize(serializer) serializer.end_suite(self)
class BaseTestSuite(_TestAndSuiteHelper): """Base class for TestSuite used in runtime and by rebot.""" def __init__(self, name, source=None, parent=None): _TestAndSuiteHelper.__init__(self, name, parent) self.source = utils.abspath(source) if source else None self._id = None self.metadata = utils.NormalizedDict() self.suites = [] self.tests = [] self.critical = _Critical() self.critical_stats = Stat() self.all_stats = Stat() if parent: parent.suites.append(self) def set_name(self, name): if name: self.name = name elif self._is_multi_source_suite(): self.name = ' & '.join(suite.name for suite in self.suites) def _is_multi_source_suite(self): return self.parent is None and self.name == '' @property def id(self): if not self._id: self._find_root()._set_id() return self._id def _find_root(self): if self.parent: return self.parent._find_root() return self def _set_id(self): if not self._id: self._id = 's1' for index, suite in enumerate(self.suites): suite._id = '%s-s%s' % (self._id, index+1) suite._set_id() def set_critical_tags(self, critical, non_critical): if critical is not None or non_critical is not None: self.critical.set(critical, non_critical) self._set_critical_tags(self.critical) def _set_critical_tags(self, critical): self.critical = critical for suite in self.suites: suite._set_critical_tags(critical) for test in self.tests: test.set_criticality(critical) def set_doc(self, doc): if doc: self.doc = doc def set_metadata(self, metalist): for name, value in metalist: self.metadata[name] = value def get_metadata(self): return self.metadata.items() def get_test_count(self): count = len(self.tests) for suite in self.suites: count += suite.get_test_count() return count def get_full_message(self): """Returns suite's message including statistics message""" stat_msg = self.get_stat_message() if not self.message: return stat_msg return '%s\n\n%s' % (self.message, stat_msg) def get_stat_message(self): ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats) atotal, aend, apass, afail = self._get_counts(self.all_stats) return ('%d critical test%s, %d passed, %d failed\n' '%d test%s total, %d passed, %d failed' % (ctotal, cend, cpass, cfail, atotal, aend, apass, afail)) def _get_counts(self, stat): ending = utils.plural_or_not(stat.total) return stat.total, ending, stat.passed, stat.failed def set_status(self): """Sets status and statistics based on subsuite and test statuses. Can/should be used when statuses have been changed somehow. """ self.status = self._set_stats() def _set_stats(self): self.critical_stats = Stat() self.all_stats = Stat() for suite in self.suites: suite.set_status() self._add_suite_to_stats(suite) for test in self.tests: self._add_test_to_stats(test) return self._get_status() def _get_status(self): return 'PASS' if not self.critical_stats.failed else 'FAIL' def _add_test_to_stats(self, test): self.all_stats.add_test(test) if test.critical == 'yes': self.critical_stats.add_test(test) def _add_suite_to_stats(self, suite): self.critical_stats.add_stat(suite.critical_stats) self.all_stats.add_stat(suite.all_stats) def suite_teardown_failed(self, message=None): if message: self._set_teardown_fail_msg(message) self.critical_stats.fail_all() self.all_stats.fail_all() self.status = self._get_status() sub_message = 'Teardown of the parent suite failed.' for suite in self.suites: suite.suite_teardown_failed(sub_message) for test in self.tests: test.suite_teardown_failed(sub_message) def set_tags(self, tags): if tags: for test in self.tests: test.tags = utils.normalize_tags(test.tags + tags) for suite in self.suites: suite.set_tags(tags) def filter(self, suites=None, tests=None, includes=None, excludes=None, zero_tests_ok=False): if suites or tests: self.filter_by_names(suites, tests, zero_tests_ok) if includes or excludes: self.filter_by_tags(includes, excludes, zero_tests_ok) def filter_by_names(self, suites=None, tests=None, zero_tests_ok=False): suites = [([], name.split('.')) for name in suites or []] tests = tests or [] if not self._filter_by_names(suites, tests) and not zero_tests_ok: self._raise_no_tests_filtered_by_names(suites, tests) def _filter_by_names(self, suites, tests): suites = self._filter_suite_names(suites) self.suites = [suite for suite in self.suites if suite._filter_by_names(suites, tests)] if not suites: self.tests = [test for test in self.tests if tests == [] or any(utils.matches_any(name, tests, ignore=['_']) for name in [test.name, test.longname])] else: self.tests = [] return bool(self.suites or self.tests) def _filter_suite_names(self, suites): try: return [self._filter_suite_name(p, s) for p, s in suites] except StopIteration: return [] def _filter_suite_name(self, parent, suite): if utils.matches(self.name, suite[0], ignore=['_']): if len(suite) == 1: raise StopIteration('Match found') return (parent + [suite[0]], suite[1:]) return ([], parent + suite) def _raise_no_tests_filtered_by_names(self, suites, tests): tests = utils.seq2str(tests, lastsep=' or ') suites = utils.seq2str(['.'.join(p + s) for p, s in suites], lastsep=' or ') if not suites: msg = 'test cases named %s.' % tests elif not tests: msg = 'test suites named %s.' % suites else: msg = 'test cases %s in suites %s.' % (tests, suites) raise DataError("Suite '%s' contains no %s" % (self.name, msg)) def filter_by_tags(self, includes=None, excludes=None, zero_tests_ok=False): includes = includes or [] excludes = excludes or [] if not self._filter_by_tags(includes, excludes) and not zero_tests_ok: self._raise_no_tests_filtered_by_tags(includes, excludes) def _filter_by_tags(self, incls, excls): self.suites = [suite for suite in self.suites if suite._filter_by_tags(incls, excls)] self.tests = [test for test in self.tests if test.is_included(incls, excls)] return bool(self.suites or self.tests) def _raise_no_tests_filtered_by_tags(self, incls, excls): incl = utils.seq2str(incls) excl = utils.seq2str(excls) msg = "Suite '%s' with " % self.name if incl: msg += 'includes %s ' % incl if excl: msg += 'and ' if excl: msg += 'excludes %s ' % excl raise DataError(msg + 'contains no test cases.') def set_runmode(self, runmode): runmode = runmode.upper() if runmode == 'EXITONFAILURE': self._run_mode_exit_on_failure = True elif runmode == 'SKIPTEARDOWNONEXIT': self._run_mode_skip_teardowns_on_exit = True elif runmode == 'DRYRUN': self._run_mode_dry_run = True elif runmode == 'RANDOM:TEST': random.shuffle(self.tests) elif runmode == 'RANDOM:SUITE': random.shuffle(self.suites) elif runmode == 'RANDOM:ALL': random.shuffle(self.suites) random.shuffle(self.tests) else: return for suite in self.suites: suite.set_runmode(runmode) def set_options(self, settings): self.set_tags(settings['SetTag']) self.filter(settings['SuiteNames'], settings['TestNames'], settings['Include'], settings['Exclude'], settings['RunEmptySuite']) self.set_name(settings['Name']) self.set_doc(settings['Doc']) self.set_metadata(settings['Metadata']) self.set_critical_tags(settings['Critical'], settings['NonCritical']) self._return_status_rc = not settings['NoStatusRC'] if 'RunMode' in settings: map(self.set_runmode, settings['RunMode']) if 'RemoveKeywords' in settings: self.remove_keywords(settings['RemoveKeywords']) def serialize(self, serializer): serializer.start_suite(self) if self.setup is not None: self.setup.serialize(serializer) if self.teardown is not None: self.teardown.serialize(serializer) for suite in self.suites: suite.serialize(serializer) for test in self.tests: test.serialize(serializer) serializer.end_suite(self) @property def return_code(self): rc = min(self.critical_stats.failed, 250) return rc if self._return_status_rc else 0
class BaseTestSuite(_TestAndSuiteHelper): """Base class for TestSuite used in runtime and by rebot.""" def __init__(self, name, source=None, parent=None): _TestAndSuiteHelper.__init__(self, name, parent) self.source = source is not None and utils.normpath(source) or None self.metadata = utils.NormalizedDict() self.suites = [] self.tests = [] self.critical = _Critical() self.critical_stats = Stat() self.all_stats = Stat() if parent: parent.suites.append(self) def set_name(self, name): if name: self.name = name elif not self.parent and self.name == '': # MultiSourceSuite self.name = ' & '.join([suite.name for suite in self.suites]) def set_critical_tags(self, critical, non_critical): if critical is not None or non_critical is not None: self.critical.set(critical, non_critical) self._set_critical_tags(self.critical) def _set_critical_tags(self, critical): self.critical = critical for suite in self.suites: suite._set_critical_tags(critical) for test in self.tests: test.set_criticality(critical) def set_doc(self, doc): if doc is not None: self.doc = doc def set_metadata(self, metalist): for metastr in metalist: try: name, value = metastr.split(':', 1) except ValueError: name, value = metastr, '' self.metadata[name] = value def get_metadata(self, html=False): names = sorted(self.metadata.keys()) values = [self.metadata[n] for n in names] if html: values = [utils.html_escape(v, formatting=True) for v in values] return zip(names, values) def get_test_count(self): count = len(self.tests) for suite in self.suites: count += suite.get_test_count() return count def get_full_message(self, html=False): """Returns suite's message including statistics message""" stat_msg = self.get_stat_message(html) if self.message == '': return stat_msg if not html: return '%s\n\n%s' % (self.message, stat_msg) return '%s<br /><br />%s' % (utils.html_escape(self.message), stat_msg) def get_stat_message(self, html=False): ctotal, cend, cpass, cfail = self._get_counts(self.critical_stats) atotal, aend, apass, afail = self._get_counts(self.all_stats) msg = ('%%d critical test%%s, %%d passed, %(cfail)s%%d failed%(end)s\n' '%%d test%%s total, %%d passed, %(afail)s%%d failed%(end)s') if html: msg = msg.replace(' ', ' ').replace('\n', '<br />') msg = msg % { 'cfail': '<span%s>' % (cfail and ' class="fail"' or ''), 'afail': '<span%s>' % (afail and ' class="fail"' or ''), 'end': '</span>' } else: msg = msg % {'cfail': '', 'afail': '', 'end': ''} return msg % (ctotal, cend, cpass, cfail, atotal, aend, apass, afail) def _get_counts(self, stat): total = stat.passed + stat.failed ending = utils.plural_or_not(total) return total, ending, stat.passed, stat.failed def set_status(self): """Sets status and statistics based on subsuite and test statuses. Can/should be used when statuses have been changed somehow. """ self._set_stats() self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL' def _set_stats(self): self.critical_stats = Stat() self.all_stats = Stat() for suite in self.suites: suite.set_status() self._add_suite_to_stats(suite) for test in self.tests: self._add_test_to_stats(test) def _add_test_to_stats(self, test): self.all_stats.add_test(test) if test.critical == 'yes': self.critical_stats.add_test(test) def _add_suite_to_stats(self, suite): self.critical_stats.add_stat(suite.critical_stats) self.all_stats.add_stat(suite.all_stats) def suite_teardown_failed(self, message=None): if message is not None: self._set_teardown_fail_msg(message) self.critical_stats.fail_all() self.all_stats.fail_all() self.status = self.critical_stats.failed == 0 and 'PASS' or 'FAIL' sub_message = 'Teardown of the parent suite failed.' for suite in self.suites: suite.suite_teardown_failed(sub_message) for test in self.tests: test.suite_teardown_failed(sub_message) def set_tags(self, tags): if tags: for test in self.tests: test.tags = utils.normalize_tags(test.tags + tags) for suite in self.suites: suite.set_tags(tags) def filter(self, suites=None, tests=None, includes=None, excludes=None): self.filter_by_names(suites, tests) self.filter_by_tags(includes, excludes) def filter_by_names(self, suites=None, tests=None): suites = [([], name.split('.')) for name in suites or []] tests = tests or [] if (suites or tests) and not self._filter_by_names(suites, tests): self._raise_no_tests_filtered_by_names(suites, tests) def _filter_by_names(self, suites, tests): suites = self._filter_suite_names(suites) self.suites = [ suite for suite in self.suites if suite._filter_by_names(suites, tests) ] if not suites: self.tests = [ test for test in self.tests if tests == [] or utils.matches_any(test.name, tests, ignore=['_']) ] else: self.tests = [] return self.suites or self.tests def _filter_suite_names(self, suites): try: return [self._filter_suite_name(p, s) for p, s in suites] except StopIteration: return [] def _filter_suite_name(self, parent, suite): if utils.matches(self.name, suite[0], ignore=['_']): if len(suite) == 1: raise StopIteration('Match found') return (parent + [suite[0]], suite[1:]) return ([], parent + suite) def _raise_no_tests_filtered_by_names(self, suites, tests): tests = utils.seq2str(tests, lastsep=' or ') suites = utils.seq2str(['.'.join(p + s) for p, s in suites], lastsep=' or ') if not suites: msg = 'test cases named %s.' % tests elif not tests: msg = 'test suites named %s.' % suites else: msg = 'test cases %s in suites %s.' % (tests, suites) raise DataError("Suite '%s' contains no %s" % (self.name, msg)) def filter_by_tags(self, includes=None, excludes=None): if not (includes or excludes): return if not includes: includes = [] if not excludes: excludes = [] if not self._filter_by_tags(includes, excludes): self._raise_no_tests_filtered_by_tags(includes, excludes) def _filter_by_tags(self, incls, excls): self.suites = [ suite for suite in self.suites if suite._filter_by_tags(incls, excls) ] self.tests = [ test for test in self.tests if test.is_included(incls, excls) ] return len(self.suites) + len(self.tests) > 0 def _raise_no_tests_filtered_by_tags(self, incls, excls): incl = utils.seq2str(incls) excl = utils.seq2str(excls) msg = "Suite '%s' with " % self.name if incl: msg += 'includes %s ' % incl if excl: msg += 'and ' if excl: msg += 'excludes %s ' % excl raise DataError(msg + 'contains no test cases.') def set_runmode(self, runmode): runmode = runmode.upper() if runmode == 'EXITONFAILURE': self._run_mode_exit_on_failure = True elif runmode == 'SKIPTEARDOWNONEXIT': self._run_mode_skip_teardowns_on_exit = True elif runmode == 'DRYRUN': self._run_mode_dry_run = True elif runmode == 'RANDOM:TEST': random.shuffle(self.tests) elif runmode == 'RANDOM:SUITE': random.shuffle(self.suites) elif runmode == 'RANDOM:ALL': random.shuffle(self.suites) random.shuffle(self.tests) else: return for suite in self.suites: suite.set_runmode(runmode) def set_options(self, settings): self.set_tags(settings['SetTag']) self.filter(settings['SuiteNames'], settings['TestNames'], settings['Include'], settings['Exclude']) self.set_name(settings['Name']) self.set_doc(settings['Doc']) self.set_metadata(settings['Metadata']) self.set_critical_tags(settings['Critical'], settings['NonCritical']) try: for runmode in settings['RunMode']: self.set_runmode(runmode) except (KeyError, AttributeError): # Only applicable when running tcs pass if not self.suites: settings['SplitOutputs'] = -2 try: self.remove_keywords(settings['RemoveKeywords']) except (KeyError, AttributeError): # Only applicable with Rebot pass def serialize(self, serializer): serializer.start_suite(self) if self.setup is not None: self.setup.serialize(serializer) if self.teardown is not None: self.teardown.serialize(serializer) for suite in self.suites: suite.serialize(serializer) for test in self.tests: test.serialize(serializer) serializer.end_suite(self)