def test_criticality(self):
for c in range(10):
crit = ['c%s' % (i + 1) for i in range(c)]
for n in range(10):
nonc = ['n%s' % (i + 1) for i in range(n)]
builder = TagStatisticsBuilder(Criticality(crit, nonc))
assert_equal([s.name for s in builder.stats], crit + nonc),
def _verify_combined_statistics(self, comb_tags, test_tags,
expected_count):
builder = TagStatisticsBuilder(Criticality(),
combined=[(comb_tags, 'name')])
builder._add_to_combined_statistics(TestCase(tags=test_tags))
assert_equals([s.total for s in builder.stats if s.combined],
[expected_count])
def test_exclude(self):
for excl, tags in self._incl_excl_data:
builder = TagStatisticsBuilder(Criticality(), [], excl)
builder.add_test(TestCase(status='PASS', tags=tags))
matcher = MultiMatcher(excl)
expected = [tag for tag in tags if not matcher.match(tag)]
assert_equals([s.name for s in builder.stats], sorted(expected))
def test_tags_equal_to_critical_or_non_critical_pattern_are_not_removed(
self):
builder = TagStatisticsBuilder(Criticality('sORry'))
builder.add_test(TestCase(tags=['sorry']))
builder.add_test(TestCase(tags=['s OR ry']))
assert_equal([(s.name, s.info, s.total) for s in builder.stats],
[('s OR ry', 'critical', 0), ('sorry', '', 2)])
def test_include(self):
for incl, tags in self._incl_excl_data:
builder = TagStatisticsBuilder(Criticality(), incl, [])
builder.add_test(TestCase(status='PASS', tags=tags))
matcher = MultiMatcher(incl, match_if_no_patterns=True)
expected = [tag for tag in tags if matcher.match(tag)]
assert_equals([s.name for s in builder.stats], sorted(expected))
def test_include(self):
for incl, tags in self._incl_excl_data:
builder = TagStatisticsBuilder(Criticality(), incl, [])
builder.add_test(TestCase(status='PASS', tags=tags))
expected = [
tag for tag in tags
if incl == [] or any(utils.matches(tag, i) for i in incl)
]
assert_equals([s.name for s in builder.stats], sorted(expected))
def test_iter(self):
builder = TagStatisticsBuilder(Criticality())
assert_equals(list(builder.stats), [])
builder.add_test(TestCase())
assert_equals(list(builder.stats), [])
builder.add_test(TestCase(tags=['a']))
assert_equals(len(list(builder.stats)), 1)
builder.add_test(TestCase(tags=['A', 'B']))
assert_equals(len(list(builder.stats)), 2)
def test_tags_equal_to_critical_or_non_critical_are_removed_in_iter(self):
builder = TagStatisticsBuilder(Criticality('smoke', 'not-ready'),
combined=[('xxx', )])
builder.add_test(TestCase(tags=['smoke']))
builder.add_test(TestCase(tags=['SMOKE', 'not-ready', 'xxx']))
builder.add_test(TestCase(tags=['_ _ S M O K E _ _', 'X_X_X', 'YYY']))
assert_equal([(s.name, s.info, s.total) for s in builder.stats],
[('smoke', 'critical', 3),
('not-ready', 'non-critical', 1), ('xxx', 'combined', 2),
('xxx', '', 2), ('YYY', '', 1)])
def set_criticality(self, critical_tags=None, non_critical_tags=None):
"""Sets which tags are considered critical and which non-critical.
Tags can be given as lists of strings or, when giving only one,
as single strings. This information is used by tests to determine
are they considered critical or not.
Criticality can be set only to the top level test suite.
"""
if self.parent:
raise TypeError('Criticality can only be set to top level suite')
self._criticality = Criticality(critical_tags, non_critical_tags)
def test_combine_with_name(self):
for comb_tags, expected_name in [([], ''),
([('t1&t2', 'my name')], 'my name'),
([('t1NOTt3', 'Others')], 'Others'),
([('1:2&2:3', 'nAme')], 'nAme'),
([('3*', '')], '3*'),
([('4NOT5', 'Some new name')],
'Some new name')]:
builder = TagStatisticsBuilder(Criticality(), combined=comb_tags)
assert_equals(bool(list(builder.stats)), expected_name != '')
if expected_name:
assert_equals([s.name for s in builder.stats], [expected_name])
def test_sorting(self):
builder = TagStatisticsBuilder(Criticality(['c2', 'c1'], ['n*']),
combined=[('c*', ''),
('xxx', 'a title')])
builder.add_test(TestCase(tags=['c1', 'c2', 't1']))
builder.add_test(TestCase(tags=['c1', 'n2', 't2']))
builder.add_test(TestCase(tags=['n1', 'n2', 't1', 't3']))
assert_equals([(s.name, s.info, s.total) for s in builder.stats],
[('c1', 'critical', 2), ('c2', 'critical', 1),
('n1', 'non-critical', 1), ('n2', 'non-critical', 2),
('a title', 'combined', 0), ('c*', 'combined', 2),
('t1', '', 2), ('t2', '', 1), ('t3', '', 1)])
def test_include_and_exclude(self):
for incl, excl, tags, exp in [
([], [], ['t0', 't1', 't2'], ['t0', 't1', 't2']),
(['t1'], ['t2'], ['t0', 't1', 't2'], ['t1']),
(['t?'], ['t2'], ['t0', 't1', 't2', 'x'], ['t0', 't1']),
(['t?'], ['*2'], ['t0', 't1', 't2', 'x2'], ['t0', 't1']),
(['t1', 't2'], ['t2'], ['t0', 't1', 't2'], ['t1']),
(['t1', 't2', 't3', 'not'], ['t2',
't0'], ['t0', 't1', 't2', 't3',
'x'], ['t1', 't3'])
]:
builder = TagStatisticsBuilder(Criticality(), incl, excl)
builder.add_test(TestCase(status='PASS', tags=tags))
assert_equals([s.name for s in builder.stats], exp),
def set_criticality(self, critical_tags=None, non_critical_tags=None):
"""Sets which tags are considered critical and which non-critical.
:param critical_tags: Tags or patterns considered critical. See
the documentation of the ``--critical`` option for more details.
:param non_critical_tags: Tags or patterns considered non-critical. See
the documentation of the ``--noncritical`` option for more details.
Tags can be given as lists of strings or, when giving only one,
as single strings. This information is used by tests to determine
are they considered critical or not.
Criticality can be set only to the root test suite.
"""
if self.parent is not None:
raise ValueError('Criticality can only be set to the root suite.')
self._criticality = Criticality(critical_tags, non_critical_tags)
def test_critical_and_combined(self):
builder = TagStatisticsBuilder(Criticality('crit', 'nonc'),
combined=[('non*', )],
docs=[('crit', 'critical doc'),
('non?', 'not so critical doc')])
crit, nonc, comb = builder.stats
self._verify_stats(crit, 'critical doc', 0, critical=True)
self._verify_stats(nonc, 'not so critical doc', 0, non_critical=True)
self._verify_stats(comb, 'not so critical doc', 0, combined='non*')
builder.add_test(TestCase(tags=['xxx', 'crit']))
builder.add_test(TestCase(tags=['Crit', 'NoNo', 'NoNc']))
crit, nonc, comb, nono, xxx = builder.stats
self._verify_stats(crit, 'critical doc', 2, critical=True)
self._verify_stats(nonc, 'not so critical doc', 1, non_critical=True)
self._verify_stats(comb, 'not so critical doc', 1, combined='non*')
self._verify_stats(nono, 'not so critical doc', 1)
self._verify_stats(xxx, '', 1)
def test_combine(self):
# This is more like an acceptance test than a unit test ...
for comb_tags, tests_tags, crit_tags in [
(['t1&t2'], [['t1', 't2', 't3'], ['t1', 't3']], []),
(['1&2&3'], [['1', '2', '3'], ['1', '2', '3', '4']], ['1', '2']),
(['1&2', '1&3'], [['1', '2', '3'], ['1', '3'], ['1']], ['1']),
(['t*'], [['t1', 'x', 'y'], ['tee', 'z'], ['t']], ['x']),
(['t?&s'], [['t1', 's'], ['tt', 's', 'u'], ['tee', 's'],
['s']], []),
(['t*&s', '*'], [['s', 't', 'u'], ['tee', 's'], [], ['x']], []),
(['tNOTs'], [['t', 'u'], ['t', 's']], []),
(['tNOTs', 't&s', 'tNOTsNOTu', 't&sNOTu'], [['t', 'u'], ['t', 's'],
['s', 't', 'u'], ['t'],
['t', 'v']], ['t']),
(['nonex'], [['t1'], ['t1,t2'], []], [])
]:
# 1) Create tag stats
builder = TagStatisticsBuilder(Criticality(crit_tags),
combined=[(t, '')
for t in comb_tags])
all_tags = []
for tags in tests_tags:
builder.add_test(TestCase(status='PASS', tags=tags), )
all_tags.extend(tags)
# 2) Actual values
names = [stat.name for stat in builder.stats]
# 3) Expected values
exp_crit = []
exp_noncr = []
for tag in Tags(all_tags):
if tag in crit_tags:
exp_crit.append(tag)
else:
exp_noncr.append(tag)
exp_names = exp_crit + sorted(comb_tags) + exp_noncr
# 4) Verify names (match counts were already verified)
assert_equals(names, exp_names)
def test_iter_with_combine(self):
builder = TagStatisticsBuilder(Criticality(),
combined=[('x*', 'title')])
assert_equals(len(list(builder.stats)), 1)
builder.add_test(TestCase(tags=['xxx', 'yyy']))
assert_equals(len(list(builder.stats)), 3)
def test_combine_with_same_name_as_existing_tag(self):
builder = TagStatisticsBuilder(Criticality(),
combined=[('x*', 'name')])
builder.add_test(TestCase(tags=['name', 'another']))
assert_equals([(s.name, s.combined) for s in builder.stats],
[('name', 'x*'), ('another', ''), ('name', '')])