def test_nested_structure(self):
suite = TestSuite()
suite.set_criticality(critical_tags=['crit'])
suite.keywords = [Keyword(type='setup'), Keyword(type='teardown')]
K1 = self._verify_keyword(suite.keywords[0], type=1)
K2 = self._verify_keyword(suite.keywords[1], type=2)
suite.suites = [TestSuite()]
suite.suites[0].tests = [TestCase(tags=['crit', 'xxx'])]
t = self._verify_test(suite.suites[0].tests[0], tags=('crit', 'xxx'))
suite.tests = [TestCase(), TestCase(status='PASS')]
S1 = self._verify_suite(suite.suites[0],
status=0, tests=(t,), stats=(1, 0, 1, 0))
suite.tests[0].keywords = [Keyword(type=Keyword.FOR_LOOP_TYPE), Keyword()]
suite.tests[0].keywords[0].keywords = [Keyword(type=Keyword.FOR_ITEM_TYPE)]
suite.tests[0].keywords[0].messages = [Message()]
k = self._verify_keyword(suite.tests[0].keywords[0].keywords[0], type=4)
m = self._verify_message(suite.tests[0].keywords[0].messages[0])
k1 = self._verify_keyword(suite.tests[0].keywords[0],
type=3, keywords=(k,), messages=(m,))
suite.tests[0].keywords[1].messages = [Message(), Message('msg', level='TRACE')]
m1 = self._verify_message(suite.tests[0].keywords[1].messages[0])
m2 = self._verify_message(suite.tests[0].keywords[1].messages[1], 'msg', level=0)
k2 = self._verify_keyword(suite.tests[0].keywords[1], messages=(m1, m2))
T1 = self._verify_test(suite.tests[0], critical=0, keywords=(k1, k2))
T2 = self._verify_test(suite.tests[1], critical=0, status=1)
self._verify_suite(suite, status=0, keywords=(K1, K2), suites=(S1,),
tests=(T1, T2), stats=(3, 1, 1, 0))
self._verify_min_message_level('TRACE')
def test_nested_structure(self):
suite = TestSuite()
suite.set_criticality(critical_tags=['crit'])
suite.keywords = [Keyword(type='setup'), Keyword(type='teardown')]
K1 = self._verify_keyword(suite.keywords[0], type=1)
K2 = self._verify_keyword(suite.keywords[1], type=2)
suite.suites = [TestSuite()]
suite.suites[0].tests = [TestCase(tags=['crit', 'xxx'])]
t = self._verify_test(suite.suites[0].tests[0], tags=('crit', 'xxx'))
suite.tests = [TestCase(), TestCase(status='PASS')]
S1 = self._verify_suite(suite.suites[0],
status=0, tests=(t,), stats=(1, 0, 1, 0))
suite.tests[0].keywords = [Keyword(type=Keyword.FOR_LOOP_TYPE), Keyword()]
suite.tests[0].keywords[0].keywords = [Keyword(type=Keyword.FOR_ITEM_TYPE)]
suite.tests[0].keywords[0].messages = [Message()]
k = self._verify_keyword(suite.tests[0].keywords[0].keywords[0], type=4)
m = self._verify_message(suite.tests[0].keywords[0].messages[0])
k1 = self._verify_keyword(suite.tests[0].keywords[0],
type=3, keywords=(k,), messages=(m,))
suite.tests[0].keywords[1].messages = [Message(), Message('msg', level='TRACE')]
m1 = self._verify_message(suite.tests[0].keywords[1].messages[0])
m2 = self._verify_message(suite.tests[0].keywords[1].messages[1], 'msg', level=0)
k2 = self._verify_keyword(suite.tests[0].keywords[1], messages=(m1, m2))
T1 = self._verify_test(suite.tests[0], critical=0, keywords=(k1, k2))
T2 = self._verify_test(suite.tests[1], critical=0, status=1)
self._verify_suite(suite, status=0, keywords=(K1, K2), suites=(S1,),
tests=(T1, T2), stats=(3, 1, 1, 0))
self._verify_min_message_level('TRACE')
def _get_suite(self):
ts = lambda s, ms=0: '20120816 16:09:%02d.%03d' % (s, ms)
suite = TestSuite(name='root', starttime=ts(0), endtime=ts(42))
suite.set_criticality(critical_tags=['t2'])
sub1 = TestSuite(name='sub1', starttime=ts(0), endtime=ts(10))
sub2 = TestSuite(name='sub2')
suite.suites = [sub1, sub2]
sub1.tests = [
TestCase(tags=['t1', 't2'],
status='PASS',
starttime=ts(0),
endtime=ts(1, 500)),
TestCase(tags=['t1', 't3'],
status='FAIL',
starttime=ts(2),
endtime=ts(3, 499))
]
sub2.tests = [
TestCase(tags=['t1', 't2'],
status='PASS',
starttime=ts(10),
endtime=ts(30))
]
sub2.suites.create(name='below suite stat level')\
.tests.create(tags=['t1'], status='FAIL', starttime=ts(30), endtime=ts(40))
return suite
def start_suite(self, suite):
variables = GLOBAL_VARIABLES.copy()
variables.set_from_variable_table(suite.variables)
result = TestSuite(source=suite.source,
name=suite.name,
doc=suite.doc,
metadata=suite.metadata,
starttime=get_timestamp())
if not self.result:
result.set_criticality(self._settings.critical_tags,
self._settings.non_critical_tags)
self.result = Result(root_suite=result)
self.result.configure(status_rc=self._settings.status_rc,
stat_config=self._settings.statistics_config)
else:
self._suite.suites.append(result)
ns = Namespace(result, variables, self._variables,
suite.user_keywords, suite.imports)
EXECUTION_CONTEXTS.start_suite(ns, self._output, self._settings.dry_run)
self._context.set_suite_variables(result)
if not (self._suite_status and self._suite_status.failures):
ns.handle_imports()
variables.resolve_delayed()
result.doc = self._resolve_setting(result.doc)
result.metadata = [(self._resolve_setting(n), self._resolve_setting(v))
for n, v in result.metadata.items()]
self._context.set_suite_variables(result)
self._suite = result
self._suite_status = SuiteStatus(self._suite_status,
self._settings.exit_on_failure,
self._settings.skip_teardown_on_exit)
self._output.start_suite(ModelCombiner(suite, self._suite))
self._run_setup(suite.keywords.setup, self._suite_status)
self._executed_tests = NormalizedDict(ignore='_')
def start_suite(self, suite):
variables = GLOBAL_VARIABLES.copy()
variables.set_from_variable_table(suite.variables)
result = TestSuite(source=suite.source,
name=suite.name,
doc=suite.doc,
metadata=suite.metadata,
starttime=get_timestamp())
if not self.result:
result.set_criticality(self._settings.critical_tags,
self._settings.non_critical_tags)
self.result = Result(root_suite=result)
self.result.configure(status_rc=self._settings.status_rc,
stat_config=self._settings.statistics_config)
else:
self._suite.suites.append(result)
ns = Namespace(result, variables, self._variables, suite.user_keywords,
suite.imports)
EXECUTION_CONTEXTS.start_suite(ns, self._output,
self._settings.dry_run)
self._context.set_suite_variables(result)
if not (self._suite_status and self._suite_status.failures):
ns.handle_imports()
variables.resolve_delayed()
result.doc = self._resolve_setting(result.doc)
result.metadata = [(self._resolve_setting(n), self._resolve_setting(v))
for n, v in result.metadata.items()]
self._context.set_suite_variables(result)
self._suite = result
self._suite_status = SuiteStatus(self._suite_status,
self._settings.exit_on_failure,
self._settings.skip_teardown_on_exit)
self._output.start_suite(ModelCombiner(suite, self._suite))
self._run_setup(suite.keywords.setup, self._suite_status)
self._executed_tests = NormalizedDict(ignore='_')
def test_criticality_set_for_child_suites_earlier_is_ignored(self):
self.suite.set_criticality('use', 'us')
sub2 = TestSuite()
sub2.set_criticality('ignore', 'these')
self.suite.suites.append(sub2)
self._verify_criticality(self.suite, ['use'], ['us'])
self._verify_criticality(self.sub, ['use'], ['us'])
self._verify_criticality(sub2, ['use'], ['us'])
def _create_nested_suite_with_tests(self):
suite = TestSuite()
suite.set_criticality([], ['nc'])
suite.suites = [
self._create_suite_with_tests(),
self._create_suite_with_tests()
]
return suite
def test_criticality_set_for_child_suites_earlier_is_ignored(self):
self.suite.set_criticality('use', 'us')
sub2 = TestSuite()
sub2.set_criticality('ignore', 'these')
self.suite.suites.append(sub2)
self._verify_criticality(self.suite, ['use'], ['us'])
self._verify_criticality(self.sub, ['use'], ['us'])
self._verify_criticality(sub2, ['use'], ['us'])
def test_passed(self):
suite = TestSuite()
assert_true(suite.passed)
suite.tests.create(status='PASS')
assert_true(suite.passed)
suite.tests.create(status='FAIL', tags='tag')
assert_false(suite.passed)
suite.set_criticality(non_critical_tags='tag')
assert_true(suite.passed)
def test_passed(self):
suite = TestSuite()
assert_true(suite.passed)
suite.tests.create(status='PASS')
assert_true(suite.passed)
suite.tests.create(status='FAIL', tags='tag')
assert_false(suite.passed)
suite.set_criticality(non_critical_tags='tag')
assert_true(suite.passed)
def _create_suite_with_tests(self):
suite = TestSuite()
suite.set_criticality([], ['nc'])
suite.tests = [TestCase(status='PASS'),
TestCase(status='PASS', tags='nc'),
TestCase(status='PASS'),
TestCase(status='FAIL'),
TestCase(status='FAIL', tags='nc')]
return suite
def test_remove_passed_does_now_remove_setup_and_teardown_from_suite_with_noncritical_failure(self):
suite = TestSuite()
suite.set_criticality([], ['non'])
suite.keywords.create(type=SETUP).messages.create(message='some')
suite.keywords.create(type=TEARDOWN).keywords.create()
suite.tests.create(status='FAIL', tags='non')
assert_equal(suite.status, 'PASS')
self._remove_passed(suite)
assert_equal(len(suite.keywords.setup.messages), 1)
assert_equal(len(suite.keywords.teardown.keywords), 1)
def test_remove_passed_does_now_remove_setup_and_teardown_from_suite_with_noncritical_failure(self):
suite = TestSuite()
suite.set_criticality([], ['non'])
suite.keywords.create(type=SETUP).messages.create(message='some')
suite.keywords.create(type=TEARDOWN).keywords.create()
suite.tests.create(status='FAIL', tags='non')
assert_equal(suite.status, 'PASS')
self._remove_passed(suite)
assert_equal(len(suite.keywords.setup.messages), 1)
assert_equal(len(suite.keywords.teardown.keywords), 1)
def _create_suite_with_tests(self):
suite = TestSuite()
suite.set_criticality([], ['nc'])
suite.tests = [
TestCase(status='PASS'),
TestCase(status='PASS', tags='nc'),
TestCase(status='PASS'),
TestCase(status='FAIL'),
TestCase(status='FAIL', tags='nc')
]
return suite
def generate_suite():
suite = TestSuite(name='Root Suite')
suite.set_criticality(critical_tags=['smoke'])
s1 = suite.suites.create(name='First Sub Suite')
s2 = suite.suites.create(name='Second Sub Suite')
s11 = s1.suites.create(name='Sub Suite 1_1')
s12 = s1.suites.create(name='Sub Suite 1_2')
s13 = s1.suites.create(name='Sub Suite 1_3')
s21 = s2.suites.create(name='Sub Suite 2_1')
s11.tests = [TestCase(status='PASS'), TestCase(status='FAIL', tags=['t1'])]
s12.tests = [TestCase(status='PASS', tags=['t_1','t2',]),
TestCase(status='PASS', tags=['t1','smoke']),
TestCase(status='FAIL', tags=['t1','t2','t3','smoke'])]
s13.tests = [TestCase(status='PASS', tags=['t1','t 2','smoke'])]
s21.tests = [TestCase(status='FAIL', tags=['t3','Smoke'])]
return suite
def setUp(self):
ts = '20120816 00:00:'
suite = TestSuite(starttime=ts+'00.000', endtime=ts+'59.999')
suite.suites = [
TestSuite(starttime=ts+'00.000', endtime=ts+'30.000'),
TestSuite(starttime=ts+'30.000', endtime=ts+'42.042')
]
suite.suites[0].tests = [
TestCase(starttime=ts+'00.000', endtime=ts+'00.001', tags=['t1']),
TestCase(starttime=ts+'00.001', endtime=ts+'01.001', tags=['t1', 't2'])
]
suite.suites[1].tests = [
TestCase(starttime=ts+'30.000', endtime=ts+'40.000', tags=['t1', 't2', 't3'])
]
suite.set_criticality(critical_tags=['t2'])
self.stats = Statistics(suite, tag_stat_combine=[('?2', 'combined')])
def _get_suite(self):
ts = lambda s, ms=0: '20120816 16:09:%02d.%03d' % (s, ms)
suite = TestSuite(name='root', starttime=ts(0), endtime=ts(42))
suite.set_criticality(critical_tags=['t2'])
sub1 = TestSuite(name='sub1', starttime=ts(0), endtime=ts(10))
sub2 = TestSuite(name='sub2')
suite.suites = [sub1, sub2]
sub1.tests = [
TestCase(tags=['t1', 't2'], status='PASS', starttime=ts(0), endtime=ts(1, 500)),
TestCase(tags=['t1', 't3'], status='FAIL', starttime=ts(2), endtime=ts(3, 499))
]
sub2.tests = [
TestCase(tags=['t1', 't2'], status='PASS', starttime=ts(10), endtime=ts(30))
]
sub2.suites.create(name='below suite stat level')\
.tests.create(tags=['t1'], status='FAIL', starttime=ts(30), endtime=ts(40))
return suite
def start_suite(self, suite):
with open('Excute.txt', 'w') as f:
f.write('TS=')
f.write(str(suite.name))
f.write('\n')
self._output.library_listeners.new_suite_scope()
result = TestSuite(source=suite.source,
name=suite.name,
doc=suite.doc,
metadata=suite.metadata,
starttime=get_timestamp())
if not self.result:
result.set_criticality(self._settings.critical_tags,
self._settings.non_critical_tags)
self.result = Result(root_suite=result)
self.result.configure(status_rc=self._settings.status_rc,
stat_config=self._settings.statistics_config)
else:
self._suite.suites.append(result)
self._suite = result
self._suite_status = SuiteStatus(self._suite_status,
self._settings.exit_on_failure,
self._settings.exit_on_error,
self._settings.skip_teardown_on_exit)
ns = Namespace(self._variables, result, suite.resource)
ns.start_suite()
ns.variables.set_from_variable_table(suite.resource.variables)
EXECUTION_CONTEXTS.start_suite(result, ns, self._output,
self._settings.dry_run)
self._context.set_suite_variables(result)
if not self._suite_status.failures:
ns.handle_imports()
ns.variables.resolve_delayed()
result.doc = self._resolve_setting(result.doc)
result.metadata = [(self._resolve_setting(n), self._resolve_setting(v))
for n, v in result.metadata.items()]
self._context.set_suite_variables(result)
self._output.start_suite(ModelCombiner(suite, result,
tests=suite.tests,
suites=suite.suites,
test_count=suite.test_count))
self._output.register_error_listener(self._suite_status.error_occurred)
self._run_setup(suite.keywords.setup, self._suite_status)
self._executed_tests = NormalizedDict(ignore='_')
def start_suite(self, suite):
self._output.library_listeners.new_suite_scope()
result = TestSuite(source=suite.source,
name=suite.name,
doc=suite.doc,
metadata=suite.metadata,
starttime=get_timestamp(),
rpa=self._settings.rpa)
if not self.result:
result.set_criticality(self._settings.critical_tags,
self._settings.non_critical_tags)
self.result = Result(root_suite=result, rpa=self._settings.rpa)
self.result.configure(status_rc=self._settings.status_rc,
stat_config=self._settings.statistics_config)
else:
self._suite.suites.append(result)
self._suite = result
self._suite_status = SuiteStatus(self._suite_status,
self._settings.exit_on_failure,
self._settings.exit_on_error,
self._settings.skip_teardown_on_exit)
ns = Namespace(self._variables, result, suite.resource)
ns.start_suite()
ns.variables.set_from_variable_table(suite.resource.variables)
EXECUTION_CONTEXTS.start_suite(result, ns, self._output,
self._settings.dry_run)
self._context.set_suite_variables(result)
if not self._suite_status.failures:
ns.handle_imports()
ns.variables.resolve_delayed()
result.doc = self._resolve_setting(result.doc)
result.metadata = [(self._resolve_setting(n), self._resolve_setting(v))
for n, v in result.metadata.items()]
self._context.set_suite_variables(result)
self._output.start_suite(
ModelCombiner(suite,
result,
tests=suite.tests,
suites=suite.suites,
test_count=suite.test_count))
self._output.register_error_listener(self._suite_status.error_occurred)
self._run_setup(suite.keywords.setup, self._suite_status)
self._executed_tests = NormalizedDict(ignore='_')
class TestCriticality(unittest.TestCase):
def setUp(self):
self.suite = TestSuite()
self.sub = self.suite.suites.create()
def test_default_criticality(self):
self._verify_criticality(self.suite, [], [])
self._verify_criticality(self.sub, [], [])
def test_set_criticality(self):
self.suite.set_criticality(['c1', 'c2'], 'non')
self._verify_criticality(self.suite, ['c1', 'c2'], ['non'])
self._verify_criticality(self.sub, ['c1', 'c2'], ['non'])
def test_cannot_set_criticality_for_child_suites(self):
assert_raises_with_msg(
ValueError, 'Criticality can only be set to the root suite.',
self.sub.set_criticality)
def test_criticality_set_for_child_suites_earlier_is_ignored(self):
self.suite.set_criticality('use', 'us')
sub2 = TestSuite()
sub2.set_criticality('ignore', 'these')
self.suite.suites.append(sub2)
self._verify_criticality(self.suite, ['use'], ['us'])
self._verify_criticality(self.sub, ['use'], ['us'])
self._verify_criticality(sub2, ['use'], ['us'])
def test_critical_with_parent(self):
suite = TestSuite()
assert_equal(suite.tests.create().critical, True)
suite.set_criticality(critical_tags=['crit'])
assert_equal(suite.tests.create().critical, False)
assert_equal(suite.tests.create(tags=['crit']).critical, True)
def test_critical_without_parent(self):
assert_equal(TestCase().critical, True)
def _verify_criticality(self, suite, crit, non_crit):
assert_equal([str(t) for t in suite.criticality.critical_tags], crit)
assert_equal([str(t) for t in suite.criticality.non_critical_tags],
non_crit)
class TestCriticality(unittest.TestCase):
def setUp(self):
self.suite = TestSuite()
self.sub = self.suite.suites.create()
def test_default_criticality(self):
self._verify_criticality(self.suite, [], [])
self._verify_criticality(self.sub, [], [])
def test_set_criticality(self):
self.suite.set_criticality(['c1', 'c2'], 'non')
self._verify_criticality(self.suite, ['c1', 'c2'], ['non'])
self._verify_criticality(self.sub, ['c1', 'c2'], ['non'])
def test_cannot_set_criticality_for_child_suites(self):
assert_raises(TypeError, self.sub.set_criticality)
def test_criticality_set_for_child_suites_earlier_is_ignored(self):
self.suite.set_criticality('use', 'us')
sub2 = TestSuite()
sub2.set_criticality('ignore', 'these')
self.suite.suites.append(sub2)
self._verify_criticality(self.suite, ['use'], ['us'])
self._verify_criticality(self.sub, ['use'], ['us'])
self._verify_criticality(sub2, ['use'], ['us'])
def test_critical_with_parent(self):
suite = TestSuite()
assert_equal(suite.tests.create().critical, True)
suite.set_criticality(critical_tags=['crit'])
assert_equal(suite.tests.create().critical, False)
assert_equal(suite.tests.create(tags=['crit']).critical, True)
def test_critical_without_parent(self):
assert_equal(TestCase().critical, True)
def _verify_criticality(self, suite, crit, non_crit):
assert_equal([str(t) for t in suite.criticality.critical_tags], crit)
assert_equal([str(t) for t in suite.criticality.non_critical_tags], non_crit)
def test_critical_with_parent(self):
suite = TestSuite()
assert_equal(suite.tests.create().critical, True)
suite.set_criticality(critical_tags=['crit'])
assert_equal(suite.tests.create().critical, False)
assert_equal(suite.tests.create(tags=['crit']).critical, True)
def _create_nested_suite_with_tests(self):
suite = TestSuite()
suite.set_criticality([], ['nc'])
suite.suites = [self._create_suite_with_tests(),
self._create_suite_with_tests()]
return suite
def test_critical_with_parent(self):
suite = TestSuite()
assert_equal(suite.tests.create().critical, True)
suite.set_criticality(critical_tags=['crit'])
assert_equal(suite.tests.create().critical, False)
assert_equal(suite.tests.create(tags=['crit']).critical, True)
