def test_iteration_events(ctx):
observer = ctx.observer(E.TEST_ITERATION_STARTED, E.TEST_ITERATION_ENDED)
script = ctx.test(DummyTest)
data = DummyData().with_iteration(
IterationDataBuilder().with_name('new name')
.with_description('new description')
.with_tags('tag1', 'tag2')
.with_data(foo='bar')
.build()).with_iteration(IterationDataBuilder().with_data(fail='ouch').build())
script.execute(data)
it1_start, it1_end, it2_start, it2_end = observer.events
assert it1_start.event_type == E.TEST_ITERATION_STARTED
assert it1_start.data == it1_end.data == {'foo': 'bar'}
assert it1_start.test_script == it1_end.test_script == script
assert it1_start.data_provider == it1_end.data_provider == data
assert it1_start.timestamp == it1_end.start_time
assert it1_end.event_type == E.TEST_ITERATION_ENDED
assert it1_end.status == 'PASS'
assert it1_end.duration == it1_end.timestamp - it1_end.start_time
assert it1_end.exception == (None, None, None)
assert it1_start.iteration.name == it1_end.iteration.name == 'new name'
assert it1_start.iteration.description == it1_end.iteration.description == 'new description'
assert it1_start.iteration.tags == it1_end.iteration.tags == {'tag1', 'tag2'}
assert it2_start.iteration.name is None and it2_end.iteration.name is None
assert it2_start.iteration.description is None and it2_end.iteration.description is None
assert it2_start.iteration.tags is None and it2_end.iteration.tags is None
def test_iteration_events(ctx):
test = ctx.test(DummyTest)
data = DummyData()
anon_iteration = IterationDataBuilder().build()
name_iteration = IterationDataBuilder().with_name('My Iteration').build()
desc_iteration = IterationDataBuilder().with_description(
'blah blah').build()
full_iteration = IterationDataBuilder().with_name(
'My Iteration').with_description('blah blah').build()
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, anon_iteration))
assert out.getvalue().strip() == '[ITERATION]'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, name_iteration))
assert out.getvalue().strip() == '[ITERATION] My Iteration'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, desc_iteration))
assert out.getvalue().strip() == '[ITERATION]: blah blah'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, full_iteration))
assert out.getvalue().strip() == '[ITERATION] My Iteration: blah blah'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, anon_iteration,
milliseconds(0), Status.PASS, None))
assert out.getvalue().strip() == '[ITERATION - PASS] (0 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, name_iteration,
milliseconds(-999), Status.FAIL, None))
assert out.getvalue().strip() == '[ITERATION - FAIL] My Iteration (999 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, full_iteration,
milliseconds(-1567), Status.SKIP, None))
assert out.getvalue().strip() == '[ITERATION - SKIP] My Iteration (1.57 s)'
def test_access_data_in_phase_events(ctx):
observer = ctx.observer(*(ALL_PHASE_START_EVENTS + ALL_PHASE_END_EVENTS))
observer.hook(lambda e: e.data['foo'].append('bar'), *ALL_PHASE_START_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_setup_data(foo=[])
.with_iteration(IterationDataBuilder().with_data(foo=[]).build())
.with_tear_down_data(foo=[]))
assert all(e.data['foo'] == ['bar'] for e in observer.events)
def test_skip_exceptions_not_skipping_suite():
suite = DummySuite()
observer = suite.observer(E.TEST_ENDED, E.SUITE_ENDED)
suite.add_test(DummyTest)
suite.add_test(DummyTest, DummyData().with_setup_data(skip='skipped'))
suite.add_test(
DummyTest,
DummyData().with_iteration(
IterationDataBuilder().build()).with_iteration(
IterationDataBuilder().with_data(
skip='skipped').build()).with_iteration(
IterationDataBuilder().build()))
suite.add_test(DummyTest)
suite.execute()
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_ENDED] * 4 + [E.SUITE_ENDED]
assert [e.status for e in observer.events] == [
Status.PASS, Status.SKIP, Status.PASS, Status.PASS, Status.PASS
]
def test_skip_iteration_do_not_affect_results(ctx):
observer = ctx.observer(*ALL_END_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_iteration(IterationDataBuilder().with_data(foo='bar').build())
.with_iteration(IterationDataBuilder().with_data(skip='wip').build())
.with_iteration(IterationDataBuilder().with_data(foo='baz').build()))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_SETUP_ENDED,
E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED,
E.TEST_TEARDOWN_ENDED,
E.TEST_ENDED]
setup, it1, it2, it3, tear_down, test = observer.events
assert setup.status == Status.PASS
assert it1.status == Status.PASS
assert it2.status == Status.SKIP
assert it2.exception[1].reason == 'wip'
assert it3.status == Status.PASS
assert tear_down.status == Status.PASS
assert test.status == Status.PASS
assert test.exceptions == [it2.exception]
def test_failed_iteration_do_not_affect_other_iterations(ctx):
observer = ctx.observer(*ALL_END_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_iteration(IterationDataBuilder().with_data(foo='bar').build())
.with_iteration(IterationDataBuilder().with_data(fail='oops').build())
.with_iteration(IterationDataBuilder().with_data(foo='baz').build()))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_SETUP_ENDED,
E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED,
E.TEST_TEARDOWN_ENDED,
E.TEST_ENDED]
setup, it1, it2, it3, tear_down, test = observer.events
assert setup.status == Status.PASS
assert it1.status == Status.PASS
assert it2.status == Status.FAIL
assert str(it2.exception[1]) == 'oops'
assert it3.status == Status.PASS
assert tear_down.status == Status.PASS
assert test.status == Status.FAIL
assert test.exceptions == [it2.exception]
def test_suite_status_fail_if_one_test_fails():
suite = DummySuite()
observer = suite.observer(E.TEST_ENDED, E.SUITE_ENDED)
suite.add_test(DummyTest) # passes
suite.add_test(DummyTest,
DummyData().with_setup_data(skip='skipped')) # skipped
suite.add_test(
DummyTest,
DummyData().with_iteration(
IterationDataBuilder().with_data(fail='oops').build()) # fails
)
suite.add_test(DummyTest) # passes
suite.execute()
assert [e.status for e in observer.events] == [
Status.PASS, Status.SKIP, Status.FAIL, Status.PASS, Status.FAIL
]
def test_report_summary(ctx):
separator = '-' * 64
test1 = ctx.test(DummyTest)
data1 = DummyData().with_name('test 1')
test1.execute(data1)
test1_iterations = [Status.PASS, Status.SKIP, Status.FAIL, Status.PASS]
test2 = ctx.test(DummyTest)
data2 = DummyData().with_name('test 2')
test2.execute(data2)
test2_iterations = [
Status.FAIL, Status.FAIL, Status.SKIP, Status.PASS, Status.SKIP
]
cases = [{
't': test1,
'd': data1,
'i': test1_iterations,
's': Status.PASS,
'e': []
}, {
't':
test2,
'd':
data2,
'i':
test2_iterations,
's':
Status.FAIL,
'e':
[(ValueError, ValueError('oops'),
TracebackBuilder().with_frame('file.py', 'raise error',
4).with_frame('other.py', '5/0',
8).build()),
(ValueError, ValueError('oops'),
TracebackBuilder().with_frame('oops.py', 'raise exc', 5).build())]
}]
iteration = IterationDataBuilder().with_name(
'My Iteration').with_description('blah blah').build()
with event_logger_context(ctx) as (publisher, out):
for c in cases:
publisher.notify(E.TestStartedEvent(c['t'], c['d']))
for i in c['i']:
publisher.notify(
E.TestIterationEndedEvent(c['t'], c['d'], iteration,
milliseconds(-10), i, None))
publisher.notify(
E.TestEndedEvent(c['t'], c['d'], milliseconds(0), c['s'],
c['e']))
out.seek(0)
out.truncate()
publisher.notify(E.SuiteEndedEvent(ctx, milliseconds(0), Status.PASS))
value = out.getvalue()
print(value)
assert value.strip().split('\n') == [
separator,
'Execution Summary',
separator,
'',
'[PASS] test 1: 4 iteration(s) (2 pass - 1 fail - 1 skip)',
'[FAIL] test 2: 5 iteration(s) (1 pass - 2 fail - 2 skip)',
"ValueError('oops',)",
' File "file.py", line 4, in raise error',
' File "other.py", line 8, in 5/0',
'',
"ValueError('oops',)",
' File "oops.py", line 5, in raise exc',
]