def test_suite_status_skip_if_all_tests_skipped():
suite = DummySuite()
observer = suite.observer(E.SUITE_ENDED)
suite.add_test(DummyTest, DummyData().with_setup_data(skip='skipped'))
suite.add_test(DummyTest, DummyData().with_setup_data(skip='skipped'))
suite.execute()
end_event = observer.last_event
assert end_event.status == Status.SKIP
def test_iteration_events(ctx):
observer = ctx.observer(E.TEST_ITERATION_STARTED, E.TEST_ITERATION_ENDED)
script = ctx.test(DummyTest)
data = DummyData().with_iteration(
IterationDataBuilder().with_name('new name')
.with_description('new description')
.with_tags('tag1', 'tag2')
.with_data(foo='bar')
.build()).with_iteration(IterationDataBuilder().with_data(fail='ouch').build())
script.execute(data)
it1_start, it1_end, it2_start, it2_end = observer.events
assert it1_start.event_type == E.TEST_ITERATION_STARTED
assert it1_start.data == it1_end.data == {'foo': 'bar'}
assert it1_start.test_script == it1_end.test_script == script
assert it1_start.data_provider == it1_end.data_provider == data
assert it1_start.timestamp == it1_end.start_time
assert it1_end.event_type == E.TEST_ITERATION_ENDED
assert it1_end.status == 'PASS'
assert it1_end.duration == it1_end.timestamp - it1_end.start_time
assert it1_end.exception == (None, None, None)
assert it1_start.iteration.name == it1_end.iteration.name == 'new name'
assert it1_start.iteration.description == it1_end.iteration.description == 'new description'
assert it1_start.iteration.tags == it1_end.iteration.tags == {'tag1', 'tag2'}
assert it2_start.iteration.name is None and it2_end.iteration.name is None
assert it2_start.iteration.description is None and it2_end.iteration.description is None
assert it2_start.iteration.tags is None and it2_end.iteration.tags is None
def test_suite_status_fail_if_one_test_fails():
suite = DummySuite()
observer = suite.observer(E.TEST_ENDED, E.SUITE_ENDED)
suite.add_test(DummyTest) # passes
suite.add_test(DummyTest,
DummyData().with_setup_data(skip='skipped')) # skipped
suite.add_test(
DummyTest,
DummyData().with_iteration(
IterationDataBuilder().with_data(fail='oops').build()) # fails
)
suite.add_test(DummyTest) # passes
suite.execute()
assert [e.status for e in observer.events] == [
Status.PASS, Status.SKIP, Status.FAIL, Status.PASS, Status.FAIL
]
def test_access_data_in_phase_events(ctx):
observer = ctx.observer(*(ALL_PHASE_START_EVENTS + ALL_PHASE_END_EVENTS))
observer.hook(lambda e: e.data['foo'].append('bar'), *ALL_PHASE_START_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_setup_data(foo=[])
.with_iteration(IterationDataBuilder().with_data(foo=[]).build())
.with_tear_down_data(foo=[]))
assert all(e.data['foo'] == ['bar'] for e in observer.events)
def test_skip_exceptions_not_skipping_suite():
suite = DummySuite()
observer = suite.observer(E.TEST_ENDED, E.SUITE_ENDED)
suite.add_test(DummyTest)
suite.add_test(DummyTest, DummyData().with_setup_data(skip='skipped'))
suite.add_test(
DummyTest,
DummyData().with_iteration(
IterationDataBuilder().build()).with_iteration(
IterationDataBuilder().with_data(
skip='skipped').build()).with_iteration(
IterationDataBuilder().build()))
suite.add_test(DummyTest)
suite.execute()
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_ENDED] * 4 + [E.SUITE_ENDED]
assert [e.status for e in observer.events] == [
Status.PASS, Status.SKIP, Status.PASS, Status.PASS, Status.PASS
]
def test_test_meta_extended_from_data(ctx):
class SimpleTest(marvin.TestScript):
"""Test description"""
TAGS = ['a', 'few', 'tags']
test = ctx.test(SimpleTest)
test.execute(DummyData()
.with_name('New Name')
.with_description('New description')
.wih_tags('more', 'tags', 4))
assert test.name == 'New Name'
assert test.description == 'New description'
assert test.tags == {'a', 'few', 'tags', 'more', '4'}
def test_test_events(ctx):
observer = ctx.observer(*ALL_TEST_EVENTS)
d = DummyData()
t = ctx.test(DummyTest)
t.execute(d)
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == ALL_TEST_EVENTS
assert ctx.last_reported_status == Status.PASS
assert all(e.test_script == t for e in observer.events)
assert all(e.data_provider == d for e in observer.events)
assert all(e.duration == e.timestamp - e.start_time
for e in observer.events if e.event_type in ALL_END_EVENTS)
def test_iteration_events(ctx):
test = ctx.test(DummyTest)
data = DummyData()
anon_iteration = IterationDataBuilder().build()
name_iteration = IterationDataBuilder().with_name('My Iteration').build()
desc_iteration = IterationDataBuilder().with_description(
'blah blah').build()
full_iteration = IterationDataBuilder().with_name(
'My Iteration').with_description('blah blah').build()
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, anon_iteration))
assert out.getvalue().strip() == '[ITERATION]'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, name_iteration))
assert out.getvalue().strip() == '[ITERATION] My Iteration'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, desc_iteration))
assert out.getvalue().strip() == '[ITERATION]: blah blah'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationStartedEvent(test, data, full_iteration))
assert out.getvalue().strip() == '[ITERATION] My Iteration: blah blah'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, anon_iteration,
milliseconds(0), Status.PASS, None))
assert out.getvalue().strip() == '[ITERATION - PASS] (0 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, name_iteration,
milliseconds(-999), Status.FAIL, None))
assert out.getvalue().strip() == '[ITERATION - FAIL] My Iteration (999 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestIterationEndedEvent(test, data, full_iteration,
milliseconds(-1567), Status.SKIP, None))
assert out.getvalue().strip() == '[ITERATION - SKIP] My Iteration (1.57 s)'
def test_skip_setup_skips_everything(ctx):
observer = ctx.observer(*ALL_TEST_EVENTS)
ctx.test(DummyTest).execute(DummyData().with_setup_data(skip='move on'))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_STARTED,
E.TEST_SETUP_STARTED, E.TEST_SETUP_ENDED,
E.TEST_ENDED]
_, _, setup, test = observer.events
assert setup.status == Status.SKIP
assert setup.exception[0] == ContextSkippedException
assert setup.exception[1].reason == 'move on'
assert test.status == Status.SKIP
assert test.exceptions == [setup.exception]
def test_fail_setup_skips_iteration_but_not_tear_down(ctx):
observer = ctx.observer(*ALL_TEST_EVENTS)
ctx.test(DummyTest).execute(DummyData().with_setup_data(fail='kaput'))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_STARTED,
E.TEST_SETUP_STARTED, E.TEST_SETUP_ENDED,
E.TEST_TEARDOWN_STARTED, E.TEST_TEARDOWN_ENDED,
E.TEST_ENDED]
_, _, setup, _, tear_down, test = observer.events
assert setup.status == Status.FAIL
assert str(setup.exception[1]) == 'kaput'
assert tear_down.status == Status.PASS
assert test.status == Status.FAIL
assert test.exceptions == [setup.exception]
def test_tear_down_events(ctx):
observer = ctx.observer(E.TEST_TEARDOWN_STARTED, E.TEST_TEARDOWN_ENDED)
script = ctx.test(DummyTest)
data = DummyData().with_tear_down_data(fail='oops')
script.execute(data)
start, end = observer.events
assert start.event_type == E.TEST_TEARDOWN_STARTED
assert start.data == {'fail': 'oops'}
assert start.test_script == script
assert start.data_provider == data
assert start.timestamp == end.start_time
assert end.event_type == E.TEST_TEARDOWN_ENDED
assert end.status == 'FAIL'
assert end.data == {'fail': 'oops'}
assert end.test_script == script
assert end.data_provider == data
assert end.duration == end.timestamp - end.start_time
assert isinstance(end.exception[1], Exception)
def test_setup_teardown_events(ctx):
test = ctx.test(DummyTest)
data = DummyData()
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestSetupStartedEvent(test, data, None))
assert out.getvalue().strip() == '[SETUP]'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestTearDownStartedEvent(test, data, None))
assert out.getvalue().strip() == '[TEARDOWN]'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestSetupEndedEvent(test, data, None, milliseconds(-500),
Status.PASS, None))
assert out.getvalue().strip() == '[SETUP - PASS] (500 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestTearDownEndedEvent(test, data, None, milliseconds(-1500),
Status.FAIL, None))
assert out.getvalue().strip() == '[TEARDOWN - FAIL] (1.50 s)'
def test_skip_iteration_do_not_affect_results(ctx):
observer = ctx.observer(*ALL_END_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_iteration(IterationDataBuilder().with_data(foo='bar').build())
.with_iteration(IterationDataBuilder().with_data(skip='wip').build())
.with_iteration(IterationDataBuilder().with_data(foo='baz').build()))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_SETUP_ENDED,
E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED,
E.TEST_TEARDOWN_ENDED,
E.TEST_ENDED]
setup, it1, it2, it3, tear_down, test = observer.events
assert setup.status == Status.PASS
assert it1.status == Status.PASS
assert it2.status == Status.SKIP
assert it2.exception[1].reason == 'wip'
assert it3.status == Status.PASS
assert tear_down.status == Status.PASS
assert test.status == Status.PASS
assert test.exceptions == [it2.exception]
def test_failed_iteration_do_not_affect_other_iterations(ctx):
observer = ctx.observer(*ALL_END_EVENTS)
ctx.test(DummyTest).execute(DummyData()
.with_iteration(IterationDataBuilder().with_data(foo='bar').build())
.with_iteration(IterationDataBuilder().with_data(fail='oops').build())
.with_iteration(IterationDataBuilder().with_data(foo='baz').build()))
triggered_events = [e.event_type for e in observer.events]
assert triggered_events == [E.TEST_SETUP_ENDED,
E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED, E.TEST_ITERATION_ENDED,
E.TEST_TEARDOWN_ENDED,
E.TEST_ENDED]
setup, it1, it2, it3, tear_down, test = observer.events
assert setup.status == Status.PASS
assert it1.status == Status.PASS
assert it2.status == Status.FAIL
assert str(it2.exception[1]) == 'oops'
assert it3.status == Status.PASS
assert tear_down.status == Status.PASS
assert test.status == Status.FAIL
assert test.exceptions == [it2.exception]
def test_report_summary(ctx):
separator = '-' * 64
test1 = ctx.test(DummyTest)
data1 = DummyData().with_name('test 1')
test1.execute(data1)
test1_iterations = [Status.PASS, Status.SKIP, Status.FAIL, Status.PASS]
test2 = ctx.test(DummyTest)
data2 = DummyData().with_name('test 2')
test2.execute(data2)
test2_iterations = [
Status.FAIL, Status.FAIL, Status.SKIP, Status.PASS, Status.SKIP
]
cases = [{
't': test1,
'd': data1,
'i': test1_iterations,
's': Status.PASS,
'e': []
}, {
't':
test2,
'd':
data2,
'i':
test2_iterations,
's':
Status.FAIL,
'e':
[(ValueError, ValueError('oops'),
TracebackBuilder().with_frame('file.py', 'raise error',
4).with_frame('other.py', '5/0',
8).build()),
(ValueError, ValueError('oops'),
TracebackBuilder().with_frame('oops.py', 'raise exc', 5).build())]
}]
iteration = IterationDataBuilder().with_name(
'My Iteration').with_description('blah blah').build()
with event_logger_context(ctx) as (publisher, out):
for c in cases:
publisher.notify(E.TestStartedEvent(c['t'], c['d']))
for i in c['i']:
publisher.notify(
E.TestIterationEndedEvent(c['t'], c['d'], iteration,
milliseconds(-10), i, None))
publisher.notify(
E.TestEndedEvent(c['t'], c['d'], milliseconds(0), c['s'],
c['e']))
out.seek(0)
out.truncate()
publisher.notify(E.SuiteEndedEvent(ctx, milliseconds(0), Status.PASS))
value = out.getvalue()
print(value)
assert value.strip().split('\n') == [
separator,
'Execution Summary',
separator,
'',
'[PASS] test 1: 4 iteration(s) (2 pass - 1 fail - 1 skip)',
'[FAIL] test 2: 5 iteration(s) (1 pass - 2 fail - 2 skip)',
"ValueError('oops',)",
' File "file.py", line 4, in raise error',
' File "other.py", line 8, in 5/0',
'',
"ValueError('oops',)",
' File "oops.py", line 5, in raise exc',
]
def test_test_events(ctx):
anon_data = DummyData()
name_data = DummyData().with_name('new name')
desc_data = DummyData().with_description('new description')
full_data = DummyData().with_name('new name').with_description(
'new description')
anon_test = ctx.test(DummyTest)
anon_test.execute(anon_data)
name_test = ctx.test(DummyTest)
name_test.execute(name_data)
desc_test = ctx.test(DummyTest)
desc_test.execute(desc_data)
full_test = ctx.test(DummyTest)
full_test.execute(full_data)
separator = '-' * 64
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestStartedEvent(anon_test, anon_data))
assert out.getvalue().strip().split('\n') == [
separator, '[TEST] DummyTest - Dummy Test used for testing'
]
test = ctx.test(DummyTest)
test.execute(name_data)
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestStartedEvent(name_test, name_data))
assert out.getvalue().strip().split('\n') == [
separator, '[TEST] new name - Dummy Test used for testing'
]
test = ctx.test(DummyTest)
test.execute(desc_data)
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestStartedEvent(desc_test, desc_data))
assert out.getvalue().strip().split('\n') == [
separator, '[TEST] DummyTest - new description'
]
test = ctx.test(DummyTest)
test.execute(full_data)
with event_logger_context(ctx) as (publisher, out):
publisher.notify(E.TestStartedEvent(full_test, full_data))
assert out.getvalue().strip().split('\n') == [
separator, '[TEST] new name - new description'
]
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestEndedEvent(anon_test, anon_data, milliseconds(0),
Status.PASS, [(None, None, None)]))
assert out.getvalue().strip() == '[TEST - PASS] DummyTest (0 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestEndedEvent(name_test, name_data, milliseconds(-546),
Status.SKIP, [(None, None, None)]))
assert out.getvalue().strip() == '[TEST - SKIP] new name (546 ms)'
with event_logger_context(ctx) as (publisher, out):
publisher.notify(
E.TestEndedEvent(desc_test, desc_data, milliseconds(-1001),
Status.FAIL, [(None, None, None)]))
assert out.getvalue().strip() == '[TEST - FAIL] DummyTest (1.00 s)'
