def test_no_run_name_is_filtered(self):
"""Test that a run with no run name is filtered when running with name.
* Runs a suite with success & failure cases with no name.
* Validates that both tests were run and one succeeded.
* Runs a suite with success & failure cases with name X.
* Validates that both tests were run and one succeeded.
"""
MockTestSuite.components = (SuccessCase, FailureCase)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name=None)
self.validate_suite_data(run_data.main_test,
False,
successes=1,
fails=1)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name='run1')
self.validate_suite_data(run_data.main_test,
False,
successes=1,
fails=1)
def test_no_run_name_doesnt_filter(self):
"""Test that giving no run name doesn't filter the results.
* Runs a suite with success & failure cases with name X.
* Validates that both tests were run and one succeeded.
* Runs a suite with success & failure cases with no name.
* Validates that only one test was run (the failing test of the
previous X run) and it failed again.
"""
MockTestSuite.components = (SuccessCase, FailureCase)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name='run1')
self.validate_suite_data(run_data.main_test,
False,
successes=1,
fails=1)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name=None)
self.validate_suite_data(run_data.main_test, False, skips=1, fails=1)
def test_different_run_names(self):
"""Test run delta on runs with different run names.
* Runs a suite with success & failure cases with name X.
* Validates that both tests were run and one succeeded.
* Runs a suite with success & failure cases with name Y.
* Validates that both tests were run and one succeeded.
* Runs a suite with success & failure cases with name X.
* Validates that only one test was run (the failing test of the
previous X run) and it failed again.
"""
MockTestSuite.components = (SuccessCase, FailureCase)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name='run1')
self.validate_suite_data(run_data.main_test,
False,
successes=1,
fails=1)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name='run2')
self.validate_suite_data(run_data.main_test,
False,
successes=1,
fails=1)
run_data, = run(MockTestSuite,
delta_iterations=1,
outputs=(DBHandler.NAME, ),
run_name='run1')
self.validate_suite_data(run_data.main_test, False, skips=1, fails=1)
def test_delta_iterations(self):
"""Test run delta with iterations.
* Runs a suite with success & success-after-three-runs cases.
* Validates that in the first run, both tests were run and one
succeeded.
* Validates that in the second run, one test was run, and didn't
succeed.
* Validates that in the third run, one test was run successfully.
"""
MockTestSuite.components = (SuccessCase, FailTwiceCase)
runs_data = run(MockTestSuite,
delta_iterations=self.DELTA_ITERATIONS,
outputs=(DBHandler.NAME, ))
run_suites = [run_data.main_test for run_data in runs_data]
full_suite, first_delta_suite, second_delta_suite = run_suites
self.validate_suite_data(full_suite, False, successes=1, fails=1)
self.validate_suite_data(first_delta_suite, False, skips=1, fails=1)
self.validate_suite_data(second_delta_suite,
True,
successes=1,
skips=1)