def test_really_faster(self):
"""Test getting a simple improvement."""
faster = ComparisonResult(min, False, False, [5., 6.], [10., 10., 10.],
None, None)
self.assertEquals(faster.get_value_status(), IMPROVED)
self.assertTrue(faster.is_result_interesting())
def test_really_faster(self):
"""Test getting a simple improvement."""
faster = ComparisonResult(
min, False, False, [5., 6.], [10., 10., 10.], None, None)
self.assertEquals(faster.get_value_status(), IMPROVED)
self.assertTrue(faster.is_result_interesting())
def test_big_no_regression_flat_line_multi(self):
"""Same data, but bigger 10 + 5% variation, multisample current."""
flat = ComparisonResult(
min, False, False, [10.0606, 10.4169, 10.1859],
BIG_NUMBERS_FLAT[0:10], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_comp(self):
"""Test a real example."""
curr_samples = [0.0887, 0.0919, 0.0903]
prev = [0.0858]
uninteresting = ComparisonResult(min, False, False, curr_samples, prev)
self.assertFalse(uninteresting.is_result_interesting())
self.assertEquals(uninteresting.get_test_status(), UNCHANGED_PASS)
self.assertEquals(uninteresting.get_value_status(), UNCHANGED_PASS)
def test_comp(self):
"""Test a real example."""
curr_samples = [0.0887, 0.0919, 0.0903]
prev = [0.0858]
uninteresting = ComparisonResult(min, False, False, curr_samples, prev)
self.assertFalse(uninteresting.is_result_interesting())
self.assertEquals(uninteresting.get_test_status(), UNCHANGED_PASS)
self.assertEquals(uninteresting.get_value_status(), UNCHANGED_PASS)
def test_regressed_status(self):
"""Test getting a test status improvement."""
improved = ComparisonResult(min, True, False, None, [10.], None, None)
self.assertEquals(improved.get_test_status(), REGRESSED)
def test_noticeable_regression(self):
"""Test a big looking jump."""
regressed = ComparisonResult(
min, False, False, [10.0, 10.1], [5.0, 5.1, 4.9, 5.0], None, None)
self.assertEquals(regressed.get_value_status(), REGRESSED)
def test_handle_zero_sample(self):
for agfn in (min, median):
zeroSample = ComparisonResult(agfn, False, False,
[0.005, 0.0047, 0.0048],
[0.0, 0.01, 0.01], None, None)
self.assertEquals(zeroSample.get_value_status(), UNCHANGED_PASS)
def test_noise_bimodal_regression(self):
bimodal = ComparisonResult(min, False, False, [BM_REGS_NOISE[12]],
BM_REGS_NOISE[0:11])
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_simple_improvement(self):
"""An improvement without noise."""
flat = ComparisonResult(min, False, False, [IMP[10]], IMP[0:9])
self.assertEquals(flat.get_value_status(), IMPROVED)
def test_noise_slow_regression(self):
slow = ComparisonResult(min, False, False, [SLOW_REG_NOISE[12]],
SLOW_REG_NOISE[0:11], None, None)
self.assertEquals(slow.get_value_status(), REGRESSED)
def test_noise_bimodal(self):
"""Bimodal line with 5% noise."""
bimodal = ComparisonResult(min, False, False, [BIMODAL_NOISE[10]],
BIMODAL_NOISE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_noise_bimodal_regression(self):
bimodal = ComparisonResult(
min, False, False, [BM_REGS_NOISE[12]], BM_REGS_NOISE[0:11])
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_regressed_status(self):
"""Test getting a test status improvement."""
improved = ComparisonResult(min, True, False, None, [10.], None, None)
self.assertEquals(improved.get_test_status(), REGRESSED)
def test_noisy_regression_5(self):
"""A regression in 5% noise."""
flat = ComparisonResult(min, False, False, [12.2821], REGRESS_5[0:9],
None, None)
self.assertEquals(flat.get_value_status(), REGRESSED)
def test_improved_status(self):
"""Test getting a test status improvement."""
improved = ComparisonResult(min, False, True, [1.], None, None, None)
self.assertEquals(improved.get_test_status(), IMPROVED)
def test_slow_improvement(self):
slow = ComparisonResult(min, False, False, [SLOW_IMP[12]],
SLOW_IMP[0:11])
def test_slower(self):
"""Test getting a simple regression."""
slower = ComparisonResult(min, False, False, [10.], [5.], None, None)
self.assertEquals(slower.get_value_status(), REGRESSED)
self.assertTrue(slower.is_result_interesting())
def test_no_regression_flat_line_noise(self):
"""Now 4% noise."""
flat = ComparisonResult(
min, False, False, [1.020], FLAT_NOISE[0:10], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_bimodal_regression(self):
"""A regression in a bimodal line."""
bimodal = ComparisonResult(min, False, False, [BM_REGRESSION[12]],
BM_REGRESSION[0:11], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_keep_on_failing_status(self):
"""Test getting a repeated fail."""
improved = ComparisonResult(min, True, True, None, None, None, None)
self.assertEquals(improved.get_test_status(), UNCHANGED_FAIL)
def test_simple_improvement(self):
"""An improvement without noise."""
flat = ComparisonResult(min, False, False, [IMP[10]], IMP[0:9])
self.assertEquals(flat.get_value_status(), IMPROVED)
def test_noise_improvement(self):
"""An improvement with 5% noise."""
flat = ComparisonResult(min, False, False, [IMP_NOISE[10]],
IMP_NOISE[0:9], None, None)
self.assertEquals(flat.get_value_status(), IMPROVED)
def test_noticeable_regression(self):
"""Test a big looking jump."""
regressed = ComparisonResult(min, False, False, [10.0, 10.1],
[5.0, 5.1, 4.9, 5.0], None, None)
self.assertEquals(regressed.get_value_status(), REGRESSED)
def test_noise_bimodal_alternating(self):
"""Bimodal alternating with 5% noise."""
bimodal = ComparisonResult(min, False, False, [BM_AL_NOISE[10]],
BM_AL_NOISE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_no_regression_flat_line(self):
"""This is a flat line, it should have no changes."""
flat = ComparisonResult(min, False, False, [1.0], FLAT_LINE[0:10],
None, None)
self.assertEquals(flat.get_value_status(), UNCHANGED_PASS)
def test_noise_slow_regression(self):
slow = ComparisonResult(
min, False, False, [SLOW_REG_NOISE[12]], SLOW_REG_NOISE[0:11],
None, None)
self.assertEquals(slow.get_value_status(), REGRESSED)
def test_no_regression_flat_line_noise(self):
"""Now 4% noise."""
flat = ComparisonResult(min, False, False, [1.020], FLAT_NOISE[0:10],
None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_noise_single_spike(self):
spike = ComparisonResult(min, False, False, [NOISE_SPIKE[8]],
NOISE_SPIKE[0:7], None, None)
def test_big_no_regression_flat_line_noise(self):
"""Same data, but bigger 10 + 5% variation."""
flat = ComparisonResult(min, False, False, [10.25], FLAT_NOISE2[0:10],
None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_noise_slow_improvement(self):
slow = ComparisonResult(min, False, False, [SLOW_IMP_NOISE[12]],
SLOW_IMP_NOISE[0:11], None, None)
def test_big_no_regression_flat_line_multi(self):
"""Same data, but bigger 10 + 5% variation, multisample current."""
flat = ComparisonResult(min, False, False, [10.0606, 10.4169, 10.1859],
BIG_NUMBERS_FLAT[0:10], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_slower(self):
"""Test getting a simple regression."""
slower = ComparisonResult(min, False, False, [10.], [5.], None, None)
self.assertEquals(slower.get_value_status(), REGRESSED)
self.assertTrue(slower.is_result_interesting())
def test_simple_regression(self):
"""Flat line that jumps to another flat line."""
flat = ComparisonResult(min, False, False, [SIMPLE_REGRESSION[10]],
SIMPLE_REGRESSION[0:9], None, None)
self.assertEquals(flat.get_value_status(), REGRESSED)
def test_improved_status(self):
"""Test getting a test status improvement."""
improved = ComparisonResult(min, False, True, [1.], None, None, None)
self.assertEquals(improved.get_test_status(), IMPROVED)
def test_noisy_regression_5(self):
"""A regression in 5% noise."""
flat = ComparisonResult(min, False, False, [12.2821], REGRESS_5[0:9],
None, None)
self.assertEquals(flat.get_value_status(), REGRESSED)
def test_keep_on_failing_status(self):
"""Test getting a repeated fail."""
improved = ComparisonResult(min, True, True, None, None, None, None)
self.assertEquals(improved.get_test_status(), UNCHANGED_FAIL)
def test_noisy_regression_5_multi(self):
"""A regression in 5% noise, more current samples."""
flat = ComparisonResult(min, False, False, [12.2821, 12.2141, 12.3077],
MS_5_REG[0:9], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, REGRESSED)
def test_no_regression_flat_line(self):
"""This is a flat line, it should have no changes."""
flat = ComparisonResult(
min, False, False, [1.0], FLAT_LINE[0:10], None, None)
self.assertEquals(flat.get_value_status(), UNCHANGED_PASS)
def test_noise_improvement(self):
"""An improvement with 5% noise."""
flat = ComparisonResult(min, False, False, [IMP_NOISE[10]],
IMP_NOISE[0:9], None, None)
self.assertEquals(flat.get_value_status(), IMPROVED)
def test_big_no_regression_flat_line_noise(self):
"""Same data, but bigger 10 + 5% variation."""
flat = ComparisonResult(
min, False, False, [10.25], FLAT_NOISE2[0:10], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, UNCHANGED_PASS)
def test_bimodal(self):
"""A bimodal line, with no regressions."""
bimodal = ComparisonResult(min, False, False, [BIMODAL[10]],
BIMODAL[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_simple_regression(self):
"""Flat line that jumps to another flat line."""
flat = ComparisonResult(
min, False, False, [SIMPLE_REGRESSION[10]], SIMPLE_REGRESSION[0:9],
None, None)
self.assertEquals(flat.get_value_status(), REGRESSED)
def test_noise_bimodal(self):
"""Bimodal line with 5% noise."""
bimodal = ComparisonResult(min, False, False, [BIMODAL_NOISE[10]],
BIMODAL_NOISE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_noisy_regression_5_multi(self):
"""A regression in 5% noise, more current samples."""
flat = ComparisonResult(min, False, False, [12.2821, 12.2141, 12.3077],
MS_5_REG[0:9], None, None)
ret = flat.get_value_status()
self.assertEquals(ret, REGRESSED)
def test_bimodal_alternating(self):
"""Bimodal which sticks in a mode for a while."""
bimodal = ComparisonResult(min, False, False, [BM_ALTERNATE[10]],
BM_ALTERNATE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_bimodal(self):
"""A bimodal line, with no regressions."""
bimodal = ComparisonResult(min, False, False, [BIMODAL[10]],
BIMODAL[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_noise_bimodal_alternating(self):
"""Bimodal alternating with 5% noise."""
bimodal = ComparisonResult(min, False, False, [BM_AL_NOISE[10]],
BM_AL_NOISE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_bimodal_alternating(self):
"""Bimodal which sticks in a mode for a while."""
bimodal = ComparisonResult(min, False, False, [BM_ALTERNATE[10]],
BM_ALTERNATE[0:9], None, None)
self.assertEquals(bimodal.get_value_status(), UNCHANGED_PASS)
def test_bimodal_alternating_regression(self):
"""Bimodal alternating regression."""
bimodal = ComparisonResult(min, False, False, [BM_AL_REG[11]],
BM_AL_REG[0:10], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_bimodal_alternating_regression(self):
"""Bimodal alternating regression."""
bimodal = ComparisonResult(min, False, False, [BM_AL_REG[11]],
BM_AL_REG[0:10], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_bimodal_regression(self):
"""A regression in a bimodal line."""
bimodal = ComparisonResult(min, False, False, [BM_REGRESSION[12]],
BM_REGRESSION[0:11], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_noise_bimodal_overlapping_regression(self):
bimodal = ComparisonResult(
min, False, False, [BM_REG_OVER_NOISE[12]],
BM_REG_OVER_NOISE[0:11], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_noise_bimodal_overlapping_regression(self):
bimodal = ComparisonResult(min, False, False, [BM_REG_OVER_NOISE[12]],
BM_REG_OVER_NOISE[0:11], None, None)
self.assertEquals(bimodal.get_value_status(), REGRESSED)
def test_handle_zero_sample(self):
for agfn in (min, median):
zeroSample = ComparisonResult(
agfn, False, False, [0.005, 0.0047, 0.0048], [0.0, 0.01, 0.01],
None, None)
self.assertEquals(zeroSample.get_value_status(), UNCHANGED_PASS)
def test_single_spike(self):
spike = ComparisonResult(min, False, False, [SPIKE[11]], SPIKE[0:10],
None, None)