def test_regression_threshold():
steps = [(0, 1, 1.0, 1.0, 0.0),
(1, 2, 1.1, 1.1, 0.0),
(2, 3, 2.0, 2.0, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.05)
assert latest == 2
assert best == 1
assert pos == [(0, 1, 1.0, 1.1), (1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.8)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=1.1)
assert latest == None
assert best == None
assert pos == None
steps = [(0, 1, 1.0, 1.0, 0.0),
(1, 2, 1.3, 1.3, 0.0),
(2, 3, 1.1, 1.1, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2)
assert latest == None
assert best == None
assert pos == None
def test_regression_threshold():
steps = [(0, 1, 1.0, 1.0, 0.0), (1, 2, 1.1, 1.1, 0.0),
(2, 3, 2.0, 2.0, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.05)
assert latest == 2
assert best == 1
assert pos == [(0, 1, 1.0, 1.1), (1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.9)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=1.1)
assert latest == None
assert best == None
assert pos == None
steps = [(0, 1, 1.0, 1.0, 0.0), (1, 2, 1.3, 1.3, 0.0),
(2, 3, 1.1, 1.1, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2)
assert latest == None
assert best == None
assert pos == None
def test_regression_min_size():
steps = [(0, 2, 1.0, 1.0, 0.0), (2, 3, 0.0, 0.0, 0.0),
(3, 5, 1.0, 1.0, 0.0)]
latest, best, pos = detect_regressions(steps)
assert latest is None and best is None and pos is None
latest, best, pos = detect_regressions(steps, min_size=1)
assert latest == 1.0
assert best == 0.0
assert pos == [(2, 3, 0.0, 1.0)]
steps = [(0, 2, 1.0, 1.0, 0.0), (2, 3, 0.0, 0.0, 0.0),
(3, 5, 2.0, 1.0, 0.0)]
latest, best, pos = detect_regressions(steps)
assert latest == 2.0
assert best == 0.0
assert pos == [(2, 3, 0.0, 2.0)]
def test_detect_regressions(use_rangemedian):
try:
np.random.seed(1234)
except NameError:
# work around a bug in old pypy/pytest
pytest.skip("test needs numpy")
return
for seed in [1234, 5678, 8901, 2345]:
np.random.seed(seed)
t = np.arange(4000)
y = 0.7 * np.random.rand(t.size)
y -= 0.3 * (t >= 1000)
y += 2.0 * (t >= 3234 + (seed % 123))
y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))
y = y.tolist()
y[123] = None
y[1234] = np.nan
steps = detect_steps(y)
steps_lr = [(l, r) for l, r, _, _, _ in steps]
k = steps[0][1]
assert 990 <= k <= 1010
assert steps_lr == [(0, k), (k, 3234 + (seed % 123)),
(3234 + (seed % 123), 3264 + (seed % 53)),
(3264 + (seed % 53), 3350),
(3350, 3500 + (seed % 71)),
(3500 + (seed % 71), 3700), (3700, 4000)]
steps_v = [x[2] for x in steps]
assert np.allclose(steps_v, [0.35, 0.05, 2.05, 4.05, 1.15, 4.05, 2.05],
rtol=0.3)
# The expected mean error is 0.7 <|U(0,1) - 1/2|> = 0.7/4
steps_err = [x[4] for x in steps]
assert np.allclose(steps_err, [0.7 / 4] * 7, rtol=0.3)
# Check detect_regressions
new_value, best_value, regression_pos = detect_regressions(steps)
assert regression_pos == [
(3233 + (seed % 123), (3233 + (seed % 123) + 1), steps_v[1],
steps_v[2]),
(3499 + (seed % 71), 3499 + (seed % 71) + 1, steps_v[4],
steps_v[5])
]
assert np.allclose(best_value, 0.7 / 2 - 0.3, rtol=0.3, atol=0)
assert np.allclose(new_value, 0.7 / 2 - 0.3 + 2, rtol=0.3, atol=0)
def test_detect_regressions(use_rangemedian):
try:
np.random.seed(1234)
except NameError:
# work around a bug in old pypy/pytest
pytest.skip("test needs numpy")
return
for seed in [1234, 5678, 8901, 2345]:
np.random.seed(seed)
t = np.arange(4000)
y = 0.7 * np.random.rand(t.size)
y -= 0.3 * (t >= 1000)
y += 2.0 * (t >= 3234 + (seed % 123))
y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))
y = y.tolist()
y[123] = None
y[1234] = np.nan
steps = detect_steps(y)
steps_lr = [(l, r) for l, r, _, _, _ in steps]
k = steps[0][1]
assert 990 <= k <= 1010
assert steps_lr == [(0, k),
(k, 3234 + (seed % 123)),
(3234 + (seed % 123), 3264 + (seed % 53)),
(3264 + (seed % 53), 3350),
(3350, 3500 + (seed % 71)),
(3500 + (seed % 71), 3700),
(3700, 4000)]
steps_v = [x[2] for x in steps]
assert np.allclose(steps_v, [0.35, 0.05, 2.05, 4.05, 1.15, 4.05, 2.05], rtol=0.3)
# The expected mean error is 0.7 <|U(0,1) - 1/2|> = 0.7/4
steps_err = [x[4] for x in steps]
assert np.allclose(steps_err, [0.7/4]*7, rtol=0.3)
# Check detect_regressions
new_value, best_value, regression_pos = detect_regressions(steps)
assert regression_pos == [(3233 + (seed % 123), (3233 + (seed % 123) + 1), steps_v[1], steps_v[2]),
(3499 + (seed % 71), 3499 + (seed % 71) + 1, steps_v[4], steps_v[5])]
assert np.allclose(best_value, 0.7/2 - 0.3, rtol=0.3, atol=0)
assert np.allclose(new_value, 0.7/2 - 0.3 + 2, rtol=0.3, atol=0)
def test_detect_regressions():
for seed in [1234, 5678, 8901, 2345]:
np.random.seed(seed)
t = np.arange(4000)
y = 0.7 * np.random.rand(t.size)
y -= 0.3 * (t >= 1000)
y += 2.0 * (t >= 3234 + (seed % 123))
y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))
y = y.tolist()
y[123] = None
y[1234] = np.nan
new_value, jump_pos, best_value = detect_regressions(y)
assert jump_pos == [3233 + (seed % 123), 3499 + (seed % 71)]
assert np.allclose(best_value, 0.7/2 - 0.3, rtol=0.3, atol=0)
assert np.allclose(new_value, 0.7/2 - 0.3 + 2, rtol=0.3, atol=0)
def test_detect_regressions():
for seed in [1234, 5678, 8901, 2345]:
np.random.seed(seed)
t = np.arange(4000)
y = 0.7 * np.random.rand(t.size)
y -= 0.3 * (t >= 1000)
y += 2.0 * (t >= 3234 + (seed % 123))
y += 2.0 * ((t >= 3264 + (seed % 53)) & (t < 3700))
y -= 2.7 * ((t >= 3350) & (t < 3500 + (seed % 71)))
y = y.tolist()
y[123] = None
y[1234] = np.nan
new_value, jump_pos, best_value = detect_regressions(y)
assert jump_pos == [3233 + (seed % 123), 3499 + (seed % 71)]
assert np.allclose(best_value, 0.7 / 2 - 0.3, rtol=0.3, atol=0)
assert np.allclose(new_value, 0.7 / 2 - 0.3 + 2, rtol=0.3, atol=0)
def test_regression_threshold():
steps = [(0, 1, 1.0, 1.0, 0.0),
(1, 2, 1.1, 1.1, 0.0),
(2, 3, 2.0, 2.0, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.05, min_size=1)
assert latest == 2
assert best == 1
assert pos == [(0, 1, 1.0, 1.1), (1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2, min_size=1)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=0.8, min_size=1)
assert latest == 2
assert best == 1
assert pos == [(1, 2, 1.1, 2.0)]
latest, best, pos = detect_regressions(steps, threshold=1.1, min_size=1)
assert latest == None
assert best == None
assert pos == None
steps = [(0, 1, 1.0, 1.0, 0.0),
(1, 2, 1.3, 1.3, 0.0),
(2, 3, 1.1, 1.1, 0.0)]
latest, best, pos = detect_regressions(steps, threshold=0.2, min_size=1)
assert latest == None
assert best == None
assert pos == None
# Gradual change should result to a regression detected somewhere,
# even if the individual steps are smaller than the threshold
steps = [(0, 1, 1.0, 1.0, 0.0),
(1, 2, 1.04, 1.04, 0.0),
(2, 3, 1.08, 1.08, 0.0),]
latest, best, pos = detect_regressions(steps, threshold=0.05)
assert pos == [(0, 1, 1.0, 1.04)]
def time_detect_regressions(self):
step_detect.detect_regressions(self.y)
def time_detect_regressions(self):
step_detect.detect_regressions(self.y)
def time_detect_regressions(self):
steps = step_detect.detect_steps(self.y)
step_detect.detect_regressions(steps)
def time_detect_regressions(self):
steps = step_detect.detect_steps(self.y)
step_detect.detect_regressions(steps)
