def test3_one_line_tests(self):
assert PccfPerformanceMetrics([10]).f1_score([Roi(10, 1)]) == 1
assert PccfPerformanceMetrics([10, 12]).f1_score([Roi(10,
5)]) == 1.0 / 1.5
assert PccfPerformanceMetrics([10,
20]).f1_score([Roi(9, 5),
Roi(17, 1)]) == 1.0 / 2
def test_roi_setters():
roi0 = Roi(10)
roi0.radius = 5
assert roi0.center == 10
assert roi0.left == 5
roi0.center = 20
assert roi0.center == 20
assert roi0.radius == 5
assert roi0.right == 25
def test1(self):
perf = PccfPerformanceMetrics(changes=[10, 11, 20, 30])
assert perf.tps([Roi(10, 2), Roi(35, 1)]) == 1
assert perf.fns([Roi(10, 2), Roi(35, 1)]) == 3
assert perf.fps([Roi(10, 2), Roi(35, 1)]) == 1
assert perf.f1_score([Roi(10, 2),
Roi(35, 1)]) == 1.0 / (1.0 + 0.5 * (3.0 + 1.0))
assert PccfPerformanceMetrics([10, 20]).fns([Roi(9, 5),
Roi(17, 1)]) == 1
def rois_temperature_signal():
radius = 200
mu = 1400
rois = [Roi(changes_temperature_indices[0], radius=radius)]
for k in range(5):
rois.append(Roi(changes_temperature_indices[0] + (k + 1) * mu, radius))
logger.info("Temp. sig. ROIs")
for r in rois:
print(r)
return rois
def test_tps(self):
assert fn_tps([10, 20, 30], [1]) == []
assert fn_tps([10, 20, 30], [9]) == []
assert fn_tps([10, 20, 30],
[11, 12, 13, 20, 21, 29, 34]) == [11, 20, 34]
with pytest.raises(ValueError):
assert fn_tps([10, 20, 30], [22, 25], [Roi(20, 2)]) == [22]
assert fn_tps([10, 20, 30], [19, 21, 22], [Roi(20, 2)]) == [21]
assert DetectionsPerformanceMetrics([10, 20, 30], [19, 21, 22],
[Roi(20, 2)]).tps() == 1
assert fn_tps([10, 20, 30], [7, 9, 12, 18, 20, 21],
[Roi(10, 3), Roi(20, 2)]) == [12, 20]
def test_paper_examples(self):
changes = [100, 200]
detections = [80, 90, 110, 120, 216]
detections_roi = [80, 90, 110, 120, 216]
roi = [Roi(100, 15), Roi(200, 15)]
assert fn_tps(changes, detections) == [110, 216]
assert fn_fps(changes, detections) == [80, 90, 120]
assert fn_fns(changes, detections) == []
f1_score = fn_f1score_vec(len(fn_tps(changes, detections)),
len(fn_fps(changes, detections)),
len(fn_fns(changes, detections)))
assert f1_score == 2.0 / (2.0 + 0.5 * (3 + 0))
def test_cusum_pccf_one_change():
sig = np.concatenate((np.random.randn(100), np.random.randn(100) + 2.1),
axis=0)
rois = [Roi(100, 3)]
for theta in np.linspace(0.1, 10, 100):
r = cusum_pccf(sig, theta, rois)
assert all(rois[0].left <= e <= rois[0].right for e in r.detections)
def test_fns_behaviour():
"""
Catches cases when detection is after ROI and no detections inside.
And => FN.
"""
n = 10
half_length = 100
change = half_length
changes = [change]
rois = [Roi(change, 5)]
count_pccf_fns = 0
count_cusum_fns = 0
for _ in range(n):
for theta in np.linspace(0.1, 10, 30):
sig = np.concatenate((np.random.rand(half_length) * 1.1,
np.random.rand(half_length) * 1.1 + 2.0),
axis=0)
r_cusum = cusum(sig, theta)
r_pccf = cusum_pccf(sig, theta, rois)
fns_cusum = fn_fns(changes, r_cusum.detections)
fns_pccf = fn_fns(changes, r_pccf.detections)
count_cusum_fns += len(fns_cusum)
count_pccf_fns += len(fns_pccf)
if len(fns_pccf) > 0:
print(fns_cusum, fns_pccf, ' -- ', r_pccf.detections,
r_cusum.detections)
assert count_pccf_fns > count_cusum_fns
def test_detection_delay_behaviour_cusum_pccf():
"""
Test that when we increase threshold (sensitivity)
then detection delay for Pccf Cusum decreases and becomes nan.
Also test that if delay == nan then tps_num == 0
"""
n_points = 100
sigma = 1.0
mu0 = 0.0
mu1 = 1.1
sig = np.concatenate((
randn(n_points) * sigma + mu0,
randn(n_points) * sigma + mu1,
),
axis=0)
changes = [100]
max_theta = 40
theta_vec = np.linspace(1, max_theta,
100) # gen_theta_vec(1, 1, max_theta)
rois = [Roi(100, radius=25)]
cusum_pccf_ref = make_cusum_pccf(rois)
delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \
collect_performance(cusum_pccf_ref, sig, changes, theta_vec, rois=rois)
for i, d in enumerate(delays_dyn):
if i == 0:
continue
else:
assert d >= delays_dyn[i] or math.isnan(d)
if math.isnan(d):
assert tps_dyn[i] == 0
def test_two_detectors_outputs():
change = 30
threshold = 1.0
sig = np.concatenate(
(np.random.rand(change) * 1.1, np.random.rand(change) * 1.1 + 2.0),
axis=0)
r_stat = cusum(sig, threshold)
r_dyn = cusum_pccf(sig, threshold, [Roi(change, 3)])
r_obj_stat = Detector(sig, threshold).run()
r_obj_dyn = Detector(sig, threshold, [Roi(change, 3)]).run()
assert r_stat.detections == r_obj_stat.detections
assert r_dyn.detections == r_obj_dyn.detections
np.testing.assert_almost_equal(r_stat.statistic, r_obj_stat.statistic)
np.testing.assert_almost_equal(r_dyn.statistic, r_obj_dyn.statistic)
def roi_intervals(self, n: int, starting_point=0) -> List[Roi]:
"""
k: number of changes to predict
"""
if self.radius is not None:
return [
Roi(self.mu * (i + 1) + starting_point, self.radius)
for i in range(n)
]
else:
raise TypeError("pccf: provide radius")
def test_fps(self):
assert fn_fps([10, 20, 30], [1]) == [1]
assert DetectionsPerformanceMetrics([10, 20, 30], [1]).fps() == 1
assert fn_fps([10, 20, 30], [11]) == []
assert fn_fps([10, 20, 30], []) == []
assert fn_fps([10, 20, 30], [0]) == [0]
assert fn_fps([10, 20, 30], [2, 11, 12]) == [2, 12]
assert fn_fps([10, 20, 30], [2, 11, 12, 31, 35, 37]) == [2, 12, 35, 37]
with pytest.raises(ValueError):
assert fn_fps([10, 20, 30], [1, 15, 16, 21],
[Roi(20, 3)]) == [1, 15, 16]
assert fn_fps([10], [5], [Roi(10, 3)]) == [5]
assert fn_fps([10], [16], [Roi(10, 3)]) == [16]
assert fn_fps([10], [11, 16], [Roi(10, 3)]) == [16]
assert fn_fps([10], [13, 16], [Roi(10, 3)]) == [16]
assert fn_fps([10], [14, 16], [Roi(10, 3)]) == [14, 16]
assert fn_fps([10], [9], [Roi(10, 3)]) == [9]
assert fn_fps([10], [7, 8, 9], [Roi(10, 3)]) == [7, 8, 9]
assert fn_fps([10], [11], [Roi(10, 3)]) == []
assert fn_fps([10], [11, 12, 13], [Roi(10, 5)]) == [12, 13]
def test_collect_performance():
n_points = 15
ROI_RADIUS = 4
MIN_THETA = 0.1
MAX_THETA = 7.0
THETA_STEP = 0.7
changes = [n_points + 1]
np.random.seed(11)
mu0 = 0.0
theta_vec = [0.1, 0.8, 1.5, 2.2, 2.9, 3.6, 4.3, 5.0, 5.7, 6.4, 7.5]
sig = [
1.92440022, -0.3146803, -0.53302165, -2.91865042, -0.00911309,
-0.3515945, -0.5902923, 0.34694294, 0.46315579, -1.17216328,
-0.97486364, -0.52330684, 0.75865054, 0.61731139, -1.43610336,
0.86857721, 2.91052113, 3.83209748, 2.0658174, 1.34820871, 3.30519267,
1.7594657, 2.89832745, 3.8039788, 2.7930878, 2.18084256, 2.90549849,
1.39316707, 1.90409751, 1.46864998
]
cusum_single_ref = make_cusum_single_ref(mu0)
rois = [Roi(n_points, ROI_RADIUS)]
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, rois[0])
delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \
collect_performance(cusum_single_ref, sig, changes, theta_vec)
delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \
collect_performance(cusum_single_pccf_ref, sig, changes, theta_vec,
rois=rois)
np.testing.assert_equal(
delays_stat,
[np.nan, np.nan, np.nan, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0])
assert tps_stat == [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
assert fps_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
assert fns_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
assert f1_stat == list([
tp / (tp + 0.5 * (fp + fn))
for (tp, fp, fn) in zip(tps_stat, fps_stat, fns_stat)
])
assert tps_dyn == [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]
assert fns_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]
# assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] - old behaviour.
# After the fix we do not detect outside ROI
assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
np.testing.assert_equal(
delays_dyn,
[1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, np.nan, np.nan])
def pccf_rois_adhoc(changes_list: List[int], delta: int = 10):
"""
NOTE: To find best Pccf use collect_performance() results.
Ad-hoc because change will be included into ROIs aposteriori,
so that we don't need to run search / optimization.
Given PCCF ROI locations add widths - return list of Roi objects
"""
pccf_predictions, mu = pccf_optimal_locations(changes_list)
rois = []
for c, p in zip(changes_list, pccf_predictions):
rois.append(Roi(p, radius=abs(p - c)))
return rois
def test_collect_arl():
mu0 = 0.0
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, Roi(100,
radius=25))
n_points = 100
sigma = 1.1
mu1 = 1.1
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0)
theta_vec = np.linspace(0.01, 150.0, 300)
arls_stat = collect_arl(cusum_single_ref, sig, theta_vec)
arls_dyn = collect_arl(cusum_single_pccf_ref, sig, theta_vec)
assert len(arls_dyn) == len(arls_stat)
def __init__(self):
self.n = 501
self.roi = Roi(230, 300)
self.chp_loc = 251
self.mu0 = 0.0
self.mu1 = 2.0
self.sigma = 1.1
self.signal = np.concatenate(
(
np.random.randn(self.chp_loc - 1) * self.sigma + self.mu0,
np.random.randn(self.n - self.chp_loc + 1) * self.sigma +
self.mu1,
),
axis=0,
)
def test_fns(self):
assert fn_fns([10, 20, 30], [1]) == [10, 20, 30]
assert DetectionsPerformanceMetrics([10, 20, 30], [1]).fns() == 3
assert fn_fns([10, 20, 30], []) == [10, 20, 30]
assert fn_fns([10, 20, 30], [11]) == [20, 30]
assert fn_fns([10, 20, 30], [11, 29]) == [30]
assert fn_fns([150], [11]) == [150]
assert fn_fns([150], [151]) == []
with pytest.raises(ValueError):
assert fn_fns([10], [16], [Roi(10, 3)]) == [10]
assert fn_fns([10], [13], [Roi(10, 3)]) == []
assert fn_fns([10], [9], [Roi(10, 3)]) == [10]
assert fn_fns([10], [7, 8, 9], [Roi(10, 3)]) == [10]
assert fn_fns([10], [7, 8, 9, 11], [Roi(10, 3)]) == []
assert fn_fns([10, 20], [7, 8, 9, 11], [Roi(10, 3)]) == [20]
assert fn_fns([10, 20, 30], [7, 8, 9, 11], [Roi(10, 3)]) == [20, 30]
def test_cusum_single_nan_outputs():
mu0 = 0.0
n_points = 150
sigma = 1.1
mu1 = 2.1
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0,
Roi(n_points + 1, 15))
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0)
r_stat = cusum_single_ref(sig, 1)
r_dyn = cusum_single_pccf_ref(sig, 1)
perf_stat = tp_fp_fn_delays([n_points], r_stat.detections)
perf_dyn = tp_fp_fn_delays([n_points], r_dyn.detections)
assert np.isnan(perf_stat.delays[0])
assert np.isnan(perf_dyn.delays[0])
r_dyn2 = cusum_single_pccf_ref(sig, 2000)
perf_dyn2 = tp_fp_fn_delays([n_points], r_dyn.detections)
assert np.isnan(perf_dyn2.delays[0])
def test_cusum_single_pccf_ref():
"""
Test for detection location
"""
mu0 = 0.0
n_points = 150
ROI_RADIUS = 5
cusum_single_pccf_ref = make_cusum_single_pccf_ref(
mu0, Roi(n_points + 1, ROI_RADIUS))
sigma = 1.1
mu1 = 2.1
for _ in range(30):
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1),
axis=0)
r = cusum_single_pccf_ref(sig, 20)
if len(r.detections) > 0 and not np.isnan(r.detections[0]):
assert n_points - ROI_RADIUS <= r.detections[0] <= n_points + \
ROI_RADIUS + 1
def test_delays(self):
assert fn_delays([351, 701], [341, 691, 706]) == [340, 5]
assert fn_delays([351, 701], [341, 691, 706], rois=[Roi(701,
10)]) == [5]
assert fn_delays([10, 20, 30], [1]) == [np.nan]
assert fn_delays([10, 20, 30], []) == []
assert fn_delays([10, 20, 30], [1, 2, 3]) == [np.nan, np.nan, np.nan]
assert fn_delays([10, 20, 30], [1, 10]) == [0]
assert fn_delays([10, 20, 30], [1, 11]) == [1]
assert fn_delays([10, 20, 30], [10, 20, 30]) == [0, 0, 0]
assert fn_delays([10, 20, 30], []) == []
assert fn_delays([10, 20, 30], [11, 12, 13]) == [1]
print(
fn_delays([351, 701], [
4, 6, 23, 35, 38, 51, 68, 76, 86, 90, 97, 113, 128, 132, 158,
172, 176, 186, 198, 208, 227, 235, 247, 255, 263, 308, 314,
324, 330, 350, 360, 373, 395, 400, 414, 420, 427, 444, 473,
477, 485, 495, 503, 522, 532, 544, 547, 583, 602, 617, 631,
636, 642, 655, 662, 675, 686, 691, 700, 724, 738, 746, 751,
767, 777, 783, 787, 793, 815, 837, 879, 883, 888, 899, 912,
927, 958, 968, 988, 1009, 1021, 1032, 1041, 1047
]))
def test_cusum_pccf_multi_changes():
sig_earth_quakes = [
13.0, 14.0, 8.0, 10.0, 16.0, 26.0, 32.0, 27.0, 18.0, 32.0, 36.0, 24.0,
22.0, 23.0, 22.0, 18.0, 25.0, 21.0, 21.0, 14.0, 8.0, 11.0, 14.0, 23.0,
18.0, 17.0, 19.0, 20.0, 22.0, 19.0, 13.0, 26.0, 13.0, 14.0, 22.0, 24.0,
21.0, 22.0, 26.0, 21.0, 23.0, 24.0, 27.0, 41.0, 31.0, 27.0, 35.0, 26.0,
28.0, 36.0, 39.0, 21.0, 17.0, 22.0, 17.0, 19.0, 15.0, 34.0, 10.0, 15.0,
22.0, 18.0, 15.0, 20.0, 15.0, 22.0, 19.0, 16.0, 30.0, 27.0, 29.0, 23.0,
20.0, 16.0, 21.0, 21.0, 25.0, 16.0, 18.0, 15.0, 18.0, 14.0, 10.0, 15.0,
8.0, 15.0, 6.0, 11.0, 8.0, 7.0, 13.0, 10.0, 23.0, 16.0, 15.0, 25.0,
22.0, 20.0, 16.0
]
sig_changes = [20.0, 40.0, 53.0, 69.0, 79.0, 95.0]
# rois = list(pccf_rois_adhoc(sig_changes, 3))
rois = [
Roi(20, 3),
Roi(36, 7),
Roi(52, 2),
Roi(68, 4),
Roi(84, 8),
Roi(100, 8)
]
changes_in_rois = []
# Check if all changes are within ROIs
for change in sig_changes:
for roi in rois:
if roi.left <= change <= roi.right:
changes_in_rois.append(change)
assert set(changes_in_rois) == set(sig_changes)
cusum_pccf_ref = make_cusum_pccf(rois)
all_detections = []
for theta in np.linspace(0.01, 40, 100):
detections = cusum_pccf_ref(sig_earth_quakes, theta).detections
all_detections.extend(detections)
all_detections = list(set(all_detections))
print(all_detections)
detections_with_rois = []
# Check if all CDEs are within ROIs
for cde in all_detections:
for roi in rois:
if roi.left <= cde <= roi.right:
detections_with_rois.append(cde)
print(f"left: {roi.left} < cde: {cde} > right: {roi.right} |"
f" center: {roi.center}")
assert set(detections_with_rois) == set(all_detections)
def test_detections_out_rois(self):
assert events_outside_rois([1, 2, 3, 4, 5, 6, 7], [Roi(5, 3)]) == [1]
r = events_outside_rois([1, 6, 9, 14, 19, 21, 24, 25],
[Roi(10, 3), Roi(20, 4)])
assert r == [1, 6, 14, 25]
def test_roi_obj(self):
roi = Roi(100, 2)
assert roi.left == 98
assert roi.right == 102
def test_pccf_predictions():
pccf = Pccf(mu=10, radius=3)
rois = pccf.roi_intervals(n=1)
assert rois == [Roi(10, 3)]
rois = pccf.roi_intervals(n=2)
assert rois == [Roi(10, 3), Roi(20, 3)]
def test_roi_objects():
r = Roi(9, 3)
assert 9 in r
assert 6 in r
assert 12 in r
assert 13 not in r
def test_roi_obj():
r1 = Roi(10, radius=3)
assert r1.left == 7
assert r1.right == 13
r2 = Roi(12, radius=3)
assert r2 in r1
assert Roi(10, 1) in Roi(12, 1)
assert Roi(10, 1) not in Roi(13, 1)
assert Roi.from_left_right(5, 10) in Roi.from_left_right(9, 20)
assert Roi.from_left_right(5, 10) not in Roi.from_left_right(11, 20)
assert Roi.from_left_right(5, 10).radius == 2.5
def results_example(
sig_params=SignalSettings(0.0, 1.1, 1.0, 100), theta=0.5,
output_fig=None):
"""
Maybe the most important code / simulation.
See sample of signal and detections to check what is being measured in
simulation.
"""
mu0 = sig_params.mu0
mu1 = sig_params.mu1
sigma = sig_params.sigma
half_len = sig_params.half_len
changes = [half_len]
rois = [Roi(half_len, ROI_RADIUS_ARTIFICIAL_SIGNAL)]
sig = np.concatenate(
(randn(half_len) * sigma + mu0, randn(half_len) * sigma + mu1), axis=0)
r_stat = cusum(sig, theta)
r_dyn = cusum_pccf(sig, theta, rois)
stat_tps = fn_tps(changes, r_stat.detections)
stat_fps = fn_fps(changes, r_stat.detections)
stat_fns = fn_fns(changes, r_stat.detections)
pccf_tps = fn_tps(changes, r_dyn.detections, rois)
pccf_fps = fn_fps(changes, r_dyn.detections, rois)
pccf_fns = fn_fns(changes, r_dyn.detections, rois)
logger.info(f"""
\nStart simulation example. Repeat several times to see difference in performance.
Input signal settings: mu0 = {sig_params.mu0}, mu1 = {sig_params.mu1}, sigma = {sig_params.sigma}, change = {half_len}, length = {len(sig)}
Prediction interval : roi = {rois[0]}
Detector's threshold : theta = {theta}
""")
if output_fig:
fig = plt.figure(111, figsize=(6, 4))
ax1 = fig.add_subplot(111)
ax1.plot(sig, color='black')
plt.savefig(output_fig, format='eps')
logger.info(f"Save signal sample into {output_fig}")
else:
print("CUSUM")
print(" {0:11s} {1:}".format("Detections", r_stat.detections))
print(" {0:11s} {1:}".format('Delays',
fn_delays(changes, r_stat.detections)))
print(" {0:11s} {1:}".format('FPs', stat_fps))
print(" {0:11s} {1:}".format('FNs', stat_fns))
print(" {0:11s} {1:}".format('TPs', stat_tps))
print(" {0:11s} {1:.2f}".format(
'F1',
f1_score(tps_count=len(stat_tps),
fps_count=len(stat_fps),
fns_count=len(stat_fns))))
print('\nPCCF')
print(" {0:11s} {1:}".format("Detections", r_dyn.detections))
print(" {0:11s} {1:}".format(
'Delays', fn_delays(changes, r_dyn.detections, rois)))
print(" {0:11s} {1:}".format('FPs', pccf_fps))
print(" {0:11s} {1:}".format('FNs', pccf_fns))
print(" {0:11s} {1:}".format('TPs', pccf_tps))
print(" {0:11s} {1:.2f}".format(
'F1',
f1_score(tps_count=len(pccf_tps),
fps_count=len(pccf_fps),
fns_count=len(pccf_fns))))
print('\n')
