def test_collect_performance():
n_points = 15
ROI_RADIUS = 4
MIN_THETA = 0.1
MAX_THETA = 7.0
THETA_STEP = 0.7
changes = [n_points + 1]
np.random.seed(11)
mu0 = 0.0
theta_vec = [0.1, 0.8, 1.5, 2.2, 2.9, 3.6, 4.3, 5.0, 5.7, 6.4, 7.5]
sig = [
1.92440022, -0.3146803, -0.53302165, -2.91865042, -0.00911309,
-0.3515945, -0.5902923, 0.34694294, 0.46315579, -1.17216328,
-0.97486364, -0.52330684, 0.75865054, 0.61731139, -1.43610336,
0.86857721, 2.91052113, 3.83209748, 2.0658174, 1.34820871, 3.30519267,
1.7594657, 2.89832745, 3.8039788, 2.7930878, 2.18084256, 2.90549849,
1.39316707, 1.90409751, 1.46864998
]
cusum_single_ref = make_cusum_single_ref(mu0)
rois = [Roi(n_points, ROI_RADIUS)]
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, rois[0])
delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \
collect_performance(cusum_single_ref, sig, changes, theta_vec)
delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \
collect_performance(cusum_single_pccf_ref, sig, changes, theta_vec,
rois=rois)
np.testing.assert_equal(
delays_stat,
[np.nan, np.nan, np.nan, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0])
assert tps_stat == [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
assert fps_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
assert fns_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
assert f1_stat == list([
tp / (tp + 0.5 * (fp + fn))
for (tp, fp, fn) in zip(tps_stat, fps_stat, fns_stat)
])
assert tps_dyn == [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]
assert fns_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]
# assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] - old behaviour.
# After the fix we do not detect outside ROI
assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
np.testing.assert_equal(
delays_dyn,
[1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, np.nan, np.nan])
def test_collect_arl():
mu0 = 0.0
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, Roi(100,
radius=25))
n_points = 100
sigma = 1.1
mu1 = 1.1
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0)
theta_vec = np.linspace(0.01, 150.0, 300)
arls_stat = collect_arl(cusum_single_ref, sig, theta_vec)
arls_dyn = collect_arl(cusum_single_pccf_ref, sig, theta_vec)
assert len(arls_dyn) == len(arls_stat)
def run_simulation_arl_perf(signal_settings: SignalSettings,
save_path):
n_sim = 100
n_points = signal_settings.half_len
roi = Roi(signal_settings.half_len, ROI_RADIUS_ARTIFICIAL_SIGNAL)
sigma = signal_settings.sigma
mu0 = signal_settings.mu0
mu1 = signal_settings.mu1
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, roi)
detector_fn = cusum_single_ref
detector_pccf_fn = cusum_single_pccf_ref
theta_vec = np.linspace(0.01, 40.0, 100)
arl_vec_stat = []
arl_vec_dyn = []
for _ in tqdm(range(n_sim)):
sig = np.concatenate((randn(n_points) * sigma + mu0,
randn(n_points) * sigma + mu1,
), axis=0)
arl_stat = collect_arl(detector_fn, sig, theta_vec)
arl_dyn = collect_arl(detector_pccf_fn, sig, theta_vec)
arl_vec_stat.append(arl_stat)
arl_vec_dyn.append(arl_dyn)
m_arl_stat = np.vstack(arl_vec_stat)
m_arl_dyn = np.vstack(arl_vec_dyn)
count_nans_as_zero = False
# If ARL is NaN then we disregard it
avg_arl_stat = average_results_matrix_rows(m_arl_stat, count_nans_as_zero=count_nans_as_zero)
avg_arl_dyn = average_results_matrix_rows(m_arl_dyn, count_nans_as_zero=count_nans_as_zero)
# avg_arl_stat = np.sum(m_arl_stat, axis=0)/m_arl_stat.shape[0]
# avg_arl_dyn = np.nansum(m_arl_dyn, axis=0)/m_arl_dyn.shape[0]
results = {'theta_vec': list(theta_vec),
'avg_arl_stat': list(avg_arl_stat),
'avg_arl_dyn': list(avg_arl_dyn),
}
with open(save_path, 'w') as f:
json.dump(results, f, indent=4)
logger.info(f"Save into {save_path}")
return save_path
def test_cusum_single_ref():
"""
Test for detection location
"""
mu0 = 0.0
cusum_single_ref = make_cusum_single_ref(mu0)
n_points = 150
sigma = 1.1
mu1 = 2.1
detections = []
for _ in range(30):
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1),
axis=0)
r = cusum_single_ref(sig, 20)
detections.append(r.detections[0])
cde_average = np.mean(detections)
assert 140 <= cde_average <= 170
def test_cusum_single_nan_outputs():
mu0 = 0.0
n_points = 150
sigma = 1.1
mu1 = 2.1
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0,
Roi(n_points + 1, 15))
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0)
r_stat = cusum_single_ref(sig, 1)
r_dyn = cusum_single_pccf_ref(sig, 1)
perf_stat = tp_fp_fn_delays([n_points], r_stat.detections)
perf_dyn = tp_fp_fn_delays([n_points], r_dyn.detections)
assert np.isnan(perf_stat.delays[0])
assert np.isnan(perf_dyn.delays[0])
r_dyn2 = cusum_single_pccf_ref(sig, 2000)
perf_dyn2 = tp_fp_fn_delays([n_points], r_dyn.detections)
assert np.isnan(perf_dyn2.delays[0])
def test_tps_num_behaviour_cusum_single():
n_points = 100
sigma = 1.0
mu0 = 0.0
mu1 = 1.1
sig = np.concatenate((
randn(n_points) * sigma + mu0,
randn(n_points) * sigma + mu1,
),
axis=0)
changes = [100]
max_theta = 20
theta_vec = np.linspace(0.001, max_theta,
100) # gen_theta_vec(0.001, 0.1, max_theta)
cusum_loc = make_cusum_single_ref(mu0)
delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \
collect_performance(cusum_loc, sig, changes, theta_vec)
for i, d in enumerate(tps_stat):
if i > 0:
assert d >= tps_stat[i - 1]
def run_artificial_signal_avg(max_theta=10,
signal_settings: SignalSettings = None,
one_change_point=True,
with_resetting=False):
"""
The same as run_artificial_signal() in performance_detection_signals.py but
averaged results and 1 change.
"""
n_sim = N_SIM
n_points = signal_settings.half_len
sigma = signal_settings.sigma
mu0 = signal_settings.mu0
mu1 = signal_settings.mu1
if one_change_point:
changes = [n_points]
else:
changes = [n_points, 2 * n_points]
rois = [Roi(change, ROI_RADIUS_ARTIFICIAL_SIGNAL) for change in changes]
tps_vec_stat = []
tps_vec_dyn = []
fns_vec_stat = []
fns_vec_dyn = []
fps_vec_stat = []
fps_vec_dyn = []
delays_vec_stat = []
delays_vec_dyn = []
f1_vec_stat = []
f1_vec_dyn = []
theta_vec = list(np.linspace(0.1, max_theta, 100))
if with_resetting:
detector_fn = cusum
cusum_pccf_ref = make_cusum_pccf(rois)
detector_pccf_fn = cusum_pccf_ref
else:
cusum_single_ref = make_cusum_single_ref(mu0)
cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, rois[0])
detector_fn = cusum_single_ref
detector_pccf_fn = cusum_single_pccf_ref
for _ in tqdm(range(n_sim)):
if one_change_point:
sig = np.concatenate((
randn(n_points) * sigma + mu0,
randn(n_points) * sigma + mu1,
),
axis=0)
else:
sig = np.concatenate(
(randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1,
randn(n_points) * sigma + mu0),
axis=0)
# Vectors with counts
delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \
collect_performance(detector_fn, sig, changes, theta_vec)
delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \
collect_performance(detector_pccf_fn, sig, changes, theta_vec, rois=rois)
delays_vec_stat.append(delays_stat)
delays_vec_dyn.append(delays_dyn)
fps_vec_stat.append(fps_stat)
fps_vec_dyn.append(fps_dyn)
fns_vec_stat.append(fns_stat)
fns_vec_dyn.append(fns_dyn)
tps_vec_stat.append(tps_stat)
tps_vec_dyn.append(tps_dyn)
f1_vec_stat.append(f1_stat)
f1_vec_dyn.append(f1_dyn)
delays_stat_m = np.vstack(delays_vec_stat)
delays_dyn_m = np.vstack(delays_vec_dyn)
fps_stat_m = np.vstack(fps_vec_stat)
fps_dyn_m = np.vstack(fps_vec_dyn)
fns_stat_m = np.vstack(fns_vec_stat)
fns_dyn_m = np.vstack(fns_vec_dyn)
tps_stat_m = np.vstack(tps_vec_stat)
tps_dyn_m = np.vstack(tps_vec_dyn)
f1_stat_m = np.vstack(f1_vec_stat)
f1_dyn_m = np.vstack(f1_vec_dyn)
averaged_delays_stat = average_results_matrix_rows(delays_stat_m)
averaged_delays_dyn = average_results_matrix_rows(delays_dyn_m)
averaged_fps_stat = average_results_matrix_rows(fps_stat_m,
count_nans_as_zero=False)
averaged_fps_dyn = average_results_matrix_rows(fps_dyn_m,
count_nans_as_zero=False)
averaged_fns_stat = average_results_matrix_rows(fns_stat_m,
count_nans_as_zero=False)
averaged_fns_dyn = average_results_matrix_rows(fns_dyn_m,
count_nans_as_zero=False)
averaged_tps_stat = average_results_matrix_rows(tps_stat_m,
count_nans_as_zero=False)
averaged_tps_dyn = average_results_matrix_rows(tps_dyn_m,
count_nans_as_zero=False)
averaged_f1_stat = average_results_matrix_rows(f1_stat_m,
count_nans_as_zero=False)
averaged_f1_dyn = average_results_matrix_rows(f1_dyn_m,
count_nans_as_zero=False)
# Normalize
# averaged_fps_stat /= len(sig)
# averaged_fps_dyn /= len(sig)
# averaged_fns_stat /= len(changes)
# averaged_fns_dyn /= len(changes)
# averaged_tps_stat /= len(changes)
# averaged_tps_dyn /= len(changes)
# #
# averaged_f1_stat /= len(changes)
# averaged_f1_dyn /= len(changes)
results = SimulationResults(theta_vec, averaged_delays_stat,
averaged_delays_dyn, averaged_tps_stat,
averaged_tps_dyn, averaged_fps_stat,
averaged_fps_dyn, averaged_fns_stat,
averaged_fns_dyn, averaged_f1_stat,
averaged_f1_dyn)
return results