def test_collect_performance(): n_points = 15 ROI_RADIUS = 4 MIN_THETA = 0.1 MAX_THETA = 7.0 THETA_STEP = 0.7 changes = [n_points + 1] np.random.seed(11) mu0 = 0.0 theta_vec = [0.1, 0.8, 1.5, 2.2, 2.9, 3.6, 4.3, 5.0, 5.7, 6.4, 7.5] sig = [ 1.92440022, -0.3146803, -0.53302165, -2.91865042, -0.00911309, -0.3515945, -0.5902923, 0.34694294, 0.46315579, -1.17216328, -0.97486364, -0.52330684, 0.75865054, 0.61731139, -1.43610336, 0.86857721, 2.91052113, 3.83209748, 2.0658174, 1.34820871, 3.30519267, 1.7594657, 2.89832745, 3.8039788, 2.7930878, 2.18084256, 2.90549849, 1.39316707, 1.90409751, 1.46864998 ] cusum_single_ref = make_cusum_single_ref(mu0) rois = [Roi(n_points, ROI_RADIUS)] cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, rois[0]) delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \ collect_performance(cusum_single_ref, sig, changes, theta_vec) delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \ collect_performance(cusum_single_pccf_ref, sig, changes, theta_vec, rois=rois) np.testing.assert_equal( delays_stat, [np.nan, np.nan, np.nan, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0]) assert tps_stat == [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] assert fps_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] assert fns_stat == [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] assert f1_stat == list([ tp / (tp + 0.5 * (fp + fn)) for (tp, fp, fn) in zip(tps_stat, fps_stat, fns_stat) ]) assert tps_dyn == [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0] assert fns_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] # assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] - old behaviour. # After the fix we do not detect outside ROI assert fps_dyn == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] np.testing.assert_equal( delays_dyn, [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, np.nan, np.nan])
def test_collect_arl(): mu0 = 0.0 cusum_single_ref = make_cusum_single_ref(mu0) cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, Roi(100, radius=25)) n_points = 100 sigma = 1.1 mu1 = 1.1 sig = np.concatenate( (randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0) theta_vec = np.linspace(0.01, 150.0, 300) arls_stat = collect_arl(cusum_single_ref, sig, theta_vec) arls_dyn = collect_arl(cusum_single_pccf_ref, sig, theta_vec) assert len(arls_dyn) == len(arls_stat)
def run_simulation_arl_perf(signal_settings: SignalSettings, save_path): n_sim = 100 n_points = signal_settings.half_len roi = Roi(signal_settings.half_len, ROI_RADIUS_ARTIFICIAL_SIGNAL) sigma = signal_settings.sigma mu0 = signal_settings.mu0 mu1 = signal_settings.mu1 cusum_single_ref = make_cusum_single_ref(mu0) cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, roi) detector_fn = cusum_single_ref detector_pccf_fn = cusum_single_pccf_ref theta_vec = np.linspace(0.01, 40.0, 100) arl_vec_stat = [] arl_vec_dyn = [] for _ in tqdm(range(n_sim)): sig = np.concatenate((randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1, ), axis=0) arl_stat = collect_arl(detector_fn, sig, theta_vec) arl_dyn = collect_arl(detector_pccf_fn, sig, theta_vec) arl_vec_stat.append(arl_stat) arl_vec_dyn.append(arl_dyn) m_arl_stat = np.vstack(arl_vec_stat) m_arl_dyn = np.vstack(arl_vec_dyn) count_nans_as_zero = False # If ARL is NaN then we disregard it avg_arl_stat = average_results_matrix_rows(m_arl_stat, count_nans_as_zero=count_nans_as_zero) avg_arl_dyn = average_results_matrix_rows(m_arl_dyn, count_nans_as_zero=count_nans_as_zero) # avg_arl_stat = np.sum(m_arl_stat, axis=0)/m_arl_stat.shape[0] # avg_arl_dyn = np.nansum(m_arl_dyn, axis=0)/m_arl_dyn.shape[0] results = {'theta_vec': list(theta_vec), 'avg_arl_stat': list(avg_arl_stat), 'avg_arl_dyn': list(avg_arl_dyn), } with open(save_path, 'w') as f: json.dump(results, f, indent=4) logger.info(f"Save into {save_path}") return save_path
def test_cusum_single_ref(): """ Test for detection location """ mu0 = 0.0 cusum_single_ref = make_cusum_single_ref(mu0) n_points = 150 sigma = 1.1 mu1 = 2.1 detections = [] for _ in range(30): sig = np.concatenate( (randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0) r = cusum_single_ref(sig, 20) detections.append(r.detections[0]) cde_average = np.mean(detections) assert 140 <= cde_average <= 170
def test_cusum_single_nan_outputs(): mu0 = 0.0 n_points = 150 sigma = 1.1 mu1 = 2.1 cusum_single_ref = make_cusum_single_ref(mu0) cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, Roi(n_points + 1, 15)) sig = np.concatenate( (randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1), axis=0) r_stat = cusum_single_ref(sig, 1) r_dyn = cusum_single_pccf_ref(sig, 1) perf_stat = tp_fp_fn_delays([n_points], r_stat.detections) perf_dyn = tp_fp_fn_delays([n_points], r_dyn.detections) assert np.isnan(perf_stat.delays[0]) assert np.isnan(perf_dyn.delays[0]) r_dyn2 = cusum_single_pccf_ref(sig, 2000) perf_dyn2 = tp_fp_fn_delays([n_points], r_dyn.detections) assert np.isnan(perf_dyn2.delays[0])
def test_tps_num_behaviour_cusum_single(): n_points = 100 sigma = 1.0 mu0 = 0.0 mu1 = 1.1 sig = np.concatenate(( randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1, ), axis=0) changes = [100] max_theta = 20 theta_vec = np.linspace(0.001, max_theta, 100) # gen_theta_vec(0.001, 0.1, max_theta) cusum_loc = make_cusum_single_ref(mu0) delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \ collect_performance(cusum_loc, sig, changes, theta_vec) for i, d in enumerate(tps_stat): if i > 0: assert d >= tps_stat[i - 1]
def run_artificial_signal_avg(max_theta=10, signal_settings: SignalSettings = None, one_change_point=True, with_resetting=False): """ The same as run_artificial_signal() in performance_detection_signals.py but averaged results and 1 change. """ n_sim = N_SIM n_points = signal_settings.half_len sigma = signal_settings.sigma mu0 = signal_settings.mu0 mu1 = signal_settings.mu1 if one_change_point: changes = [n_points] else: changes = [n_points, 2 * n_points] rois = [Roi(change, ROI_RADIUS_ARTIFICIAL_SIGNAL) for change in changes] tps_vec_stat = [] tps_vec_dyn = [] fns_vec_stat = [] fns_vec_dyn = [] fps_vec_stat = [] fps_vec_dyn = [] delays_vec_stat = [] delays_vec_dyn = [] f1_vec_stat = [] f1_vec_dyn = [] theta_vec = list(np.linspace(0.1, max_theta, 100)) if with_resetting: detector_fn = cusum cusum_pccf_ref = make_cusum_pccf(rois) detector_pccf_fn = cusum_pccf_ref else: cusum_single_ref = make_cusum_single_ref(mu0) cusum_single_pccf_ref = make_cusum_single_pccf_ref(mu0, rois[0]) detector_fn = cusum_single_ref detector_pccf_fn = cusum_single_pccf_ref for _ in tqdm(range(n_sim)): if one_change_point: sig = np.concatenate(( randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1, ), axis=0) else: sig = np.concatenate( (randn(n_points) * sigma + mu0, randn(n_points) * sigma + mu1, randn(n_points) * sigma + mu0), axis=0) # Vectors with counts delays_stat, fps_stat, fns_stat, tps_stat, f1_stat = \ collect_performance(detector_fn, sig, changes, theta_vec) delays_dyn, fps_dyn, fns_dyn, tps_dyn, f1_dyn = \ collect_performance(detector_pccf_fn, sig, changes, theta_vec, rois=rois) delays_vec_stat.append(delays_stat) delays_vec_dyn.append(delays_dyn) fps_vec_stat.append(fps_stat) fps_vec_dyn.append(fps_dyn) fns_vec_stat.append(fns_stat) fns_vec_dyn.append(fns_dyn) tps_vec_stat.append(tps_stat) tps_vec_dyn.append(tps_dyn) f1_vec_stat.append(f1_stat) f1_vec_dyn.append(f1_dyn) delays_stat_m = np.vstack(delays_vec_stat) delays_dyn_m = np.vstack(delays_vec_dyn) fps_stat_m = np.vstack(fps_vec_stat) fps_dyn_m = np.vstack(fps_vec_dyn) fns_stat_m = np.vstack(fns_vec_stat) fns_dyn_m = np.vstack(fns_vec_dyn) tps_stat_m = np.vstack(tps_vec_stat) tps_dyn_m = np.vstack(tps_vec_dyn) f1_stat_m = np.vstack(f1_vec_stat) f1_dyn_m = np.vstack(f1_vec_dyn) averaged_delays_stat = average_results_matrix_rows(delays_stat_m) averaged_delays_dyn = average_results_matrix_rows(delays_dyn_m) averaged_fps_stat = average_results_matrix_rows(fps_stat_m, count_nans_as_zero=False) averaged_fps_dyn = average_results_matrix_rows(fps_dyn_m, count_nans_as_zero=False) averaged_fns_stat = average_results_matrix_rows(fns_stat_m, count_nans_as_zero=False) averaged_fns_dyn = average_results_matrix_rows(fns_dyn_m, count_nans_as_zero=False) averaged_tps_stat = average_results_matrix_rows(tps_stat_m, count_nans_as_zero=False) averaged_tps_dyn = average_results_matrix_rows(tps_dyn_m, count_nans_as_zero=False) averaged_f1_stat = average_results_matrix_rows(f1_stat_m, count_nans_as_zero=False) averaged_f1_dyn = average_results_matrix_rows(f1_dyn_m, count_nans_as_zero=False) # Normalize # averaged_fps_stat /= len(sig) # averaged_fps_dyn /= len(sig) # averaged_fns_stat /= len(changes) # averaged_fns_dyn /= len(changes) # averaged_tps_stat /= len(changes) # averaged_tps_dyn /= len(changes) # # # averaged_f1_stat /= len(changes) # averaged_f1_dyn /= len(changes) results = SimulationResults(theta_vec, averaged_delays_stat, averaged_delays_dyn, averaged_tps_stat, averaged_tps_dyn, averaged_fps_stat, averaged_fps_dyn, averaged_fns_stat, averaged_fns_dyn, averaged_f1_stat, averaged_f1_dyn) return results