def generate_metric_values(self, prediction_dir, splice_save_dir):
alg_splice_size = {}
metrics = [MeanError, AdTP, AdFP, AdTN, AdFN]
for alg_name, alg_func in algs_with_name().items():
mas = MockAlgStore(prediction_dir + alg_name)
alg_splice_size[alg_name] = {}
print(alg_name)
# need to be adapted to the available values
for noise_level in np.linspace(0, 2, 5):
noise_str = str(noise_level).replace(".", "-")
alg_splice_size[alg_name][noise_level] = {}
for splice_size in range(3, 11, 2):
reg = "[a-zA-Z]_{}_{}.*_[0-9]+\\.atr".format(noise_str, splice_size)
loaded_data = read_ann_files(mas, reg)
alg_splice_size[alg_name][noise_level][splice_size] = {}
with cf.ProcessPoolExecutor(max_workers=len(metrics)) as pool:
met_per_beats = list(pool.map(evaluate_algorithm_with_metric, metrics,
list([loaded_data] * len(metrics))))
for met_idx, met_per_beat in enumerate(met_per_beats):
reduced_metric = reduce(join, met_per_beat.values())
alg_splice_size[alg_name][noise_level][splice_size][metrics[met_idx].__abbrev__] = reduced_metric.compute()
print(alg_splice_size)
for alg in alg_splice_size:
os.makedirs(splice_save_dir, exist_ok=True)
for noise_level, noise_vals in alg_splice_size[alg].items():
write_strs = {}
for spli, vals in noise_vals.items():
for metric, metric_value in vals.items():
write_strs.setdefault(metric, "")
write_strs[metric] += "{} {}\n".format(spli, metric_value)
for metrics_abbrev, write_str in write_strs.items():
with open(splice_save_dir + "/{}-{}-{}.dat".format(metrics_abbrev, alg, noise_level), "w") as splice_file:
splice_file.write(write_str)
def test_different_confusion_matrix_tresholds(self):
algorthim_metrics = []
for alg_num, alg_store in enumerate(self.alg_stores):
print(alg_store.alg_name)
loaded_data = list(read_ann_files(alg_store))
print(len(loaded_data), "loaded data")
me = evaluate_algorithm_with_metric(MeanError, loaded_data)
mse = evaluate_algorithm_with_metric(MeanSquaredError, loaded_data)
mae = evaluate_algorithm_with_metric(MeanAbsoluteError, loaded_data)
gtme = evaluate_algorithm_with_metric(MeanErrorGt, loaded_data)
gtmse = evaluate_algorithm_with_metric(MeanSquaredErrorGt, loaded_data)
gtmae = evaluate_algorithm_with_metric(MeanAbsoluteErrorGt, loaded_data)
combined_metrics = []
combined_metrics.append(reduce(join, me.values()))
combined_metrics.append(reduce(join, gtme.values()))
combined_metrics.append(reduce(join, mse.values()))
combined_metrics.append(reduce(join, gtmse.values()))
combined_metrics.append(reduce(join, mae.values()))
combined_metrics.append(reduce(join, gtmae.values()))
combined_metrics = list(map(lambda x: x.compute(), combined_metrics))
print(combined_metrics)
algorthim_metrics.append(combined_metrics)
for ic in range(6):
alg_data = ""
for ia, alg_met in enumerate(algorthim_metrics):
alg_data += "{} {}\n".format(ia, alg_met[ic])
with open("data/latex_data/reg-met{}.dat".format(ic), "w+") as data_file:
data_file.write(alg_data)
def test_different_confusion_matrix_tresholds(self):
for alg_num, alg_store in enumerate(self.alg_stores):
print(alg_store.alg_name)
sum_tp = [0] * 10
sum_fp = [0] * 10
sum_tn = [0] * 10
sum_fn = [0] * 10
loaded_data = list(read_ann_files(alg_store))
print(len(loaded_data), "loaded data")
tps = evaluate_algorithm_with_metric(TP, loaded_data)
fps = evaluate_algorithm_with_metric(FP, loaded_data)
tns = evaluate_algorithm_with_metric(TN, loaded_data)
fns = evaluate_algorithm_with_metric(FN, loaded_data)
for data_type in tps:
tp_vals = tps[data_type].compute()
fp_vals = fps[data_type].compute()
tn_vals = tns[data_type].compute()
fn_vals = fns[data_type].compute()
for i in range(len(tp_vals)):
sum_tp[i] += tp_vals[i]
sum_fp[i] += fp_vals[i]
sum_tn[i] += tn_vals[i]
sum_fn[i] += fn_vals[i]
grouped = list(map(sum, zip(sum_tp, sum_tn, sum_fn, sum_fp)))
self.save_sliding_window_metrics(sum_tp, grouped, alg_num, "TP")
self.save_sliding_window_metrics(sum_fp, grouped, alg_num, "FP")
self.save_sliding_window_metrics(sum_tn, grouped, alg_num, "TN")
self.save_sliding_window_metrics(sum_fn, grouped, alg_num, "FN")
def test_predictions_not_empty(self):
alg_splice_size = {}
for alg_name, alg_func in algs_with_name().items():
mas = MockAlgStore("alg_pred2/" + alg_name)
alg_splice_size[alg_name] = {}
print(alg_name)
for splice_size in range(1, 22, 2):
reg = "[a-zA-Z]_0-0_{}_[0-9]+\\.atr".format(splice_size)
loaded_data = read_ann_files(mas, reg)
def test_confusion_matrix_treshold(self):
for alg_store in self.alg_stores:
print(alg_store.alg_name)
sum_tp = 0
sum_fp = 0
sum_tn = 0
sum_fn = 0
loaded_data = list(read_ann_files(alg_store))
print(len(loaded_data), "loaded data")
tps = evaluate_algorithm_with_metric(TP, loaded_data)
fps = evaluate_algorithm_with_metric(FP, loaded_data)
tns = evaluate_algorithm_with_metric(TN, loaded_data)
fns = evaluate_algorithm_with_metric(FN, loaded_data)
for data_type in tps:
tp_vals = tps[data_type].compute()
fp_vals = fps[data_type].compute()
tn_vals = tns[data_type].compute()
fn_vals = fns[data_type].compute()
max_vals = min(zip(range(len(tp_vals)), tp_vals, fp_vals,
tn_vals, fn_vals),
key=lambda x: (x[2]**2 + x[4]**2)**0.5)
# print("1", data_type, max_vals)
max_vals = max(zip(range(len(tp_vals)), tp_vals, fp_vals,
tn_vals, fn_vals),
key=lambda x: x[1] + x[3] - x[2] - x[4])
# print("2", data_type, max_vals)
max_vals = max(zip(range(len(tp_vals)), tp_vals, fp_vals,
tn_vals, fn_vals),
key=lambda x: x[1])
# print("3", data_type, max_vals)
max_vals = min(zip(range(len(tp_vals)), tp_vals, fp_vals,
tn_vals, fn_vals),
key=lambda x: ((x[1] - sum(x[1:]))**2 +
(x[3] - sum(x[1:]))**2)**0.5)
# print("4", data_type, max_vals)
max_vals = min(
zip(range(len(tp_vals)), tp_vals, fp_vals, tn_vals,
fn_vals),
key=lambda x:
((x[1] / max(x[1] + x[4], 0.000001) - 1)**2 +
(x[3] / max(x[3] + x[2], 0.000001) - 1)**2)**0.5)
# print("5", data_type, max_vals)
sum_tp += tp_vals[-1]
sum_fp += fp_vals[-1]
sum_tn += tn_vals[-1]
sum_fn += fn_vals[-1]
# print("")
print("All", sum_tp, sum_fp, sum_tn, sum_fn)
def test_gqrs_hopping_window(self):
for alg_idx, alg_store in enumerate(self.alg_stores):
if alg_store.alg_name != "gqrs":
continue
print(alg_store.alg_name)
steps = 6
sum_tp = [[0 for _ in range(steps)] for _ in range(10)]
sum_fp = [[0 for _ in range(steps)] for _ in range(10)]
sum_tn = [[0 for _ in range(steps)] for _ in range(10)]
sum_fn = [[0 for _ in range(steps)] for _ in range(10)]
loaded_data = list(read_ann_files(alg_store))
print(len(loaded_data), "loaded data")
tps = evaluate_algorithm_with_metric(HopTP, loaded_data)
fps = evaluate_algorithm_with_metric(HopFP, loaded_data)
tns = evaluate_algorithm_with_metric(HopTN, loaded_data)
fns = evaluate_algorithm_with_metric(HopFN, loaded_data)
for data_type in tps:
tp_vals = tps[data_type].compute()
fp_vals = fps[data_type].compute()
tn_vals = tns[data_type].compute()
fn_vals = fns[data_type].compute()
for i in range(len(tp_vals)):
for j in range(len(tp_vals[i])):
sum_tp[i][j] += tp_vals[i][j]
sum_fp[i][j] += fp_vals[i][j]
sum_tn[i][j] += tn_vals[i][j]
sum_fn[i][j] += fn_vals[i][j]
a = [0] * len(sum_tp[0])
for j in range(len(sum_tp[0])):
a[j] += max(
sum_tp[0][j] + sum_fp[0][j] + sum_tn[0][j] + sum_fn[0][j],
0.000001)
print("TP")
self.save_hopping_metrics(sum_tp, a, alg_idx, "TP", False)
print("FP")
self.save_hopping_metrics(sum_fp, a, alg_idx, "FP", False)
print("TN")
self.save_hopping_metrics(sum_tn, a, alg_idx, "TN", False)
print("FN")
self.save_hopping_metrics(sum_fn, a, alg_idx, "FN", False)
def test_different_confusion_matrix_tresholds(self):
for alg_num, alg_store in enumerate(self.alg_stores):
print(alg_store.alg_name)
loaded_data = list(read_ann_files(alg_store))
print(len(loaded_data), "loaded data")
me = evaluate_algorithm_with_metric(MeanError, loaded_data)
distances = []
for data_type in me:
distances.extend(me[data_type].distance_to_true)
bins = [
-1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5,
0.6, 0.7
]
hist = np.histogram(distances, bins=bins)
print(list(hist[0]))
print(list(hist[1]))
file_data = ""
for i in range(len(hist[0])):
file_data += "{} {}\n".format(hist[1][i], hist[0][i])
with open("data/latex_data/offset-al{}.dat".format(alg_num),
"w+") as data_file:
data_file.write(file_data)