def compute_psds(self):
logging.info("Compute psds scores")
##########
# Optional but recommended
##########
# Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased.
n_thresholds = 50
out_nb_frames_1s = self.sample_rate / self.hop_size / self.pooling_time_ratio
# median_window = max(int(0.45 * out_nb_frames_1s), 1)
post_processing_fn = [functools.partial(median_filt_1d, filt_span=3)]
# Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9
list_thresholds = np.arange(1 / (n_thresholds * 2), 1,
1 / n_thresholds)
prediction_dfs = {}
for threshold in list_thresholds:
prediction_dfs[threshold] = self.get_prediction_dataframe(
post_processing=post_processing_fn,
threshold=threshold,
binarization_type="global_threshold",
)
pred_thresh = []
for key in prediction_dfs:
pred_thresh.append(prediction_dfs[key])
if len(pred_thresh) == 1:
pred_thresh = pred_thresh[0]
# save predictions
(self.output_dir / "predictions_thresh").mkdir(exist_ok=True)
for th, pred_df in zip(list_thresholds, pred_thresh):
pred_df.to_csv(
self.output_dir / "predictions_thresh" / f"{th}.csv",
index=False,
sep="\t",
float_format="%.3f",
)
psds = compute_psds_from_operating_points(pred_thresh,
self.validation_df,
self.durations_validation)
psds_score(psds, filename_roc_curves=self.output_dir / "psds_roc.png")
# Preds with only one value
valid_predictions = get_predictions(crnn,
validation_dataloader,
many_hot_encoder.decode_strong,
pooling_time_ratio,
median_window=median_window,
save_predictions=predicitons_fname)
compute_metrics(valid_predictions, validation_labels_df,
durations_validation)
# ##########
# Optional but recommended
# ##########
# Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased.
n_thresholds = 50
# Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9
list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds)
pred_ss_thresh = get_predictions(crnn,
validation_dataloader,
many_hot_encoder.decode_strong,
pooling_time_ratio,
thresholds=list_thresholds,
median_window=median_window,
save_predictions=predicitons_fname)
psds = compute_psds_from_operating_points(pred_ss_thresh,
validation_labels_df,
durations_validation)
psds_score(psds,
filename_roc_curves=os.path.join(saved_pred_dir,
"figures/psds_roc.png"))
single_predictions = get_predictions(
params["model"],
params["dataloader"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path)
compute_metrics(single_predictions, groundtruth, durations)
# ##########
# Optional but recommended
# ##########
# Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased.
n_thresholds = 50
# Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9
list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds)
pred_ss_thresh = get_predictions(
params["model"],
params["dataloader"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
thresholds=list_thresholds,
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path)
psds = compute_psds_from_operating_points(pred_ss_thresh, groundtruth,
durations)
fname_roc = None
if f_args.save_predictions_path is not None:
fname_roc = osp.splitext(f_args.save_predictions_path)[0] + "_roc.png"
psds_score(psds, filename_roc_curves=fname_roc)
if __name__ == '__main__':
groundtruth_path = "../dataset/metadata/validation/validation.tsv"
durations_path = "../dataset/metadata/validation/validation_durations.tsv"
# If durations do not exists, audio dir is needed
groundtruth_audio_path = "../dataset/audio/validation"
base_prediction_path = "stored_data/MeanTeacher_with_synthetic/predictions/baseline_validation"
groundtruth = pd.read_csv(groundtruth_path, sep="\t")
if osp.exists(durations_path):
meta_dur_df = pd.read_csv(durations_path, sep='\t')
else:
meta_dur_df = generate_tsv_wav_durations(groundtruth_audio_path,
durations_path)
# Evaluate a single prediction
single_predictions = pd.read_csv(base_prediction_path + ".tsv", sep="\t")
compute_metrics(single_predictions, groundtruth, meta_dur_df)
# Evaluate predictions with multiple thresholds (better). Need a list of predictions.
prediction_list_path = glob.glob(osp.join(base_prediction_path, "*.tsv"))
list_predictions = []
for fname in prediction_list_path:
pred_df = pd.read_csv(fname, sep="\t")
list_predictions.append(pred_df)
psds = compute_psds_from_operating_points(list_predictions, groundtruth,
meta_dur_df)
psds_score(psds,
filename_roc_curves=osp.join(base_prediction_path,
"figures/psds_roc.png"))
gt_df_feat_ss = dataset.initialize_and_get_df(f_args.groundtruth_tsv, gt_audio_dir, f_args.base_dir_ss,
pattern_ss="_events", nb_files=f_args.nb_files,
keep_sources=keep_sources)
params = _load_state_vars(expe_state, gt_df_feat_ss, median_window)
alpha_norm = 1
# Preds with only one value (note that in comparison of TestModel, here we do not use a dataloader)
single_predictions = get_predictions_ss_late_integration(params["model"], params["dataload"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path,
alpha=alpha_norm)
compute_metrics(single_predictions, groundtruth, durations)
# ##########
# Optional but recommended
# ##########
# Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased.
n_thresholds = 50
# Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9
thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds)
pred_ss_thresh = get_predictions_ss_late_integration(params["model"], params["dataload"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
thresholds=thresholds,
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path)
psds = compute_psds_from_operating_points(pred_ss_thresh, groundtruth, durations)
psds_score(psds, filename_roc_curves=osp.splitext(f_args.save_predictions_path)[0] + "_roc.png")