def compute_psds(self): logging.info("Compute psds scores") ########## # Optional but recommended ########## # Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased. n_thresholds = 50 out_nb_frames_1s = self.sample_rate / self.hop_size / self.pooling_time_ratio # median_window = max(int(0.45 * out_nb_frames_1s), 1) post_processing_fn = [functools.partial(median_filt_1d, filt_span=3)] # Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9 list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds) prediction_dfs = {} for threshold in list_thresholds: prediction_dfs[threshold] = self.get_prediction_dataframe( post_processing=post_processing_fn, threshold=threshold, binarization_type="global_threshold", ) pred_thresh = [] for key in prediction_dfs: pred_thresh.append(prediction_dfs[key]) if len(pred_thresh) == 1: pred_thresh = pred_thresh[0] # save predictions (self.output_dir / "predictions_thresh").mkdir(exist_ok=True) for th, pred_df in zip(list_thresholds, pred_thresh): pred_df.to_csv( self.output_dir / "predictions_thresh" / f"{th}.csv", index=False, sep="\t", float_format="%.3f", ) psds = compute_psds_from_operating_points(pred_thresh, self.validation_df, self.durations_validation) psds_score(psds, filename_roc_curves=self.output_dir / "psds_roc.png")
# Preds with only one value valid_predictions = get_predictions(crnn, validation_dataloader, many_hot_encoder.decode_strong, pooling_time_ratio, median_window=median_window, save_predictions=predicitons_fname) compute_metrics(valid_predictions, validation_labels_df, durations_validation) # ########## # Optional but recommended # ########## # Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased. n_thresholds = 50 # Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9 list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds) pred_ss_thresh = get_predictions(crnn, validation_dataloader, many_hot_encoder.decode_strong, pooling_time_ratio, thresholds=list_thresholds, median_window=median_window, save_predictions=predicitons_fname) psds = compute_psds_from_operating_points(pred_ss_thresh, validation_labels_df, durations_validation) psds_score(psds, filename_roc_curves=os.path.join(saved_pred_dir, "figures/psds_roc.png"))
single_predictions = get_predictions( params["model"], params["dataloader"], params["many_hot_encoder"].decode_strong, params["pooling_time_ratio"], median_window=params["median_window"], save_predictions=f_args.save_predictions_path) compute_metrics(single_predictions, groundtruth, durations) # ########## # Optional but recommended # ########## # Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased. n_thresholds = 50 # Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9 list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds) pred_ss_thresh = get_predictions( params["model"], params["dataloader"], params["many_hot_encoder"].decode_strong, params["pooling_time_ratio"], thresholds=list_thresholds, median_window=params["median_window"], save_predictions=f_args.save_predictions_path) psds = compute_psds_from_operating_points(pred_ss_thresh, groundtruth, durations) fname_roc = None if f_args.save_predictions_path is not None: fname_roc = osp.splitext(f_args.save_predictions_path)[0] + "_roc.png" psds_score(psds, filename_roc_curves=fname_roc)
if __name__ == '__main__': groundtruth_path = "../dataset/metadata/validation/validation.tsv" durations_path = "../dataset/metadata/validation/validation_durations.tsv" # If durations do not exists, audio dir is needed groundtruth_audio_path = "../dataset/audio/validation" base_prediction_path = "stored_data/MeanTeacher_with_synthetic/predictions/baseline_validation" groundtruth = pd.read_csv(groundtruth_path, sep="\t") if osp.exists(durations_path): meta_dur_df = pd.read_csv(durations_path, sep='\t') else: meta_dur_df = generate_tsv_wav_durations(groundtruth_audio_path, durations_path) # Evaluate a single prediction single_predictions = pd.read_csv(base_prediction_path + ".tsv", sep="\t") compute_metrics(single_predictions, groundtruth, meta_dur_df) # Evaluate predictions with multiple thresholds (better). Need a list of predictions. prediction_list_path = glob.glob(osp.join(base_prediction_path, "*.tsv")) list_predictions = [] for fname in prediction_list_path: pred_df = pd.read_csv(fname, sep="\t") list_predictions.append(pred_df) psds = compute_psds_from_operating_points(list_predictions, groundtruth, meta_dur_df) psds_score(psds, filename_roc_curves=osp.join(base_prediction_path, "figures/psds_roc.png"))