def compute_psds_from_operating_points(
prediction_dfs,
ground_truth_file,
durations_file,
dtc_threshold=0.5,
gtc_threshold=0.5,
cttc_threshold=0.3,
alpha_ct=0,
alpha_st=0,
max_efpr=100,
save_dir=None,
):
gt = pd.read_csv(ground_truth_file, sep="\t")
durations = pd.read_csv(durations_file, sep="\t")
psds_eval = PSDSEval(
ground_truth=gt,
metadata=durations,
dtc_threshold=dtc_threshold,
gtc_threshold=gtc_threshold,
cttc_threshold=cttc_threshold,
)
for i, k in enumerate(prediction_dfs.keys()):
det = prediction_dfs[k]
# see issue https://github.com/audioanalytic/psds_eval/issues/3
det["index"] = range(1, len(det) + 1)
det = det.set_index("index")
psds_eval.add_operating_point(det,
info={
"name": f"Op {i + 1:02d}",
"threshold": k
})
psds_score = psds_eval.psds(alpha_ct=alpha_ct,
alpha_st=alpha_st,
max_efpr=max_efpr)
if save_dir is not None:
os.makedirs(save_dir, exist_ok=True)
pred_dir = os.path.join(
save_dir,
f"predictions_dtc{dtc_threshold}_gtc{gtc_threshold}_cttc{cttc_threshold}",
)
os.makedirs(pred_dir, exist_ok=True)
for k in prediction_dfs.keys():
prediction_dfs[k].to_csv(
os.path.join(pred_dir, f"predictions_th_{k:.2f}.tsv"),
sep="\t",
index=False,
)
plot_psd_roc(
psds_score,
filename=os.path.join(save_dir,
f"PSDS_ct{alpha_ct}_st{alpha_st}_100.png"),
)
return psds_score.value
def psds_score(psds, filename_roc_curves=None):
""" add operating points to PSDSEval object and compute metrics
Args:
psds: psds.PSDSEval object initialized with the groundtruth corresponding to the predictions
filename_roc_curves: str, the base filename of the roc curve to be computed
"""
try:
psds_score = psds.psds(alpha_ct=0, alpha_st=0, max_efpr=100)
logger.info(f"\nPSD-Score (0, 0, 100): {psds_score.value:.5f}")
psds_ct_score = psds.psds(alpha_ct=1, alpha_st=0, max_efpr=100)
logger.info(f"\nPSD-Score (1, 0, 100): {psds_ct_score.value:.5f}")
psds_macro_score = psds.psds(alpha_ct=0, alpha_st=1, max_efpr=100)
logger.info(f"\nPSD-Score (0, 1, 100): {psds_macro_score.value:.5f}")
if filename_roc_curves is not None:
if osp.dirname(filename_roc_curves) != "":
os.makedirs(osp.dirname(filename_roc_curves), exist_ok=True)
base, ext = osp.splitext(filename_roc_curves)
plot_psd_roc(psds_score, filename=f"{base}_0_0_100{ext}")
plot_psd_roc(psds_ct_score, filename=f"{base}_1_0_100{ext}")
plot_psd_roc(psds_score, filename=f"{base}_0_1_100{ext}")
except psds_eval.psds.PSDSEvalError as e:
logger.error("psds score did not work ....")
logger.error(e)
gtc_threshold = 0.5
cttc_threshold = 0.3
alpha_ct = 0.0
alpha_st = 0.0
max_efpr = 100
# Load metadata and ground truth tables
data_dir = os.path.join(os.path.dirname(__file__), "data")
ground_truth_csv = os.path.join(data_dir, "dcase2019t4_gt.csv")
metadata_csv = os.path.join(data_dir, "dcase2019t4_meta.csv")
gt_table = pd.read_csv(ground_truth_csv, sep="\t")
meta_table = pd.read_csv(metadata_csv, sep="\t")
# Instantiate PSDSEval
psds_eval = PSDSEval(dtc_threshold, gtc_threshold, cttc_threshold,
ground_truth=gt_table, metadata=meta_table)
# Add the operating points
for i, th in enumerate(np.arange(0.1, 1.1, 0.1)):
csv_file = os.path.join(data_dir, f"baseline_{th:.1f}.csv")
det_t = pd.read_csv(os.path.join(csv_file), sep="\t")
psds_eval.add_operating_point(det_t)
print(f"\rOperating point {i+1} added", end=" ")
# Calculate the PSD-Score
psds = psds_eval.psds(alpha_ct, alpha_st, max_efpr)
print(f"\nPSD-Score: {psds.value:.5f}")
# Plot the PSD-ROC
plot_psd_roc(psds)