def test_unknown_class_constraint_check():
metadata = pd.read_csv(os.path.join(DATADIR, "test.metadata"), sep="\t")
det1 = pd.read_csv(os.path.join(DATADIR, "test_1.det"), sep="\t")
gt = pd.read_csv(os.path.join(DATADIR, "test_1.gt"), sep="\t")
info1 = {"name": "test_1", "threshold1": 1}
psds_eval = PSDSEval(dtc_threshold=0.5, gtc_threshold=0.5,
cttc_threshold=0.3, ground_truth=gt,
metadata=metadata)
psds_eval.add_operating_point(det1, info=info1)
constraints = pd.DataFrame([
{"class_name": "class1", "constraint": "tpr", "value": 1.}])
with pytest.raises(PSDSEvalError,
match="Unknown class: class1"):
psds_eval.select_operating_points_per_class(constraints,
alpha_ct=1., beta=1.)
def test_impossible_constraint_check():
metadata = pd.read_csv(os.path.join(DATADIR, "test.metadata"), sep="\t")
det1 = pd.read_csv(os.path.join(DATADIR, "test_1.det"), sep="\t")
gt = pd.read_csv(os.path.join(DATADIR, "test_1.gt"), sep="\t")
info1 = {"name": "test_1", "threshold1": 1}
psds_eval = PSDSEval(dtc_threshold=0.5,
gtc_threshold=0.5,
cttc_threshold=0.3,
ground_truth=gt,
metadata=metadata)
psds_eval.add_operating_point(det1, info=info1)
constraints = pd.DataFrame([{
"class_name": "c2",
"constraint": "fpr",
"value": 11.
}, {
"class_name": "c1",
"constraint": "tpr",
"value": 1.1
}])
chosen_op_points = \
psds_eval.select_operating_points_per_class(constraints, alpha_ct=1.,
beta=1.)
assert np.isnan(chosen_op_points.TPR[0]), \
"NaN value is not returned for 0, 0 operating point"
assert np.isnan(chosen_op_points.TPR[1]), \
"NaN value is not returned for non-existing operating point"
def test_retrieve_desired_operating_point():
"""Check if operating points can be found with requested constraints"""
metadata = pd.read_csv(os.path.join(DATADIR, "test.metadata"), sep="\t")
det1 = pd.read_csv(os.path.join(DATADIR, "test_1.det"), sep="\t")
det2 = pd.read_csv(os.path.join(DATADIR, "test_2.det"), sep="\t")
gt = pd.read_csv(os.path.join(DATADIR, "test_1.gt"), sep="\t")
info1 = {"name": "test_1", "threshold1": 1}
info2 = {"name": "test_2", "threshold2": 0}
psds_eval = PSDSEval(dtc_threshold=0.5, gtc_threshold=0.5,
cttc_threshold=0.3, ground_truth=gt,
metadata=metadata)
psds_eval.add_operating_point(det1, info=info1)
psds_eval.add_operating_point(det2, info=info2)
constraints = pd.DataFrame([
{"class_name": "c1", "constraint": "tpr", "value": 1.},
{"class_name": "c1", "constraint": "tpr", "value": 0.8},
{"class_name": "c2", "constraint": "fpr", "value": 13.},
{"class_name": "c3", "constraint": "efpr", "value": 240.},
{"class_name": "c3", "constraint": "efpr", "value": 26.},
{"class_name": "c1", "constraint": "fscore", "value": np.nan}])
chosen_op_points = \
psds_eval.select_operating_points_per_class(constraints, alpha_ct=1.,
beta=1.)
assert chosen_op_points.name[0] == "test_1", \
"Correct operating point is not chosen for tpr criteria with equality"
assert chosen_op_points.name[1] == "test_1", \
"Correct operating point is not chosen for tpr criteria with " \
"inequality"
assert chosen_op_points.name[2] == "test_1", \
"Correct operating point is not chosen for fpr criteria with " \
"inequality"
assert chosen_op_points.name[3] == "test_1", \
"Correct operating point is not chosen for efpr criteria with " \
"equality"
assert chosen_op_points.name[4] == "test_1", \
"Correct operating point is not chosen for efpr criteria with " \
"inequality"
assert chosen_op_points.name[5] == "test_1", \
"Correct operating point is not chosen for fscore criteria"
assert chosen_op_points.Fscore[5] == pytest.approx(2./3.), \
"Correct operating point is not chosen for fscore criteria"
class_constraints = list()
# find the op. point with minimum eFPR and TPR >= 0.6 for the first class
class_constraints.append({"class_name": psds_eval.class_names[0],
"constraint": "tpr",
"value": 0.6})
# find the op. point with maximum TPR and FPR <= 50 for the second class
class_constraints.append({"class_name": psds_eval.class_names[1],
"constraint": "fpr",
"value": 50})
# find the op. point with maximum TPR and eFPR <= 50 for the third class
class_constraints.append({"class_name": psds_eval.class_names[2],
"constraint": "efpr",
"value": 50})
# find the op. point with maximum f1-score for the fourth class
class_constraints.append({"class_name": psds_eval.class_names[3],
"constraint": "fscore",
"value": None})
class_constraints_table = pd.DataFrame(class_constraints)
selected_ops = psds_eval.select_operating_points_per_class(
class_constraints_table, alpha_ct=1., beta=1.)
for k in range(len(class_constraints)):
print(f"For class {class_constraints_table.class_name[k]}, the best "
f"op. point with {class_constraints_table.constraint[k]} ~ "
f"{class_constraints_table.value[k]}:")
print(f"\tProbability Threshold: {selected_ops.threshold[k]}, "
f"TPR: {selected_ops.TPR[k]:.2f}, "
f"FPR: {selected_ops.FPR[k]:.2f}, "
f"eFPR: {selected_ops.eFPR[k]:.2f}, "
f"F1-score: {selected_ops.Fscore[k]:.2f}")
