def evaluate(self,
gallery,
probe,
evaluation=None,
impostors=None,
plot=None,
verbose=1,
save=None,
use_precomputed=False,
**kw):
"""
Evaluate wrapped model
:param iterable gallery: List of gallery samples
:param iterable probe: List of probe samples
:param evaluation: Existing Evaluation to update. If None, new Evaluation will be created.
:type evaluation: Evaluation or None
:param impostors: Dataset for impostor verification attempts in evaluation. If None, full G/P testing will be used.
:type impostors: Dataset or None
:param plot: Plotting function, taking a tuple (x, y, figure). If None, will not plot.
:type plot: Callable or None
:param int verbose: Verbosity level
:param save: File to save distance matrix info to
:type save: str or None
:param bool use_precomputed: Whether to load saved distance matrix info. If True, should also pass save name.
:param kw: Keyword arguments to pass to :py:evaluation.compute_error_rates
:return: Evaluation updated with newly computed metrics
:rtype: Evaluation
"""
self._verbose = verbose
if not evaluation:
evaluation = Evaluation()
# Load dist_matrix and imp_matrix info
if use_precomputed and save and os.path.isfile(save):
self._print(f"Reading info from {save}.")
with open(save, 'rb') as f:
dist_matrix = pickle.load(f)
imp_matrix = pickle.load(f)
g_classes = pickle.load(f)
p_classes = pickle.load(f)
imp_classes = pickle.load(f)
# Compute dist_matrix and imp_matrix info
else:
dist_matrix, imp_matrix = self.dist_and_imp_matrix(
gallery, probe, impostors)
g_classes = self.classes(gallery)
p_classes = self.classes(probe)
imp_classes = self.classes(impostors)
# Save dist_matrix and imp_matrix info
if save and not (use_precomputed and os.path.isfile(save)):
self._print(f"Saving info to {save}.")
dir = os.path.dirname(save)
if dir and not os.path.isdir(dir):
os.makedirs(dir)
with open(save, 'wb') as f:
pickle.dump(dist_matrix, f)
pickle.dump(imp_matrix, f)
pickle.dump(g_classes, f)
pickle.dump(p_classes, f)
pickle.dump(imp_classes, f)
# Get FAR and FRR
far, frr, threshold = evaluation.compute_error_rates(
dist_matrix,
g_classes,
p_classes,
impostor_matrix=imp_matrix,
impostor_classes=imp_classes,
**kw)
# EER
eer = evaluation.update_eer()
self._print(f"EER: {eer}")
if plot:
plot(threshold, far, figure='EER')
plot(threshold, frr, figure='EER', label=None)
# AUC
auc = evaluation.update_auc()
self._print(f"AUC: {auc}")
if plot:
plot(far, 1 - frr, figure='ROC Curve')
plot(far, 1 - frr, figure='Semilog ROC Curve')
# VER@1FAR and [email protected]
ver1far = evaluation.update_ver1far()
self._print(f"VER@1FAR: {ver1far}")
ver01far = evaluation.update_ver01far()
self._print(f"[email protected]: {ver01far}")
return evaluation
