def _get_best_thres_ary(self, pd_prob_mat, gt_mat):
thres_ary = []
for k in range(len(self.lbs)):
f_val_max = -np.inf
best_thres = None
for thres in np.arange(0., 1. + 1e-6, 0.01):
prec = eva.precision(pd_prob_mat[:, k], gt_mat[:, k], thres)
rec = eva.recall(pd_prob_mat[:, k], gt_mat[:, k], thres)
f_val = eva.f_value(prec, rec)
if f_val > f_val_max:
f_val_max = f_val
best_thres = thres
thres_ary.append(best_thres)
return thres_ary
def get_stats_from_prob_mat(self, pd_prob_mat, gt_mat, thres_ary):
"""Get stats from prob_mat and ground truth mat.
Args:
pd_prob_mat: ndarray, (n_clips, n_labels)
gt_prob_mat: ndarray, (n_clips, n_labels)
thres_ary: list of float | 'auto' | 'no_need'.
Returns:
stat.
"""
n_lbs = len(self.lbs)
#print("pd_prob_mat", pd_prob_mat, "gt_mat", gt_mat)#todo
stat = {}
if type(thres_ary) is list:
stat['thres_ary'] = thres_ary
elif thres_ary == 'auto':
thres_ary = self._get_best_thres_ary(pd_prob_mat, gt_mat)
stat['thres_ary'] = thres_ary
elif thres_ary == 'no_need':
thres_ary = [0.5] * len(self.lbs)
stat['thres_ary'] = ['no_need'] * len(self.lbs)
else:
raise Exception("thres_ary type wrong!")
pd_digit_mat = self._get_digit_mat_from_thres_ary(
pd_prob_mat, thres_ary)
# overall stat
#print("pd_prob_mat", pd_prob_mat, "pd_prob_mat.flatten()", pd_prob_mat.flatten(), "gt_mat", gt_mat, "gt_mat.flatten()", gt_mat.flatten(), "finn")#todo likely there is still an issue with .flatten and sizes. Should look into this
eer = eva.eer(pd_prob_mat.flatten(), gt_mat.flatten())
auc = eva.roc_auc(pd_prob_mat.flatten(), gt_mat.flatten())
(tp, fn, fp, tn) = eva.tp_fn_fp_tn(pd_digit_mat, gt_mat, 0.5)
prec = eva.precision(pd_digit_mat, gt_mat, 0.5)
rec = eva.recall(pd_digit_mat, gt_mat, 0.5)
f_val = eva.f_value(prec, rec)
stat['overall'] = {
'tp': tp,
'fn': fn,
'fp': fp,
'tn': tn,
'precision': prec,
'recall': rec,
'f_value': f_val,
'eer': eer,
'auc': auc
}
# element-wise stat
stat['event_wise'] = {}
for k in range(len(self.lbs)):
eer = eva.eer(pd_prob_mat[:, k], gt_mat[:, k])
auc = eva.roc_auc(pd_prob_mat[:, k], gt_mat[:, k])
(tp, fn, fp, tn) = eva.tp_fn_fp_tn(pd_digit_mat[:, k],
gt_mat[:, k], 0.5)
prec = eva.precision(pd_digit_mat[:, k], gt_mat[:, k], 0.5)
rec = eva.recall(pd_digit_mat[:, k], gt_mat[:, k], 0.5)
f_val = eva.f_value(prec, rec)
stat['event_wise'][self.lbs[k]] = {
'tp': tp,
'fn': fn,
'fp': fp,
'tn': tn,
'precision': prec,
'recall': rec,
'f_value': f_val,
'eer': eer,
'auc': auc
}
return stat
def get_stats_from_prob_mat_list(self, pd_prob_mat_list, gt_mat_list,
thres_ary):
"""Get stats from prob_mat_list and gt_mat_list.
Args:
pd_prob_mat_list: list of ndarray.
gt_mat_list: list of ndarray.
thres_ary: list of float | 'no_need'.
Returns:
stat.
"""
stat = {}
if type(thres_ary) is list:
stat['thres_ary'] = thres_ary
elif thres_ary == 'no_need':
thres_ary = [0.5] * len(self.lbs)
stat['thres_ary'] = ['no_need'] * len(self.lbs)
else:
raise Exception("Incorrect thres_ary!")
N = len(pd_prob_mat_list)
pd_digit_mat_list = []
for n in range(N):
pd_prob_mat = pd_prob_mat_list[n]
pd_digit_mat = self._get_digit_mat_from_thres_ary(
pd_prob_mat, thres_ary)
pd_digit_mat_list.append(pd_digit_mat)
dict_list = []
for n in range(N):
pd_digit_mat = pd_digit_mat_list[n]
gt_mat = gt_mat_list[n]
dict = self._get_stats_from_digit_mat(pd_digit_mat, gt_mat)
dict_list.append(dict)
# Overall stat
stat['overall'] = {}
stat['overall']['tp'] = 0
stat['overall']['fn'] = 0
stat['overall']['fp'] = 0
stat['overall']['tn'] = 0
error_rate_ary = []
for n in range(N):
stat['overall']['tp'] += dict_list[n]['overall']['tp']
stat['overall']['fn'] += dict_list[n]['overall']['fn']
stat['overall']['fp'] += dict_list[n]['overall']['fp']
stat['overall']['tn'] += dict_list[n]['overall']['tn']
error_rate_ary.append(dict_list[n]['overall']['error_rate'])
stat['overall']['precision'] = self._precision(stat['overall']['tp'],
stat['overall']['fn'],
stat['overall']['fp'],
stat['overall']['tn'])
stat['overall']['recall'] = self._recall(stat['overall']['tp'],
stat['overall']['fn'],
stat['overall']['fp'],
stat['overall']['tn'])
stat['overall']['f_value'] = eva.f_value(stat['overall']['precision'],
stat['overall']['recall'])
stat['overall']['error_rate'] = np.mean(np.array(error_rate_ary))
# Event wise stat
stat['event_wise'] = {}
for lb in self.lbs:
stat['event_wise'][lb] = {}
stat['event_wise'][lb]['tp'] = 0
stat['event_wise'][lb]['fn'] = 0
stat['event_wise'][lb]['fp'] = 0
stat['event_wise'][lb]['tn'] = 0
for lb in self.lbs:
for n in range(N):
stat['event_wise'][lb]['tp'] += dict_list[n]['event_wise'][lb][
'tp']
stat['event_wise'][lb]['fn'] += dict_list[n]['event_wise'][lb][
'fn']
stat['event_wise'][lb]['fp'] += dict_list[n]['event_wise'][lb][
'fp']
stat['event_wise'][lb]['tn'] += dict_list[n]['event_wise'][lb][
'tn']
stat['event_wise'][lb]['precision'] = self._precision(
stat['event_wise'][lb]['tp'], stat['event_wise'][lb]['fn'],
stat['event_wise'][lb]['fp'], stat['event_wise'][lb]['tn'])
stat['event_wise'][lb]['recall'] = self._recall(
stat['event_wise'][lb]['tp'], stat['event_wise'][lb]['fn'],
stat['event_wise'][lb]['fp'], stat['event_wise'][lb]['tn'])
stat['event_wise'][lb]['f_value'] = eva.f_value(
stat['event_wise'][lb]['precision'],
stat['event_wise'][lb]['recall'])
return stat