def bac_metric(solution, prediction, task=BINARY_CLASSIFICATION):
"""
Compute the normalized balanced accuracy.
The binarization and
the normalization differ for the multi-label and multi-class case.
:param solution:
:param prediction:
:param task:
:return:
"""
label_num = solution.shape[1]
score = np.zeros(label_num)
bin_prediction = binarize_predictions(prediction, task)
[tn, fp, tp, fn] = acc_stat(solution, bin_prediction)
# Bounding to avoid division by 0
eps = 1e-15
tp = sp.maximum(eps, tp)
pos_num = sp.maximum(eps, tp + fn)
tpr = tp / pos_num # true positive rate (sensitivity)
if (task != MULTICLASS_CLASSIFICATION) or (label_num == 1):
tn = sp.maximum(eps, tn)
neg_num = sp.maximum(eps, tn + fp)
tnr = tn / neg_num # true negative rate (specificity)
bac = 0.5 * (tpr + tnr)
base_bac = 0.5 # random predictions for binary case
else:
bac = tpr
base_bac = 1. / label_num # random predictions for multiclass case
bac = mv_mean(bac) # average over all classes
# Normalize: 0 for random, 1 for perfect
score = (bac - base_bac) / sp.maximum(eps, (1 - base_bac))
return score
def f1_metric(solution, prediction, task=BINARY_CLASSIFICATION):
"""
Compute the normalized f1 measure.
The binarization differs
for the multi-label and multi-class case.
A non-weighted average over classes is taken.
The score is normalized.
:param solution:
:param prediction:
:param task:
:return:
"""
label_num = solution.shape[1]
score = np.zeros(label_num)
bin_prediction = binarize_predictions(prediction, task)
[tn, fp, tp, fn] = acc_stat(solution, bin_prediction)
# Bounding to avoid division by 0
eps = 1e-15
true_pos_num = sp.maximum(eps, tp + fn)
found_pos_num = sp.maximum(eps, tp + fp)
tp = sp.maximum(eps, tp)
tpr = tp / true_pos_num # true positive rate (recall)
ppv = tp / found_pos_num # positive predictive value (precision)
arithmetic_mean = 0.5 * sp.maximum(eps, tpr + ppv)
# Harmonic mean:
f1 = tpr * ppv / arithmetic_mean
# Average over all classes
f1 = mv_mean(f1)
# Normalize: 0 for random, 1 for perfect
if (task != MULTICLASS_CLASSIFICATION) or (label_num == 1):
# How to choose the "base_f1"?
# For the binary/multilabel classification case, one may want to predict all 1.
# In that case tpr = 1 and ppv = frac_pos. f1 = 2 * frac_pos / (1+frac_pos)
# frac_pos = mvmean(solution.ravel())
# base_f1 = 2 * frac_pos / (1+frac_pos)
# or predict random values with probability 0.5, in which case
# base_f1 = 0.5
# the first solution is better only if frac_pos > 1/3.
# The solution in which we predict according to the class prior frac_pos gives
# f1 = tpr = ppv = frac_pos, which is worse than 0.5 if frac_pos<0.5
# So, because the f1 score is used if frac_pos is small (typically <0.1)
# the best is to assume that base_f1=0.5
base_f1 = 0.5
# For the multiclass case, this is not possible (though it does not make much sense to
# use f1 for multiclass problems), so the best would be to assign values at random to get
# tpr=ppv=frac_pos, where frac_pos=1/label_num
else:
base_f1 = 1. / label_num
score = (f1 - base_f1) / sp.maximum(eps, (1 - base_f1))
return score
def log_loss(solution, prediction, task=BINARY_CLASSIFICATION):
"""Log loss for binary and multiclass."""
[sample_num, label_num] = solution.shape
eps = 1e-15
pred = np.copy(prediction
) # beware: changes in prediction occur through this
sol = np.copy(solution)
if (task == MULTICLASS_CLASSIFICATION) and (label_num > 1):
# Make sure the lines add up to one for multi-class classification
norma = np.sum(prediction, axis=1)
for k in range(sample_num):
pred[k, :] /= sp.maximum(norma[k], eps)
# Make sure there is a single label active per line for multi-class
# classification
sol = binarize_predictions(solution, task=MULTICLASS_CLASSIFICATION)
# For the base prediction, this solution is ridiculous in the
# multi-label case
# Bounding of predictions to avoid log(0),1/0,...
pred = sp.minimum(1 - eps, sp.maximum(eps, pred))
# Compute the log loss
pos_class_log_loss = -mv_mean(sol * np.log(pred), axis=0)
if (task != MULTICLASS_CLASSIFICATION) or (label_num == 1):
# The multi-label case is a bunch of binary problems.
# The second class is the negative class for each column.
neg_class_log_loss = - mv_mean((1 - sol) * np.log(1 - pred), axis=0)
log_loss = pos_class_log_loss + neg_class_log_loss
# Each column is an independent problem, so we average.
# The probabilities in one line do not add up to one.
# log_loss = mvmean(log_loss)
# print('binary {}'.format(log_loss))
# In the multilabel case, the right thing i to AVERAGE not sum
# We return all the scores so we can normalize correctly later on
else:
# For the multiclass case the probabilities in one line add up one.
log_loss = pos_class_log_loss
# We sum the contributions of the columns.
log_loss = np.sum(log_loss)
# print('multiclass {}'.format(log_loss))
return log_loss
def acc_metric(solution, prediction, task=BINARY_CLASSIFICATION):
"""
Compute the accuracy.
Get the accuracy stats
acc = (tpr + fpr) / (tn + fp + tp + fn)
Normalize, so 1 is the best and zero mean random...
:param solution:
:param prediction:
:param task:
:return:
"""
label_num = solution.shape[1]
bin_predictions = binarize_predictions(prediction, task)
tn, fp, tp, fn = acc_stat(solution, bin_predictions)
# Bounding to avoid division by 0
eps = np.float(1e-15)
tp = np.sum(tp)
fp = np.sum(fp)
tn = np.sum(tn)
fn = np.sum(fn)
if (task != MULTICLASS_CLASSIFICATION) or (label_num == 1):
accuracy = (np.sum(tp) + np.sum(tn)) / (
np.sum(tp) + np.sum(fp) + np.sum(tn) + np.sum(fn)
)
else:
accuracy = np.sum(tp) / (np.sum(tp) + np.sum(fp))
if (task != MULTICLASS_CLASSIFICATION) or (label_num == 1):
base_accuracy = 0.5 # random predictions for binary case
else:
base_accuracy = 1. / label_num
# Normalize: 0 for random, 1 for perfect
score = (accuracy - base_accuracy) / sp.maximum(eps, (1 - base_accuracy))
return score