def f1_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the normalized f1 measure. The binarization differs for the multi-label and multi-class case. A non-weighted average over classes is taken. The score is normalized. :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution( solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('f1_metric does not support task type %s' % task) bin_prediction = binarize_predictions(prediction, task) # Bounding to avoid division by 0 eps = 1e-15 fn = np.sum(np.multiply(solution, (1 - bin_prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_prediction), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_prediction), axis=0, dtype=float) true_pos_num = sp.maximum(eps, tp + fn) found_pos_num = sp.maximum(eps, tp + fp) tp = sp.maximum(eps, tp) tpr = tp / true_pos_num # true positive rate (recall) ppv = tp / found_pos_num # positive predictive value (precision) arithmetic_mean = 0.5 * sp.maximum(eps, tpr + ppv) # Harmonic mean: f1 = tpr * ppv / arithmetic_mean # Average over all classes f1 = np.mean(f1) # Normalize: 0 for random, 1 for perfect if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): # How to choose the "base_f1"? # For the binary/multilabel classification case, one may want to predict all 1. # In that case tpr = 1 and ppv = frac_pos. f1 = 2 * frac_pos / (1+frac_pos) # frac_pos = mvmean(solution.ravel()) # base_f1 = 2 * frac_pos / (1+frac_pos) # or predict random values with probability 0.5, in which case # base_f1 = 0.5 # the first solution is better only if frac_pos > 1/3. # The solution in which we predict according to the class prior frac_pos gives # f1 = tpr = ppv = frac_pos, which is worse than 0.5 if frac_pos<0.5 # So, because the f1 score is used if frac_pos is small (typically <0.1) # the best is to assume that base_f1=0.5 base_f1 = 0.5 # For the multiclass case, this is not possible (though it does not make much sense to # use f1 for multiclass problems), so the best would be to assign values at random to get # tpr=ppv=frac_pos, where frac_pos=1/label_num elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] base_f1 = 1. / label_num score = (f1 - base_f1) / sp.maximum(eps, (1 - base_f1)) return score
def acc_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the accuracy. Get the accuracy stats acc = (tpr + fpr) / (tn + fp + tp + fn) Normalize, so 1 is the best and zero mean random... :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution( solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('acc_metric does not support task type %s' % task) bin_predictions = binarize_predictions(prediction, task) tn = np.sum(np.multiply((1 - solution), (1 - bin_predictions)), axis=0, dtype=float) fn = np.sum(np.multiply(solution, (1 - bin_predictions)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_predictions), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_predictions), axis=0, dtype=float) # Bounding to avoid division by 0, 1e-7 because of float32 eps = np.float(1e-7) """ tp = np.sum(tp) fp = np.sum(fp) tn = np.sum(tn) fn = np.sum(fn) """ if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): accuracy = (np.sum(tp) + np.sum(tn)) / (np.sum(tp) + np.sum(fp) + np.sum(tn) + np.sum(fn)) elif task == MULTICLASS_CLASSIFICATION: accuracy = np.sum(tp) / (np.sum(tp) + np.sum(fp)) if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): base_accuracy = 0.5 # random predictions for binary case elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] base_accuracy = 1. / label_num # Normalize: 0 for random, 1 for perfect score = (accuracy - base_accuracy) / sp.maximum(eps, (1 - base_accuracy)) return score
def bac_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the normalized balanced accuracy. The binarization and the normalization differ for the multi-label and multi-class case. :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution( solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('bac_metric does not support task type %s' % task) bin_prediction = binarize_predictions(prediction, task) fn = np.sum(np.multiply(solution, (1 - bin_prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_prediction), axis=0, dtype=float) # Bounding to avoid division by 0 eps = 1e-15 tp = sp.maximum(eps, tp) pos_num = sp.maximum(eps, tp + fn) tpr = tp / pos_num # true positive rate (sensitivity) if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): tn = np.sum(np.multiply((1 - solution), (1 - bin_prediction)), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_prediction), axis=0, dtype=float) tn = sp.maximum(eps, tn) neg_num = sp.maximum(eps, tn + fp) tnr = tn / neg_num # true negative rate (specificity) bac = 0.5 * (tpr + tnr) base_bac = 0.5 # random predictions for binary case elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] bac = tpr base_bac = 1. / label_num # random predictions for multiclass case bac = np.mean(bac) # average over all classes # Normalize: 0 for random, 1 for perfect score = (bac - base_bac) / sp.maximum(eps, (1 - base_bac)) return score
def f1_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the normalized f1 measure. The binarization differs for the multi-label and multi-class case. A non-weighted average over classes is taken. The score is normalized. :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = solution.reshape((-1, 1)) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution(solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('f1_metric does not support task type %s' % task) bin_prediction = binarize_predictions(prediction, task) # Bounding to avoid division by 0 eps = 1e-15 fn = np.sum(np.multiply(solution, (1 - bin_prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_prediction), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_prediction), axis=0, dtype=float) true_pos_num = sp.maximum(eps, tp + fn) found_pos_num = sp.maximum(eps, tp + fp) tp = sp.maximum(eps, tp) tpr = tp / true_pos_num # true positive rate (recall) ppv = tp / found_pos_num # positive predictive value (precision) arithmetic_mean = 0.5 * sp.maximum(eps, tpr + ppv) # Harmonic mean: f1 = tpr * ppv / arithmetic_mean # Average over all classes f1 = np.mean(f1) # Normalize: 0 for random, 1 for perfect if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): # How to choose the "base_f1"? # For the binary/multilabel classification case, one may want to predict all 1. # In that case tpr = 1 and ppv = frac_pos. f1 = 2 * frac_pos / (1+frac_pos) # frac_pos = mvmean(solution.ravel()) # base_f1 = 2 * frac_pos / (1+frac_pos) # or predict random values with probability 0.5, in which case # base_f1 = 0.5 # the first solution is better only if frac_pos > 1/3. # The solution in which we predict according to the class prior frac_pos gives # f1 = tpr = ppv = frac_pos, which is worse than 0.5 if frac_pos<0.5 # So, because the f1 score is used if frac_pos is small (typically <0.1) # the best is to assume that base_f1=0.5 base_f1 = 0.5 # For the multiclass case, this is not possible (though it does not make much sense to # use f1 for multiclass problems), so the best would be to assign values at random to get # tpr=ppv=frac_pos, where frac_pos=1/label_num elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] base_f1 = 1. / label_num score = (f1 - base_f1) / sp.maximum(eps, (1 - base_f1)) return score
def acc_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the accuracy. Get the accuracy stats acc = (tpr + fpr) / (tn + fp + tp + fn) Normalize, so 1 is the best and zero mean random... :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = solution.reshape((-1, 1)) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape ) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution(solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('acc_metric does not support task type %s' % task) bin_predictions = binarize_predictions(prediction, task) tn = np.sum(np.multiply((1 - solution), (1 - bin_predictions)), axis=0, dtype=float) fn = np.sum(np.multiply(solution, (1 - bin_predictions)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_predictions), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_predictions), axis=0, dtype=float) # Bounding to avoid division by 0, 1e-7 because of float32 eps = np.float(1e-7) tp = np.sum(tp) fp = np.sum(fp) tn = np.sum(tn) fn = np.sum(fn) if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): accuracy = (np.sum(tp) + np.sum(tn)) / ( np.sum(tp) + np.sum(fp) + np.sum(tn) + np.sum(fn) ) elif task == MULTICLASS_CLASSIFICATION: accuracy = np.sum(tp) / (np.sum(tp) + np.sum(fp)) if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): base_accuracy = 0.5 # random predictions for binary case elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] base_accuracy = 1. / label_num # Normalize: 0 for random, 1 for perfect score = (accuracy - base_accuracy) / sp.maximum(eps, (1 - base_accuracy)) return score
def bac_metric(solution, prediction, task=BINARY_CLASSIFICATION): """ Compute the normalized balanced accuracy. The binarization and the normalization differ for the multi-label and multi-class case. :param solution: :param prediction: :param task: :return: """ if task == BINARY_CLASSIFICATION: if len(solution.shape) == 1: # Solution won't be touched - no copy solution = solution.reshape((-1, 1)) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = solution.reshape((-1, 1)) else: raise ValueError('Solution.shape %s' % solution.shape) if len(prediction.shape) == 2: if prediction.shape[1] > 2: raise ValueError('A prediction array with probability values ' 'for %d classes is not a binary ' 'classification problem' % prediction.shape[1]) # Prediction will be copied into a new binary array - no copy prediction = prediction[:, 1].reshape((-1, 1)) else: raise ValueError('Invalid prediction shape %s' % prediction.shape) elif task == MULTICLASS_CLASSIFICATION: if len(solution.shape) == 1: solution = create_multiclass_solution(solution, prediction) elif len(solution.shape) == 2: if solution.shape[1] > 1: raise ValueError('Solution array must only contain one class ' 'label, but contains %d' % solution.shape[1]) else: solution = create_multiclass_solution(solution.reshape((-1, 1)), prediction) else: raise ValueError('Solution.shape %s' % solution.shape) elif task == MULTILABEL_CLASSIFICATION: pass else: raise NotImplementedError('bac_metric does not support task type %s' % task) bin_prediction = binarize_predictions(prediction, task) fn = np.sum(np.multiply(solution, (1 - bin_prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, bin_prediction), axis=0, dtype=float) # Bounding to avoid division by 0 eps = 1e-15 tp = sp.maximum(eps, tp) pos_num = sp.maximum(eps, tp + fn) tpr = tp / pos_num # true positive rate (sensitivity) if task in (BINARY_CLASSIFICATION, MULTILABEL_CLASSIFICATION): tn = np.sum(np.multiply((1 - solution), (1 - bin_prediction)), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), bin_prediction), axis=0, dtype=float) tn = sp.maximum(eps, tn) neg_num = sp.maximum(eps, tn + fp) tnr = tn / neg_num # true negative rate (specificity) bac = 0.5 * (tpr + tnr) base_bac = 0.5 # random predictions for binary case elif task == MULTICLASS_CLASSIFICATION: label_num = solution.shape[1] bac = tpr base_bac = 1. / label_num # random predictions for multiclass case bac = np.mean(bac) # average over all classes # Normalize: 0 for random, 1 for perfect score = (bac - base_bac) / sp.maximum(eps, (1 - base_bac)) return score