def matthews_corrcoef(y_true, y_pred, sample_weight=None): from sklearn.metrics.classification import ( _check_targets, LabelEncoder, confusion_matrix, ) y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type not in {'binary', 'multiclass'}: raise ValueError('%s is not supported' % y_type) lb = LabelEncoder() lb.fit(np.hstack([y_true, y_pred])) y_true = lb.transform(y_true) y_pred = lb.transform(y_pred) C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight) t_sum = C.sum(axis=1) p_sum = C.sum(axis=0) n_correct = np.trace(C) n_samples = p_sum.sum() cov_ytyp = n_correct * n_samples - np.dot(t_sum, p_sum) cov_ypyp = n_samples**2 - np.dot(p_sum, p_sum) cov_ytyt = n_samples**2 - np.dot(t_sum, t_sum) mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp) if np.isnan(mcc): return 0.0 else: return mcc
def balanced_accuracy_score(y_true, y_pred, sample_weight=None): """Compute the balanced accuracy The balanced accuracy is used in binary classification problems to deal with imbalanced datasets. It is defined as the arithmetic mean of sensitivity (true positive rate) and specificity (true negative rate), or the average recall obtained on either class. It is also equal to the ROC AUC score given binary inputs. The best value is 1 and the worst value is 0. Read more in the :ref:`User Guide <balanced_accuracy_score>`. Parameters ---------- y_true : 1d array-like Ground truth (correct) target values. y_pred : 1d array-like Estimated targets as returned by a classifier. sample_weight : array-like of shape = [n_samples], optional Sample weights. Returns ------- balanced_accuracy : float. The average of sensitivity and specificity y_type, y_true, y_pred = _check_targets(y_true, y_pred) """ y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type != 'binary': raise ValueError('Balanced accuracy is only meaningful ' 'for binary classification problems.') # simply wrap the ``recall_score`` function return recall_score(y_true, y_pred, pos_label=None, average='macro', sample_weight=sample_weight)
def score(self, X, y): """ Generates a 2D array where each row is the count of the predicted classes and each column is the true class Parameters ---------- X : ndarray or DataFrame of shape n x m A matrix of n instances with m features y : ndarray or Series of length n An array or series of target or class values Returns ------- score_ : float Global accuracy score """ # Must be computed before calling super # We're relying on predict to raise NotFitted y_pred = self.predict(X) y_type, y_true, y_pred = _check_targets(y, y_pred) if y_type not in ("binary", "multiclass"): raise YellowbrickValueError("{} is not supported".format(y_type)) # Get the indices of the unique labels indices = unique_labels(y_true, y_pred) labels = self._labels() # Call super to compute self.score_ and verify classes try: super(ClassPredictionError, self).score(X, y) except ModelError as e: # raise visualizer-specific errors if labels is not None and len(labels) < len(indices): raise NotImplementedError( "filtering classes is currently not supported" ) else: raise e # Ensure all labels are used if labels is not None and len(labels) > len(indices): raise ModelError( "y and y_pred contain zero values for one of the specified classes" ) # Create a table of predictions whose rows are the true classes # and whose columns are the predicted classes; each element # is the count of predictions for that class that match the true # value of that class. self.predictions_ = np.array( [ [(y_pred[y == label_t] == label_p).sum() for label_p in indices] for label_t in indices ] ) self.draw() return self.score_
def _balanced_accuracy(solution, prediction): y_type, solution, prediction = _check_targets(solution, prediction) if y_type not in ["binary", "multiclass", 'multilabel-indicator']: raise ValueError("{0} is not supported".format(y_type)) if y_type == 'binary': # Do not transform into any multiclass representation max_value = max(np.max(solution), np.max(prediction)) min_value = min(np.min(solution), np.min(prediction)) if max_value == min_value: return 1.0 solution = (solution - min_value) / (max_value - min_value) prediction = (prediction - min_value) / (max_value - min_value) elif y_type == 'multiclass': # Need to create a multiclass solution and a multiclass predictions max_class = int(np.max((np.max(solution), np.max(prediction)))) solution_binary = np.zeros((len(solution), max_class + 1)) prediction_binary = np.zeros((len(prediction), max_class + 1)) for i in range(len(solution)): solution_binary[i, int(solution[i])] = 1 prediction_binary[i, int(prediction[i])] = 1 solution = solution_binary prediction = prediction_binary elif y_type == 'multilabel-indicator': solution = solution.toarray() prediction = prediction.toarray() else: raise NotImplementedError('bac_metric does not support task type %s' % y_type) fn = np.sum(np.multiply(solution, (1 - prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, prediction), axis=0, dtype=float) # Bounding to avoid division by 0 eps = 1e-15 tp = sp.maximum(eps, tp) pos_num = sp.maximum(eps, tp + fn) tpr = tp / pos_num # true positive rate (sensitivity) if y_type in ('binary', 'multilabel-indicator'): tn = np.sum(np.multiply((1 - solution), (1 - prediction)), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), prediction), axis=0, dtype=float) tn = sp.maximum(eps, tn) neg_num = sp.maximum(eps, tn + fp) tnr = tn / neg_num # true negative rate (specificity) bac = 0.5 * (tpr + tnr) elif y_type == 'multiclass': label_num = solution.shape[1] bac = tpr else: raise ValueError(y_type) return np.mean(bac) # average over all classes
def balanced_accuracy_score(y_true, y_pred, balance=0.5): """Balanced accuracy classification score. The formula for the balanced accuracy score :: balanced accuracy = balance * TP/(TP + FP) + (1 - balance) * TN/(TN + FN) Because it needs true/false negative/positive notion it only supports binary classification. The `balance` parameter determines the weight of sensitivity in the combined score. ``balance -> 1`` lends more weight to sensitiviy, while ``balance -> 0`` favors specificity (``balance = 1`` considers only sensitivity, ``balance = 0`` only specificity). Read more in the :ref:`User Guide <balanced_accuracy_score>`. Parameters ---------- y_true : 1d array-like, or label indicator array / sparse matrix Ground truth (correct) labels. y_pred : 1d array-like, or label indicator array / sparse matrix Predicted labels, as returned by a classifier. balance : float between 0 and 1. Weight associated with the sensitivity (or recall) against specificty in final score. Returns ------- score : float See also -------- accuracy_score References ---------- .. [1] `Wikipedia entry for the accuracy and precision <http://en.wikipedia.org/wiki/Accuracy_and_precision>` Examples -------- >>> import numpy as np >>> from sklearn.metrics import balanced_accuracy_score >>> y_pred = [0, 0, 1] >>> y_true = [0, 1, 1] >>> balanced_accuracy_score(y_true, y_pred) 0.75 >>> y_pred = ["cat", "cat", "ant"] >>> y_true = ["cat", "ant", "ant"] >>> balanced_accuracy_score(y_true, y_pred) 0.75 """ if balance < 0. or 1. < balance: raise ValueError("balance has to be between 0 and 1") y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type is not "binary": raise ValueError("%s is not supported" % y_type) cm = confusion_matrix(y_true, y_pred) neg, pos = cm.sum(axis=1, dtype='float') tn, tp = np.diag(cm) sensitivity = tp / pos specificity = tn / neg return balance * sensitivity + (1 - balance) * specificity
def balanced_accuracy(solution, prediction): y_type, solution, prediction = _check_targets(solution, prediction) if y_type not in ["binary", "multiclass", 'multilabel-indicator']: raise ValueError(f"{y_type} is not supported") if y_type == 'binary': # Do not transform into any multiclass representation pass elif y_type == 'multiclass': n = len(solution) unique_sol, encoded_sol = np.unique(solution, return_inverse=True) unique_pred, encoded_pred = np.unique(prediction, return_inverse=True) classes = np.unique(np.concatenate((unique_sol, unique_pred))) map_sol = np.array([np.where(classes == c)[0][0] for c in unique_sol]) map_pred = np.array( [np.where(classes == c)[0][0] for c in unique_pred]) # one hot encoding sol_ohe = np.zeros((n, len(classes))) pred_ohe = np.zeros((n, len(classes))) sol_ohe[np.arange(n), map_sol[encoded_sol]] = 1 pred_ohe[np.arange(n), map_pred[encoded_pred]] = 1 solution = sol_ohe prediction = pred_ohe elif y_type == 'multilabel-indicator': solution = solution.toarray() prediction = prediction.toarray() else: raise NotImplementedError( f'bac_metric does not support task type {y_type}') fn = np.sum(np.multiply(solution, (1 - prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, prediction), axis=0, dtype=float) # Bounding to avoid division by 0 eps = 1e-15 tp = np.maximum(eps, tp) pos_num = np.maximum(eps, tp + fn) tpr = tp / pos_num # true positive rate (sensitivity) if y_type in ('binary', 'multilabel-indicator'): tn = np.sum(np.multiply((1 - solution), (1 - prediction)), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), prediction), axis=0, dtype=float) tn = np.maximum(eps, tn) neg_num = np.maximum(eps, tn + fp) tnr = tn / neg_num # true negative rate (specificity) bac = 0.5 * (tpr + tnr) elif y_type == 'multiclass': bac = tpr else: raise ValueError(y_type) return np.mean(bac) # average over all classes
def _assert_binary(y1, y2=None): if y2 is None: y2 = y1 y_type, _, _ = _check_targets(y1, y2) if y_type != 'binary': raise ValueError('y_true and y_pred must be binary.')
def balanced_accuracy(solution, prediction): y_type, solution, prediction = _check_targets(solution, prediction) if y_type not in ["binary", "multiclass", 'multilabel-indicator']: raise ValueError("{0} is not supported".format(y_type)) if y_type == 'binary': # Do not transform into any multiclass representation pass elif y_type == 'multiclass': # Need to create a multiclass solution and a multiclass predictions max_class = int(np.max((np.max(solution), np.max(prediction)))) solution_binary = np.zeros((len(solution), max_class + 1)) prediction_binary = np.zeros((len(prediction), max_class + 1)) for i in range(len(solution)): solution_binary[i, int(solution[i])] = 1 prediction_binary[i, int(prediction[i])] = 1 solution = solution_binary prediction = prediction_binary elif y_type == 'multilabel-indicator': solution = solution.toarray() prediction = prediction.toarray() else: raise NotImplementedError('bac_metric does not support task type %s' % y_type) fn = np.sum(np.multiply(solution, (1 - prediction)), axis=0, dtype=float) tp = np.sum(np.multiply(solution, prediction), axis=0, dtype=float) # Bounding to avoid division by 0 eps = 1e-15 tp = sp.maximum(eps, tp) pos_num = sp.maximum(eps, tp + fn) tpr = tp / pos_num # true positive rate (sensitivity) if y_type in ('binary', 'multilabel-indicator'): tn = np.sum(np.multiply((1 - solution), (1 - prediction)), axis=0, dtype=float) fp = np.sum(np.multiply((1 - solution), prediction), axis=0, dtype=float) tn = sp.maximum(eps, tn) neg_num = sp.maximum(eps, tn + fp) tnr = tn / neg_num # true negative rate (specificity) bac = 0.5 * (tpr + tnr) elif y_type == 'multiclass': label_num = solution.shape[1] bac = tpr else: raise ValueError(y_type) return np.mean(bac) # average over all classes
def score(self, X, y, **kwargs): """ Generates a 2D array where each row is the count of the predicted classes and each column is the true class Parameters ---------- X : ndarray or DataFrame of shape n x m A matrix of n instances with m features y : ndarray or Series of length n An array or series of target or class values Returns ------- score_ : float Global accuracy score """ # We're relying on predict to raise NotFitted y_pred = self.predict(X) y_type, y_true, y_pred = _check_targets(y, y_pred) if y_type not in ("binary", "multiclass"): raise YellowbrickValueError("%s is not supported" % y_type) indices = unique_labels(y_true, y_pred) if len(self.classes_) > len(indices): raise ModelError("y and y_pred contain zero values " "for one of the specified classes") elif len(self.classes_) < len(indices): raise NotImplementedError("filtering classes is " "currently not supported") # Create a table of predictions whose rows are the true classes # and whose columns are the predicted classes; each element # is the count of predictions for that class that match the true # value of that class. self.predictions_ = np.array([ [ (y_pred[y == label_t] == label_p).sum() for label_p in indices ] for label_t in indices ]) self.draw() self.score_ = self.estimator.score(X, y) return self.score_
def confusion_matrix_2017_new(y_true, y_pred, labels=None, sample_weight=None): y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type not in ("binary", "multiclass"): raise ValueError("%s is not supported" % y_type) if labels is None: labels = unique_labels(y_true, y_pred) else: labels = np.asarray(labels) if np.all([l not in y_true for l in labels]): raise ValueError("At least one label specified must be in y_true") if sample_weight is None: sample_weight = np.ones(y_true.shape[0], dtype=np.int64) else: sample_weight = np.asarray(sample_weight) check_consistent_length(sample_weight, y_true, y_pred) n_labels = labels.size # If labels are not consecitive integers starting from zero, then # yt, yp must be converted into index form need_index_conversion = not ( labels.dtype.kind in {'i', 'u', 'b'} and labels.min() == 0 and np.all(np.diff(labels) == 1) and y_true.min() >= 0 and y_pred.min() >= 0 ) if need_index_conversion: label_to_ind = dict((y, x) for x, y in enumerate(labels)) y_pred = np.array([label_to_ind.get(x, n_labels + 1) for x in y_pred]) y_true = np.array([label_to_ind.get(x, n_labels + 1) for x in y_true]) # eliminate items in y_true, y_pred not in labels isvalid = np.logical_and(y_pred < n_labels, y_true < n_labels) if not np.all(isvalid): y_pred = y_pred[isvalid] y_true = y_true[isvalid] # also eliminate weights of eliminated items sample_weight = sample_weight[isvalid] # Choose the accumulator dtype to always have high precision if sample_weight.dtype.kind in {'i', 'u', 'b'}: dtype = np.int64 else: dtype = np.float64 CM = coo_matrix((sample_weight, (y_true, y_pred)), shape=(n_labels, n_labels), dtype=dtype, ).toarray() return CM
def balanced_accuracy_score(y_true, y_pred, sample_weight=None): """Compute the balanced accuracy The balanced accuracy is used in binary classification problems to deal with imbalanced datasets. It is defined as the arithmetic mean of sensitivity (true positive rate) and specificity (true negative rate), or the average recall obtained on either class. It is also equal to the ROC AUC score given binary inputs. The best value is 1 and the worst value is 0. Read more in the :ref:`User Guide <balanced_accuracy_score>`. Parameters ---------- y_true : 1d array-like Ground truth (correct) target values. y_pred : 1d array-like Estimated targets as returned by a classifier. sample_weight : array-like of shape = [n_samples], optional Sample weights. Returns ------- balanced_accuracy : float. The average of sensitivity and specificity See also -------- recall_score, roc_auc_score References ---------- .. [1] Brodersen, K.H.; Ong, C.S.; Stephan, K.E.; Buhmann, J.M. (2010). The balanced accuracy and its posterior distribution. Proceedings of the 20th International Conference on Pattern Recognition, 3121-24. Examples -------- >>> from sklearn.metrics import balanced_accuracy_score >>> y_true = [0, 1, 0, 0, 1, 0] >>> y_pred = [0, 1, 0, 0, 0, 1] >>> balanced_accuracy_score(y_true, y_pred) 0.625 """ y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type != 'binary': raise ValueError('Balanced accuracy is only meaningful ' 'for binary classification problems.') # simply wrap the ``recall_score`` function return recall_score(y_true, y_pred, pos_label=None, average='macro', sample_weight=sample_weight)
def score(self, X, y, **kwargs): """ Generates a 2D array where each row is the count of the predicted classes and each column is the true class Parameters ---------- X : ndarray or DataFrame of shape n x m A matrix of n instances with m features y : ndarray or Series of length n An array or series of target or class values Returns ------- score_ : float Global accuracy score """ # We're relying on predict to raise NotFitted y_pred = self.predict(X) y_type, y_true, y_pred = _check_targets(y, y_pred) if y_type not in ("binary", "multiclass"): raise YellowbrickValueError("%s is not supported" % y_type) indices = unique_labels(y_true, y_pred) if len(self.classes_) > len(indices): raise ModelError("y and y_pred contain zero values " "for one of the specified classes") elif len(self.classes_) < len(indices): raise NotImplementedError("filtering classes is " "currently not supported") # Create a table of predictions whose rows are the true classes # and whose columns are the predicted classes; each element # is the count of predictions for that class that match the true # value of that class. self.predictions_ = np.array([[(y_pred[y == label_t] == label_p).sum() for label_p in indices] for label_t in indices]) self.draw() self.score_ = self.estimator.score(X, y) return self.score_
def accuracy_rounding_score(y_true, y_pred, normalize=True, sample_weight=None): for p in range(0, y_pred.size): labels = [ 500., 1000., 1500., 2000., 2500., 3000., 3500., 4000., 4500., 5000., 6000., 7000., 8000., 9000., 10000., 12500., 15000. ] y_pred[p] = min(labels, key=lambda x: abs(x - y_pred[p])) # Compute accuracy for each possible representation y_type, y_true, y_pred = _check_targets(y_true, y_pred) check_consistent_length(y_true, y_pred, sample_weight) if y_type.startswith('multilabel'): differing_labels = count_nonzero(y_true - y_pred, axis=1) score = differing_labels == 0 else: score = y_true == y_pred return _weighted_sum(score, sample_weight, normalize)
def degree_of_agreement(y_true, y_pred, normalize=True, sample_weight=None): # Compute accuracy for each possible representation y_type, y_true, y_pred = _check_targets(y_true, y_pred) check_consistent_length(y_true, y_pred, sample_weight) if y_type.startswith('multilabel'): with np.errstate(divide='ignore', invalid='ignore'): # oddly, we may get an "invalid" rather than a "divide" error here pred_and_true = count_nonzero( y_true.multiply(y_pred), axis=1) # cez inters eez len_true = count_nonzero(y_true) not_ytrue = 1 - y_true pred_and_nottrue = count_nonzero(not_ytrue.multiply(y_pred), axis=1) len_nottrue = count_nonzero(not_ytrue) pred_or_true = count_nonzero(y_true + y_pred, axis=1) # compute the doa statistic score = pred_and_true / len_true - pred_and_nottrue / len_nottrue # score = pred_and_true / pred_or_true print(score) score[pred_or_true == 0.0] = 1.0 print(score) else: # oddly, we may get an "invalid" rather than a "divide" error here pred_and_true = np.count_nonzero( y_true * y_pred, axis=0) # cez inters eez len_true = np.count_nonzero(y_true) not_ytrue = np.subtract(1, y_true) pred_and_nottrue = np.count_nonzero(not_ytrue * y_pred, axis=0) len_nottrue = np.count_nonzero(not_ytrue) # compute the doa statistic score = pred_and_true / len_true - pred_and_nottrue / len_nottrue return _weighted_sum(score, sample_weight, normalize)
def test__check_targets(): """Check that _check_targets correctly merges target types, squeezes output and fails if input lengths differ.""" IND = 'multilabel-indicator' SEQ = 'multilabel-sequences' MC = 'multiclass' BIN = 'binary' CNT = 'continuous' MMC = 'multiclass-multioutput' MCN = 'continuous-multioutput' # all of length 3 EXAMPLES = [ (IND, np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])), # must not be considered binary (IND, np.array([[0, 1], [1, 0], [1, 1]])), (SEQ, [[2, 3], [1], [3]]), (MC, [2, 3, 1]), (BIN, [0, 1, 1]), (CNT, [0., 1.5, 1.]), (MC, np.array([[2], [3], [1]])), (BIN, np.array([[0], [1], [1]])), (CNT, np.array([[0.], [1.5], [1.]])), (MMC, np.array([[0, 2], [1, 3], [2, 3]])), (MCN, np.array([[0.5, 2.], [1.1, 3.], [2., 3.]])), ] # expected type given input types, or None for error # (types will be tried in either order) EXPECTED = { (IND, IND): IND, (SEQ, SEQ): IND, (MC, MC): MC, (BIN, BIN): BIN, (IND, SEQ): None, (MC, SEQ): None, (BIN, SEQ): None, (MC, IND): None, (BIN, IND): None, (BIN, MC): MC, # Disallowed types (CNT, CNT): None, (MMC, MMC): None, (MCN, MCN): None, (IND, CNT): None, (SEQ, CNT): None, (MC, CNT): None, (BIN, CNT): None, (MMC, CNT): None, (MCN, CNT): None, (IND, MMC): None, (SEQ, MMC): None, (MC, MMC): None, (BIN, MMC): None, (MCN, MMC): None, (IND, MCN): None, (SEQ, MCN): None, (MC, MCN): None, (BIN, MCN): None, } for (type1, y1), (type2, y2) in product(EXAMPLES, repeat=2): try: expected = EXPECTED[type1, type2] except KeyError: expected = EXPECTED[type2, type1] if expected is None: assert_raises(ValueError, _check_targets, y1, y2) if type1 != type2: assert_raise_message( ValueError, "Can't handle mix of {0} and {1}".format(type1, type2), _check_targets, y1, y2) else: if type1 not in (BIN, MC, SEQ, IND): assert_raise_message(ValueError, "{0} is not supported".format(type1), _check_targets, y1, y2) else: merged_type, y1out, y2out = _check_targets(y1, y2) assert_equal(merged_type, expected) if merged_type.startswith('multilabel'): assert_equal(y1out.format, 'csr') assert_equal(y2out.format, 'csr') else: assert_array_equal(y1out, np.squeeze(y1)) assert_array_equal(y2out, np.squeeze(y2)) assert_raises(ValueError, _check_targets, y1[:-1], y2)
def sensitivity_specificity_support(y_true, y_pred, labels=None, pos_label=1, average=None, warn_for=('sensitivity', 'specificity'), sample_weight=None): """Compute sensitivity, specificity, and support for each class The sensitivity is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of true positives and ``fn`` the number of false negatives. The sensitivity quantifies the ability to avoid false negatives_[1]. The specificity is the ratio ``tn / (tn + fp)`` where ``tn`` is the number of true negatives and ``fn`` the number of false negatives. The specificity quantifies the ability to avoid false positives_[1]. The support is the number of occurrences of each class in ``y_true``. If ``pos_label is None`` and in binary classification, this function returns the average sensitivity and specificity if ``average`` is one of ``'weighted'``. Read more in the :ref:`User Guide <sensitivity_specificity>`. Parameters ---------- y_true : ndarray, shape (n_samples, ) Ground truth (correct) target values. y_pred : ndarray, shape (n_samples, ) Estimated targets as returned by a classifier. labels : list, optional The set of labels to include when ``average != 'binary'``, and their order if ``average is None``. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in ``y_true`` and ``y_pred`` are used in sorted order. pos_label : str or int, optional (default=1) The class to report if ``average='binary'`` and the data is binary. If the data are multiclass, this will be ignored; setting ``labels=[pos_label]`` and ``average != 'binary'`` will report scores for that label only. average : str or None, optional (default=None) If ``None``, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data: ``'binary'``: Only report results for the class specified by ``pos_label``. This is applicable only if targets (``y_{true,pred}``) are binary. ``'micro'``: Calculate metrics globally by counting the total true positives, false negatives and false positives. ``'macro'``: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. ``'weighted'``: Calculate metrics for each label, and find their average, weighted by support (the number of true instances for each label). This alters 'macro' to account for label imbalance; it can result in an F-score that is not between precision and recall. ``'samples'``: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification where this differs from :func:`accuracy_score`). warn_for : tuple or set, for internal use This determines which warnings will be made in the case that this function is being used to return only one of its metrics. sample_weight : ndarray, shape (n_samples, ) Sample weights. Returns ------- sensitivity : float (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) specificity : float (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) support : int (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) The number of occurrences of each label in ``y_true``. References ---------- .. [1] `Wikipedia entry for the Sensitivity and specificity <https://en.wikipedia.org/wiki/Sensitivity_and_specificity>`_ Examples -------- >>> import numpy as np >>> from imblearn.metrics import sensitivity_specificity_support >>> y_true = np.array(['cat', 'dog', 'pig', 'cat', 'dog', 'pig']) >>> y_pred = np.array(['cat', 'pig', 'dog', 'cat', 'cat', 'dog']) >>> sensitivity_specificity_support(y_true, y_pred, average='macro') (0.33333333333333331, 0.66666666666666663, None) >>> sensitivity_specificity_support(y_true, y_pred, average='micro') (0.33333333333333331, 0.66666666666666663, None) >>> sensitivity_specificity_support(y_true, y_pred, average='weighted') (0.33333333333333331, 0.66666666666666663, None) """ average_options = (None, 'micro', 'macro', 'weighted', 'samples') if average not in average_options and average != 'binary': raise ValueError('average has to be one of ' + str(average_options)) y_type, y_true, y_pred = _check_targets(y_true, y_pred) present_labels = unique_labels(y_true, y_pred) if average == 'binary': if y_type == 'binary': if pos_label not in present_labels: if len(present_labels) < 2: # Only negative labels return (0., 0., 0) else: raise ValueError("pos_label=%r is not a valid label: %r" % (pos_label, present_labels)) labels = [pos_label] else: raise ValueError("Target is %s but average='binary'. Please " "choose another average setting." % y_type) elif pos_label not in (None, 1): warnings.warn( "Note that pos_label (set to %r) is ignored when " "average != 'binary' (got %r). You may use " "labels=[pos_label] to specify a single positive class." % (pos_label, average), UserWarning) if labels is None: labels = present_labels n_labels = None else: n_labels = len(labels) labels = np.hstack( [labels, np.setdiff1d(present_labels, labels, assume_unique=True)]) # Calculate tp_sum, pred_sum, true_sum ### if y_type.startswith('multilabel'): raise ValueError('imblearn does not support multilabel') elif average == 'samples': raise ValueError("Sample-based precision, recall, fscore is " "not meaningful outside multilabel " "classification. See the accuracy_score instead.") else: le = LabelEncoder() le.fit(labels) y_true = le.transform(y_true) y_pred = le.transform(y_pred) sorted_labels = le.classes_ # labels are now from 0 to len(labels) - 1 -> use bincount tp = y_true == y_pred tp_bins = y_true[tp] if sample_weight is not None: tp_bins_weights = np.asarray(sample_weight)[tp] else: tp_bins_weights = None if len(tp_bins): tp_sum = np.bincount( tp_bins, weights=tp_bins_weights, minlength=len(labels)) else: # Pathological case true_sum = pred_sum = tp_sum = np.zeros(len(labels)) if len(y_pred): pred_sum = np.bincount( y_pred, weights=sample_weight, minlength=len(labels)) if len(y_true): true_sum = np.bincount( y_true, weights=sample_weight, minlength=len(labels)) # Compute the true negative tn_sum = y_true.size - (pred_sum + true_sum - tp_sum) # Retain only selected labels indices = np.searchsorted(sorted_labels, labels[:n_labels]) tp_sum = tp_sum[indices] true_sum = true_sum[indices] pred_sum = pred_sum[indices] tn_sum = tn_sum[indices] if average == 'micro': tp_sum = np.array([tp_sum.sum()]) pred_sum = np.array([pred_sum.sum()]) true_sum = np.array([true_sum.sum()]) tn_sum = np.array([tn_sum.sum()]) # Finally, we have all our sufficient statistics. Divide! # with np.errstate(divide='ignore', invalid='ignore'): # Divide, and on zero-division, set scores to 0 and warn: # Oddly, we may get an "invalid" rather than a "divide" error # here. specificity = _prf_divide(tn_sum, tn_sum + pred_sum - tp_sum, 'specificity', 'predicted', average, warn_for) sensitivity = _prf_divide(tp_sum, true_sum, 'sensitivity', 'true', average, warn_for) # Average the results if average == 'weighted': weights = true_sum if weights.sum() == 0: return 0, 0, None elif average == 'samples': weights = sample_weight else: weights = None if average is not None: assert average != 'binary' or len(specificity) == 1 specificity = np.average(specificity, weights=weights) sensitivity = np.average(sensitivity, weights=weights) true_sum = None # return no support return sensitivity, specificity, true_sum
def sensitivity_specificity_support(y_true, y_pred, labels=None, pos_label=1, average=None, warn_for=('sensitivity', 'specificity'), sample_weight=None): """Compute sensitivity, specificity, and support for each class The sensitivity is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of true positives and ``fn`` the number of false negatives. The sensitivity quantifies the ability to avoid false negatives_[1]. The specificity is the ratio ``tn / (tn + fp)`` where ``tn`` is the number of true negatives and ``fn`` the number of false negatives. The specificity quantifies the ability to avoid false positives_[1]. The support is the number of occurrences of each class in ``y_true``. If ``pos_label is None`` and in binary classification, this function returns the average sensitivity and specificity if ``average`` is one of ``'weighted'``. Parameters ---------- y_true : ndarray, shape (n_samples, ) Ground truth (correct) target values. y_pred : ndarray, shape (n_samples, ) Estimated targets as returned by a classifier. labels : list, optional The set of labels to include when ``average != 'binary'``, and their order if ``average is None``. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in ``y_true`` and ``y_pred`` are used in sorted order. pos_label : str or int, optional (default=1) The class to report if ``average='binary'`` and the data is binary. If the data are multiclass, this will be ignored; setting ``labels=[pos_label]`` and ``average != 'binary'`` will report scores for that label only. average : str or None, optional (default=None) If ``None``, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data: ``'binary'``: Only report results for the class specified by ``pos_label``. This is applicable only if targets (``y_{true,pred}``) are binary. ``'micro'``: Calculate metrics globally by counting the total true positives, false negatives and false positives. ``'macro'``: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. ``'weighted'``: Calculate metrics for each label, and find their average, weighted by support (the number of true instances for each label). This alters 'macro' to account for label imbalance; it can result in an F-score that is not between precision and recall. ``'samples'``: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification where this differs from :func:`accuracy_score`). warn_for : tuple or set, for internal use This determines which warnings will be made in the case that this function is being used to return only one of its metrics. sample_weight : ndarray, shape (n_samples, ) Sample weights. Returns ------- sensitivity : float (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) specificity : float (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) support : int (if ``average`` = None) or ndarray, \ shape (n_unique_labels, ) The number of occurrences of each label in ``y_true``. Examples -------- References ---------- .. [1] `Wikipedia entry for the Sensitivity and specificity <https://en.wikipedia.org/wiki/Sensitivity_and_specificity>`_ """ average_options = (None, 'micro', 'macro', 'weighted', 'samples') if average not in average_options and average != 'binary': raise ValueError('average has to be one of ' + str(average_options)) y_type, y_true, y_pred = _check_targets(y_true, y_pred) present_labels = unique_labels(y_true, y_pred) if average == 'binary': if y_type == 'binary': if pos_label not in present_labels: if len(present_labels) < 2: # Only negative labels return (0., 0., 0) else: raise ValueError("pos_label=%r is not a valid label: %r" % (pos_label, present_labels)) labels = [pos_label] else: raise ValueError("Target is %s but average='binary'. Please " "choose another average setting." % y_type) elif pos_label not in (None, 1): warnings.warn( "Note that pos_label (set to %r) is ignored when " "average != 'binary' (got %r). You may use " "labels=[pos_label] to specify a single positive class." % (pos_label, average), UserWarning) if labels is None: labels = present_labels n_labels = None else: n_labels = len(labels) labels = np.hstack( [labels, np.setdiff1d(present_labels, labels, assume_unique=True)]) # Calculate tp_sum, pred_sum, true_sum ### if y_type.startswith('multilabel'): raise ValueError('imblearn does not support multilabel') elif average == 'samples': raise ValueError("Sample-based precision, recall, fscore is " "not meaningful outside multilabel " "classification. See the accuracy_score instead.") else: le = LabelEncoder() le.fit(labels) y_true = le.transform(y_true) y_pred = le.transform(y_pred) sorted_labels = le.classes_ # labels are now from 0 to len(labels) - 1 -> use bincount tp = y_true == y_pred tp_bins = y_true[tp] if sample_weight is not None: tp_bins_weights = np.asarray(sample_weight)[tp] else: tp_bins_weights = None if len(tp_bins): tp_sum = np.bincount(tp_bins, weights=tp_bins_weights, minlength=len(labels)) else: # Pathological case true_sum = pred_sum = tp_sum = np.zeros(len(labels)) if len(y_pred): pred_sum = np.bincount(y_pred, weights=sample_weight, minlength=len(labels)) if len(y_true): true_sum = np.bincount(y_true, weights=sample_weight, minlength=len(labels)) # Compute the true negative tn_sum = y_true.size - (pred_sum + true_sum - tp_sum) # Retain only selected labels indices = np.searchsorted(sorted_labels, labels[:n_labels]) tp_sum = tp_sum[indices] true_sum = true_sum[indices] pred_sum = pred_sum[indices] tn_sum = tn_sum[indices] if average == 'micro': tp_sum = np.array([tp_sum.sum()]) pred_sum = np.array([pred_sum.sum()]) true_sum = np.array([true_sum.sum()]) tn_sum = np.array([tn_sum.sum()]) # Finally, we have all our sufficient statistics. Divide! # with np.errstate(divide='ignore', invalid='ignore'): # Divide, and on zero-division, set scores to 0 and warn: # Oddly, we may get an "invalid" rather than a "divide" error # here. specificity = _prf_divide(tn_sum, tn_sum + pred_sum - tp_sum, 'specificity', 'predicted', average, warn_for) sensitivity = _prf_divide(tp_sum, true_sum, 'sensitivity', 'true', average, warn_for) # Average the results if average == 'weighted': weights = true_sum if weights.sum() == 0: return 0, 0, None elif average == 'samples': weights = sample_weight else: weights = None if average is not None: assert average != 'binary' or len(specificity) == 1 specificity = np.average(specificity, weights=weights) sensitivity = np.average(sensitivity, weights=weights) true_sum = None # return no support return sensitivity, specificity, true_sum
def confusion_matrix_2021(y_true, y_pred, *, labels=None, sample_weight=None, normalize=None): y_type, y_true, y_pred = _check_targets(y_true, y_pred) if y_type not in ("binary", "multiclass"): raise ValueError("%s is not supported" % y_type) if labels is None: labels = unique_labels(y_true, y_pred) else: labels = np.asarray(labels) n_labels = labels.size if n_labels == 0: raise ValueError("'labels' should contains at least one label.") elif y_true.size == 0: return np.zeros((n_labels, n_labels), dtype=int) elif np.all([l not in y_true for l in labels]): raise ValueError("At least one label specified must be in y_true") if sample_weight is None: sample_weight = np.ones(y_true.shape[0], dtype=np.int64) else: sample_weight = np.asarray(sample_weight) check_consistent_length(y_true, y_pred, sample_weight) if normalize not in ['true', 'pred', 'all', None]: raise ValueError("normalize must be one of {'true', 'pred', " "'all', None}") n_labels = labels.size label_to_ind = {y: x for x, y in enumerate(labels)} # convert yt, yp into index y_pred = np.array([label_to_ind.get(x, n_labels + 1) for x in y_pred]) y_true = np.array([label_to_ind.get(x, n_labels + 1) for x in y_true]) # intersect y_pred, y_true with labels, eliminate items not in labels ind = np.logical_and(y_pred < n_labels, y_true < n_labels) y_pred = y_pred[ind] y_true = y_true[ind] # also eliminate weights of eliminated items sample_weight = sample_weight[ind] # Choose the accumulator dtype to always have high precision if sample_weight.dtype.kind in {'i', 'u', 'b'}: dtype = np.int64 else: dtype = np.float64 cm = coo_matrix((sample_weight, (y_true, y_pred)), shape=(n_labels, n_labels), dtype=dtype, ).toarray() with np.errstate(all='ignore'): if normalize == 'true': cm = cm / cm.sum(axis=1, keepdims=True) elif normalize == 'pred': cm = cm / cm.sum(axis=0, keepdims=True) elif normalize == 'all': cm = cm / cm.sum() cm = np.nan_to_num(cm) return cm
def test__check_targets(): # Check that _check_targets correctly merges target types, squeezes # output and fails if input lengths differ. IND = 'multilabel-indicator' SEQ = 'multilabel-sequences' MC = 'multiclass' BIN = 'binary' CNT = 'continuous' MMC = 'multiclass-multioutput' MCN = 'continuous-multioutput' # all of length 3 EXAMPLES = [ (IND, np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])), # must not be considered binary (IND, np.array([[0, 1], [1, 0], [1, 1]])), (SEQ, [[2, 3], [1], [3]]), (MC, [2, 3, 1]), (BIN, [0, 1, 1]), (CNT, [0., 1.5, 1.]), (MC, np.array([[2], [3], [1]])), (BIN, np.array([[0], [1], [1]])), (CNT, np.array([[0.], [1.5], [1.]])), (MMC, np.array([[0, 2], [1, 3], [2, 3]])), (MCN, np.array([[0.5, 2.], [1.1, 3.], [2., 3.]])), ] # expected type given input types, or None for error # (types will be tried in either order) EXPECTED = { (IND, IND): IND, (SEQ, SEQ): IND, (MC, MC): MC, (BIN, BIN): BIN, (IND, SEQ): None, (MC, SEQ): None, (BIN, SEQ): None, (MC, IND): None, (BIN, IND): None, (BIN, MC): MC, # Disallowed types (CNT, CNT): None, (MMC, MMC): None, (MCN, MCN): None, (IND, CNT): None, (SEQ, CNT): None, (MC, CNT): None, (BIN, CNT): None, (MMC, CNT): None, (MCN, CNT): None, (IND, MMC): None, (SEQ, MMC): None, (MC, MMC): None, (BIN, MMC): None, (MCN, MMC): None, (IND, MCN): None, (SEQ, MCN): None, (MC, MCN): None, (BIN, MCN): None, } for (type1, y1), (type2, y2) in product(EXAMPLES, repeat=2): try: expected = EXPECTED[type1, type2] except KeyError: expected = EXPECTED[type2, type1] if expected is None: assert_raises(ValueError, _check_targets, y1, y2) if type1 != type2: assert_raise_message( ValueError, "Can't handle mix of {0} and {1}".format(type1, type2), _check_targets, y1, y2) else: if type1 not in (BIN, MC, SEQ, IND): assert_raise_message(ValueError, "{0} is not supported".format(type1), _check_targets, y1, y2) else: merged_type, y1out, y2out = _check_targets(y1, y2) assert_equal(merged_type, expected) if merged_type.startswith('multilabel'): assert_equal(y1out.format, 'csr') assert_equal(y2out.format, 'csr') else: assert_array_equal(y1out, np.squeeze(y1)) assert_array_equal(y2out, np.squeeze(y2)) assert_raises(ValueError, _check_targets, y1[:-1], y2)
def precision_recall_fscore_support(y_true, y_pred, beta=1.0, labels=None, pos_label=1, average=None, warn_for=('precision', 'recall', 'f-score'), sample_weight=None): """Compute precision, recall, F-measure and support for each class The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of true positives and ``fp`` the number of false positives. The precision is intuitively the ability of the classifier not to label as positive a sample that is negative. The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of true positives and ``fn`` the number of false negatives. The recall is intuitively the ability of the classifier to find all the positive samples. The F-beta score can be interpreted as a weighted harmonic mean of the precision and recall, where an F-beta score reaches its best value at 1 and worst score at 0. The F-beta score weights recall more than precision by a factor of ``beta``. ``beta == 1.0`` means recall and precision are equally important. The support is the number of occurrences of each class in ``y_true``. If ``pos_label is None`` and in binary classification, this function returns the average precision, recall and F-measure if ``average`` is one of ``'micro'``, ``'macro'``, ``'weighted'`` or ``'samples'``. Read more in the :ref:`User Guide <precision_recall_f_measure_metrics>`. Parameters ---------- y_true : 1d array-like, or label indicator array / sparse matrix Ground truth (correct) target values. y_pred : 1d array-like, or label indicator array / sparse matrix Estimated targets as returned by a classifier. beta : float, 1.0 by default The strength of recall versus precision in the F-score. labels : list, optional The set of labels to include when ``average != 'binary'``, and their order if ``average is None``. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in ``y_true`` and ``y_pred`` are used in sorted order. pos_label : str or int, 1 by default The class to report if ``average='binary'`` and the data is binary. If the data are multiclass or multilabel, this will be ignored; setting ``labels=[pos_label]`` and ``average != 'binary'`` will report scores for that label only. average : string, [None (default), 'binary', 'micro', 'macro', 'samples', \ 'weighted'] If ``None``, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data: ``'binary'``: Only report results for the class specified by ``pos_label``. This is applicable only if targets (``y_{true,pred}``) are binary. ``'micro'``: Calculate metrics globally by counting the total true positives, false negatives and false positives. ``'macro'``: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. ``'weighted'``: Calculate metrics for each label, and find their average, weighted by support (the number of true instances for each label). This alters 'macro' to account for label imbalance; it can result in an F-score that is not between precision and recall. ``'samples'``: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification where this differs from :func:`accuracy_score`). warn_for : tuple or set, for internal use This determines which warnings will be made in the case that this function is being used to return only one of its metrics. sample_weight : array-like of shape = [n_samples], optional Sample weights. Returns ------- precision : float (if average is not None) or array of float, shape =\ [n_unique_labels] recall : float (if average is not None) or array of float, , shape =\ [n_unique_labels] fbeta_score : float (if average is not None) or array of float, shape =\ [n_unique_labels] support : int (if average is not None) or array of int, shape =\ [n_unique_labels] The number of occurrences of each label in ``y_true``. References ---------- .. [1] `Wikipedia entry for the Precision and recall <https://en.wikipedia.org/wiki/Precision_and_recall>`_ .. [2] `Wikipedia entry for the F1-score <https://en.wikipedia.org/wiki/F1_score>`_ .. [3] `Discriminative Methods for Multi-labeled Classification Advances in Knowledge Discovery and Data Mining (2004), pp. 22-30 by Shantanu Godbole, Sunita Sarawagi <http://www.godbole.net/shantanu/pubs/multilabelsvm-pakdd04.pdf>`_ Examples -------- >>> from sklearn.metrics import precision_recall_fscore_support >>> y_true = np.array(['cat', 'dog', 'pig', 'cat', 'dog', 'pig']) >>> y_pred = np.array(['cat', 'pig', 'dog', 'cat', 'cat', 'dog']) >>> precision_recall_fscore_support(y_true, y_pred, average='macro') ... # doctest: +ELLIPSIS (0.22..., 0.33..., 0.26..., None) >>> precision_recall_fscore_support(y_true, y_pred, average='micro') ... # doctest: +ELLIPSIS (0.33..., 0.33..., 0.33..., None) >>> precision_recall_fscore_support(y_true, y_pred, average='weighted') ... # doctest: +ELLIPSIS (0.22..., 0.33..., 0.26..., None) It is possible to compute per-label precisions, recalls, F1-scores and supports instead of averaging: >>> precision_recall_fscore_support(y_true, y_pred, average=None, ... labels=['pig', 'dog', 'cat']) ... # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE (array([0. , 0. , 0.66...]), array([0., 0., 1.]), array([0. , 0. , 0.8]), array([2, 2, 2])) """ average_options = (None, 'micro', 'macro', 'weighted', 'samples') if average not in average_options and average != 'binary': raise ValueError('average has to be one of ' + str(average_options)) if beta <= 0: raise ValueError("beta should be >0 in the F-beta score") y_type, y_true, y_pred = _check_targets(y_true, y_pred) check_consistent_length(y_true, y_pred, sample_weight) present_labels = unique_labels(y_true, y_pred) if average == 'binary': if y_type == 'binary': if pos_label not in present_labels: if len(present_labels) < 2: # Only negative labels return (0., 0., 0., 0) else: raise ValueError("pos_label=%r is not a valid label: %r" % (pos_label, present_labels)) labels = [pos_label] else: raise ValueError("Target is %s but average='binary'. Please " "choose another average setting." % y_type) elif pos_label not in (None, 1): warnings.warn( "Note that pos_label (set to %r) is ignored when " "average != 'binary' (got %r). You may use " "labels=[pos_label] to specify a single positive class." % (pos_label, average), UserWarning) if labels is None: labels = present_labels n_labels = None else: n_labels = len(labels) labels = np.hstack( [labels, np.setdiff1d(present_labels, labels, assume_unique=True)]) # Calculate tp_sum, pred_sum, true_sum ### if y_type.startswith('multilabel'): sum_axis = 1 if average == 'samples' else 0 # All labels are index integers for multilabel. # Select labels: if not np.all(labels == present_labels): if np.max(labels) > np.max(present_labels): raise ValueError('All labels must be in [0, n labels). ' 'Got %d > %d' % (np.max(labels), np.max(present_labels))) if np.min(labels) < 0: raise ValueError('All labels must be in [0, n labels). ' 'Got %d < 0' % np.min(labels)) if n_labels is not None: y_true = y_true[:, labels[:n_labels]] y_pred = y_pred[:, labels[:n_labels]] # calculate weighted counts true_and_pred = y_true.multiply(y_pred) tp_sum = count_nonzero(true_and_pred, axis=sum_axis, sample_weight=sample_weight) pred_sum = count_nonzero(y_pred, axis=sum_axis, sample_weight=sample_weight) true_sum = count_nonzero(y_true, axis=sum_axis, sample_weight=sample_weight) elif average == 'samples': raise ValueError("Sample-based precision, recall, fscore is " "not meaningful outside multilabel " "classification. See the accuracy_score instead.") else: le = LabelEncoder() le.fit(labels) y_true = le.transform(y_true) y_pred = le.transform(y_pred) sorted_labels = le.classes_ # labels are now from 0 to len(labels) - 1 -> use bincount tp = y_true == y_pred tp_bins = y_true[tp] if sample_weight is not None: tp_bins_weights = np.asarray(sample_weight)[tp] else: tp_bins_weights = None if len(tp_bins): tp_sum = np.bincount(tp_bins, weights=tp_bins_weights, minlength=len(labels)) else: # Pathological case true_sum = pred_sum = tp_sum = np.zeros(len(labels)) if len(y_pred): pred_sum = np.bincount(y_pred, weights=sample_weight, minlength=len(labels)) if len(y_true): true_sum = np.bincount(y_true, weights=sample_weight, minlength=len(labels)) # Retain only selected labels indices = np.searchsorted(sorted_labels, labels[:n_labels]) tp_sum = tp_sum[indices] true_sum = true_sum[indices] pred_sum = pred_sum[indices] if average == 'micro': tp_sum = np.array([tp_sum.sum()]) pred_sum = np.array([pred_sum.sum()]) true_sum = np.array([true_sum.sum()]) # Finally, we have all our sufficient statistics. Divide! # beta2 = beta**2 with np.errstate(divide='ignore', invalid='ignore'): # Divide, and on zero-division, set scores to 0 and warn: # Oddly, we may get an "invalid" rather than a "divide" error # here. precision = _prf_divide(tp_sum, pred_sum, 'precision', 'predicted', average, warn_for) recall = _prf_divide(tp_sum, true_sum, 'recall', 'true', average, warn_for) # Don't need to warn for F: either P or R warned, or tp == 0 where pos # and true are nonzero, in which case, F is well-defined and zero f_score = ((1 + beta2) * precision * recall / (beta2 * precision + recall)) f_score[tp_sum == 0] = 0.0 # Average the results if average == 'weighted': weights = true_sum if weights.sum() == 0: return 0, 0, 0, None elif average == 'samples': weights = sample_weight else: weights = None if average is not None: assert average != 'binary' or len(precision) == 1 precision = np.average(precision, weights=weights) recall = np.average(recall, weights=weights) f_score = np.average(f_score, weights=weights) true_sum = None # return no support return precision, recall, f_score, true_sum
def test__check_targets(): # Check that _check_targets correctly merges target types, squeezes # output and fails if input lengths differ. IND = 'multilabel-indicator' MC = 'multiclass' BIN = 'binary' CNT = 'continuous' MMC = 'multiclass-multioutput' MCN = 'continuous-multioutput' # all of length 3 EXAMPLES = [ (IND, np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])), # must not be considered binary (IND, np.array([[0, 1], [1, 0], [1, 1]])), (MC, [2, 3, 1]), (BIN, [0, 1, 1]), (CNT, [0., 1.5, 1.]), (MC, np.array([[2], [3], [1]])), (BIN, np.array([[0], [1], [1]])), (CNT, np.array([[0.], [1.5], [1.]])), (MMC, np.array([[0, 2], [1, 3], [2, 3]])), (MCN, np.array([[0.5, 2.], [1.1, 3.], [2., 3.]])), ] # expected type given input types, or None for error # (types will be tried in either order) EXPECTED = { (IND, IND): IND, (MC, MC): MC, (BIN, BIN): BIN, (MC, IND): None, (BIN, IND): None, (BIN, MC): MC, # Disallowed types (CNT, CNT): None, (MMC, MMC): None, (MCN, MCN): None, (IND, CNT): None, (MC, CNT): None, (BIN, CNT): None, (MMC, CNT): None, (MCN, CNT): None, (IND, MMC): None, (MC, MMC): None, (BIN, MMC): None, (MCN, MMC): None, (IND, MCN): None, (MC, MCN): None, (BIN, MCN): None, } for (type1, y1), (type2, y2) in product(EXAMPLES, repeat=2): try: expected = EXPECTED[type1, type2] except KeyError: expected = EXPECTED[type2, type1] if expected is None: assert_raises(ValueError, _check_targets, y1, y2) if type1 != type2: assert_raise_message( ValueError, "Can't handle mix of {0} and {1}".format(type1, type2), _check_targets, y1, y2) else: if type1 not in (BIN, MC, IND): assert_raise_message(ValueError, "{0} is not supported".format(type1), _check_targets, y1, y2) else: merged_type, y1out, y2out = _check_targets(y1, y2) assert_equal(merged_type, expected) if merged_type.startswith('multilabel'): assert_equal(y1out.format, 'csr') assert_equal(y2out.format, 'csr') else: assert_array_equal(y1out, np.squeeze(y1)) assert_array_equal(y2out, np.squeeze(y2)) assert_raises(ValueError, _check_targets, y1[:-1], y2) # Make sure seq of seq is not supported y1 = [(1, 2,), (0, 2, 3)] y2 = [(2,), (0, 2,)] msg = ('You appear to be using a legacy multi-label data representation. ' 'Sequence of sequences are no longer supported; use a binary array' ' or sparse matrix instead.') assert_raise_message(ValueError, msg, _check_targets, y1, y2)
def multilabel_confusion_matrix(y_true, y_pred, sample_weight=None, labels=None, samplewise=False): """Compute a confusion matrix for each class or sample .. versionadded:: 0.21 Compute class-wise (default) or sample-wise (samplewise=True) multilabel confusion matrix to evaluate the accuracy of a classification, and output confusion matrices for each class or sample. In multilabel confusion matrix :math:`MCM`, the count of true negatives is :math:`MCM_{:,0,0}`, false negatives is :math:`MCM_{:,1,0}`, true positives is :math:`MCM_{:,1,1}` and false positives is :math:`MCM_{:,0,1}`. Multiclass data will be treated as if binarized under a one-vs-rest transformation. Returned confusion matrices will be in the order of sorted unique labels in the union of (y_true, y_pred). Read more in the :ref:`User Guide <multilabel_confusion_matrix>`. Parameters ---------- y_true : 1d array-like, or label indicator array / sparse matrix of shape (n_samples, n_outputs) or (n_samples,) Ground truth (correct) target values. y_pred : 1d array-like, or label indicator array / sparse matrix of shape (n_samples, n_outputs) or (n_samples,) Estimated targets as returned by a classifier sample_weight : array-like of shape = (n_samples,), optional Sample weights labels : array-like A list of classes or column indices to select some (or to force inclusion of classes absent from the data) samplewise : bool, default=False In the multilabel case, this calculates a confusion matrix per sample Returns ------- multi_confusion : array, shape (n_outputs, 2, 2) A 2x2 confusion matrix corresponding to each output in the input. When calculating class-wise multi_confusion (default), then n_outputs = n_labels; when calculating sample-wise multi_confusion (samplewise=True), n_outputs = n_samples. If ``labels`` is defined, the results will be returned in the order specified in ``labels``, otherwise the results will be returned in sorted order by default. See also -------- confusion_matrix Notes ----- The multilabel_confusion_matrix calculates class-wise or sample-wise multilabel confusion matrices, and in multiclass tasks, labels are binarized under a one-vs-rest way; while confusion_matrix calculates one confusion matrix for confusion between every two classes. Examples -------- Multilabel-indicator case: >>> import numpy as np >>> from sklearn.metrics import multilabel_confusion_matrix >>> y_true = np.array([[1, 0, 1], ... [0, 1, 0]]) >>> y_pred = np.array([[1, 0, 0], ... [0, 1, 1]]) >>> multilabel_confusion_matrix(y_true, y_pred) array([[[1, 0], [0, 1]], <BLANKLINE> [[1, 0], [0, 1]], <BLANKLINE> [[0, 1], [1, 0]]]) Multiclass case: >>> y_true = ["cat", "ant", "cat", "cat", "ant", "bird"] >>> y_pred = ["ant", "ant", "cat", "cat", "ant", "cat"] >>> multilabel_confusion_matrix(y_true, y_pred, ... labels=["ant", "bird", "cat"]) array([[[3, 1], [0, 2]], <BLANKLINE> [[5, 0], [1, 0]], <BLANKLINE> [[2, 1], [1, 2]]]) """ y_type, y_true, y_pred = _check_targets(y_true, y_pred) if sample_weight is not None: sample_weight = column_or_1d(sample_weight) check_consistent_length(y_true, y_pred, sample_weight) if y_type not in ("binary", "multiclass", "multilabel-indicator"): raise ValueError("%s is not supported" % y_type) present_labels = unique_labels(y_true, y_pred) if labels is None: labels = present_labels n_labels = None else: n_labels = len(labels) labels = np.hstack( [labels, np.setdiff1d(present_labels, labels, assume_unique=True)]) if y_true.ndim == 1: if samplewise: raise ValueError("Samplewise metrics are not available outside of " "multilabel classification.") le = LabelEncoder() le.fit(labels) y_true = le.transform(y_true) y_pred = le.transform(y_pred) sorted_labels = le.classes_ # labels are now from 0 to len(labels) - 1 -> use bincount tp = y_true == y_pred tp_bins = y_true[tp] if sample_weight is not None: tp_bins_weights = np.asarray(sample_weight)[tp] else: tp_bins_weights = None if len(tp_bins): tp_sum = np.bincount(tp_bins, weights=tp_bins_weights, minlength=len(labels)) else: # Pathological case true_sum = pred_sum = tp_sum = np.zeros(len(labels)) if len(y_pred): pred_sum = np.bincount(y_pred, weights=sample_weight, minlength=len(labels)) if len(y_true): true_sum = np.bincount(y_true, weights=sample_weight, minlength=len(labels)) # Retain only selected labels indices = np.searchsorted(sorted_labels, labels[:n_labels]) tp_sum = tp_sum[indices] true_sum = true_sum[indices] pred_sum = pred_sum[indices] else: sum_axis = 1 if samplewise else 0 # All labels are index integers for multilabel. # Select labels: if not np.array_equal(labels, present_labels): if np.max(labels) > np.max(present_labels): raise ValueError('All labels must be in [0, n labels) for ' 'multilabel targets. ' 'Got %d > %d' % (np.max(labels), np.max(present_labels))) if np.min(labels) < 0: raise ValueError('All labels must be in [0, n labels) for ' 'multilabel targets. ' 'Got %d < 0' % np.min(labels)) if n_labels is not None: y_true = y_true[:, labels[:n_labels]] y_pred = y_pred[:, labels[:n_labels]] # calculate weighted counts true_and_pred = y_true.multiply(y_pred) tp_sum = count_nonzero(true_and_pred, axis=sum_axis, sample_weight=sample_weight) pred_sum = count_nonzero(y_pred, axis=sum_axis, sample_weight=sample_weight) true_sum = count_nonzero(y_true, axis=sum_axis, sample_weight=sample_weight) fp = pred_sum - tp_sum fn = true_sum - tp_sum tp = tp_sum if sample_weight is not None and samplewise: sample_weight = np.array(sample_weight) tp = np.array(tp) fp = np.array(fp) fn = np.array(fn) tn = sample_weight * y_true.shape[1] - tp - fp - fn elif sample_weight is not None: tn = sum(sample_weight) - tp - fp - fn elif samplewise: tn = y_true.shape[1] - tp - fp - fn else: tn = y_true.shape[0] - tp - fp - fn return np.array([tn, fp, fn, tp]).T.reshape(-1, 2, 2)
def sk_clf_report(y_true: list, y_pred: list, labels: list = None, target_names: list = None, sample_weight: list = None, digits: int = 2, output_dict: bool = False) -> str: """Build a text report showing the main classification metrics Read more in the :ref:`User Guide <classification_report>`. Parameters ---------- y_true : 1d array-like, or label indicator array / sparse matrix Ground truth (correct) target values. y_pred : 1d array-like, or label indicator array / sparse matrix Estimated targets as returned by a classifier. labels : array, shape = [n_labels] Optional list of label indices to include in the report. target_names : list of strings Optional display names matching the labels (same order). sample_weight : array-like of shape = [n_samples], optional Sample weights. digits : int Number of digits for formatting output floating point values. When ``output_dict`` is ``True``, this will be ignored and the returned values will not be rounded. output_dict : bool (default = False) If True, return output as dict Returns ------- report : string / dict Text summary of the precision, recall, F1 score for each class. Dictionary returned if output_dict is True. Dictionary has the following structure:: {'label 1': {'precision':0.5, 'recall':1.0, 'f1-score':0.67, 'support':1}, 'label 2': { ... }, ... } The reported averages include micro average (averaging the total true positives, false negatives and false positives), macro average (averaging the unweighted mean per label), weighted average (averaging the support-weighted mean per label) and sample average (only for multilabel classification). See also :func:`precision_recall_fscore_support` for more details on averages. Note that in binary classification, recall of the positive class is also known as "sensitivity"; recall of the negative class is "specificity". Examples -------- >>> from sklearn.metrics import classification_report >>> y_true = [0, 1, 2, 2, 2] >>> y_pred = [0, 0, 2, 2, 1] >>> target_names = ['class 0', 'class 1', 'class 2'] >>> print(classification_report(y_true, y_pred, target_names=target_names)) precision recall f1-score support <BLANKLINE> class 0 0.50 1.00 0.67 1 class 1 0.00 0.00 0.00 1 class 2 1.00 0.67 0.80 3 <BLANKLINE> micro avg 0.60 0.60 0.60 5 macro avg 0.50 0.56 0.49 5 weighted avg 0.70 0.60 0.61 5 <BLANKLINE> """ y_type, y_true, y_pred = _check_targets(y_true, y_pred) labels_given = True if labels is None: labels = unique_labels(y_true, y_pred) labels_given = False else: labels = np.asarray(labels) if target_names is not None and len(labels) != len(target_names): if labels_given: warnings.warn( "labels size, {0}, does not match size of target_names, {1}" .format(len(labels), len(target_names)) ) else: raise ValueError( "Number of classes, {0}, does not match size of " "target_names, {1}. Try specifying the labels " "parameter".format(len(labels), len(target_names)) ) if target_names is None: target_names = [u'%s' % l for l in labels] headers = ["precision", "recall", "f1-score", "support"] # compute per-class results without averaging p, r, f1, s = precision_recall_fscore_support(y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight) rows = zip(target_names, p, r, f1, s) if y_type.startswith('multilabel'): average_options = ('micro', 'macro', 'weighted', 'samples') else: average_options = ('micro', 'macro', 'weighted') if output_dict: report_dict = {label[0]: label[1:] for label in rows} for label, scores in report_dict.items(): report_dict[label] = dict(zip(headers, [i.item() for i in scores])) else: longest_last_line_heading = 'weighted avg' name_width = max(len(cn) for cn in target_names) width = max(name_width, len(longest_last_line_heading), digits) head_fmt = u'{:>{width}s} ' + u' {:>9}' * len(headers) report = head_fmt.format(u'', *headers, width=width) report += u'\n\n' row_fmt = u'{:>{width}s} ' + u' {:>9.{digits}f}' * 3 + u' {:>9}\n' for row in rows: report += row_fmt.format(*row, width=width, digits=digits) report += u'\n' # compute all applicable averages for average in average_options: line_heading = average + ' avg' # compute averages with specified averaging method avg_p, avg_r, avg_f1, _ = precision_recall_fscore_support( y_true, y_pred, labels=labels, average=average, sample_weight=sample_weight) avg = [avg_p, avg_r, avg_f1, np.sum(s)] if output_dict: report_dict[line_heading] = dict( zip(headers, [i.item() for i in avg])) else: report += row_fmt.format(line_heading, *avg, width=width, digits=digits) if output_dict: return report_dict else: return report