def test_encode_util(values, expected): uniques = _encode(values) assert_array_equal(uniques, expected) uniques, encoded = _encode(values, encode=True) assert_array_equal(uniques, expected) assert_array_equal(encoded, np.array([1, 0, 2, 0, 2])) _, encoded = _encode(values, uniques, encode=True) assert_array_equal(encoded, np.array([1, 0, 2, 0, 2]))
def test_encode_check_unknown(): # test for the check_unknown parameter of _encode() uniques = np.array([1, 2, 3]) values = np.array([1, 2, 3, 4]) # Default is True, raise error with pytest.raises(ValueError, match='y contains previously unseen labels'): _encode(values, uniques, encode=True, check_unknown=True) # dont raise error if False _encode(values, uniques, encode=True, check_unknown=False) # parameter is ignored for object dtype uniques = np.array(['a', 'b', 'c'], dtype=object) values = np.array(['a', 'b', 'c', 'd'], dtype=object) with pytest.raises(ValueError, match='y contains previously unseen labels'): _encode(values, uniques, encode=True, check_unknown=False)
def multiclass_roc_auc_score( y_true, y_score, labels, multi_class, average, sample_weight=None, invalid_proba_tolerance: float = 1e-6, ): """Multiclass roc auc score (copied from sklearn) Parameters ---------- y_true : array-like of shape (n_samples,) True multiclass labels. y_score : array-like of shape (n_samples, n_classes) Target scores corresponding to probability estimates of a sample belonging to a particular class labels : array, shape = [n_classes] or None, optional (default=None) List of labels to index ``y_score`` used for multiclass. If ``None``, the lexical order of ``y_true`` is used to index ``y_score``. multi_class : string, 'ovr' or 'ovo' Determines the type of multiclass configuration to use. ``'ovr'``: Calculate metrics for the multiclass case using the one-vs-rest approach. ``'ovo'``: Calculate metrics for the multiclass case using the one-vs-one approach. average : 'macro' or 'weighted', optional (default='macro') Determines the type of averaging performed on the pairwise binary metric scores ``'macro'``: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. Classes are assumed to be uniformly distributed. ``'weighted'``: Calculate metrics for each label, taking into account the prevalence of the classes. sample_weight : array-like of shape (n_samples,), default=None Sample weights. :param invalid_proba_tolerance: float in [0, 1] The proportion of samples that can eventually be ignored if their class scores do not sum up to 1. """ # validation of the input y_score are_close = np.isclose(1, y_score.sum(axis=1)) # I added this try-except to deal with cases where a very small amount of voxels have an issue # to sum the probabilities to 1, which might happen (probably, i suppose) because I use float16 instead of 64 try: if not np.all(are_close): raise ValueError( "Target scores need to be probabilities for multiclass " "roc_auc, i.e. they should sum up to 1.0 over classes") except ValueError as ex: logger.exception(ex) assert 0 <= invalid_proba_tolerance <= 1, f"{invalid_proba_tolerance=}" nsamples_not_close = int((~are_close).sum()) percentage_samples_not_close = nsamples_not_close / are_close.size logger.warning( f"{nsamples_not_close=} ({percentage_samples_not_close=:.7%})") if percentage_samples_not_close > invalid_proba_tolerance: raise ValueError( f"Too many samples are not close 1 {nsamples_not_close=} {percentage_samples_not_close=:.7%} {invalid_proba_tolerance=:.7%}." ) else: logger.warning( f"The amount of probabilities not summing up to 1 will be tolerated " f"{percentage_samples_not_close=:.7%} {invalid_proba_tolerance=:.7%}. " f"The bad samples will be ignored!") y_true = y_true[are_close] y_score = y_score[are_close, :] # validation for multiclass parameter specifications average_options = ("macro", "weighted") if average not in average_options: raise ValueError("average must be one of {0} for " "multiclass problems".format(average_options)) multiclass_options = ("ovo", "ovr") if multi_class not in multiclass_options: raise ValueError("multi_class='{0}' is not supported " "for multiclass ROC AUC, multi_class must be " "in {1}".format(multi_class, multiclass_options)) from sklearn.utils import column_or_1d from sklearn.preprocessing._label import _encode from sklearn.metrics._base import _average_multiclass_ovo_score from sklearn.preprocessing import label_binarize from sklearn.metrics._ranking import _binary_roc_auc_score from sklearn.metrics._base import _average_binary_score if labels is not None: labels = column_or_1d(labels) classes = _encode(labels) if len(classes) != len(labels): raise ValueError("Parameter 'labels' must be unique") if not np.array_equal(classes, labels): raise ValueError("Parameter 'labels' must be ordered") if len(classes) != y_score.shape[1]: raise ValueError( "Number of given labels, {0}, not equal to the number " "of columns in 'y_score', {1}".format(len(classes), y_score.shape[1])) if len(np.setdiff1d(y_true, classes)): raise ValueError( "'y_true' contains labels not in parameter 'labels'") else: classes = _encode(y_true) if len(classes) != y_score.shape[1]: raise ValueError( "Number of classes in y_true not equal to the number of " "columns in 'y_score'") if multi_class == "ovo": if sample_weight is not None: raise ValueError("sample_weight is not supported " "for multiclass one-vs-one ROC AUC, " "'sample_weight' must be None in this case.") _, y_true_encoded = _encode(y_true, uniques=classes, encode=True) # Hand & Till (2001) implementation (ovo) return _average_multiclass_ovo_score(_binary_roc_auc_score, y_true_encoded, y_score, average=average) else: # ovr is same as multi-label y_true_multilabel = label_binarize(y_true, classes=classes) return _average_binary_score(_binary_roc_auc_score, y_true_multilabel, y_score, average, sample_weight=sample_weight)