def __call__(self, y_true: np.array, y_pred: np.array) -> float: """ Calculate discounted cumulative gain (dcg). Relevance is positive real values or binary values. Example: >>> y_true = [0, 1, 2, 0] >>> y_pred = [0.4, 0.2, 0.5, 0.7] >>> DiscountedCumulativeGain(1)(y_true, y_pred) 0.0 >>> round(DiscountedCumulativeGain(k=-1)(y_true, y_pred), 2) 0.0 >>> round(DiscountedCumulativeGain(k=2)(y_true, y_pred), 2) 2.73 >>> round(DiscountedCumulativeGain(k=3)(y_true, y_pred), 2) 2.73 >>> type(DiscountedCumulativeGain(k=1)(y_true, y_pred)) <class 'float'> :param y_true: The ground true label of each document. :param y_pred: The predicted scores of each document. :return: Discounted cumulative gain. """ if self._k <= 0: return 0. coupled_pair = sort_and_couple(y_true, y_pred) result = 0. for i, (label, score) in enumerate(coupled_pair): if i >= self._k: break if label > self._threshold: result += (math.pow(2., label) - 1.) / math.log(2. + i) return result
def __call__(self, y_true: np.array, y_pred: np.array) -> float: """ Calculate precision@k. Example: >>> y_true = [0, 0, 0, 1] >>> y_pred = [0.2, 0.4, 0.3, 0.1] >>> Precision(k=1)(y_true, y_pred) 0.0 >>> Precision(k=2)(y_true, y_pred) 0.0 >>> Precision(k=4)(y_true, y_pred) 0.25 >>> Precision(k=5)(y_true, y_pred) 0.2 :param y_true: The ground true label of each document. :param y_pred: The predicted scores of each document. :return: Precision @ k :raises: ValueError: len(r) must be >= k. """ if self._k <= 0: raise ValueError(f"k must be greater than 0." f"{self._k} received.") coupled_pair = sort_and_couple(y_true, y_pred) precision = 0.0 for idx, (label, score) in enumerate(coupled_pair): if idx >= self._k: break if label > self._threshold: precision += 1. return precision / self._k
def __call__(self, y_true: np.array, y_pred: np.array) -> float: """ Calculate mean average precision. This metric works better with BINARY labels rather than ORDINAL labels. Example: >>> y_true = [0, 1, 0, 0] >>> y_pred = [0.1, 0.6, 0.2, 0.3] >>> MeanAveragePrecision()(y_true, y_pred) 1.0 :param y_true: The ground true label of each document. :param y_pred: The predicted scores of each document. :return: Mean average precision. """ result = 0. pos = 0 coupled_pair = sort_and_couple(y_true, y_pred) for idx, (label, score) in enumerate(coupled_pair): if label > self._threshold: pos += 1. result += pos / (idx + 1.) if pos == 0: return 0. else: return result / pos
def __call__(self, y_true: np.array, y_pred: np.array) -> float: """ Calculate precision@k. This metric works better with BINARY labels rather than ORDINAL labels. Example: >>> y_true = [0, 0, 0, 1] >>> y_pred = [0.2, 0.4, 0.3, 0.1] >>> Precision(k=1)(y_true, y_pred) 0.0 >>> Precision(k=2)(y_true, y_pred) 0.0 >>> Precision(k=4)(y_true, y_pred) 0.25 >>> Precision(k=5)(y_true, y_pred) 0.2 :param y_true: The ground true label of each document. Usually 0 or 1 (irrelevant or relevant) :param y_pred: The predicted scores of each document. Relevance scores predicted by a model. The order of y_pred items is random. They will be sorted inside this function. :return: Precision @ k :raises: ValueError: len(r) must be >= k. """ if self._k <= 0: raise ValueError(f"k must be greater than 0." f"{self._k} received.") coupled_pair = sort_and_couple(y_true, y_pred) precision = 0.0 for idx, (label, score) in enumerate(coupled_pair): if idx >= self._k: break if label > self._threshold: precision += 1. return precision / self._k
def __call__(self, y_true, y_pred): coupled_pair = sort_and_couple(y_true, y_pred) for idx, (label, pred) in enumerate(coupled_pair): if idx+1>self._k: return 0 if label > 0: return 1. / (idx + 1) return 0.
def __call__(self, y_true: np.array, y_pred: np.array) -> float: """ Calculate reciprocal of the rank of the first relevant item. Example: >>> import numpy as np >>> y_pred = np.asarray([0.2, 0.3, 0.7, 1.0]) >>> y_true = np.asarray([1, 0, 0, 0]) >>> MeanReciprocalRank()(y_true, y_pred) 0.25 :param y_true: The ground true label of each document. :param y_pred: The predicted scores of each document. :return: Mean reciprocal rank. """ coupled_pair = sort_and_couple(y_true, y_pred) for idx, (label, pred) in enumerate(coupled_pair): if label > self._threshold: return 1. / (idx + 1) return 0.
def test_sort_and_couple(): l = [0, 1, 2] s = [0.1, 0.4, 0.2] c = sort_and_couple(l, s) assert (c == np.array([(1, 0.4), (2, 0.2), (0, 0.1)])).all()