def __call__(self, y_true: np.array, y_pred: np.array) -> float:
"""
Calculate discounted cumulative gain (dcg).
Relevance is positive real values or binary values.
Example:
>>> y_true = [0, 1, 2, 0]
>>> y_pred = [0.4, 0.2, 0.5, 0.7]
>>> DiscountedCumulativeGain(1)(y_true, y_pred)
0.0
>>> round(DiscountedCumulativeGain(k=-1)(y_true, y_pred), 2)
0.0
>>> round(DiscountedCumulativeGain(k=2)(y_true, y_pred), 2)
2.73
>>> round(DiscountedCumulativeGain(k=3)(y_true, y_pred), 2)
2.73
>>> type(DiscountedCumulativeGain(k=1)(y_true, y_pred))
<class 'float'>
:param y_true: The ground true label of each document.
:param y_pred: The predicted scores of each document.
:return: Discounted cumulative gain.
"""
if self._k <= 0:
return 0.
coupled_pair = sort_and_couple(y_true, y_pred)
result = 0.
for i, (label, score) in enumerate(coupled_pair):
if i >= self._k:
break
if label > self._threshold:
result += (math.pow(2., label) - 1.) / math.log(2. + i)
return result
def __call__(self, y_true: np.array, y_pred: np.array) -> float:
"""
Calculate precision@k.
Example:
>>> y_true = [0, 0, 0, 1]
>>> y_pred = [0.2, 0.4, 0.3, 0.1]
>>> Precision(k=1)(y_true, y_pred)
0.0
>>> Precision(k=2)(y_true, y_pred)
0.0
>>> Precision(k=4)(y_true, y_pred)
0.25
>>> Precision(k=5)(y_true, y_pred)
0.2
:param y_true: The ground true label of each document.
:param y_pred: The predicted scores of each document.
:return: Precision @ k
:raises: ValueError: len(r) must be >= k.
"""
if self._k <= 0:
raise ValueError(f"k must be greater than 0."
f"{self._k} received.")
coupled_pair = sort_and_couple(y_true, y_pred)
precision = 0.0
for idx, (label, score) in enumerate(coupled_pair):
if idx >= self._k:
break
if label > self._threshold:
precision += 1.
return precision / self._k
def __call__(self, y_true: np.array, y_pred: np.array) -> float:
"""
Calculate mean average precision.
This metric works better with BINARY labels rather than ORDINAL labels.
Example:
>>> y_true = [0, 1, 0, 0]
>>> y_pred = [0.1, 0.6, 0.2, 0.3]
>>> MeanAveragePrecision()(y_true, y_pred)
1.0
:param y_true: The ground true label of each document.
:param y_pred: The predicted scores of each document.
:return: Mean average precision.
"""
result = 0.
pos = 0
coupled_pair = sort_and_couple(y_true, y_pred)
for idx, (label, score) in enumerate(coupled_pair):
if label > self._threshold:
pos += 1.
result += pos / (idx + 1.)
if pos == 0:
return 0.
else:
return result / pos
def __call__(self, y_true: np.array, y_pred: np.array) -> float:
"""
Calculate precision@k.
This metric works better with BINARY labels rather than ORDINAL labels.
Example:
>>> y_true = [0, 0, 0, 1]
>>> y_pred = [0.2, 0.4, 0.3, 0.1]
>>> Precision(k=1)(y_true, y_pred)
0.0
>>> Precision(k=2)(y_true, y_pred)
0.0
>>> Precision(k=4)(y_true, y_pred)
0.25
>>> Precision(k=5)(y_true, y_pred)
0.2
:param y_true: The ground true label of each document. Usually 0 or 1 (irrelevant or relevant)
:param y_pred: The predicted scores of each document. Relevance scores predicted by a model. The order of y_pred items is random. They will be sorted inside this function.
:return: Precision @ k
:raises: ValueError: len(r) must be >= k.
"""
if self._k <= 0:
raise ValueError(f"k must be greater than 0."
f"{self._k} received.")
coupled_pair = sort_and_couple(y_true, y_pred)
precision = 0.0
for idx, (label, score) in enumerate(coupled_pair):
if idx >= self._k:
break
if label > self._threshold:
precision += 1.
return precision / self._k
def __call__(self, y_true, y_pred):
coupled_pair = sort_and_couple(y_true, y_pred)
for idx, (label, pred) in enumerate(coupled_pair):
if idx+1>self._k:
return 0
if label > 0:
return 1. / (idx + 1)
return 0.
def __call__(self, y_true: np.array, y_pred: np.array) -> float:
"""
Calculate reciprocal of the rank of the first relevant item.
Example:
>>> import numpy as np
>>> y_pred = np.asarray([0.2, 0.3, 0.7, 1.0])
>>> y_true = np.asarray([1, 0, 0, 0])
>>> MeanReciprocalRank()(y_true, y_pred)
0.25
:param y_true: The ground true label of each document.
:param y_pred: The predicted scores of each document.
:return: Mean reciprocal rank.
"""
coupled_pair = sort_and_couple(y_true, y_pred)
for idx, (label, pred) in enumerate(coupled_pair):
if label > self._threshold:
return 1. / (idx + 1)
return 0.
def test_sort_and_couple():
l = [0, 1, 2]
s = [0.1, 0.4, 0.2]
c = sort_and_couple(l, s)
assert (c == np.array([(1, 0.4), (2, 0.2), (0, 0.1)])).all()