def test_top_selector_find_top_k_binary_k_larger_than_length():
ts = TopSelector()
tst = TopSelectorTorch()
with pytest.raises(ValueError):
ts.find_top_k_binary(values, values.shape[1])
with pytest.raises(ValueError):
tst.find_top_k_binary(torch.from_numpy(values), values.shape[1])
def test_top_selector_find_top_k_binary_values_none():
ts = TopSelector()
tst = TopSelectorTorch()
with pytest.raises(TypeError):
ts.find_top_k_binary(None, k)
with pytest.raises(TypeError):
tst.find_top_k_binary(None, k)
def test_top_selector_find_top_k_binary_k_negative():
ts = TopSelector()
tst = TopSelectorTorch()
with pytest.raises(ValueError):
ts.find_top_k_binary(values, -1)
with pytest.raises(ValueError):
tst.find_top_k_binary(torch.from_numpy(values), -1)
def test_top_selector_find_top_k_binary_k_not_int():
ts = TopSelector()
tst = TopSelectorTorch()
with pytest.raises(TypeError):
ts.find_top_k_binary(values, 2.5)
with pytest.raises(TypeError):
tst.find_top_k_binary(torch.from_numpy(values), 2.5)
def test_top_selector_find_top_k_binary_correct_case_zeros():
ts = TopSelector()
tst = TopSelectorTorch()
values = np.zeros((3, 3))
ground_truth = np.zeros((3, 3), dtype=bool)
assert np.all(ts.find_top_k_binary(values, k) == ground_truth)
assert torch.equal(tst.find_top_k_binary(torch.from_numpy(values), k),
torch.from_numpy(ground_truth))
def test_top_selector_find_top_k_binary_correct_case():
ts = TopSelector()
tst = TopSelectorTorch()
values = np.array([[0.4, 0.7, 0.1], [0.1, 0.3, 0.6], [0.02, 0.25, 0.2]])
ground_truth = np.array([[True, True, False], [False, True, True],
[False, True, True]])
assert np.all(ts.find_top_k_binary(values, k) == ground_truth)
assert torch.equal(tst.find_top_k_binary(torch.from_numpy(values), k),
torch.from_numpy(ground_truth))
def __init__(self, k):
"""Inits HitRatioAtK with its k value.
k must be greater than 0.
Raises:
TypeError: The k value is not an integer or is not set.
ValueError: The k value is smaller than 1.
"""
super().__init__('Hit Ratio', k)
self._top_selector = TopSelector()
def __init__(self, k, revenue):
"""Inits RevenueAtK with its k value and revenue.
k must be greater than 0.
Raises:
TypeError: The k value is not an integer or is not set. The revenue
is not set or is of the incorrect type. The revenue
contains something other than floats and/or integers
ValueError: The k value is smaller than 1. The revenue is empty.
"""
super().__init__('Revenue', k)
if revenue is None:
raise TypeError('Argument: revenue must be set.')
elif not isinstance(revenue, np.ndarray) or revenue.ndim != 1:
raise TypeError('Argument: revenue must be a 1D numpy array.')
elif revenue.size == 0:
raise ValueError('Argument: revenue must not be an empty array.')
elif not all(
isinstance(i, (np.floating, float, np.integer, int))
for i in revenue):
raise TypeError('All elements of argument: revenue must be' +
' of type int or float.')
self._revenue = revenue
self._top_selector = TopSelector()
def test_top_selector_find_top_k_binary_values_not_2D():
ts = TopSelector()
tst = TopSelectorTorch()
with pytest.raises(ValueError):
ts.find_top_k_binary(np.empty((2, 3, 3)), k)
with pytest.raises(ValueError):
ts.find_top_k_binary(np.empty(3), k)
with pytest.raises(ValueError):
tst.find_top_k_binary(torch.from_numpy(np.empty((2, 3, 3))), k)
with pytest.raises(ValueError):
tst.find_top_k_binary(torch.from_numpy(np.empty(3)), k)
class HitRatioAtK(MetricAtK):
"""HitRatioAtK class. Inherits the MetricAtK class.
The HitRatioAtK is used to calculate the hit ratio metric.
Attributes:
_top_selector: A class used to extract top results used in hit ratio
calculations.
"""
def __init__(self, k):
"""Inits HitRatioAtK with its k value.
k must be greater than 0.
Raises:
TypeError: The k value is not an integer or is not set.
ValueError: The k value is smaller than 1.
"""
super().__init__('Hit Ratio', k)
self._top_selector = TopSelector()
def evaluate(self, y_true, y_pred):
"""Evaluates the given predictions with the hit ratio metric.
Calculates the hit ratio on the passed predicted and true values at k.
Args:
y_true: A PyTorch tensor of true values. Only one value per row
can be > 0!
y_pred: A PyTorch tensor of predicted values.
Returns:
Will return a float with the calculated hit ratio value. The hit
ratio is defined as follows:
math::
HR@K = \\frac{Number of Hits @ K}{Number of Ground Truth Items(=1)}
This is then averaged over all sets of predictions/ground truths
(users).
From:
https://www.comp.nus.edu.sg/~kanmy/papers/cikm15-trirank-cr.pdf
Raises:
TypeError: An error occured while accessing the arguments -
one of the arguments is NoneType.
ValueError: An error occured when checking the dimensions of the
y_pred and y_true arguments. One or both are not a 2D arrays,
or they are 2D but of different sizes along those dimensions.
If y_true has more than one true value per row this error
is raised. This is also raised if the output is not in [0,1].
"""
self._check_input(y_true, y_pred)
y_true = y_true.cpu().numpy()
y_pred = y_pred.cpu().numpy()
self._check_args_numpy(y_pred, y_true)
# Check only one ground truth value = 1 per row in y_true.
y_true[y_true > 0] = 1
y_true[y_true < 0] = 0
for x in np.sum(y_true, axis=1):
if x != 1:
raise ValueError('Incorrect format of argument: y_true. \
Input must have only one true value \
per row.')
y_pred_binary = self._top_selector.find_top_k_binary(y_pred, self._k)
y_true_binary = (y_true > 0)
result = (np.logical_and(y_true_binary, y_pred_binary)
.sum(axis=1)).astype(np.float32).mean()
if not (0 <= result <= 1):
raise ValueError('The output of HitRatioAtK.evaluate \
must be in [0,1]')
return result
class RevenueAtK(MetricAtK):
"""RevenueAtK class. Inherits the MetricAtK class.
The RevenueAtK is used to calculate the revenue metric.
Attributes:
_revenue: The revenue of items on which the metric will be calculated
_top_selector: A class used to extract top results used in revenue
calculations.
"""
def __init__(self, k, revenue):
"""Inits RevenueAtK with its k value and revenue.
k must be greater than 0.
Raises:
TypeError: The k value is not an integer or is not set. The revenue
is not set or is of the incorrect type. The revenue
contains something other than floats and/or integers
ValueError: The k value is smaller than 1. The revenue is empty.
"""
super().__init__('Revenue', k)
if revenue is None:
raise TypeError('Argument: revenue must be set.')
elif not isinstance(revenue, np.ndarray) or revenue.ndim != 1:
raise TypeError('Argument: revenue must be a 1D numpy array.')
elif revenue.size == 0:
raise ValueError('Argument: revenue must not be an empty array.')
elif not all(
isinstance(i, (np.floating, float, np.integer, int))
for i in revenue):
raise TypeError('All elements of argument: revenue must be' +
' of type int or float.')
self._revenue = revenue
self._top_selector = TopSelector()
def evaluate(self, y_true, y_pred):
"""Evaluates the given predictions with the revenue metric.
Calculates the revenue on the passed predicted and true values at k.
Args:
y_true: A PyTorch tensor of true values.
y_pred: A PyTorch tensor of predicted values.
Returns:
Will return a float with the calculated revenue value. The revenue
is defined as the revenue of the relevant predicted values.
math:: Revenue@K = \\sum_{i \\in R_{relevant & recommended}}value_i
Raises:
TypeError: An error occured while accessing the arguments -
one of the arguments is NoneType.
ValueError: An error occured when checking the dimensions of the
y_pred and y_true arguments. One or both are not a 2D arrays,
or they are 2D but of different sizes along those dimensions.
Also occurs if passed arguments are not the same along the
first dimension as the revenue stored in the object.
"""
self._check_input(y_true, y_pred)
y_true = y_true.cpu().numpy()
y_pred = y_pred.cpu().numpy()
self._check_args_numpy(y_pred, y_true)
if y_pred.shape[1] != len(self._revenue):
raise ValueError('Arguments must have axis 1 of the same size as\
the revenue.')
y_pred_binary = self._top_selector.find_top_k_binary(y_pred, self._k)
y_true_binary = (y_true > 0)
tmp = np.logical_and(y_true_binary, y_pred_binary)
revenue = 0
for i in range(tmp.shape[0]):
revenue += np.sum(self._revenue[tmp[i]])
return revenue / float(tmp.shape[0])
class PrecisionAtK(MetricAtK):
"""PrecisionAtK class. Inherits the MetricAtK class.
The PrecisionAtK is used to calculate the precision metric.
Attributes:
_top_selector: A class used to extract top results used in precision
calculations.
"""
def __init__(self, k):
"""Inits PrecisionAtK with its k value.
k must be greater than 0.
Raises:
TypeError: The k value is not an integer or is not set.
ValueError: The k value is smaller than 1.
"""
super().__init__('Precision', k)
self._top_selector = TopSelector()
def evaluate(self, y_true, y_pred):
"""Evaluates the given predictions with the precision metric.
Calculates the precision on the passed predicted and true values at k.
Args:
y_true: A PyTorch tensor of true values.
y_pred: A PyTorch tensor of predicted values.
Returns:
Will return a float with the calculated precision value. The
precision for one set of predictions is defined as follows:
Precision@K = (# of recommended items @k that are relevant) /
(# of recommended items @k)
math::
Precision@K = \\frac{n_{relevant & recommended}}{min(k,
n_{recommended})}
Raises:
TypeError: An error occured while accessing the arguments -
one of the arguments is NoneType.
ValueError: An error occured when checking the dimensions of the
y_pred and y_true arguments. One or both are not a 2D arrays,
or they are 2D but of different sizes along those dimensions.
This is also raised if the output is not in [0,1].
"""
self._check_input(y_true, y_pred)
y_true = y_true.cpu().numpy()
y_pred = y_pred.cpu().numpy()
self._check_args_numpy(y_pred, y_true)
y_pred_binary = self._top_selector.find_top_k_binary(y_pred, self._k)
y_true_binary = (y_true > 0)
tmp = (np.logical_and(y_true_binary,
y_pred_binary).sum(axis=1)).astype(np.float32)
precision = tmp / np.minimum(
self._k, np.maximum(1,
np.ones_like(y_pred_binary).sum(axis=1)))
result = precision.mean()
if not (0 <= result <= 1):
raise ValueError('The output of PrecisionAtK.evaluate \
must be in [0,1]')
return result