def testComputePRAtRanksWorks(self):
# Define inputs.
positive_ranks = np.array([0, 2, 5])
desired_pr_ranks = np.array([1, 5, 10])
# Run tested function.
precisions, recalls = dataset.ComputePRAtRanks(positive_ranks,
desired_pr_ranks)
# Define expected outputs.
expected_precisions = [1.0, 0.4, 0.5]
expected_recalls = [0.333333, 0.666667, 1.0]
# Compare actual versus expected.
self.assertAllClose(precisions, expected_precisions)
self.assertAllClose(recalls, expected_recalls)
def ComputeMetrics(sorted_index_ids, ground_truth, desired_pr_ranks):
"""Computes metrics for retrieval results on the Revisited datasets.
If there are no valid ground-truth index images for a given query, the metric
results for the given query (`average_precisions`, `precisions` and `recalls`)
are set to NaN, and they are not taken into account when computing the
aggregated metrics (`mean_average_precision`, `mean_precisions` and
`mean_recalls`) over all queries.
Args:
sorted_index_ids: Integer NumPy array of shape [#queries, #index_images].
For each query, contains an array denoting the most relevant index images,
sorted from most to least relevant.
ground_truth: List containing ground-truth information for dataset. Each
entry is a dict corresponding to the ground-truth information for a query.
The dict has keys 'ok' and 'junk', mapping to a NumPy array of integers.
desired_pr_ranks: List of integers containing the desired precision/recall
ranks to be reported. Eg, if precision@1/recall@1 and
precision@10/recall@10 are desired, this should be set to [1, 10]. The
largest item should be <= #index_images.
Returns:
mean_average_precision: Mean average precision (float).
mean_precisions: Mean precision @ `desired_pr_ranks` (NumPy array of
floats, with shape [len(desired_pr_ranks)]).
mean_recalls: Mean recall @ `desired_pr_ranks` (NumPy array of floats, with
shape [len(desired_pr_ranks)]).
average_precisions: Average precision for each query (NumPy array of floats,
with shape [#queries]).
precisions: Precision @ `desired_pr_ranks`, for each query (NumPy array of
floats, with shape [#queries, len(desired_pr_ranks)]).
recalls: Recall @ `desired_pr_ranks`, for each query (NumPy array of
floats, with shape [#queries, len(desired_pr_ranks)]).
Raises:
ValueError: If largest desired PR rank in `desired_pr_ranks` >
#index_images.
"""
num_queries, num_index_images = sorted_index_ids.shape
num_desired_pr_ranks = len(desired_pr_ranks)
sorted_desired_pr_ranks = sorted(desired_pr_ranks)
if sorted_desired_pr_ranks[-1] > num_index_images:
raise ValueError(
'Requested PR ranks up to %d, however there are only %d images' %
(sorted_desired_pr_ranks[-1], num_index_images))
# Instantiate all outputs, then loop over each query and gather metrics.
mean_average_precision = 0.0
mean_precisions = np.zeros([num_desired_pr_ranks])
mean_recalls = np.zeros([num_desired_pr_ranks])
average_precisions = np.zeros([num_queries])
precisions = np.zeros([num_queries, num_desired_pr_ranks])
recalls = np.zeros([num_queries, num_desired_pr_ranks])
num_empty_gt_queries = 0
for i in range(num_queries):
index_ground_truth = np.array(ground_truth[i])
if not index_ground_truth.size:
average_precisions[i] = float('nan')
precisions[i, :] = float('nan')
recalls[i, :] = float('nan')
num_empty_gt_queries += 1
continue
positive_ranks = np.arange(num_index_images)[np.in1d(sorted_index_ids[i],
index_ground_truth)]
average_precisions[i] = dataset.ComputeAveragePrecision(positive_ranks)
precisions[i, :], recalls[i, :] = dataset.ComputePRAtRanks(positive_ranks, desired_pr_ranks)
mean_average_precision += average_precisions[i]
mean_precisions += precisions[i, :]
mean_recalls += recalls[i, :]
# Normalize aggregated metrics by number of queries.
num_valid_queries = num_queries - num_empty_gt_queries
mean_average_precision /= num_valid_queries
mean_precisions /= num_valid_queries
mean_recalls /= num_valid_queries
return (mean_average_precision, mean_precisions, mean_recalls,
average_precisions, precisions, recalls)