def testReadRetrievalPredictionsWorks(self):
# Define inputs.
file_path = os.path.join(tf.test.get_temp_dir(),
'retrieval_predictions.csv')
with tf.gfile.GFile(file_path, 'w') as f:
f.write('id,images\n')
f.write('0123456789abcdef,fedcba9876543250 \n')
f.write('0423456789abcdef,fedcba9876543260\n')
f.write('0223456789abcdef,fedcba9876543210 fedcba9876543200 '
'fedcba9876543220\n')
f.write('\n')
f.write('0523456789abcdef,\n')
public_ids = ['0223456789abcdef']
private_ids = ['0323456789abcdef', '0423456789abcdef']
ignored_ids = ['0123456789abcdef', '0523456789abcdef']
# Run tested function.
public_predictions, private_predictions = dataset_file_io.ReadPredictions(
file_path, public_ids, private_ids, ignored_ids,
dataset_file_io.RETRIEVAL_TASK_ID)
# Define expected results.
expected_public_predictions = {
'0223456789abcdef':
['fedcba9876543210', 'fedcba9876543200', 'fedcba9876543220']
}
expected_private_predictions = {
'0423456789abcdef': ['fedcba9876543260']
}
# Compare actual and expected results.
self.assertEqual(public_predictions, expected_public_predictions)
self.assertEqual(private_predictions, expected_private_predictions)
def main(argv):
if len(argv) > 1:
raise RuntimeError('Too many command-line arguments.')
# Read solution.
print('Reading solution...')
public_solution, private_solution, ignored_ids = dataset_file_io.ReadSolution(
cmd_args.solution_path, dataset_file_io.RETRIEVAL_TASK_ID)
print('done!')
# Read predictions.
print('Reading predictions...')
public_predictions, private_predictions = dataset_file_io.ReadPredictions(
cmd_args.predictions_path, set(public_solution.keys()),
set(private_solution.keys()), set(ignored_ids),
dataset_file_io.RETRIEVAL_TASK_ID)
print('done!')
# Mean average precision.
print('**********************************************')
print('(Public) Mean Average Precision: %f' %
metrics.MeanAveragePrecision(public_predictions, public_solution))
print('(Private) Mean Average Precision: %f' %
metrics.MeanAveragePrecision(private_predictions, private_solution))
# Mean precision@k.
print('**********************************************')
public_precisions = 100.0 * metrics.MeanPrecisions(public_predictions,
public_solution)
private_precisions = 100.0 * metrics.MeanPrecisions(private_predictions,
private_solution)
print('(Public) Mean precisions: P@1: %.2f, P@5: %.2f, P@10: %.2f, '
'P@50: %.2f, P@100: %.2f' %
(public_precisions[0], public_precisions[4], public_precisions[9],
public_precisions[49], public_precisions[99]))
print('(Private) Mean precisions: P@1: %.2f, P@5: %.2f, P@10: %.2f, '
'P@50: %.2f, P@100: %.2f' %
(private_precisions[0], private_precisions[4], private_precisions[9],
private_precisions[49], private_precisions[99]))
# Mean/median position of first correct.
print('**********************************************')
public_mean_position, public_median_position = metrics.MeanMedianPosition(
public_predictions, public_solution)
private_mean_position, private_median_position = metrics.MeanMedianPosition(
private_predictions, private_solution)
print('(Public) Mean position: %.2f, median position: %.2f' %
(public_mean_position, public_median_position))
print('(Private) Mean position: %.2f, median position: %.2f' %
(private_mean_position, private_median_position))
def main(argv):
if len(argv) > 1:
raise RuntimeError('Too many command-line arguments.')
# Read solution.
print('Reading solution...')
public_solution, private_solution, ignored_ids = dataset_file_io.ReadSolution(
cmd_args.solution_path, dataset_file_io.RECOGNITION_TASK_ID)
print('done!')
# Read predictions.
print('Reading predictions...')
public_predictions, private_predictions = dataset_file_io.ReadPredictions(
cmd_args.predictions_path, set(public_solution.keys()),
set(private_solution.keys()), set(ignored_ids),
dataset_file_io.RECOGNITION_TASK_ID)
print('done!')
# Global Average Precision.
print('**********************************************')
print('(Public) Global Average Precision: %f' %
metrics.GlobalAveragePrecision(public_predictions, public_solution))
print(
'(Private) Global Average Precision: %f' %
metrics.GlobalAveragePrecision(private_predictions, private_solution))
# Global Average Precision ignoring non-landmark queries.
print('**********************************************')
print(
'(Public) Global Average Precision ignoring non-landmark queries: %f'
% metrics.GlobalAveragePrecision(public_predictions,
public_solution,
ignore_non_gt_test_images=True))
print(
'(Private) Global Average Precision ignoring non-landmark queries: %f'
% metrics.GlobalAveragePrecision(private_predictions,
private_solution,
ignore_non_gt_test_images=True))
# Top-1 accuracy.
print('**********************************************')
print('(Public) Top-1 accuracy: %.2f' %
(100.0 * metrics.Top1Accuracy(public_predictions, public_solution)))
print(
'(Private) Top-1 accuracy: %.2f' %
(100.0 * metrics.Top1Accuracy(private_predictions, private_solution)))
def testReadRecognitionPredictionsWorks(self):
# Define inputs.
file_path = os.path.join(tf.test.get_temp_dir(),
'recognition_predictions.csv')
with tf.gfile.GFile(file_path, 'w') as f:
f.write('id,landmarks\n')
f.write('0123456789abcdef,12 0.1 \n')
f.write('0423456789abcdef,0 19.0\n')
f.write('0223456789abcdef,\n')
f.write('\n')
f.write('0523456789abcdef,14 0.01\n')
public_ids = ['0123456789abcdef', '0223456789abcdef']
private_ids = ['0423456789abcdef']
ignored_ids = ['0323456789abcdef', '0523456789abcdef']
# Run tested function.
public_predictions, private_predictions = dataset_file_io.ReadPredictions(
file_path, public_ids, private_ids, ignored_ids,
dataset_file_io.RECOGNITION_TASK_ID)
# Define expected results.
expected_public_predictions = {
'0123456789abcdef': {
'class': 12,
'score': 0.1
}
}
expected_private_predictions = {
'0423456789abcdef': {
'class': 0,
'score': 19.0
}
}
# Compare actual and expected results.
self.assertEqual(public_predictions, expected_public_predictions)
self.assertEqual(private_predictions, expected_private_predictions)
