def test_precision_evaluator(eval_at, expected): matches_ids = [0, 1, 2, 3, 4] desired_ids = [1, 0, 20, 30, 40] evaluator = PrecisionEvaluator(eval_at=eval_at) assert evaluator.evaluate(actual=matches_ids, desired=desired_ids) == expected np.testing.assert_almost_equal(evaluator.mean, expected)
def test_evaluate_assert_doc_groundtruth_structure( simple_chunk_rank_evaluate_driver, eval_request_with_unmatching_struct): simple_chunk_rank_evaluate_driver.attach( executor=PrecisionEvaluator(eval_at=2), pea=None) simple_chunk_rank_evaluate_driver.eval_request = eval_request_with_unmatching_struct with pytest.raises(AssertionError): simple_chunk_rank_evaluate_driver()
def test_ranking_evaluate_runningavg_driver(runningavg_rank_evaluate_driver, ground_truth_pairs): runningavg_rank_evaluate_driver.attach(executor=PrecisionEvaluator(eval_at=2), runtime=None) runningavg_rank_evaluate_driver._apply_all(ground_truth_pairs) for pair in ground_truth_pairs: doc = pair.doc assert len(doc.evaluations) == 1 assert doc.evaluations[0].op_name == 'PrecisionEvaluator@2' assert doc.evaluations[0].value == 1.0
def test_precision_evaluator_no_groundtruth(): matches_ids = [0, 1, 2, 3, 4] desired_ids = [] evaluator = PrecisionEvaluator(eval_at=2) assert evaluator.evaluate(actual=matches_ids, desired=desired_ids) == 0.0 assert evaluator._running_stats._n == 1 np.testing.assert_almost_equal(evaluator.mean, 0.0)
def test_ranking_evaluate_driver(simple_rank_evaluate_driver, ground_truth_pairs): simple_rank_evaluate_driver.attach(executor=PrecisionEvaluator(eval_at=2), pea=None) simple_rank_evaluate_driver._apply_all(ground_truth_pairs) for pair in ground_truth_pairs: doc = pair.doc assert len(doc.evaluations) == 1 assert doc.evaluations[0].op_name == 'SimpleRankEvaluateDriver-Precision@2' assert doc.evaluations[0].value == 1.0
def test_precision_evaluator(eval_at, expected): matches_ids = [0, 1, 2, 3, 4] groundtruth_ids = [1, 0, 20, 30, 40] evaluator = PrecisionEvaluator(eval_at=eval_at) assert evaluator.evaluate(matches_ids=matches_ids, groundtruth_ids=groundtruth_ids) == expected assert evaluator.num_documents == 1 assert evaluator.sum == expected assert evaluator.avg == expected
def test_precision_evaluator_no_groundtruth(): matches_ids = [0, 1, 2, 3, 4] groundtruth_ids = [] evaluator = PrecisionEvaluator(eval_at=2) assert evaluator.evaluate(matches_ids=matches_ids, groundtruth_ids=groundtruth_ids) == 0.0 assert evaluator.num_documents == 1 assert evaluator.sum == 0.0 assert evaluator.avg == 0.0
def test_precision_evaluator_average(eval_at, expected_first): matches_ids = [[0, 1, 2, 3, 4], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1]] desired_ids = [[1, 0, 20, 30, 40], [1, 0, 20, 30, 40], [1, 0, 20, 30, 40]] evaluator = PrecisionEvaluator(eval_at=eval_at) assert evaluator.evaluate(actual=matches_ids[0], desired=desired_ids[0]) == expected_first assert evaluator.evaluate(actual=matches_ids[1], desired=desired_ids[1]) == 0.0 assert evaluator.evaluate(actual=matches_ids[2], desired=desired_ids[2]) == 0.0 assert evaluator._running_stats._n == 3 np.testing.assert_almost_equal(evaluator.mean, expected_first / 3)
def test_ranking_evaluate_driver_matches_in_chunks(simple_chunk_rank_evaluate_driver, eval_request): # this test proves that we can evaluate matches at chunk level, # proving that the driver can traverse in a parallel way docs and groundtruth simple_chunk_rank_evaluate_driver.attach(executor=PrecisionEvaluator(eval_at=2), pea=None) simple_chunk_rank_evaluate_driver.eval_request = eval_request simple_chunk_rank_evaluate_driver() assert len(eval_request.docs) == len(eval_request.groundtruths) assert len(eval_request.docs) == 10 for doc in eval_request.docs: assert len(doc.evaluations) == 0 # evaluation done at chunk level assert len(doc.chunks) == 1 chunk = doc.chunks[0] assert len(chunk.evaluations) == 1 # evaluation done at chunk level assert chunk.evaluations[0].op_name == 'SimpleChunkRankEvaluateDriver-Precision@2' assert chunk.evaluations[0].value == 1.0
def test_precision_evaluator_average(eval_at, expected_first): matches_ids = [[0, 1, 2, 3, 4], [-1, -1, -1, -1, -1], [-1, -1, -1, -1, -1]] groundtruth_ids = [[1, 0, 20, 30, 40], [1, 0, 20, 30, 40], [1, 0, 20, 30, 40]] evaluator = PrecisionEvaluator(eval_at=eval_at) assert evaluator.evaluate( matches_ids=matches_ids[0], groundtruth_ids=groundtruth_ids[0]) == expected_first assert evaluator.evaluate(matches_ids=matches_ids[1], groundtruth_ids=groundtruth_ids[1]) == 0.0 assert evaluator.evaluate(matches_ids=matches_ids[2], groundtruth_ids=groundtruth_ids[2]) == 0.0 assert evaluator.num_documents == 3 assert evaluator.sum == expected_first assert evaluator.avg == expected_first / 3