Exemple #1
0
    def test_mae_on_interval_with_unsorted_values(self):
        annotations = [
            RegressionAnnotation('identifier', -1),
            RegressionAnnotation('identifier',  2),
            RegressionAnnotation('identifier',  1)
        ]
        predictions = [
            RegressionPrediction('identifier', 1),
            RegressionPrediction('identifier', 3),
            RegressionPrediction('identifier', 1)
        ]
        config = [{'type': 'mae_on_interval', 'intervals': [2.0, 0.0, 4.0]}]
        expected = EvaluationResult(
            pytest.approx([0.0, 0.0, 1.0, 0.0]),
            None,
            'mae_on_interval',
            'mae_on_interval',
            None,
            None,
            {
                'postfix': ' ', 'scale': 1,
                'names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'],
                'orig_names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'],
                'calculate_mean': False,
                'target': 'higher-worse'
            },
            None
        )
        dispatcher = MetricsExecutor(config, None)

        dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
            assert evaluation_result == expected
    def test_classification_accuracy_result_for_batch_2_with_not_ordered_ids(
            self):
        annotations = [
            ClassificationAnnotation('identifier', 3),
            ClassificationAnnotation('identifier1', 1)
        ]
        predictions = [
            ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0]),
            ClassificationPrediction('identifier2', [1.0, 1.0, 1.0, 4.0])
        ]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
        metric_result = dispatcher.update_metrics_on_batch([42, 17],
                                                           annotations,
                                                           predictions)
        expected_metric_result = [
            PerImageMetricResult('accuracy', 'accuracy', 1.0, 'higher-better'),
            PerImageMetricResult('accuracy', 'accuracy', 0.0, 'higher-better')
        ]
        assert len(metric_result) == 2
        assert 42 in metric_result
        assert len(metric_result[42]) == 1
        assert metric_result[42][0] == expected_metric_result[0]
        assert 17 in metric_result
        assert len(metric_result[17]) == 1
        assert metric_result[17][0] == expected_metric_result[1]
Exemple #3
0
 def test_classification_per_class_accuracy_particual_prediction(self):
     annotation = [
         ClassificationAnnotation('identifier_1', 1),
         ClassificationAnnotation('identifier_2', 0),
         ClassificationAnnotation('identifier_3', 0)
     ]
     prediction = [
         ClassificationPrediction('identifier_1', [1.0, 2.0]),
         ClassificationPrediction('identifier_2', [2.0, 1.0]),
         ClassificationPrediction('identifier_3', [1.0, 5.0])
     ]
     config = {
         'annotation': 'mocked',
         'metrics': [{
             'type': 'accuracy_per_class',
             'top_k': 1
         }]
     }
     dataset = DummyDataset(label_map={0: '0', 1: '1'})
     dispatcher = MetricsExecutor(config, dataset)
     dispatcher.update_metrics_on_batch(annotation, prediction)
     for _, evaluation_result in dispatcher.iterate_metrics(
             annotation, prediction):
         assert evaluation_result.name == 'accuracy_per_class'
         assert len(evaluation_result.evaluated_value) == 2
         assert evaluation_result.evaluated_value[0] == pytest.approx(0.5)
         assert evaluation_result.evaluated_value[1] == pytest.approx(1.0)
         assert evaluation_result.reference_value is None
         assert evaluation_result.threshold is None
Exemple #4
0
    def test_mae_with_positive_diff_between_annotation_and_prediction(self):
        annotations = [
            RegressionAnnotation('identifier', 3),
            RegressionAnnotation('identifier2', 1)
        ]
        predictions = [
            RegressionPrediction('identifier', 1),
            RegressionPrediction('identifier2', -3)
        ]
        config = [{'type': 'mae'}]
        expected = EvaluationResult(
            pytest.approx([3.0, 1.0]), None, 'mae', 'mae', None, {
                'postfix': ' ',
                'scale': 1,
                'names': ['mean', 'std'],
                'calculate_mean': False,
                'target': 'higher-worse'
            })
        dispatcher = MetricsExecutor(config, None)

        dispatcher.update_metrics_on_batch(range(len(annotations)),
                                           annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(
                annotations, predictions):
            assert evaluation_result == expected
    def test_accuracy_with_wrong_annotation_type_raise_config_error_exception(self):
        annotations = [DetectionAnnotation('identifier', 3)]
        predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
    def test_classification_per_class_accuracy_prediction_top3(self):
        annotation = [
            ClassificationAnnotation('identifier_1', 1),
            ClassificationAnnotation('identifier_2', 1)
        ]
        prediction = [
            ClassificationPrediction('identifier_1', [1.0, 2.0, 3.0, 4.0]),
            ClassificationPrediction('identifier_2', [2.0, 1.0, 3.0, 4.0])
        ]
        dataset = DummyDataset(label_map={0: '0', 1: '1', 2: '2', 3: '3'})
        dispatcher = MetricsExecutor([{
            'type': 'accuracy_per_class',
            'top_k': 3
        }], dataset)

        dispatcher.update_metrics_on_batch(range(len(annotation)), annotation,
                                           prediction)

        for _, evaluation_result in dispatcher.iterate_metrics(
                annotation, prediction):
            assert evaluation_result.name == 'accuracy_per_class'
            assert len(evaluation_result.evaluated_value) == 4
            assert evaluation_result.evaluated_value[0] == pytest.approx(0.0)
            assert evaluation_result.evaluated_value[1] == pytest.approx(0.5)
            assert evaluation_result.evaluated_value[2] == pytest.approx(0.0)
            assert evaluation_result.evaluated_value[3] == pytest.approx(0.0)
            assert evaluation_result.reference_value is None
            assert evaluation_result.threshold is None
    def test_complete_accuracy_with_container_sources(self):
        annotations = [
            ContainerAnnotation(
                {'a': ClassificationAnnotation('identifier', 3)})
        ]
        predictions = [
            ContainerPrediction({
                'p':
                ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
            })
        ]
        config = [{
            'type': 'accuracy',
            'top_k': 1,
            'annotation_source': 'a',
            'prediction_source': 'p'
        }]

        dispatcher = MetricsExecutor(config, None)
        dispatcher.update_metrics_on_batch(range(len(annotations)),
                                           annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(
                annotations, predictions):
            assert evaluation_result.name == 'accuracy'
            assert evaluation_result.evaluated_value == pytest.approx(1.0)
            assert evaluation_result.reference_value is None
            assert evaluation_result.threshold is None
    def test_classification_accuracy_result_for_batch_1_with_2_metrics(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [
            ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
        ]

        dispatcher = MetricsExecutor([{
            'name': 'top1',
            'type': 'accuracy',
            'top_k': 1
        }, {
            'name': 'top3',
            'type': 'accuracy',
            'top_k': 3
        }], None)
        metric_result = dispatcher.update_metrics_on_batch(
            range(len(annotations)), annotations, predictions)
        expected_metric_result = [
            PerImageMetricResult('top1', 'accuracy', 1.0, 'higher-better'),
            PerImageMetricResult('top3', 'accuracy', 1.0, 'higher-better')
        ]
        assert len(metric_result) == 1
        assert 0 in metric_result
        assert len(metric_result[0]) == 2
        assert metric_result[0][0] == expected_metric_result[0]
        assert metric_result[0][1] == expected_metric_result[1]
Exemple #9
0
 def test_update_metric_result(self):
     dataset = multi_class_dataset()
     annotations = make_segmentation_representation(np.array([[1, 2, 3, 2, 3], [0, 0, 0, 0, 0]]), True)
     predictions = make_segmentation_representation(np.array([[1, 0, 3, 0, 0], [0, 0, 0, 0, 0]]), False)
     dispatcher = MetricsExecutor(create_config(self.name), dataset)
     metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
     assert metric_result[0][0].result == 0.5125
    def test_accuracy_with_unsupported_annotation_type_as_annotation_source_for_container_raises_config_error(self):
        annotations = [ContainerAnnotation({'annotation': DetectionAnnotation('identifier', 3)})]
        predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1, 'annotation_source': 'annotation'}], None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
Exemple #11
0
    def test_mae_on_interval_default_all_not_in_range_not_ignore_out_of_range(self):
        annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2)]
        predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 2)]
        expected = EvaluationResult(
            pytest.approx([2.0, 0.0, 0.0, 0.0]),
            None,
            'mae_on_interval',
            'mae_on_interval',
            None,
            None,
            {
                'postfix': ' ',
                'scale': 1,
                'names': ['mean: < 0.0', 'std: < 0.0', 'mean: > 1.0', 'std: > 1.0'],
                'calculate_mean': False,
                'target': 'higher-worse',
                'orig_names': ['mean: < 0.0', 'std: < 0.0', 'mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0', 'mean: > 1.0', 'std: > 1.0']
            },
            None
        )
        config = [{'type': 'mae_on_interval', 'end': 1, 'ignore_values_not_in_interval': False}]
        dispatcher = MetricsExecutor(config, None)

        dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
            assert evaluation_result == expected
    def test_mae_on_interval_values_in_range(self):
        annotations = [
            RegressionAnnotation('identifier', 0.5),
            RegressionAnnotation('identifier', 0.5)
        ]
        predictions = [
            RegressionPrediction('identifier', 1),
            RegressionPrediction('identifier', 0.25)
        ]
        config = [{'type': 'mae_on_interval', 'end': 1}]
        expected = EvaluationResult(
            pytest.approx([0.375, 0.125]), None, 'mae_on_interval',
            'mae_on_interval', None, {
                'postfix': ' ',
                'scale': 1,
                'names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0'],
                'calculate_mean': False
            })
        dispatcher = MetricsExecutor(config, None)

        dispatcher.update_metrics_on_batch(annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(
                annotations, predictions):
            assert evaluation_result == expected
Exemple #13
0
 def test_multi_class_not_matched(self):
     annotations = make_segmentation_representation(np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]), True)
     predictions = make_segmentation_representation(np.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 1]]), False)
     dispatcher = MetricsExecutor(create_config(self.name), multi_class_dataset())
     dispatcher.update_metrics_on_batch(annotations, predictions)
     expected = generate_expected_result(0.0, self.name)
     for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
         assert evaluation_result == expected
Exemple #14
0
 def test_multi_class(self):
     annotations = make_segmentation_representation(np.array([[1, 0, 3, 0, 0], [0, 0, 0, 0, 0]]), True)
     predictions = make_segmentation_representation(np.array([[1, 2, 3, 2, 3], [0, 0, 0, 0, 0]]), False)
     dispatcher = MetricsExecutor(create_config(self.name), multi_class_dataset())
     dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
     expected = generate_expected_result((5.0+1.0+1.0)/(8.0+1.0+1.0), self.name)
     for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
         assert evaluation_result == expected
    def test_accuracy_on_annotation_container_with_several_suitable_representations_config_value_error_exception(self):
        annotations = [ContainerAnnotation({'annotation1': ClassificationAnnotation('identifier', 3), 'annotation2': ClassificationAnnotation('identifier', 3)})]
        predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
        config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1}]}

        dispatcher = MetricsExecutor(config, None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
    def test_accuracy_on_container_with_wrong_annotation_source_name_raise_config_error_exception(self):
        annotations = [ContainerAnnotation({'annotation': ClassificationAnnotation('identifier', 3)})]
        predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
        config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1, 'annotation_source': 'a'}]}

        dispatcher = MetricsExecutor(config, None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
    def test_accuracy_with_unsupported_prediction_in_container_raise_config_error_exception(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ContainerPrediction({'prediction': DetectionPrediction('identifier', [1.0, 1.0, 1.0, 4.0])})]
        config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1}]}

        dispatcher = MetricsExecutor(config, None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
Exemple #18
0
    def test_update_mse_metric_result(self):
        annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)]
        predictions = [RegressionPrediction('identifier', 5), RegressionPrediction('identifier2', 5)]
        config = [{'type': 'mse'}]
        dispatcher = MetricsExecutor(config, None)

        metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
        assert metric_result[0][0].result == 4
        assert metric_result[1][0].result == 16
Exemple #19
0
    def test_update_normed_error(self):
        config = [{'type': 'normed_error'}]
        annotations = [FacialLandmarksAnnotation('identifier', np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]))]
        annotations[0].metadata.update({'left_eye': 0, 'right_eye': 1})
        predictions = [FacialLandmarksPrediction('identifier', np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]))]
        dispatcher = MetricsExecutor(config, None)

        metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
        assert metric_result[0][0].result == 0
Exemple #20
0
    def test_update_rmse_on_interval_metric(self):
        config = [{'type': 'rmse_on_interval', 'intervals': [0.0, 2.0, 4.0]}]
        annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)]
        predictions = [RegressionPrediction('identifier', 5), RegressionPrediction('identifier2', 5)]
        dispatcher = MetricsExecutor(config, None)

        metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
        assert metric_result[0][0].result == 2
        assert metric_result[1][0].result == 4
Exemple #21
0
 def test_multi_class(self):
     dataset = multi_class_dataset()
     annotations = make_segmentation_representation(np.array([[1, 2, 3, 2, 3], [0, 0, 0, 0, 0]]), True)
     predictions = make_segmentation_representation(np.array([[1, 0, 3, 0, 0], [0, 0, 0, 0, 0]]), False)
     dispatcher = MetricsExecutor(create_config(self.name), dataset)
     dispatcher.update_metrics_on_batch(annotations, predictions)
     expected = generate_expected_result([1.0, 1.0, 0.0, 0.5], self.name, dataset.labels)
     for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
         assert evaluation_result == expected
    def test_config_vector_presenter(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
        config = [{'type': 'accuracy', 'top_k': 1, 'presenter': 'print_vector'}]
        dispatcher = MetricsExecutor(config, None)
        dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)

        for presenter, _ in dispatcher.iterate_metrics(annotations, predictions):
            assert isinstance(presenter, VectorPrintPresenter)
Exemple #23
0
 def test_one_class(self):
     annotations = make_segmentation_representation(np.array([[0, 0], [0, 0]]), True)
     predictions = make_segmentation_representation(np.array([[0, 0], [0, 0]]), False)
     dataset = single_class_dataset()
     dispatcher = MetricsExecutor(create_config(self.name), dataset)
     dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
     expected = generate_expected_result([1.0], self.name, {0: dataset.labels[0]})
     for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
         assert evaluation_result == expected
    def test_mse_with_positive_diff_bitween_annotation_and_prediction(self):
        annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)]
        predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier2', -3)]
        config = {'annotation': 'mocked', 'metrics': [{'type': 'mse'}]}
        expected = EvaluationResult(pytest.approx([10.0, 6.0]), None, 'mse', None, {'postfix': ' ', 'scale': 1, 'names': ['mean', 'std'], 'calculate_mean': False})
        dispatcher = MetricsExecutor(config, None)
        dispatcher.update_metrics_on_batch(annotations, predictions)

        for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
            assert evaluation_result == expected
Exemple #25
0
 def test_one_class_update_metric_result(self):
     annotations = make_segmentation_representation(
         np.array([[0, 0], [0, 0]]), True)
     predictions = make_segmentation_representation(
         np.array([[0, 0], [0, 0]]), False)
     dispatcher = MetricsExecutor(create_config(self.name),
                                  single_class_dataset())
     metric_result = dispatcher.update_metrics_on_batch(
         range(len(annotations)), annotations, predictions)
     assert metric_result[0][0].result == 1
    def test_accuracy_on_prediction_container_with_several_suitable_representations_raise_config_error_exception(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ContainerPrediction({
            'prediction1': ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0]),
            'prediction2': ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
        })]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
        with pytest.raises(ConfigError):
            dispatcher.update_metrics_on_batch(annotations, predictions)
    def test_threshold_is_10_by_config(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 3, 'threshold': 10}], None)

        for _, evaluation_result in dispatcher.iterate_metrics([annotations], [predictions]):
            assert evaluation_result.name == 'accuracy'
            assert evaluation_result.evaluated_value == 0.0
            assert evaluation_result.reference_value is None
            assert evaluation_result.threshold == 10
    def test_zero_accuracy_top_3(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]

        dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 3}], None)

        for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
            assert evaluation_result.name == 'accuracy'
            assert evaluation_result.evaluated_value == 0.0
            assert evaluation_result.reference_value is None
            assert evaluation_result.threshold is None
 def test_mae_on_interval_default_all_missed(self):
     annotations = [RegressionAnnotation('identifier', -2)]
     predictions = [RegressionPrediction('identifier', 1)]
     config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'end': 1}]}
     expected = EvaluationResult(pytest.approx([0.0]), None, 'mae_on_interval', None, {'postfix': ' ', 'scale': 1, 'names': [], 'calculate_mean': False})
     dispatcher = MetricsExecutor(config, None)
     dispatcher.update_metrics_on_batch(annotations, predictions)
     with pytest.warns(UserWarning) as warnings:
         for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
             assert len(warnings) == 1
             assert evaluation_result == expected
    def test_reference_is_10_by_config(self):
        annotations = [ClassificationAnnotation('identifier', 3)]
        predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]
        config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 3, 'reference': 10}]}

        dispatcher = MetricsExecutor(config, None)

        for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
            assert evaluation_result.name == 'accuracy'
            assert evaluation_result.evaluated_value == 0.0
            assert evaluation_result.reference_value == 10
            assert evaluation_result.threshold is None