def test_classification_per_class_accuracy_particual_prediction(self):
annotation = [
ClassificationAnnotation('identifier_1', 1),
ClassificationAnnotation('identifier_2', 0),
ClassificationAnnotation('identifier_3', 0)
]
prediction = [
ClassificationPrediction('identifier_1', [1.0, 2.0]),
ClassificationPrediction('identifier_2', [2.0, 1.0]),
ClassificationPrediction('identifier_3', [1.0, 5.0])
]
config = {
'annotation': 'mocked',
'metrics': [{
'type': 'accuracy_per_class',
'top_k': 1
}]
}
dataset = DummyDataset(label_map={0: '0', 1: '1'})
dispatcher = MetricsExecutor(config, dataset)
dispatcher.update_metrics_on_batch(annotation, prediction)
for _, evaluation_result in dispatcher.iterate_metrics(
annotation, prediction):
assert evaluation_result.name == 'accuracy_per_class'
assert len(evaluation_result.evaluated_value) == 2
assert evaluation_result.evaluated_value[0] == pytest.approx(0.5)
assert evaluation_result.evaluated_value[1] == pytest.approx(1.0)
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_classification_per_class_accuracy_prediction_top3(self):
annotation = [
ClassificationAnnotation('identifier_1', 1),
ClassificationAnnotation('identifier_2', 1)
]
prediction = [
ClassificationPrediction('identifier_1', [1.0, 2.0, 3.0, 4.0]),
ClassificationPrediction('identifier_2', [2.0, 1.0, 3.0, 4.0])
]
dataset = DummyDataset(label_map={0: '0', 1: '1', 2: '2', 3: '3'})
dispatcher = MetricsExecutor([{
'type': 'accuracy_per_class',
'top_k': 3
}], dataset)
dispatcher.update_metrics_on_batch(range(len(annotation)), annotation,
prediction)
for _, evaluation_result in dispatcher.iterate_metrics(
annotation, prediction):
assert evaluation_result.name == 'accuracy_per_class'
assert len(evaluation_result.evaluated_value) == 4
assert evaluation_result.evaluated_value[0] == pytest.approx(0.0)
assert evaluation_result.evaluated_value[1] == pytest.approx(0.5)
assert evaluation_result.evaluated_value[2] == pytest.approx(0.0)
assert evaluation_result.evaluated_value[3] == pytest.approx(0.0)
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_classification_accuracy_result_for_batch_2_with_not_ordered_ids(
self):
annotations = [
ClassificationAnnotation('identifier', 3),
ClassificationAnnotation('identifier1', 1)
]
predictions = [
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0]),
ClassificationPrediction('identifier2', [1.0, 1.0, 1.0, 4.0])
]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
metric_result = dispatcher.update_metrics_on_batch([42, 17],
annotations,
predictions)
expected_metric_result = [
PerImageMetricResult('accuracy', 'accuracy', 1.0, 'higher-better'),
PerImageMetricResult('accuracy', 'accuracy', 0.0, 'higher-better')
]
assert len(metric_result) == 2
assert 42 in metric_result
assert len(metric_result[42]) == 1
assert metric_result[42][0] == expected_metric_result[0]
assert 17 in metric_result
assert len(metric_result[17]) == 1
assert metric_result[17][0] == expected_metric_result[1]
def test_accuracy_on_annotation_container_with_several_suitable_representations_config_value_error_exception(self):
annotations = [ContainerAnnotation({
'annotation1': ClassificationAnnotation('identifier', 3),
'annotation2': ClassificationAnnotation('identifier', 3)
})]
predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(annotations, predictions)
def test_accuracy_with_wrong_prediction_type_raise_config_error_exception(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [DetectionPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(annotations, predictions)
def test_filter_annotations_unsupported_source_type_in_container_raise_type_error_exception(self):
config = [{'type': 'filter', 'annotation_source': 'annotation', 'labels': ['to_be_filtered']}]
annotation = ContainerAnnotation({'annotation': ClassificationAnnotation()})
executor = PostprocessingExecutor(config)
with pytest.raises(TypeError):
postprocess_data(executor, [annotation], [None])
def test_classification_accuracy_result_for_batch_1_with_2_metrics(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
]
dispatcher = MetricsExecutor([{
'name': 'top1',
'type': 'accuracy',
'top_k': 1
}, {
'name': 'top3',
'type': 'accuracy',
'top_k': 3
}], None)
metric_result = dispatcher.update_metrics_on_batch(
range(len(annotations)), annotations, predictions)
expected_metric_result = [
PerImageMetricResult('top1', 'accuracy', 1.0, 'higher-better'),
PerImageMetricResult('top3', 'accuracy', 1.0, 'higher-better')
]
assert len(metric_result) == 1
assert 0 in metric_result
assert len(metric_result[0]) == 2
assert metric_result[0][0] == expected_metric_result[0]
assert metric_result[0][1] == expected_metric_result[1]
def test_complete_accuracy_with_container_sources(self):
annotations = [
ContainerAnnotation(
{'a': ClassificationAnnotation('identifier', 3)})
]
predictions = [
ContainerPrediction({
'p':
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
})
]
config = [{
'type': 'accuracy',
'top_k': 1,
'annotation_source': 'a',
'prediction_source': 'p'
}]
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == pytest.approx(1.0)
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_accuracy_with_unsupported_prediction_type_as_prediction_source_for_container_raises_config_error(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ContainerPrediction({'prediction': DetectionPrediction('identifier', [1.0, 1.0, 1.0, 4.0])})]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1, 'prediction_source': 'prediction'}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(annotations, predictions)
def test_accuracy_on_container_with_wrong_annotation_source_name_raise_config_error_exception(self):
annotations = [ContainerAnnotation({'annotation': ClassificationAnnotation('identifier', 3)})]
predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1, 'annotation_source': 'a'}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(annotations, predictions)
def test_filter_predictions_unsupported_source_type_raise_type_error_exception(self):
config = [{'type': 'filter', 'prediction_source': 'detection_out', 'labels': ['to_be_filtered'],
'remove_filtered': False}]
prediction = ContainerPrediction({'detection_out': ClassificationAnnotation()})
executor = PostprocessingExecutor(config)
with pytest.raises(TypeError):
postprocess_data(executor, [None], [prediction])
def test_accuracy_with_unsupported_prediction_in_container_raise_config_error_exception(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ContainerPrediction({'prediction': DetectionPrediction('identifier', [1.0, 1.0, 1.0, 4.0])})]
config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1}]}
dispatcher = MetricsExecutor(config, None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(annotations, predictions)
def test_config_vector_presenter(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
config = [{'type': 'accuracy', 'top_k': 1, 'presenter': 'print_vector'}]
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
for presenter, _ in dispatcher.iterate_metrics(annotations, predictions):
assert isinstance(presenter, VectorPrintPresenter)
def test_zero_accuracy_top_3(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 3}], None)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == 0.0
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_threshold_is_10_by_config(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 3, 'threshold': 10}], None)
for _, evaluation_result in dispatcher.iterate_metrics([annotations], [predictions]):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == 0.0
assert evaluation_result.reference_value is None
assert evaluation_result.threshold == 10
def test_complete_accuracy_top_3(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ClassificationPrediction('identifier', [1.0, 3.0, 4.0, 2.0])]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 3}], None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == pytest.approx(1.0)
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_reference_is_10_by_config(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [ClassificationPrediction('identifier', [5.0, 3.0, 4.0, 1.0])]
config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 3, 'reference': 10}]}
dispatcher = MetricsExecutor(config, None)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == 0.0
assert evaluation_result.reference_value == 10
assert evaluation_result.threshold is None
def test_zero_accuracy(self):
annotation = [ClassificationAnnotation('identifier', 2)]
prediction = [ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])]
config = {'annotation': 'mocked', 'metrics': [{'type': 'accuracy', 'top_k': 1}]}
dispatcher = MetricsExecutor(config, None)
for _, evaluation_result in dispatcher.iterate_metrics([annotation], [prediction]):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == 0.0
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_classification_per_class_accuracy_fully_zero_prediction(self):
annotation = ClassificationAnnotation('identifier', 0)
prediction = ClassificationPrediction('identifier', [1.0, 2.0])
dataset = DummyDataset(label_map={0: '0', 1: '1'})
dispatcher = MetricsExecutor([{'type': 'accuracy_per_class', 'top_k': 1}], dataset)
dispatcher.update_metrics_on_batch([annotation], [prediction])
for _, evaluation_result in dispatcher.iterate_metrics([annotation], [prediction]):
assert evaluation_result.name == 'accuracy_per_class'
assert len(evaluation_result.evaluated_value) == 2
assert evaluation_result.evaluated_value[0] == pytest.approx(0.0)
assert evaluation_result.evaluated_value[1] == pytest.approx(0.0)
assert evaluation_result.reference_value is None
assert evaluation_result.threshold is None
def test_config_default_presenter(self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
]
config = {
'annotation': 'mocked',
'metrics': [{
'type': 'accuracy',
'top_k': 1
}]
}
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for presenter, _ in dispatcher.iterate_metrics(annotations,
predictions):
assert isinstance(presenter, ScalarPrintPresenter)
def _convert_annotations(test_dir, labels):
"""Create annotation representations list"""
annotations = []
# iterate over all png images in test directory
for image in test_dir.glob("*.png"):
# get file name (e.g. from /foo/bar/image.png we get image.png)
image_base = str(image.parts[-1])
# extract class name from file name
regex_match = re.match(FILE_PATTERN_REGEX, image_base)
image_label = regex_match.group(1)
# look up class index in label list
class_id = labels.index(image_label)
# create annotation representation object
annotations.append(ClassificationAnnotation(image_base, class_id))
return annotations
