def test_complete_accuracy_with_container_sources(self):
annotations = [
ContainerAnnotation(
{'a': ClassificationAnnotation('identifier', 3)})
]
predictions = [
ContainerPrediction({
'p':
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
})
]
config = [{
'type': 'accuracy',
'top_k': 1,
'annotation_source': 'a',
'prediction_source': 'p'
}]
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result.name == 'accuracy'
assert evaluation_result.evaluated_value == pytest.approx(1.0)
assert evaluation_result.reference_value is None
assert evaluation_result.abs_threshold is None
assert evaluation_result.rel_threshold is None
def test_accuracy_with_unsupported_prediction_in_container_raise_config_error_exception(
self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [
ContainerPrediction({
'prediction':
DetectionPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
})
]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
def test_accuracy_on_prediction_container_with_several_suitable_representations_raise_config_error_exception(
self):
annotations = [ClassificationAnnotation('identifier', 3)]
predictions = [
ContainerPrediction({
'prediction1':
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0]),
'prediction2':
ClassificationPrediction('identifier', [1.0, 1.0, 1.0, 4.0])
})
]
dispatcher = MetricsExecutor([{'type': 'accuracy', 'top_k': 1}], None)
with pytest.raises(ConfigError):
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)