def test_specific_format_for_scalar_presenter_with_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[0.456],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={
'scale': 0.5,
'postfix': 'km/h',
'data_format': '{:.4f}'
},
profiling_file=None)
presenter = ScalarPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
mock_write_scalar_res.assert_called_once_with(np.mean(
result.evaluated_value),
result.name,
result.reference_value,
result.abs_threshold,
result.rel_threshold,
postfix=' ',
scale=1,
result_format='{}')
def test_vector_presenter_with_vector_data_contain_one_element(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={'names': ['prediction']},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(
result.evaluated_value[0],
result.name,
None,
result.abs_threshold,
result.rel_threshold,
postfix='%',
scale=100,
value_name=result.meta['names'][0],
result_format='{:.2f}')
def test_mae_with_positive_diff_between_annotation_and_prediction(self):
annotations = [
RegressionAnnotation('identifier', 3),
RegressionAnnotation('identifier2', 1)
]
predictions = [
RegressionPrediction('identifier', 1),
RegressionPrediction('identifier2', -3)
]
config = [{'type': 'mae'}]
expected = EvaluationResult(
pytest.approx([3.0, 1.0]), None, 'mae', 'mae', None, None, {
'postfix': ' ',
'scale': 1,
'names': ['mean', 'std'],
'calculate_mean': False,
'target': 'higher-worse'
}, None)
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result == expected
def test_mae_on_interval_values_in_range(self):
annotations = [
RegressionAnnotation('identifier', 0.5),
RegressionAnnotation('identifier', 0.5)
]
predictions = [
RegressionPrediction('identifier', 1),
RegressionPrediction('identifier', 0.25)
]
config = [{'type': 'mae_on_interval', 'end': 1}]
expected = EvaluationResult(
pytest.approx([0.375, 0.125]), None, 'mae_on_interval',
'mae_on_interval', None, None, {
'postfix': ' ',
'scale': 1,
'names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0'],
'calculate_mean': False,
'target': 'higher-worse',
'orig_names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0']
}, None)
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result == expected
def test_vector_presenter_with_vector_data_with_dict_ref_without_represented_classes(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value={
'class3': 0.4,
'class4': 0.5
},
abs_threshold=None,
rel_threshold=None,
meta={
'names': ['class1', 'class2'],
'scale': [1, 1]
},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result)
calls = [
call(result.evaluated_value[0],
result.name,
None,
None,
None,
postfix='%',
scale=result.meta['scale'][0],
result_format='{:.2f}',
value_name=result.meta['names'][0]),
call(result.evaluated_value[1],
result.name,
None,
None,
None,
postfix='%',
scale=result.meta['scale'][1],
result_format='{:.2f}',
value_name=result.meta['names'][1]),
call(np.mean(result.evaluated_value),
result.name,
result.abs_threshold,
result.rel_threshold,
None,
result_format='{:.2f}',
value_name='mean',
postfix='%',
scale=1)
]
mock_write_scalar_res.assert_has_calls(calls)
def test_vector_presenter_with_vector_data_has_specific_format_with_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={
'names': ['class1', 'class2'],
'scale': 0.5,
'postfix': 'km/h',
'data_format': '{:.4f}'
},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
calls = [
call(result.evaluated_value[0],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][0],
result_format='{}'),
call(result.evaluated_value[1],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][1],
result_format='{}'),
call(np.mean(result.evaluated_value),
result.name,
result.reference_value,
result.abs_threshold,
result.rel_threshold,
value_name='mean',
postfix=' ',
scale=1,
result_format='{}')
]
mock_write_scalar_res.assert_has_calls(calls)
def test_vector_presenter_with_vector_data_has_default_format_with_ignore_formatting_compare_with_ref(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[40, 60],
reference_value=49,
abs_threshold=None,
rel_threshold=None,
meta={'names': ['class1', 'class2']},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
calls = [
call(result.evaluated_value[0],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][0],
result_format='{}'),
call(result.evaluated_value[1],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][1],
result_format='{}'),
call(np.mean(result.evaluated_value),
result.name,
result.abs_threshold,
result.rel_threshold, (1.0, 0.02040816326530612),
value_name='mean',
postfix=' ',
scale=1,
result_format='{}')
]
mock_write_scalar_res.assert_has_calls(calls)
def test_reference_value_for_scalar_presenter(self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[0.456],
reference_value=0.456,
abs_threshold=None,
rel_threshold=None,
meta={},
profiling_file=None)
presenter = ScalarPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(
np.mean(result.evaluated_value),
result.name,
result.abs_threshold,
result.rel_threshold, (0.0, 0.0),
postfix='%',
scale=100,
result_format='{:.2f}')
def test_vector_presenter_with_vector_data_contain_one_element_compare_with_reference_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'openvino.tools.accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[40],
reference_value=42,
abs_threshold=None,
rel_threshold=None,
meta={},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
mock_write_scalar_res.assert_called_once_with(
result.evaluated_value[0],
result.name,
result.abs_threshold,
result.rel_threshold, (2.0, 0.047619047619047616),
postfix=' ',
scale=1,
value_name=None,
result_format='{}')
def test_mae_on_interval_default_all_missed(self):
annotations = [RegressionAnnotation('identifier', -2)]
predictions = [RegressionPrediction('identifier', 1)]
config = [{'type': 'mae_on_interval', 'end': 1}]
expected = EvaluationResult(
pytest.approx([0.0]), None, 'mae_on_interval', 'mae_on_interval',
None, None, {
'postfix': ' ',
'scale': 1,
'names': [],
'calculate_mean': False,
'target': 'higher-worse',
'orig_names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0']
}, None)
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
with pytest.warns(UserWarning) as warnings:
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert len(warnings) == 1
assert evaluation_result == expected
def generate_expected_result(values, metric_name, labels=None):
meta = {'target': 'higher-better'}
if labels:
meta.update({'names': list(labels.values())})
return EvaluationResult(pytest.approx(values), None, metric_name, metric_name, None, None, meta, None)