def test_vector_presenter_with_vector_data_has_specific_format_with_ignore_formatting(self, mocker):
mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result') # type: MagicMock
result = EvaluationResult(
name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value=None,
threshold=None,
meta={'names': ['class1', 'class2'], 'scale': 0.5, 'postfix': 'km/h', 'data_format': '{:.4f}'}
)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
calls = [
call(
result.evaluated_value[0], result.name,
postfix=' ', scale=1, value_name=result.meta['names'][0], result_format='{}'
),
call(
result.evaluated_value[1], result.name,
postfix=' ', scale=1, value_name=result.meta['names'][1], result_format='{}'
),
call(
np.mean(np.multiply(result.evaluated_value, 1)), result.name, result.reference_value, result.threshold,
value_name='mean', postfix=' ', scale=1, result_format='{}'
)
]
mock_write_scalar_res.assert_has_calls(calls)
def test_specific_format_for_scalar_presenter_with_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(
name='vector_metric',
evaluated_value=[0.456],
reference_value=None,
threshold=None,
meta={
'scale': 0.5,
'postfix': 'km/h',
'data_format': '{:.4f}'
},
)
presenter = ScalarPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
mock_write_scalar_res.assert_called_once_with(np.mean(
result.evaluated_value),
result.name,
result.reference_value,
result.threshold,
postfix=' ',
scale=1,
result_format='{}')
def test_vector_presenter_with_scaler_data_compare_with_reference(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(
name='scalar_metric',
metric_type='metric',
evaluated_value=0.4,
reference_value=42,
abs_threshold=None,
rel_threshold=None,
meta={},
)
presenter = VectorPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(
result.evaluated_value,
result.name,
result.abs_threshold,
result.rel_threshold, (2.0, 0.047619047619047616),
postfix='%',
scale=100,
value_name=None,
result_format='{:.2f}')
def test_vector_presenter_with_vector_data_contain_one_element_compare_with_reference_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(
name='vector_metric',
metric_type='metric',
evaluated_value=[0.4],
reference_value=42,
abs_threshold=None,
rel_threshold=None,
meta={},
)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
mock_write_scalar_res.assert_called_once_with(
result.evaluated_value[0],
result.name,
result.abs_threshold,
result.rel_threshold, (2.0, 0.047619047619047616),
postfix=' ',
scale=1,
value_name=None,
result_format='{}')
def generate_expected_result(values, metric_name, labels=None):
meta = {'target': 'higher-better'}
if labels:
meta.update({'names': list(labels.values())})
return EvaluationResult(pytest.approx(values), None, metric_name,
metric_name, None, meta)
def test_default_format_for_scalar_presenter_with_ignore_formatting(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(
name='vector_metric',
metric_type='metric',
evaluated_value=[0.456],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={},
)
presenter = ScalarPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
mock_write_scalar_res.assert_called_once_with(np.mean(
result.evaluated_value),
result.name,
result.abs_threshold,
result.rel_threshold,
None,
postfix=' ',
scale=1,
result_format='{}')
def test_vector_presenter_with_vector_data_contain_one_element(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={'names': ['prediction']},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(
result.evaluated_value[0],
result.name,
None,
result.abs_threshold,
result.rel_threshold,
postfix='%',
scale=100,
value_name=result.meta['names'][0],
result_format='{:.2f}')
def test_mae_on_interval_default_all_not_in_range_not_ignore_out_of_range(self):
annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2)]
predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 2)]
expected = EvaluationResult(
pytest.approx([2.0, 0.0, 0.0, 0.0]),
None,
'mae_on_interval',
'mae_on_interval',
None,
None,
{
'postfix': ' ',
'scale': 1,
'names': ['mean: < 0.0', 'std: < 0.0', 'mean: > 1.0', 'std: > 1.0'],
'calculate_mean': False,
'target': 'higher-worse',
'orig_names': ['mean: < 0.0', 'std: < 0.0', 'mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0', 'mean: > 1.0', 'std: > 1.0']
},
None
)
config = [{'type': 'mae_on_interval', 'end': 1, 'ignore_values_not_in_interval': False}]
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result == expected
def test_mae_on_interval_values_in_range(self):
annotations = [
RegressionAnnotation('identifier', 0.5),
RegressionAnnotation('identifier', 0.5)
]
predictions = [
RegressionPrediction('identifier', 1),
RegressionPrediction('identifier', 0.25)
]
config = [{'type': 'mae_on_interval', 'end': 1}]
expected = EvaluationResult(
pytest.approx([0.375, 0.125]), None, 'mae_on_interval',
'mae_on_interval', None, {
'postfix': ' ',
'scale': 1,
'names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0'],
'calculate_mean': False
})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result == expected
def test_mae_on_interval_with_unsorted_values(self):
annotations = [
RegressionAnnotation('identifier', -1),
RegressionAnnotation('identifier', 2),
RegressionAnnotation('identifier', 1)
]
predictions = [
RegressionPrediction('identifier', 1),
RegressionPrediction('identifier', 3),
RegressionPrediction('identifier', 1)
]
config = [{'type': 'mae_on_interval', 'intervals': [2.0, 0.0, 4.0]}]
expected = EvaluationResult(
pytest.approx([0.0, 0.0, 1.0, 0.0]),
None,
'mae_on_interval',
'mae_on_interval',
None,
None,
{
'postfix': ' ', 'scale': 1,
'names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'],
'orig_names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'],
'calculate_mean': False,
'target': 'higher-worse'
},
None
)
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result == expected
def test_mae_with_positive_diff_between_annotation_and_prediction(self):
annotations = [
RegressionAnnotation('identifier', 3),
RegressionAnnotation('identifier2', 1)
]
predictions = [
RegressionPrediction('identifier', 1),
RegressionPrediction('identifier2', -3)
]
config = [{'type': 'mae'}]
expected = EvaluationResult(
pytest.approx([3.0, 1.0]), None, 'mae', 'mae', None, {
'postfix': ' ',
'scale': 1,
'names': ['mean', 'std'],
'calculate_mean': False,
'target': 'higher-worse'
})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)),
annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(
annotations, predictions):
assert evaluation_result == expected
def test_vector_presenter_with_vector_data_with_vector_scale(self, mocker):
mock_write_scalar_res = mocker.patch('accuracy_checker.presenters.write_scalar_result') # type: MagicMock
result = EvaluationResult(
name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value=None,
threshold=None,
meta={'names': ['class1', 'class2'], 'scale': [1, 2]}
)
presenter = VectorPrintPresenter()
presenter.write_result(result)
calls = [
call(
result.evaluated_value[0], result.name,
postfix='%', scale=result.meta['scale'][0], result_format='{:.2f}', value_name=result.meta['names'][0]
),
call(
result.evaluated_value[1], result.name, postfix='%',
scale=result.meta['scale'][1], result_format='{:.2f}', value_name=result.meta['names'][1]
),
call(
np.mean(np.multiply(result.evaluated_value, result.meta['scale'])), result.name, result.threshold,
None, result_format='{:.2f}', value_name='mean', postfix='%', scale=1
)
]
mock_write_scalar_res.assert_has_calls(calls)
def test_mse_with_positive_diff_bitween_annotation_and_prediction(self):
annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)]
predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier2', -3)]
config = {'annotation': 'mocked', 'metrics': [{'type': 'mse'}]}
expected = EvaluationResult(pytest.approx([10.0, 6.0]), None, 'mse', None, {'postfix': ' ', 'scale': 1, 'names': ['mean', 'std'], 'calculate_mean': False})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result == expected
def test_mae_on_interval_default_all_missed(self):
annotations = [RegressionAnnotation('identifier', -2)]
predictions = [RegressionPrediction('identifier', 1)]
config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'end': 1}]}
expected = EvaluationResult(pytest.approx([0.0]), None, 'mae_on_interval', None, {'postfix': ' ', 'scale': 1, 'names': [], 'calculate_mean': False})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
with pytest.warns(UserWarning) as warnings:
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert len(warnings) == 1
assert evaluation_result == expected
def test_mae_on_interval_with_given_interval(self):
annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2), RegressionAnnotation('identifier', 1)]
predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 3), RegressionPrediction('identifier', 1)]
config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'intervals': [0.0, 2.0, 4.0]}]}
expected = EvaluationResult(pytest.approx([0.0, 0.0, 1.0, 0.0]), None, 'mae_on_interval', None,
{'postfix': ' ', 'scale': 1,
'names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'],
'calculate_mean': False})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result == expected
def test_mae_on_interval_default_not_ignore_out_of_range(self):
annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2), RegressionAnnotation('identifier', 0.5)]
predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 2), RegressionPrediction('identifier', 1)]
config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'end': 1, 'ignore_values_not_in_interval': False}]}
expected = EvaluationResult(pytest.approx([2.0, 0.0, 0.5, 0.0, 0.0, 0.0]), None, 'mae_on_interval', None,
{'postfix': ' ', 'scale': 1, 'names': ['mean: < 0.0', 'std: < 0.0', 'mean: <= 0.0 < 1.0',
'std: <= 0.0 < 1.0', 'mean: > 1.0', 'std: > 1.0'],
'calculate_mean': False})
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(annotations, predictions)
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert evaluation_result == expected
def test_vector_presenter_with_vector_data_with_scalar_postfix(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={
'names': ['class1', 'class2'],
'postfix': '_'
},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result)
calls = [
call(result.evaluated_value[0],
result.name,
None,
None,
None,
postfix=result.meta['postfix'],
scale=100,
value_name=result.meta['names'][0],
result_format='{:.2f}'),
call(result.evaluated_value[1],
result.name,
None,
None,
None,
postfix=result.meta['postfix'],
scale=100,
value_name=result.meta['names'][1],
result_format='{:.2f}'),
call(np.mean(np.multiply(result.evaluated_value, 100)),
result.name,
result.abs_threshold,
result.rel_threshold,
None,
value_name='mean',
postfix=result.meta['postfix'],
scale=1,
result_format='{:.2f}')
]
mock_write_scalar_res.assert_has_calls(calls)
def test_vector_presenter_with_vector_data_has_default_format_with_ignore_formatting_compare_with_ref(
self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[0.4, 0.6],
reference_value=49,
abs_threshold=None,
rel_threshold=None,
meta={'names': ['class1', 'class2']},
profiling_file=None)
presenter = VectorPrintPresenter()
presenter.write_result(result, ignore_results_formatting=True)
calls = [
call(result.evaluated_value[0],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][0],
result_format='{}'),
call(result.evaluated_value[1],
result.name,
None,
None,
None,
postfix=' ',
scale=1,
value_name=result.meta['names'][1],
result_format='{}'),
call(np.mean(np.multiply(result.evaluated_value, 1)),
result.name,
result.abs_threshold,
result.rel_threshold, (1.0, 0.02040816326530612),
value_name='mean',
postfix=' ',
scale=1,
result_format='{}')
]
mock_write_scalar_res.assert_has_calls(calls)
def test_vector_presenter_with_vector_data_with_one_element(self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
res = EvaluationResult(name='scalar_metric',
evaluated_value=[0.4],
reference_value=None,
threshold=None,
meta={'names': ['prediction']})
presenter = VectorPrintPresenter()
presenter.write_result(res)
mock_write_scalar_res.assert_called_once_with(
res.evaluated_value,
res.name,
res.reference_value,
res.threshold,
postfix='%',
scale=100,
value_name=res.meta['names'][0])
def test_scalar_presenter_with_scalar_data(self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
res = EvaluationResult(
name='scalar_metric',
evaluated_value=0.1,
reference_value=None,
threshold=None,
meta={},
)
presenter = ScalarPrintPresenter()
presenter.write_result(res)
mock_write_scalar_res.assert_called_once_with(res.evaluated_value,
res.name,
res.reference_value,
res.threshold,
postfix='%',
scale=100)
def test_scalar_presenter_with_vector_data(self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(
name='vector_metric',
evaluated_value=[0.4, 0.6],
reference_value=None,
threshold=None,
meta={},
)
presenter = ScalarPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(np.mean(
result.evaluated_value),
result.name,
result.reference_value,
result.threshold,
postfix='%',
scale=100,
result_format='{:.2f}')
def test_reference_value_for_scalar_presenter(self, mocker):
mock_write_scalar_res = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='vector_metric',
metric_type='metric',
evaluated_value=[0.456],
reference_value=45.6,
abs_threshold=None,
rel_threshold=None,
meta={},
profiling_file=None)
presenter = ScalarPrintPresenter()
presenter.write_result(result)
mock_write_scalar_res.assert_called_once_with(
np.mean(result.evaluated_value),
result.name,
result.abs_threshold,
result.rel_threshold, (0.0, 0.0),
postfix='%',
scale=100,
result_format='{:.2f}')
def test_mae_on_interval_default_all_missed(self):
annotations = [RegressionAnnotation('identifier', -2)]
predictions = [RegressionPrediction('identifier', 1)]
config = [{'type': 'mae_on_interval', 'end': 1}]
expected = EvaluationResult(
pytest.approx([0.0]),
None,
'mae_on_interval',
'mae_on_interval',
None,
None,
{'postfix': ' ', 'scale': 1, 'names': [], 'calculate_mean': False, 'target': 'higher-worse', 'orig_names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0']},
None
)
dispatcher = MetricsExecutor(config, None)
dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions)
with pytest.warns(UserWarning) as warnings:
for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions):
assert len(warnings) == 1
assert evaluation_result == expected
def test_scalar_presenter_with_scalar_data(self, mocker):
mock_write_scalar_result = mocker.patch(
'accuracy_checker.presenters.write_scalar_result'
) # type: MagicMock
result = EvaluationResult(name='scalar_metric',
metric_type='metric',
evaluated_value=0.1,
reference_value=None,
abs_threshold=None,
rel_threshold=None,
meta={},
profiling_file=None)
presenter = ScalarPrintPresenter()
presenter.write_result(result)
mock_write_scalar_result.assert_called_once_with(
result.evaluated_value,
result.name,
result.abs_threshold,
result.rel_threshold,
None,
postfix='%',
scale=100,
result_format='{:.2f}')
def generate_expected_result(values, metric_name, labels=None):
meta = {'names': list(labels.values())} if labels else {}
return EvaluationResult(pytest.approx(values), None, metric_name, metric_name, None, meta)