def test_mae_on_interval_default_all_not_in_range_not_ignore_out_of_range(self): annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2)] predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 2)] expected = EvaluationResult( pytest.approx([2.0, 0.0, 0.0, 0.0]), None, 'mae_on_interval', 'mae_on_interval', None, None, { 'postfix': ' ', 'scale': 1, 'names': ['mean: < 0.0', 'std: < 0.0', 'mean: > 1.0', 'std: > 1.0'], 'calculate_mean': False, 'target': 'higher-worse', 'orig_names': ['mean: < 0.0', 'std: < 0.0', 'mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0', 'mean: > 1.0', 'std: > 1.0'] }, None ) config = [{'type': 'mae_on_interval', 'end': 1, 'ignore_values_not_in_interval': False}] dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert evaluation_result == expected
def test_mae_on_interval_with_unsorted_values(self): annotations = [ RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2), RegressionAnnotation('identifier', 1) ] predictions = [ RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 3), RegressionPrediction('identifier', 1) ] config = [{'type': 'mae_on_interval', 'intervals': [2.0, 0.0, 4.0]}] expected = EvaluationResult( pytest.approx([0.0, 0.0, 1.0, 0.0]), None, 'mae_on_interval', 'mae_on_interval', None, None, { 'postfix': ' ', 'scale': 1, 'names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'], 'orig_names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'], 'calculate_mean': False, 'target': 'higher-worse' }, None ) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert evaluation_result == expected
def test_mae_on_interval_values_in_range(self): annotations = [ RegressionAnnotation('identifier', 0.5), RegressionAnnotation('identifier', 0.5) ] predictions = [ RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 0.25) ] config = [{'type': 'mae_on_interval', 'end': 1}] expected = EvaluationResult( pytest.approx([0.375, 0.125]), None, 'mae_on_interval', 'mae_on_interval', None, { 'postfix': ' ', 'scale': 1, 'names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0'], 'calculate_mean': False }) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics( annotations, predictions): assert evaluation_result == expected
def test_mae_with_positive_diff_between_annotation_and_prediction(self): annotations = [ RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1) ] predictions = [ RegressionPrediction('identifier', 1), RegressionPrediction('identifier2', -3) ] config = [{'type': 'mae'}] expected = EvaluationResult( pytest.approx([3.0, 1.0]), None, 'mae', 'mae', None, { 'postfix': ' ', 'scale': 1, 'names': ['mean', 'std'], 'calculate_mean': False, 'target': 'higher-worse' }) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics( annotations, predictions): assert evaluation_result == expected
def test_update_rmse_on_interval_metric(self): config = [{'type': 'rmse_on_interval', 'intervals': [0.0, 2.0, 4.0]}] annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)] predictions = [RegressionPrediction('identifier', 5), RegressionPrediction('identifier2', 5)] dispatcher = MetricsExecutor(config, None) metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) assert metric_result[0][0].result == 2 assert metric_result[1][0].result == 4
def test_update_mse_metric_result(self): annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)] predictions = [RegressionPrediction('identifier', 5), RegressionPrediction('identifier2', 5)] config = [{'type': 'mse'}] dispatcher = MetricsExecutor(config, None) metric_result, _ = dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) assert metric_result[0][0].result == 4 assert metric_result[1][0].result == 16
def test_mse_with_positive_diff_bitween_annotation_and_prediction(self): annotations = [RegressionAnnotation('identifier', 3), RegressionAnnotation('identifier2', 1)] predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier2', -3)] config = {'annotation': 'mocked', 'metrics': [{'type': 'mse'}]} expected = EvaluationResult(pytest.approx([10.0, 6.0]), None, 'mse', None, {'postfix': ' ', 'scale': 1, 'names': ['mean', 'std'], 'calculate_mean': False}) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert evaluation_result == expected
def test_mae_on_interval_with_given_interval(self): annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2), RegressionAnnotation('identifier', 1)] predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 3), RegressionPrediction('identifier', 1)] config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'intervals': [0.0, 2.0, 4.0]}]} expected = EvaluationResult(pytest.approx([0.0, 0.0, 1.0, 0.0]), None, 'mae_on_interval', None, {'postfix': ' ', 'scale': 1, 'names': ['mean: <= 0.0 < 2.0', 'std: <= 0.0 < 2.0', 'mean: <= 2.0 < 4.0', 'std: <= 2.0 < 4.0'], 'calculate_mean': False}) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert evaluation_result == expected
def test_mae_on_interval_default_not_ignore_out_of_range(self): annotations = [RegressionAnnotation('identifier', -1), RegressionAnnotation('identifier', 2), RegressionAnnotation('identifier', 0.5)] predictions = [RegressionPrediction('identifier', 1), RegressionPrediction('identifier', 2), RegressionPrediction('identifier', 1)] config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'end': 1, 'ignore_values_not_in_interval': False}]} expected = EvaluationResult(pytest.approx([2.0, 0.0, 0.5, 0.0, 0.0, 0.0]), None, 'mae_on_interval', None, {'postfix': ' ', 'scale': 1, 'names': ['mean: < 0.0', 'std: < 0.0', 'mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0', 'mean: > 1.0', 'std: > 1.0'], 'calculate_mean': False}) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(annotations, predictions) for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert evaluation_result == expected
def test_mae_on_interval_default_all_missed(self): annotations = [RegressionAnnotation('identifier', -2)] predictions = [RegressionPrediction('identifier', 1)] config = {'annotation': 'mocked', 'metrics': [{'type': 'mae_on_interval', 'end': 1}]} expected = EvaluationResult(pytest.approx([0.0]), None, 'mae_on_interval', None, {'postfix': ' ', 'scale': 1, 'names': [], 'calculate_mean': False}) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(annotations, predictions) with pytest.warns(UserWarning) as warnings: for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert len(warnings) == 1 assert evaluation_result == expected
def test_mae_on_interval_default_all_missed(self): annotations = [RegressionAnnotation('identifier', -2)] predictions = [RegressionPrediction('identifier', 1)] config = [{'type': 'mae_on_interval', 'end': 1}] expected = EvaluationResult( pytest.approx([0.0]), None, 'mae_on_interval', 'mae_on_interval', None, None, {'postfix': ' ', 'scale': 1, 'names': [], 'calculate_mean': False, 'target': 'higher-worse', 'orig_names': ['mean: <= 0.0 < 1.0', 'std: <= 0.0 < 1.0']}, None ) dispatcher = MetricsExecutor(config, None) dispatcher.update_metrics_on_batch(range(len(annotations)), annotations, predictions) with pytest.warns(UserWarning) as warnings: for _, evaluation_result in dispatcher.iterate_metrics(annotations, predictions): assert len(warnings) == 1 assert evaluation_result == expected