def to_standard_metric_inputs(
extracts: types.Extracts,
include_features: bool = False,
include_transformed_features: bool = False,
include_attributions: bool = False) -> metric_types.StandardMetricInputs:
"""Verifies extract keys and converts extracts to StandardMetricInputs."""
if constants.LABELS_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the '
'configuration is setup properly to specify the name of '
'label input and that the proper extractor has been '
'configured to extract the labels from the inputs.'.format(
constants.LABELS_KEY))
if constants.PREDICTIONS_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to perform model '
'inference.'.format(constants.PREDICTIONS_KEY))
if include_features and constants.FEATURES_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to extract the features '
'from the inputs.'.format(constants.FEATURES_KEY))
if (include_transformed_features and
constants.TRANSFORMED_FEATURES_KEY not in extracts):
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to extract the transformed '
'features from the inputs.'.format(
constants.TRANSFORMED_FEATURES_KEY))
if (include_attributions and constants.ATTRIBUTIONS_KEY not in extracts):
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to extract the '
'attributions from the inputs.'.format(
constants.ATTRIBUTIONS_KEY))
return metric_types.StandardMetricInputs(extracts)
def to_standard_metric_inputs(
extracts: types.Extracts,
include_features: bool = False) -> metric_types.StandardMetricInputs:
"""Filters and converts extracts to StandardMetricInputs."""
if constants.LABELS_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the '
'configuration is setup properly to specify the name of '
'label input and that the proper extractor has been '
'configured to extract the labels from the inputs.'.format(
constants.LABELS_KEY))
if constants.PREDICTIONS_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to perform model '
'inference.'.format(constants.PREDICTIONS_KEY))
example_weights = None
if constants.EXAMPLE_WEIGHTS_KEY in extracts:
example_weights = extracts[constants.EXAMPLE_WEIGHTS_KEY]
features = None
if include_features:
if constants.FEATURES_KEY not in extracts:
raise ValueError('"{}" key not found in extracts. Check that the proper '
'extractor has been configured to extract the features '
'from the inputs.'.format(constants.FEATURES_KEY))
features = extracts[constants.FEATURES_KEY]
return metric_types.StandardMetricInputs(extracts[constants.LABELS_KEY],
extracts[constants.PREDICTIONS_KEY],
example_weights, features)
def testStandardMetricInputsWithMultipleOutputs(self):
example = metric_types.StandardMetricInputs(label={
'output1': np.array([0, 1]),
'output2': np.array([1, 1])
},
prediction={
'output1':
np.array([0, 0.5]),
'output2':
np.array([0.2, 0.8])
},
example_weight={
'output1':
np.array([0.5]),
'output2':
np.array([1.0])
})
for output in ('output1', 'output2'):
iterator = metric_util.to_label_prediction_example_weight(
example, output_name=output, flatten=False)
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label, example.label[output])
self.assertAllEqual(got_pred, example.prediction[output])
self.assertAllClose(got_example_weight,
example.example_weight[output])
def testStandardMetricInputsWithCustomLabelKeys(self):
example = metric_types.StandardMetricInputs(
labels={
'custom_label': np.array([2]),
'other_label': np.array([0])
},
predictions={'custom_prediction': np.array([0, 0.5, 0.3, 0.9])},
example_weights=np.array([1.0]))
eval_config = config_pb2.EvalConfig(model_specs=[
config_pb2.ModelSpec(label_key='custom_label',
prediction_key='custom_prediction')
])
iterator = metric_util.to_label_prediction_example_weight(
example, eval_config=eval_config)
for expected_label, expected_prediction in zip((0.0, 0.0, 1.0, 0.0),
(0.0, 0.5, 0.3, 0.9)):
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label,
np.array([expected_label]),
atol=0,
rtol=0)
self.assertAllClose(got_pred,
np.array([expected_prediction]),
atol=0,
rtol=0)
self.assertAllClose(got_example_weight,
np.array([1.0]),
atol=0,
rtol=0)
def testStandardMetricInputsWithMissingStringLabel(self):
example = metric_types.StandardMetricInputs(
label=np.array(['d']),
prediction={
'scores': np.array([0.2, 0.7, 0.1]),
'classes': np.array(['a', 'b', 'c'])
},
example_weight=np.array([1.0]))
iterator = metric_util.to_label_prediction_example_weight(example)
for expected_label, expected_prediction in zip((0.0, 0.0, 0.0),
(0.2, 0.7, 0.1)):
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label,
np.array([expected_label]),
atol=0,
rtol=0)
self.assertAllClose(got_pred,
np.array([expected_prediction]),
atol=0,
rtol=0)
self.assertAllClose(got_example_weight,
np.array([1.0]),
atol=0,
rtol=0)
def testStandardMetricInputsRequiringSingleExampleWeightRaisesError(self):
with self.assertRaises(ValueError):
example = metric_types.StandardMetricInputs(
labels=np.array([2]),
predictions=np.array([0, 0.5, 0.3, 0.9]),
example_weights=np.array([1.0, 0.0]))
next(
metric_util.to_label_prediction_example_weight(
example, require_single_example_weight=True))
def testStandardMetricInputsWithZeroWeightsToNumpyWithoutFlatten(self):
example = metric_types.StandardMetricInputs(
np.array([2]), np.array([0, 0.5, 0.3, 0.9]), np.array([0.0]))
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(example, flatten=False))
self.assertAllClose(got_label, np.array([2]))
self.assertAllClose(got_pred, np.array([0, 0.5, 0.3, 0.9]))
self.assertAllClose(got_example_weight, np.array([0.0]))
def testStandardMetricInputsWithClassWeightsRaisesErrorWithoutFlatten(self):
with self.assertRaises(ValueError):
example = metric_types.StandardMetricInputs(
np.array([2]), np.array([0, 0.5, 0.3, 0.9]), np.array([1.0]))
next(
metric_util.to_label_prediction_example_weight(
example, class_weights={
1: 0.5,
2: 0.25
}, flatten=False))
def testStandardMetricInputsWithNonScalarWeightsNoFlatten(self):
example = metric_types.StandardMetricInputs(
label=np.array([2]),
prediction=np.array([0, 0.5, 0.3, 0.9]),
example_weight=np.array([1.0, 0.0, 1.0, 1.0]))
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example, flatten=False, require_single_example_weight=False))
self.assertAllClose(got_label, np.array([2]))
self.assertAllEqual(got_pred, np.array([0, 0.5, 0.3, 0.9]))
self.assertAllClose(got_example_weight, np.array([1.0, 0.0, 1.0, 1.0]))
def testStandardMetricInputsWithZeroWeightsToNumpy(self):
example = metric_types.StandardMetricInputs(
np.array([2]), np.array([0, 0.5, 0.3, 0.9]), np.array([0.0]))
iterable = metric_util.to_label_prediction_example_weight(example)
for expected_label, expected_prediction in zip((0.0, 0.0, 1.0, 0.0),
(0.0, 0.5, 0.3, 0.9)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([0.0]))
def testStandardMetricInputsToNumpyWithoutFlatten(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([2])},
prediction={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weight={'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example, output_name='output_name', flatten=False))
self.assertAllClose(got_label, np.array([2]))
self.assertAllClose(got_pred, np.array([0, 0.5, 0.3, 0.9]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsToNumpy(self):
example = metric_types.StandardMetricInputs(
{'output_name': np.array([2])},
{'output_name': np.array([0, 0.5, 0.3, 0.9])},
{'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = (
metric_util.to_label_prediction_example_weight(
example, output_name='output_name'))
self.assertAllClose(got_label, np.array([2]))
self.assertAllClose(got_pred, np.array([0, 0.5, 0.3, 0.9]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithoutPredictions(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([0, 0.5, 0.3, 0.9])},
prediction={'output_name': np.array([])},
example_weight={'output_name': np.array([1.0])})
iterator = metric_util.to_label_prediction_example_weight(
example, output_name='output_name')
for expected_label in (0.0, 0.5, 0.3, 0.9):
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllEqual(got_pred, np.array([]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithSparseTensorValue(self):
example = metric_types.StandardMetricInputs(
tf.compat.v1.SparseTensorValue(
values=np.array([1]), indices=np.array([2]), dense_shape=(0, 1)),
np.array([0, 0.5, 0.3, 0.9]), np.array([0.0]))
iterable = metric_util.to_label_prediction_example_weight(example)
for expected_label, expected_prediction in zip((0.0, 0.0, 1.0, 0.0),
(0.0, 0.5, 0.3, 0.9)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([0.0]))
def testStandardMetricInputsWithMissingLabelsAndExampleWeights(self):
example = metric_types.StandardMetricInputs(
prediction={
'output1': np.array([0, 0.5]),
'output2': np.array([0.2, 0.8])
})
for output in ('output1', 'output2'):
iterator = metric_util.to_label_prediction_example_weight(
example, output_name=output, flatten=False, allow_none=True)
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllEqual(got_label, np.array([]))
self.assertAllEqual(got_pred, example.prediction[output])
self.assertAllEqual(got_example_weight, np.array([1.0]))
def testStandardMetricInputsToNumpy(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([2])},
prediction={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weight={'output_name': np.array([1.0])})
iterable = metric_util.to_label_prediction_example_weight(
example, output_name='output_name')
for expected_label, expected_prediction in zip((0.0, 0.0, 1.0, 0.0),
(0.0, 0.5, 0.3, 0.9)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithTopKToNumpy(self):
example = metric_types.StandardMetricInputs(
{'output_name': np.array([1])},
{'output_name': np.array([0, 0.5, 0.3, 0.9])},
{'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = (
metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(top_k=2)))
self.assertAllClose(got_label, np.array([0.0, 1.0]))
self.assertAllClose(got_pred, np.array([0.9, 0.5]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithClassIDToNumpy(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([2])},
prediction={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weight={'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(class_id=2)))
self.assertAllClose(got_label, np.array([1.0]))
self.assertAllClose(got_pred, np.array([0.3]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithTopKToNumpy(self):
example = metric_types.StandardMetricInputs(
{'output_name': np.array([1])},
{'output_name': np.array([0, 0.5, 0.3, 0.9])},
{'output_name': np.array([1.0])})
iterable = metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(top_k=2))
for expected_label, expected_prediction in zip((0.0, 1.0), (0.9, 0.5)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithMissingPredictionRaisesError(self):
example = metric_types.StandardMetricInputs(
label={
'output1': np.array([0, 1]),
'output2': np.array([1, 1])
},
prediction={'output2': np.array([0.8])},
example_weight={
'output1': np.array([0.5]),
'output2': np.array([1.0])
})
with self.assertRaisesRegex(ValueError, '"output1" key not found.*'):
next(
metric_util.to_label_prediction_example_weight(
example, output_name='output1'))
def testStandardMetricInputsWithTopKAndAggregationTypeToNumpy(self):
example = metric_types.StandardMetricInputs(
labels={'output_name': np.array([1])},
predictions={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weights={'output_name': np.array([1.0])})
iterator = metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(top_k=2),
aggregation_type=metric_types.AggregationType(micro_average=True))
for expected_label, expected_prediction in zip((1.0, 0.0), (0.5, 0.9)):
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithNonScalarWeights(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([2])},
prediction={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weight={'output_name': np.array([1.0, 0.0, 1.0, 1.0])})
iterable = metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
require_single_example_weight=False)
for expected_label, expected_prediction, expected_weight in zip(
(0.0, 0.0, 1.0, 0.0), (0.0, 0.5, 0.3, 0.9), (1.0, 0.0, 1.0, 1.0)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllEqual(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight,
np.array([expected_weight]))
def testStandardMetricInputsWithMissingExampleWeightKeyRaisesError(self):
example = metric_types.StandardMetricInputs(
label={
'output1': np.array([0, 1]),
'output2': np.array([1, 1])
},
prediction={
'output1': np.array([0.5]),
'output2': np.array([0.8])
},
example_weight={'output2': np.array([1.0])})
with self.assertRaisesRegex(
ValueError,
'unable to prepare example_weight for metric computation.*'):
next(
metric_util.to_label_prediction_example_weight(
example, output_name='output1'))
def testStandardMetricInputsWithClassWeightsWithoutFlatten(self):
example = metric_types.StandardMetricInputs(
{'output_name': np.array([1.0, 1.0, 1.0, 0.0])},
{'output_name': np.array([0, 0.5, 0.3, 0.9])},
{'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
class_weights={
0: 0.5,
2: 0.25
},
flatten=False))
self.assertAllClose(got_label, np.array([0.5, 1.0, 0.25, 0.0]))
self.assertAllClose(got_pred, np.array([0, 0.5, 0.075, 0.9]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithKToNumpy2D(self):
example = metric_types.StandardMetricInputs(
labels={'output_name': np.array([1, 2])},
predictions={
'output_name': np.array([[0, 0.5, 0.3, 0.9],
[0.1, 0.4, 0.2, 0.3]])
},
example_weights={'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(k=2),
flatten=False,
squeeze=False))
self.assertAllClose(got_label, np.array([[1], [0]]))
self.assertAllClose(got_pred, np.array([[0.5], [0.3]]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithClassWeights(self):
example = metric_types.StandardMetricInputs(
{'output_name': np.array([2])},
{'output_name': np.array([0, 0.5, 0.3, 0.9])},
{'output_name': np.array([1.0])})
iterable = metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
class_weights={
1: 0.5,
2: 0.25
})
for expected_label, expected_prediction in zip(
(0.0, 0.0, 0.25, 0.0), (0.0, 0.25, 0.075, 0.9)):
got_label, got_pred, got_example_weight = next(iterable)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def to_standard_metric_inputs(
extracts: types.Extracts,
include_labels: bool = True,
include_predictions: bool = True,
include_features: bool = False,
include_transformed_features: bool = False,
include_attributions: bool = False
) -> metric_types.StandardMetricInputs:
"""Verifies extract keys and converts extracts to StandardMetricInputs."""
if include_labels and constants.LABELS_KEY not in extracts:
raise ValueError(
f'"{constants.LABELS_KEY}" key not found in extracts. '
'Check that the configuration is setup properly to '
'specify the name of label input and that the proper '
'extractor has been configured to extract the labels from '
f'the inputs. Existing keys: {extracts.keys()}')
if include_predictions and constants.PREDICTIONS_KEY not in extracts:
raise ValueError(f'"{constants.PREDICTIONS_KEY}" key not found in '
'extracts. Check that the proper extractor has been '
'configured to perform model inference.')
if include_features and constants.FEATURES_KEY not in extracts:
raise ValueError(
f'"{constants.FEATURES_KEY}" key not found in extracts. '
'Check that the proper extractor has been configured to '
'extract the features from the inputs. Existing keys: '
f'{extracts.keys()}')
if (include_transformed_features
and constants.TRANSFORMED_FEATURES_KEY not in extracts):
raise ValueError(
f'"{constants.TRANSFORMED_FEATURES_KEY}" key not found in '
'extracts. Check that the proper extractor has been '
'configured to extract the transformed features from the '
f'inputs. Existing keys: {extracts.keys()}')
if (include_attributions and constants.ATTRIBUTIONS_KEY not in extracts):
raise ValueError(
f'"{constants.ATTRIBUTIONS_KEY}" key not found in '
'extracts. Check that the proper extractor has been '
'configured to extract the attributions from the inputs.'
f'Existing keys: {extracts.keys()}')
return metric_types.StandardMetricInputs(extracts)
def testStandardMetricInputsWithTopKToNumpyWithoutFlatten(self):
example = metric_types.StandardMetricInputs(
label={'output_name': np.array([1, 2])},
prediction={
'output_name': np.array([[0, 0.5, 0.3, 0.9],
[0.1, 0.4, 0.2, 0.3]])
},
example_weight={'output_name': np.array([1.0])})
got_label, got_pred, got_example_weight = next(
metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
sub_key=metric_types.SubKey(top_k=2),
flatten=False))
self.assertAllClose(got_label, np.array([1, 2]))
self.assertAllClose(
got_pred,
np.array([[float('-inf'), 0.5,
float('-inf'), 0.9],
[float('-inf'), 0.4,
float('-inf'), 0.3]]))
self.assertAllClose(got_example_weight, np.array([1.0]))
def testStandardMetricInputsWithClassWeights(self):
example = metric_types.StandardMetricInputs(
labels={'output_name': np.array([2])},
predictions={'output_name': np.array([0, 0.5, 0.3, 0.9])},
example_weights={'output_name': np.array([1.0])})
iterator = metric_util.to_label_prediction_example_weight(
example,
output_name='output_name',
aggregation_type=metric_types.AggregationType(micro_average=True),
class_weights={
0: 1.0,
1: 0.5,
2: 0.25,
3: 1.0
},
flatten=True)
for expected_label, expected_prediction, expected_weight in zip(
(0.0, 0.0, 1.0, 0.0), (0.0, 0.5, 0.3, 0.9), (1.0, 0.5, 0.25, 1.0)):
got_label, got_pred, got_example_weight = next(iterator)
self.assertAllClose(got_label, np.array([expected_label]))
self.assertAllClose(got_pred, np.array([expected_prediction]))
self.assertAllClose(got_example_weight,
np.array([expected_weight]))
def testWithMultiClassClassificationMultiOutput(self, add_custom_metrics):
export_dir = self._createMultiClassClassificationModel(
sequential=False,
output_names=('output_1', 'output_2'),
add_custom_metrics=add_custom_metrics)
eval_shared_model = self.createTestEvalSharedModel(
eval_saved_model_path=export_dir)
computation = keras_util.metric_computations_using_keras_saved_model(
'', eval_shared_model.model_loader, None)[0]
inputs = [
metric_types.StandardMetricInputs(
labels={
'output_1': np.array([0, 0, 1, 0, 0]),
'output_2': np.array([0, 0, 1, 0, 0]),
},
predictions={
'output_1': np.array([0.1, 0.2, 0.1, 0.25, 0.35]),
'output_2': np.array([0.1, 0.2, 0.1, 0.25, 0.35]),
},
example_weights={
'output_1': np.array([0.5]),
'output_2': np.array([0.5]),
},
input=self._makeExample(output_1=[0.1, 0.2, 0.1, 0.25, 0.35],
output_2=[0.1, 0.2, 0.1, 0.25,
0.35]).SerializeToString()),
metric_types.StandardMetricInputs(
labels={
'output_1': np.array([0, 1, 0, 0, 0]),
'output_2': np.array([0, 1, 0, 0, 0]),
},
predictions={
'output_1': np.array([0.2, 0.3, 0.05, 0.15, 0.3]),
'output_2': np.array([0.2, 0.3, 0.05, 0.15, 0.3]),
},
example_weights={
'output_1': np.array([0.7]),
'output_2': np.array([0.7]),
},
input=self._makeExample(output_1=[0.2, 0.3, 0.05, 0.15, 0.3],
output_2=[0.2, 0.3, 0.05, 0.15,
0.3]).SerializeToString()),
metric_types.StandardMetricInputs(
labels={
'output_1': np.array([0, 0, 0, 1, 0]),
'output_2': np.array([0, 0, 0, 1, 0]),
},
predictions={
'output_1': np.array([0.01, 0.2, 0.09, 0.5, 0.2]),
'output_2': np.array([0.01, 0.2, 0.09, 0.5, 0.2]),
},
example_weights={
'output_1': np.array([0.9]),
'output_2': np.array([0.9]),
},
input=self._makeExample(output_1=[0.01, 0.2, 0.09, 0.5, 0.2],
output_2=[0.01, 0.2, 0.09, 0.5,
0.2]).SerializeToString()),
metric_types.StandardMetricInputs(
labels={
'output_1': np.array([0, 1, 0, 0, 0]),
'output_2': np.array([0, 1, 0, 0, 0]),
},
predictions={
'output_1': np.array([0.3, 0.2, 0.05, 0.4, 0.05]),
'output_2': np.array([0.3, 0.2, 0.05, 0.4, 0.05]),
},
example_weights={
'output_1': np.array([0.3]),
'output_2': np.array([0.3]),
},
input=self._makeExample(output_1=[0.3, 0.2, 0.05, 0.4, 0.05],
output_2=[0.3, 0.2, 0.05, 0.4,
0.05]).SerializeToString())
]
# Unweighted:
# top_k = 2
# TP = 0 + 1 + 1 + 0 = 2
# FP = 2 + 1 + 1 + 2 = 6
# FN = 1 + 0 + 0 + 1 = 2
#
# top_k = 3
# TP = 0 + 1 + 1 + 1 = 3
# FP = 3 + 2 + 2 + 2 = 9
# FN = 1 + 0 + 0 + 0 = 1
#
# Weighted:
# top_k = 2
# TP = 0.5*0 + 0.7*1 + 0.9*1 + 0.3*0 = 1.6
# FP = 0.5*2 + 0.7*1 + 0.9*1 + 0.3*2 = 3.2
# FN = 0.5*1 + 0.7*0 + 0.9*0 + 0.3*1 = 0.8
#
# top_k = 3
# TP = 0.5*0 + 0.7*1 + 0.9*1 + 0.3*1 = 1.9
# FP = 0.5*3 + 0.7*2 + 0.9*2 + 0.3*2 = 5.3
# FN = 0.5*1 + 0.7*0 + 0.9*0 + 0.3*0 = 0.5
expected_values = {
'output_1': {
'precision@2': 2 / (2 + 6),
'precision@3': 3 / (3 + 9),
'recall@2': 2 / (2 + 2),
'recall@3': 3 / (3 + 1),
'weighted_precision@2': 1.6 / (1.6 + 3.2),
'weighted_precision@3': 1.9 / (1.9 + 5.3),
'weighted_recall@2': 1.6 / (1.6 + 0.8),
'weighted_recall@3': 1.9 / (1.9 + 0.5),
'loss': 0.77518433
},
'output_2': {
'precision@2': 2 / (2 + 6),
'precision@3': 3 / (3 + 9),
'recall@2': 2 / (2 + 2),
'recall@3': 3 / (3 + 1),
'weighted_precision@2': 1.6 / (1.6 + 3.2),
'weighted_precision@3': 1.9 / (1.9 + 5.3),
'weighted_recall@2': 1.6 / (1.6 + 0.8),
'weighted_recall@3': 1.9 / (1.9 + 0.5),
'loss': 0.77518433
},
'': {
'loss': 0.77518433 + 0.77518433
}
}
if add_custom_metrics:
expected_values['']['custom_output_1'] = 4.0
expected_values['']['custom_output_2'] = 4.0
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeMetric' >> beam.CombinePerKey(computation.combiner))
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
expected = {}
for output_name, per_output_values in expected_values.items(
):
for name, value in per_output_values.items():
sub_key = None
if '@' in name:
sub_key = metric_types.SubKey(
top_k=int(name.split('@')[1]))
key = metric_types.MetricKey(
name=name,
output_name=output_name,
sub_key=sub_key)
expected[key] = value
self.assertDictElementsAlmostEqual(got_metrics, expected)
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
