def example_count_specs(
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None,
output_weights: Optional[Dict[Text, float]] = None,
include_example_count: bool = True,
include_weighted_example_count: bool = True) -> List[config.MetricsSpec]:
"""Returns metric specs for example count and weighted example counts.
Args:
model_names: Optional list of model names (if multi-model evaluation).
output_names: Optional list of output names (if multi-output model).
output_weights: Optional output weights for creating overall metric
aggregated across outputs (if multi-output model). If a weight is not
provided for an output, it's weight defaults to 0.0 (i.e. output ignored).
include_example_count: True to add example_count metric.
include_weighted_example_count: True to add weighted_example_count metric. A
weighted example count will be added per output for multi-output models.
"""
specs = []
if include_example_count:
metric_config = _serialize_tfma_metric(example_count.ExampleCount())
specs.append(
config.MetricsSpec(metrics=[metric_config], model_names=model_names))
if include_weighted_example_count:
metric_config = _serialize_tfma_metric(
weighted_example_count.WeightedExampleCount())
specs.append(
config.MetricsSpec(
metrics=[metric_config],
model_names=model_names,
output_names=output_names,
output_weights=output_weights))
return specs
def testExampleCount(self):
metric = example_count.ExampleCount().computations()[0]
example1 = {}
example2 = {}
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (pipeline
| 'Create' >> beam.Create([example1, example2])
| 'Process' >> beam.ParDo(metric.preprocessor)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeMetric' >> beam.CombinePerKey(metric.combiner))
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
example_count_key = metric_types.MetricKey(
name='example_count')
self.assertDictElementsAlmostEqual(got_metrics,
{example_count_key: 2})
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
def example_count_specs(
model_names: Optional[List[Text]] = None,
output_names: Optional[List[Text]] = None,
include_example_count: bool = True,
include_weighted_example_count: bool = True
) -> List[config.MetricsSpec]:
"""Returns metric specs for example count and weighted example counts.
Args:
model_names: Optional list of model names (if multi-model evaluation).
output_names: Optional list of output names (if multi-output model).
include_example_count: True to add example_count metric.
include_weighted_example_count: True to add weighted_example_count metric. A
weighted example count will be added per output for multi-output models.
"""
specs = []
if include_example_count:
metric_config = _serialize_tfma_metric(example_count.ExampleCount())
specs.append(
config.MetricsSpec(metrics=[metric_config],
model_names=model_names))
if include_weighted_example_count:
metric_config = _serialize_tfma_metric(
weighted_example_count.WeightedExampleCount())
specs.append(
config.MetricsSpec(metrics=[metric_config],
model_names=model_names,
output_names=output_names))
return specs
def testExampleCount(self, model_name, output_name, example_weighted):
metric = example_count.ExampleCount().computations(
model_names=[model_name],
output_names=[output_name],
example_weighted=example_weighted)[0]
example0 = {
'labels': None,
'predictions': None,
'example_weights': [0.0]
}
example1 = {
'labels': None,
'predictions': None,
'example_weights': [0.5]
}
example2 = {
'labels': None,
'predictions': None,
'example_weights': [1.0]
}
example3 = {
'labels': None,
'predictions': None,
'example_weights': [0.7]
}
if output_name:
example0['example_weights'] = {
output_name: example0['example_weights']
}
example1['example_weights'] = {
output_name: example1['example_weights']
}
example2['example_weights'] = {
output_name: example2['example_weights']
}
example3['example_weights'] = {
output_name: example3['example_weights']
}
if model_name:
example0['example_weights'] = {
model_name: example0['example_weights']
}
example1['example_weights'] = {
model_name: example1['example_weights']
}
example2['example_weights'] = {
model_name: example2['example_weights']
}
example3['example_weights'] = {
model_name: example3['example_weights']
}
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(
[example0, example1, example2, example3])
| 'Process' >> beam.Map(metric_util.to_standard_metric_inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeMetric' >> beam.CombinePerKey(metric.combiner))
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
example_count_key = metric_types.MetricKey(
name='example_count',
model_name=model_name,
output_name=output_name,
example_weighted=example_weighted)
if example_weighted:
self.assertDictElementsAlmostEqual(
got_metrics,
{example_count_key: (0.0 + 0.5 + 1.0 + 0.7)})
else:
self.assertDictElementsAlmostEqual(
got_metrics,
{example_count_key: (1.0 + 1.0 + 1.0 + 1.0)})
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
