def _total_attributions(
absolute: bool = True,
name: Text = '',
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None,
) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions."""
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
# Make sure total_attributions is calculated.
computations = _total_attributions_computations(
absolute=absolute,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
)
private_key = computations[-1].keys[-1]
def result(
metrics: Dict[metric_types.MetricKey, Any]
) -> Dict[metric_types.AttributionsKey, Dict[Text, Union[float,
np.ndarray]]]:
"""Returns total attributions."""
return {key: metrics[private_key]}
derived_computation = metric_types.DerivedMetricComputation(keys=[key],
result=result)
computations.append(derived_computation)
return computations
def _total_attributions_computations(
absolute: bool = True,
name: Text = '',
model_name: Text = '',
output_name: Text = '',
sub_key: Optional[metric_types.SubKey] = None
) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions.
Args:
absolute: True to use absolute value when summing.
name: Metric name.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
"""
if not name:
if absolute:
name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME
else:
name = '_' + TOTAL_ATTRIBUTIONS_NAME
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=_AttributionsPreprocessor(),
combiner=_TotalAttributionsCombiner(key, absolute))
]
def _total_attributions_computations(
absolute: bool = True,
name: str = '',
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions.
Args:
absolute: True to use absolute value when summing.
name: Metric name.
eval_config: Eval config.
model_name: Optional model name (if multi-model evaluation).
output_name: Optional output name (if multi-output model type).
sub_key: Optional sub key.
example_weighted: True if example weights should be applied.
"""
if not name:
if absolute:
name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME
else:
name = '_' + TOTAL_ATTRIBUTIONS_NAME
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
return [
metric_types.MetricComputation(
keys=[key],
preprocessor=metric_types.AttributionPreprocessor(feature_keys={}),
combiner=_TotalAttributionsCombiner(key, eval_config, absolute))
]
def _mean_attributions(
absolute: bool = True,
name: str = MEAN_ATTRIBUTIONS_NAME,
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
example_weighted: bool = False,
) -> metric_types.MetricComputations:
"""Returns metric computations for mean attributions."""
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
# Make sure total_attributions is calculated.
computations = _total_attributions_computations(
absolute=absolute,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
total_attributions_key = computations[-1].keys[-1]
# Make sure example_count is calculated
computations.extend(
example_count.example_count(model_names=[model_name],
output_names=[output_name],
sub_keys=[sub_key],
example_weighted=example_weighted))
example_count_key = computations[-1].keys[-1]
def result(
metrics: Dict[metric_types.MetricKey, Any]
) -> Dict[metric_types.AttributionsKey, Dict[str, Union[float,
np.ndarray]]]:
"""Returns mean attributions."""
total_attributions = metrics[total_attributions_key]
count = metrics[example_count_key]
attributions = {}
for k, v in total_attributions.items():
if np.isclose(count, 0.0):
attributions[k] = float('nan')
else:
attributions[k] = v / count
return {key: attributions}
derived_computation = metric_types.DerivedMetricComputation(keys=[key],
result=result)
computations.append(derived_computation)
return computations
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_attributions = got[0]
self.assertEqual(got_slice_key, ())
total_attributions_key = metric_types.AttributionsKey(
name='total_attributions', sub_key=sub_key)
self.assertIn(total_attributions_key, got_attributions)
self.assertAllClose(got_attributions[total_attributions_key],
expected_values)
except AssertionError as err:
raise util.BeamAssertException(err)
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_attributions = got[0]
self.assertEqual(got_slice_key, ())
total_attributions_key = metric_types.AttributionsKey(
name='total_absolute_attributions',
model_name=model_name,
output_name=output_name)
self.assertIn(total_attributions_key, got_attributions)
self.assertDictElementsAlmostEqual(
got_attributions[total_attributions_key], expected_values)
except AssertionError as err:
raise util.BeamAssertException(err)
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_attributions = got[0]
self.assertEqual(got_slice_key, ())
mean_attributions_key = metric_types.AttributionsKey(
name='mean_attributions')
self.assertIn(mean_attributions_key, got_attributions)
self.assertDictElementsAlmostEqual(
got_attributions[mean_attributions_key], {
'feature1': 1.0 / 0.5,
'feature2': -2.0 / 0.5,
})
except AssertionError as err:
raise util.BeamAssertException(err)
def testMeanAbsoluteAttributions(self):
computation = attributions.MeanAbsoluteAttributions().computations(
)[-1]
total_absolute_attributions_key = metric_types.AttributionsKey(
name='_total_absolute_attributions')
weighted_examples_key = metric_types.MetricKey(
name='weighted_example_count')
metrics = {
total_absolute_attributions_key: {
'feature1': 1.0,
'feature2': 2.0
},
weighted_examples_key: 0.5
}
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (pipeline
| 'Create' >> beam.Create([metrics])
| 'ComputeMetric' >> beam.Map(lambda x: (
(), computation.result(x))))
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_attributions = got[0]
self.assertEqual(got_slice_key, ())
mean_attributions_key = metric_types.AttributionsKey(
name='mean_absolute_attributions')
self.assertIn(mean_attributions_key, got_attributions)
self.assertDictElementsAlmostEqual(
got_attributions[mean_attributions_key], {
'feature1': 1.0 / 0.5,
'feature2': 2.0 / 0.5,
})
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
def _total_attributions(
absolute: bool = True,
name: str = '',
eval_config: Optional[config_pb2.EvalConfig] = None,
model_name: str = '',
output_name: str = '',
sub_key: Optional[metric_types.SubKey] = None,
example_weighted: bool = False) -> metric_types.MetricComputations:
"""Returns metric computations for total attributions."""
key = metric_types.AttributionsKey(name=name,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
# Make sure total_attributions is calculated.
computations = _total_attributions_computations(
absolute=absolute,
eval_config=eval_config,
model_name=model_name,
output_name=output_name,
sub_key=sub_key,
example_weighted=example_weighted)
private_key = computations[-1].keys[-1]
def result(
metrics: Dict[metric_types.MetricKey, Any]
) -> Dict[metric_types.AttributionsKey, Dict[str, Union[float,
np.ndarray]]]:
"""Returns total attributions."""
return {key: metrics[private_key]}
derived_computation = metric_types.DerivedMetricComputation(keys=[key],
result=result)
computations.append(derived_computation)
return computations
