def _total_attributions( absolute: bool = True, name: Text = '', model_name: Text = '', output_name: Text = '', sub_key: Optional[metric_types.SubKey] = None, ) -> metric_types.MetricComputations: """Returns metric computations for total attributions.""" key = metric_types.AttributionsKey(name=name, model_name=model_name, output_name=output_name, sub_key=sub_key) # Make sure total_attributions is calculated. computations = _total_attributions_computations( absolute=absolute, model_name=model_name, output_name=output_name, sub_key=sub_key, ) private_key = computations[-1].keys[-1] def result( metrics: Dict[metric_types.MetricKey, Any] ) -> Dict[metric_types.AttributionsKey, Dict[Text, Union[float, np.ndarray]]]: """Returns total attributions.""" return {key: metrics[private_key]} derived_computation = metric_types.DerivedMetricComputation(keys=[key], result=result) computations.append(derived_computation) return computations
def _total_attributions_computations( absolute: bool = True, name: Text = '', model_name: Text = '', output_name: Text = '', sub_key: Optional[metric_types.SubKey] = None ) -> metric_types.MetricComputations: """Returns metric computations for total attributions. Args: absolute: True to use absolute value when summing. name: Metric name. model_name: Optional model name (if multi-model evaluation). output_name: Optional output name (if multi-output model type). sub_key: Optional sub key. """ if not name: if absolute: name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME else: name = '_' + TOTAL_ATTRIBUTIONS_NAME key = metric_types.AttributionsKey(name=name, model_name=model_name, output_name=output_name, sub_key=sub_key) return [ metric_types.MetricComputation( keys=[key], preprocessor=_AttributionsPreprocessor(), combiner=_TotalAttributionsCombiner(key, absolute)) ]
def _total_attributions_computations( absolute: bool = True, name: str = '', eval_config: Optional[config_pb2.EvalConfig] = None, model_name: str = '', output_name: str = '', sub_key: Optional[metric_types.SubKey] = None, example_weighted: bool = False) -> metric_types.MetricComputations: """Returns metric computations for total attributions. Args: absolute: True to use absolute value when summing. name: Metric name. eval_config: Eval config. model_name: Optional model name (if multi-model evaluation). output_name: Optional output name (if multi-output model type). sub_key: Optional sub key. example_weighted: True if example weights should be applied. """ if not name: if absolute: name = '_' + TOTAL_ABSOLUTE_ATTRIBUTIONS_NAME else: name = '_' + TOTAL_ATTRIBUTIONS_NAME key = metric_types.AttributionsKey(name=name, model_name=model_name, output_name=output_name, sub_key=sub_key, example_weighted=example_weighted) return [ metric_types.MetricComputation( keys=[key], preprocessor=metric_types.AttributionPreprocessor(feature_keys={}), combiner=_TotalAttributionsCombiner(key, eval_config, absolute)) ]
def _mean_attributions( absolute: bool = True, name: str = MEAN_ATTRIBUTIONS_NAME, eval_config: Optional[config_pb2.EvalConfig] = None, model_name: str = '', output_name: str = '', sub_key: Optional[metric_types.SubKey] = None, example_weighted: bool = False, ) -> metric_types.MetricComputations: """Returns metric computations for mean attributions.""" key = metric_types.AttributionsKey(name=name, model_name=model_name, output_name=output_name, sub_key=sub_key, example_weighted=example_weighted) # Make sure total_attributions is calculated. computations = _total_attributions_computations( absolute=absolute, eval_config=eval_config, model_name=model_name, output_name=output_name, sub_key=sub_key, example_weighted=example_weighted) total_attributions_key = computations[-1].keys[-1] # Make sure example_count is calculated computations.extend( example_count.example_count(model_names=[model_name], output_names=[output_name], sub_keys=[sub_key], example_weighted=example_weighted)) example_count_key = computations[-1].keys[-1] def result( metrics: Dict[metric_types.MetricKey, Any] ) -> Dict[metric_types.AttributionsKey, Dict[str, Union[float, np.ndarray]]]: """Returns mean attributions.""" total_attributions = metrics[total_attributions_key] count = metrics[example_count_key] attributions = {} for k, v in total_attributions.items(): if np.isclose(count, 0.0): attributions[k] = float('nan') else: attributions[k] = v / count return {key: attributions} derived_computation = metric_types.DerivedMetricComputation(keys=[key], result=result) computations.append(derived_computation) return computations
def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_attributions = got[0] self.assertEqual(got_slice_key, ()) total_attributions_key = metric_types.AttributionsKey( name='total_attributions', sub_key=sub_key) self.assertIn(total_attributions_key, got_attributions) self.assertAllClose(got_attributions[total_attributions_key], expected_values) except AssertionError as err: raise util.BeamAssertException(err)
def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_attributions = got[0] self.assertEqual(got_slice_key, ()) total_attributions_key = metric_types.AttributionsKey( name='total_absolute_attributions', model_name=model_name, output_name=output_name) self.assertIn(total_attributions_key, got_attributions) self.assertDictElementsAlmostEqual( got_attributions[total_attributions_key], expected_values) except AssertionError as err: raise util.BeamAssertException(err)
def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_attributions = got[0] self.assertEqual(got_slice_key, ()) mean_attributions_key = metric_types.AttributionsKey( name='mean_attributions') self.assertIn(mean_attributions_key, got_attributions) self.assertDictElementsAlmostEqual( got_attributions[mean_attributions_key], { 'feature1': 1.0 / 0.5, 'feature2': -2.0 / 0.5, }) except AssertionError as err: raise util.BeamAssertException(err)
def testMeanAbsoluteAttributions(self): computation = attributions.MeanAbsoluteAttributions().computations( )[-1] total_absolute_attributions_key = metric_types.AttributionsKey( name='_total_absolute_attributions') weighted_examples_key = metric_types.MetricKey( name='weighted_example_count') metrics = { total_absolute_attributions_key: { 'feature1': 1.0, 'feature2': 2.0 }, weighted_examples_key: 0.5 } with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter result = (pipeline | 'Create' >> beam.Create([metrics]) | 'ComputeMetric' >> beam.Map(lambda x: ( (), computation.result(x)))) # pylint: enable=no-value-for-parameter def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_attributions = got[0] self.assertEqual(got_slice_key, ()) mean_attributions_key = metric_types.AttributionsKey( name='mean_absolute_attributions') self.assertIn(mean_attributions_key, got_attributions) self.assertDictElementsAlmostEqual( got_attributions[mean_attributions_key], { 'feature1': 1.0 / 0.5, 'feature2': 2.0 / 0.5, }) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(result, check_result, label='result')
def _total_attributions( absolute: bool = True, name: str = '', eval_config: Optional[config_pb2.EvalConfig] = None, model_name: str = '', output_name: str = '', sub_key: Optional[metric_types.SubKey] = None, example_weighted: bool = False) -> metric_types.MetricComputations: """Returns metric computations for total attributions.""" key = metric_types.AttributionsKey(name=name, model_name=model_name, output_name=output_name, sub_key=sub_key, example_weighted=example_weighted) # Make sure total_attributions is calculated. computations = _total_attributions_computations( absolute=absolute, eval_config=eval_config, model_name=model_name, output_name=output_name, sub_key=sub_key, example_weighted=example_weighted) private_key = computations[-1].keys[-1] def result( metrics: Dict[metric_types.MetricKey, Any] ) -> Dict[metric_types.AttributionsKey, Dict[str, Union[float, np.ndarray]]]: """Returns total attributions.""" return {key: metrics[private_key]} derived_computation = metric_types.DerivedMetricComputation(keys=[key], result=result) computations.append(derived_computation) return computations