def render_slicing_attributions( result: view_types.EvalResult, slicing_column: Optional[Text] = None, slicing_spec: Optional[Union[slicer.SingleSliceSpec, config_pb2.SlicingSpec]] = None, metric_name: Optional[Text] = None, weighted_example_column: Optional[Text] = None, event_handlers: Optional[Callable[[Dict[Text, Union[Text, float]]], None]] = None, ) -> Optional[visualization.SlicingMetricsViewer]: # pytype: disable=invalid-annotation """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The tfma.SlicingSpec to filter results. If neither column nor spec is set, show overall. metric_name: Name of attributions metric to show attributions for. Optional if only one metric used. weighted_example_column: Override for the weighted example column. This can be used when different weights are applied in different aprts of the model (eg: multi-head). event_handlers: The event handlers Returns: A SlicingMetricsViewer object if in Jupyter notebook; None if in Colab. """ if slicing_spec and isinstance(slicing_spec, config_pb2.SlicingSpec): slicing_spec = slicer.SingleSliceSpec(spec=slicing_spec) data = util.get_slicing_metrics(result.attributions, slicing_column, slicing_spec) # Attributions have one additional level of indirection for the metric_name. # Filter this out using the metric_name provided. for d in data: updated_data = {} for output_name, per_output_items in d['metrics'].items(): # pytype: disable=attribute-error updated_data[output_name] = {} for sub_key, per_sub_key_items in per_output_items.items(): updated_data[output_name][sub_key] = {} if metric_name: if metric_name not in per_sub_key_items: raise ValueError( 'metric_name={} not found in {}'.format( metric_name, per_sub_key_items.keys())) updated_data[output_name][sub_key] = per_sub_key_items[ metric_name] elif len(per_sub_key_items) == 1: updated_data[output_name][sub_key] = list( per_sub_key_items.values())[0] else: raise ValueError( 'metric_name must be one of the following: {}'.format( per_sub_key_items.keys())) d['metrics'] = updated_data cfg = util.get_slicing_config(result.config, weighted_example_column) return visualization.render_slicing_metrics(data, cfg, event_handlers=event_handlers)
def render_slicing_metrics( result: view_types.EvalResult, slicing_column: Optional[Text] = None, slicing_spec: Optional[Union[slicer.SingleSliceSpec, config_pb2.SlicingSpec]] = None, weighted_example_column: Optional[Text] = None, event_handlers: Optional[Callable[[Dict[Text, Union[Text, float]]], None]] = None, ) -> Optional[visualization.SlicingMetricsViewer]: # pytype: disable=invalid-annotation """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The tfma.SlicingSpec to filter results. If neither column nor spec is set, show overall. weighted_example_column: Override for the weighted example column. This can be used when different weights are applied in different aprts of the model (eg: multi-head). event_handlers: The event handlers Returns: A SlicingMetricsViewer object if in Jupyter notebook; None if in Colab. """ if slicing_spec and isinstance(slicing_spec, config_pb2.SlicingSpec): slicing_spec = slicer.SingleSliceSpec(spec=slicing_spec) data = util.get_slicing_metrics(result.slicing_metrics, slicing_column, slicing_spec) cfg = util.get_slicing_config(result.config, weighted_example_column) return visualization.render_slicing_metrics(data, cfg, event_handlers=event_handlers)
def render_slicing_metrics( result: model_eval_lib.EvalResult, slicing_column: Optional[Text] = None, slicing_spec: Optional[SingleSliceSpec] = None, weighted_example_column: Text = None ) -> Optional[visualization.SlicingMetricsViewer]: """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The slicing spec to filter results. If neither column nor spec is set, show overall. weighted_example_column: Override for the weighted example column. This can be used when different weights are applied in different aprts of the model (eg: multi-head). Returns: A SlicingMetricsViewer object if in Jupyter notebook; None if in Colab. """ data = util.get_slicing_metrics(result.slicing_metrics, slicing_column, slicing_spec) config = util.get_slicing_config(result.config, weighted_example_column) return visualization.render_slicing_metrics(data, config)
def testFilterColumnResultBySpec(self): self.assertEqual( util.get_slicing_metrics( self._makeTestData(), slicing_spec=SingleSliceSpec(columns=[self.column_2])), [{ 'slice': self.column_c, 'metrics': self.metrics_c }])
def testGetSlicingMetrics(self): self.assertEqual( util.get_slicing_metrics(self._makeTestData(), self.column_1), [{ 'slice': self.column_a, 'metrics': self.metrics_a }, { 'slice': self.column_b, 'metrics': self.metrics_b }])
def testFilterFeatrueResultBySpec(self): self.assertEqual( util.get_slicing_metrics( self._makeTestData(), slicing_spec=SingleSliceSpec(features=[(self.column_2, self.slice_c)])), [{ 'slice': self.column_c, 'metrics': self.metrics_c }])
def testFilterColumnCrossFeatrueResultBySpec(self): self.assertEqual( util.get_slicing_metrics( self._makeTestData(), slicing_spec=SingleSliceSpec( columns=[self.column_1], features=[(self.column_2, self.slice_d)])), [{ 'slice': self.column_d, 'metrics': self.metrics_d }])
def render_slicing_metrics(result, slicing_column=None, slicing_spec=None): """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The slicing spec to filter results. If neither column nor spec is set, show overall. Returns: A SlicingMetricsViewer object if in Jupyter notebook; None if in Colab. """ data = util.get_slicing_metrics(result.slicing_metrics, slicing_column, slicing_spec) config = { 'weightedExamplesColumn': result.config.example_weight_metric_key } return visualization.render_slicing_metrics(data, config)
def render_slicing_metrics( result: model_eval_lib.EvalResult, slicing_column: Optional[Text] = None, slicing_spec: Optional[SingleSliceSpec] = None ) -> Optional[visualization.SlicingMetricsViewer]: """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The slicing spec to filter results. If neither column nor spec is set, show overall. Returns: A SlicingMetricsViewer object if in Jupyter notebook; None if in Colab. """ data = util.get_slicing_metrics(result.slicing_metrics, slicing_column, slicing_spec) config = util.get_slicing_config(result.config) return visualization.render_slicing_metrics(data, config)
def render_slicing_metrics( result, slicing_column = None, slicing_spec = None): """Renders the slicing metrics view as widget. Args: result: An tfma.EvalResult. slicing_column: The column to slice on. slicing_spec: The slicing spec to filter results. If neither column nor spec is set, show overall. Returns: A SlicingMetricsViewer object. """ view = tfma_widget.SlicingMetricsViewer() view.data = util.get_slicing_metrics(result.slicing_metrics, slicing_column, slicing_spec) view.config = { 'weightedExamplesColumn': result.config.example_weight_metric_key } return view
def testRaisesErrorWhenMoreThanOneAggregateAvailable(self): with self.assertRaises(ValueError): util.get_slicing_metrics([('', self.metrics_a), ('', self.metrics_b)])
def testRaisesErrorWhenAggregateNotAvailable(self): with self.assertRaises(ValueError): util.get_slicing_metrics([(self.column_a, self.metrics_a), (self.column_b, self.metrics_b)])
def testRaisesErrorWhenColumnNotAvailable(self): with self.assertRaises(ValueError): util.get_slicing_metrics(self._makeTestData(), 'col3')
def testGetAggregateMetrics(self): self.assertEqual(util.get_slicing_metrics(self._makeTestData()), [{ 'slice': OVERALL_SLICE_NAME, 'metrics': self.metrics_aggregate }])