def testRaiseErrorWhenBothLabelAndPlotKeyAreProvided(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), label='head_a', output_name='')
def testRaiseErrorWhenMoreThanOneMultiClassKeyAreProvided(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), top_k=3, class_id=0)
def testGetPlotDataAndConfig(self): data, eval_config = util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_1, self.slice_a)])) self.assertEqual(data, self.plots_data_a) self.assertEqual(eval_config['sliceName'], self.column_a)
def testGetPlotUsingLabel(self): data, _ = util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), label='head_a') self.assertEqual(data, self.plots_data_c2['label/head_a'])
def testFilterColumnResultBySpec(self): self.assertEqual( util.get_slicing_metrics( self._makeTestData(), slicing_spec=SingleSliceSpec(columns=[self.column_2])), [{ 'slice': self.column_c, 'metrics': self.metrics_c }])
def testFilterFeatrueCrossResultBySpec(self): self.assertEqual( util.get_slicing_metrics( self._makeTestData(), slicing_spec=SingleSliceSpec( features=[(self.column_1, self.slice_a), (self.column_2, self.slice_d)])), [{ 'slice': self.column_d, 'metrics': self.metrics_d }])
def testDisplayFullPathForTimeSeries(self): data = util.get_time_series( self._makeEvalResults(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), display_full_path=True) self.assertEqual(len(data), 2) self.assertEqual( data[1]['config'], { 'dataIdentifier': self.full_data_location_2, 'modelIdentifier': self.full_model_location_2 })
def testReplaceNaNInPlotWithNone(self): data, _ = util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_1, self.slice_c)])) self.assertEqual(data, { 'calibrationHistogramBuckets': { 'buckets': [{ 'v': 0.5 }, { 'v': None }], } })
def testFilterEvalResultsForTimeSeries(self): data = util.get_time_series( self._makeEvalResults(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), display_full_path=False) self.assertEqual(len(data), 2) self.assertEqual(data[0]['metrics'], self.metrics_c) self.assertEqual( data[0]['config'], { 'dataIdentifier': self.data_location_1, 'modelIdentifier': self.model_location_1 }) self.assertEqual(data[1]['metrics'], self.metrics_c2) self.assertEqual( data[1]['config'], { 'dataIdentifier': self.base_data_location_2, 'modelIdentifier': self.base_model_location_2 })
def find_all_slices( results: List[Tuple[slicer.SliceKeyType, Dict[Text, Any]]], slicing_spec: slicer.SingleSliceSpec ) -> List[Dict[Text, Union[Dict[Text, Any], Text]]]: """Util function that extracts slicing metrics for the named column. Args: results: A list of records. Each record is a tuple of (slice_name, {metric_name, metric_value}). slicing_spec: The spec to slice on. Returns: A list of {slice, metrics} """ data = [] for (slice_key, metric_value) in results: if slicing_spec.is_slice_applicable(slice_key): data.append({ 'slice': slicer.stringify_slice_key(slice_key), 'metrics': metric_value }) return data # pytype: disable=bad-return-type
def testRaisesErrorWhenMultipleMatchForLabel(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]), label='head_')
def testRaisesErrorWhenLabelNotProvided(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_c)]))
def testRaisesErrorWhenMultipleSlicesMatchInPlotData(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_1, self.slice_b)]))
def testRaisesErrorWhenNoMatchAvailableInPlotData(self): with self.assertRaises(ValueError): util.get_plot_data_and_config( self._makeTestPlotsData(), SingleSliceSpec(features=[(self.column_2, self.slice_a)]))
def testRaisesErrorWhenToomManyMatchesAvailableInTimeSeries(self): with self.assertRaises(ValueError): util.get_time_series( self._makeEvalResults(), SingleSliceSpec(columns=[(self.column_1)]), display_full_path=False)
def testRaisesErrorWhenNoMatchAvailableInTimeSeries(self): with self.assertRaises(ValueError): util.get_time_series( self._makeEvalResults(), SingleSliceSpec(features=[(self.column_2, self.slice_a)]), display_full_path=False)