def testRaiseErrorWhenBothLabelAndPlotKeyAreProvided(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
label='head_a',
output_name='')
def testRaiseErrorWhenMoreThanOneMultiClassKeyAreProvided(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
top_k=3,
class_id=0)
def testGetPlotDataAndConfig(self):
data, eval_config = util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_1, self.slice_a)]))
self.assertEqual(data, self.plots_data_a)
self.assertEqual(eval_config['sliceName'], self.column_a)
def testGetPlotUsingLabel(self):
data, _ = util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
label='head_a')
self.assertEqual(data, self.plots_data_c2['label/head_a'])
def testFilterColumnResultBySpec(self):
self.assertEqual(
util.get_slicing_metrics(
self._makeTestData(),
slicing_spec=SingleSliceSpec(columns=[self.column_2])), [{
'slice': self.column_c,
'metrics': self.metrics_c
}])
def testFilterFeatrueCrossResultBySpec(self):
self.assertEqual(
util.get_slicing_metrics(
self._makeTestData(),
slicing_spec=SingleSliceSpec(
features=[(self.column_1,
self.slice_a), (self.column_2, self.slice_d)])), [{
'slice': self.column_d,
'metrics': self.metrics_d
}])
def testDisplayFullPathForTimeSeries(self):
data = util.get_time_series(
self._makeEvalResults(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
display_full_path=True)
self.assertEqual(len(data), 2)
self.assertEqual(
data[1]['config'], {
'dataIdentifier': self.full_data_location_2,
'modelIdentifier': self.full_model_location_2
})
def testReplaceNaNInPlotWithNone(self):
data, _ = util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_1, self.slice_c)]))
self.assertEqual(data, {
'calibrationHistogramBuckets': {
'buckets': [{
'v': 0.5
}, {
'v': None
}],
}
})
def testFilterEvalResultsForTimeSeries(self):
data = util.get_time_series(
self._makeEvalResults(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
display_full_path=False)
self.assertEqual(len(data), 2)
self.assertEqual(data[0]['metrics'], self.metrics_c)
self.assertEqual(
data[0]['config'], {
'dataIdentifier': self.data_location_1,
'modelIdentifier': self.model_location_1
})
self.assertEqual(data[1]['metrics'], self.metrics_c2)
self.assertEqual(
data[1]['config'], {
'dataIdentifier': self.base_data_location_2,
'modelIdentifier': self.base_model_location_2
})
def find_all_slices(
results: List[Tuple[slicer.SliceKeyType,
Dict[Text, Any]]], slicing_spec: slicer.SingleSliceSpec
) -> List[Dict[Text, Union[Dict[Text, Any], Text]]]:
"""Util function that extracts slicing metrics for the named column.
Args:
results: A list of records. Each record is a tuple of (slice_name,
{metric_name, metric_value}).
slicing_spec: The spec to slice on.
Returns:
A list of {slice, metrics}
"""
data = []
for (slice_key, metric_value) in results:
if slicing_spec.is_slice_applicable(slice_key):
data.append({
'slice': slicer.stringify_slice_key(slice_key),
'metrics': metric_value
})
return data # pytype: disable=bad-return-type
def testRaisesErrorWhenMultipleMatchForLabel(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]),
label='head_')
def testRaisesErrorWhenLabelNotProvided(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_c)]))
def testRaisesErrorWhenMultipleSlicesMatchInPlotData(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_1, self.slice_b)]))
def testRaisesErrorWhenNoMatchAvailableInPlotData(self):
with self.assertRaises(ValueError):
util.get_plot_data_and_config(
self._makeTestPlotsData(),
SingleSliceSpec(features=[(self.column_2, self.slice_a)]))
def testRaisesErrorWhenToomManyMatchesAvailableInTimeSeries(self):
with self.assertRaises(ValueError):
util.get_time_series(
self._makeEvalResults(),
SingleSliceSpec(columns=[(self.column_1)]),
display_full_path=False)
def testRaisesErrorWhenNoMatchAvailableInTimeSeries(self):
with self.assertRaises(ValueError):
util.get_time_series(
self._makeEvalResults(),
SingleSliceSpec(features=[(self.column_2, self.slice_a)]),
display_full_path=False)
