def test_remove_subset_slices(self):
input_slices = [
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ('country', 'USA')),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ('country', 'UK')),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ('country', 'UK'), ('sex',
'M')),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
]
expected_slices = [
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ('country', 'UK')),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
]
self.assertCountEqual(
auto_slicing_util.remove_subset_slices(input_slices),
expected_slices)
self.assertCountEqual(auto_slicing_util.remove_subset_slices([]), [])
def test_find_top_slices(self):
statistics = text_format.Parse(
"""
datasets{
num_examples: 1500
features {
path { step: 'country' }
type: STRING
string_stats {
unique: 10
}
}
features {
path { step: 'age' }
type: INT
num_stats {
common_stats {
num_non_missing: 1500
min_num_values: 1
max_num_values: 1
}
min: 1
max: 18
histograms {
buckets {
low_value: 1
high_value: 6.0
sample_count: 500
}
buckets {
low_value: 6.0
high_value: 12.0
sample_count: 500
}
buckets {
low_value: 12.0
high_value: 18.0
sample_count: 500
}
type: QUANTILES
}
}
}
}
""", statistics_pb2.DatasetFeatureStatisticsList())
metrics = [
text_format.Parse(
"""
slice_key {
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.8 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.8 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.8 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 1500 }
lower_bound { value: 1500 }
upper_bound { value: 1500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 1500 }
upper_bound { value: 1500 }
t_distribution_value {
sample_mean { value: 1500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 1500 }
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'transformed_age'
int64_value: 1
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.4 }
lower_bound { value: 0.3737843 }
upper_bound { value: 0.6262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.3737843 }
upper_bound { value: 0.6262157 }
t_distribution_value {
sample_mean { value: 0.4 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.4 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500 }
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'transformed_age'
int64_value: 2
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.79 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.79 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.79 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'transformed_age'
int64_value: 3
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.9 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.9 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.9 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'country'
bytes_value: 'USA'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.9 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.9 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.9 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice())
]
self.assertCountEqual(
auto_slicing_util.find_top_slices(metrics,
metric_key='accuracy',
statistics=statistics,
comparison_type='LOWER'),
[
auto_slicing_util.SliceComparisonResult(
slice_key=u'age:[1.0, 6.0]',
num_examples=500.0,
slice_metric=0.4,
base_metric=0.8,
pvalue=0.0,
effect_size=4.0)
])
self.assertCountEqual(
auto_slicing_util.find_top_slices(metrics,
metric_key='accuracy',
statistics=statistics,
comparison_type='HIGHER'),
[
auto_slicing_util.SliceComparisonResult(
slice_key=u'age:[12.0, 18.0]',
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
pvalue=7.356017854191938e-70,
effect_size=0.9999999999999996),
auto_slicing_util.SliceComparisonResult(
slice_key=u'country:USA',
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
pvalue=7.356017854191938e-70,
effect_size=0.9999999999999996)
])
def test_revert_slice_keys_for_transformed_features(self):
statistics = text_format.Parse(
"""
datasets{
num_examples: 1500
features {
path { step: 'country' }
type: STRING
string_stats {
unique: 10
}
}
features {
path { step: 'age' }
type: INT
num_stats {
common_stats {
num_non_missing: 1500
min_num_values: 1
max_num_values: 1
}
min: 1
max: 18
histograms {
buckets {
low_value: 1
high_value: 6.0
sample_count: 500
}
buckets {
low_value: 6.0
high_value: 12.0
sample_count: 500
}
buckets {
low_value: 12.0
high_value: 18.0
sample_count: 500
}
type: QUANTILES
}
}
}
}
""", statistics_pb2.DatasetFeatureStatisticsList())
slices = [
auto_slicing_util.SliceComparisonResult(slice_key=(),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('transformed_age', 1), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('transformed_age', 2), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(slice_key=(('country',
'USA'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
]
expected_slices = [
auto_slicing_util.SliceComparisonResult(slice_key=(),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[1.0, 6.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(slice_key=(('country',
'USA'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.9,
raw_slice_metrics=None),
]
actual = auto_slicing_util.revert_slice_keys_for_transformed_features(
slices, statistics)
self.assertEqual(actual, expected_slices)
def test_find_top_slices(self):
input_slices = [
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=1500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.8,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0.00001,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ('country', 'USA')),
num_examples=500.0,
slice_metric=0.91,
base_metric=0.8,
p_value=0.000011,
effect_size=0.91,
raw_slice_metrics=None)
]
self.assertCountEqual(
auto_slicing_util.find_top_slices(input_slices,
rank_by='EFFECT_SIZE',
prune_subset_slices=False),
[
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ('country', 'USA')),
num_examples=500.0,
slice_metric=0.91,
base_metric=0.8,
p_value=0.000011,
effect_size=0.91,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0.00001,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=1500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.8,
raw_slice_metrics=None),
])
self.assertCountEqual(
auto_slicing_util.find_top_slices(input_slices,
rank_by='EFFECT_SIZE',
prune_subset_slices=True),
[
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0.00001,
effect_size=0.9,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=1500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.8,
raw_slice_metrics=None),
])
self.assertCountEqual(
auto_slicing_util.find_top_slices(input_slices, rank_by='PVALUE'),
[
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=1500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.8,
raw_slice_metrics=None),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0.00001,
effect_size=0.9,
raw_slice_metrics=None),
])
self.assertCountEqual(
auto_slicing_util.find_top_slices(input_slices,
min_num_examples=1000,
rank_by='EFFECT_SIZE'),
[
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[6.0, 12.0)'), ),
num_examples=1500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=0,
effect_size=0.8,
raw_slice_metrics=None),
])
def test_get_slices_as_dataframe(self):
input_slices = [
auto_slicing_util.SliceComparisonResult(
slice_key=(('native-country', 'United-States'), ),
num_examples=29170,
slice_metric=0.09,
base_metric=0.087,
p_value=0,
effect_size=0.46,
raw_slice_metrics=text_format.Parse(
"""
slice_key {
single_slice_keys {
column: "native-country"
bytes_value: "United-States"
}
}
metric_keys_and_values {
key { name: "false_positives" }
value {
bounded_value {
lower_bound { value: 1754.6514199722158 }
upper_bound { value: 2092.488580027784 }
value { value: 1923.57 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 1754.6514199722158 }
upper_bound { value: 2092.488580027784 }
t_distribution_value {
sample_mean { value: 1923.57 }
sample_standard_deviation { value: 85.13110418664061 }
sample_degrees_of_freedom { value: 99 }
unsampled_value { value: 1943.0 }
}
}
}
}
metric_keys_and_values {
key { name: "false_negatives" }
value {
bounded_value {
lower_bound { value: 3595.413107983637 }
upper_bound { value: 4195.886892016363 }
value { value: 3895.65 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 3595.413107983637 }
upper_bound { value: 4195.886892016363 }
t_distribution_value {
sample_mean { value: 3895.65 }
sample_standard_deviation { value: 151.31253252729257 }
sample_degrees_of_freedom { value: 99 }
unsampled_value { value: 3935.0 }
}
}
}
}""", metrics_for_slice_pb2.MetricsForSlice())),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[58.0, 90.0)'), ),
num_examples=2999,
slice_metric=0.09,
base_metric=0.0875,
p_value=7.8,
effect_size=0.98,
raw_slice_metrics=text_format.Parse(
"""
slice_key {
single_slice_keys {
column: "age"
bytes_value: "[58.0, 90.0)"
}
}
metric_keys_and_values {
key { name: "false_positives" }
value {
bounded_value {
lower_bound { value: 167.54646972321814 }
upper_bound { value: 236.37353027678188 }
value { value: 201.96 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 167.54646972321814 }
upper_bound { value: 236.37353027678188 }
t_distribution_value {
sample_mean { value: 201.96 }
sample_standard_deviation { value: 17.343632837435358 }
sample_degrees_of_freedom { value: 99 }
unsampled_value { value: 204.0 }
}
}
}
}
metric_keys_and_values {
key { name: "false_negatives" }
value {
bounded_value {
lower_bound { value: 486.4402337348782 }
upper_bound { value: 610.479766265122 }
value { value: 548.46 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 486.4402337348782 }
upper_bound { value: 610.479766265122 }
t_distribution_value {
sample_mean { value: 548.46 }
sample_standard_deviation { value: 31.256544914589938 }
sample_degrees_of_freedom { value: 99 }
unsampled_value { value: 554.0 }
}
}
}
}""", metrics_for_slice_pb2.MetricsForSlice()))
]
additional_metric_keys = [
metric_types.MetricKey('false_positives'),
metric_types.MetricKey('false_negatives')
]
expected_dataframe_data = [{
'Slice':
slicer_lib.stringify_slice_key(input_slices[0].slice_key),
'Size':
input_slices[0].num_examples,
'Slice metric':
input_slices[0].slice_metric,
'Base metric':
input_slices[0].base_metric,
'P-Value':
input_slices[0].p_value,
'Effect size':
input_slices[0].effect_size,
str(additional_metric_keys[0]):
1923.57,
str(additional_metric_keys[1]):
3895.65
}, {
'Slice':
slicer_lib.stringify_slice_key(input_slices[1].slice_key),
'Size':
input_slices[1].num_examples,
'Slice metric':
input_slices[1].slice_metric,
'Base metric':
input_slices[1].base_metric,
'P-Value':
input_slices[1].p_value,
'Effect size':
input_slices[1].effect_size,
str(additional_metric_keys[0]):
201.96,
str(additional_metric_keys[1]):
548.46
}]
expected_dataframe_column_labels = [
'Slice', 'Size', 'Slice metric', 'Base metric', 'P-Value',
'Effect size',
str(additional_metric_keys[0]),
str(additional_metric_keys[1])
]
expected_dataframe = pd.DataFrame(
expected_dataframe_data, columns=expected_dataframe_column_labels)
expected_dataframe.set_index('Slice', inplace=True)
actual_dataframe = auto_slicing_util.get_slices_as_dataframe(
input_slices, additional_metric_keys)
assert_frame_equal(actual_dataframe, expected_dataframe)
def test_find_significant_slices(self):
metrics = [
text_format.Parse(
"""
slice_key {
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.8 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.8 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.8 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 1500 }
lower_bound { value: 1500 }
upper_bound { value: 1500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 1500 }
upper_bound { value: 1500 }
t_distribution_value {
sample_mean { value: 1500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 1500 }
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'age'
bytes_value: '[1.0, 6.0)'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.4 }
lower_bound { value: 0.3737843 }
upper_bound { value: 0.6262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.3737843 }
upper_bound { value: 0.6262157 }
t_distribution_value {
sample_mean { value: 0.4 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.4 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500 }
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'age'
bytes_value: '[6.0, 12.0)'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.79 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.79 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.79 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'age'
bytes_value: '[12.0, 18.0)'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.9 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.9 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.9 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'country'
bytes_value: 'USA'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.9 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.9 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.9 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice()),
text_format.Parse(
"""
slice_key {
single_slice_keys {
column: 'country'
bytes_value: 'USA'
}
single_slice_keys {
column: 'age'
bytes_value: '[12.0, 18.0)'
}
}
metric_keys_and_values {
key { name: "accuracy" }
value {
bounded_value {
value { value: 0.9 }
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 0.5737843 }
upper_bound { value: 1.0262157 }
t_distribution_value {
sample_mean { value: 0.9 }
sample_standard_deviation { value: 0.1 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 0.9 }
}
}
}
}
metric_keys_and_values {
key { name: "example_count" }
value {
bounded_value {
value { value: 500 }
lower_bound { value: 500 }
upper_bound { value: 500 }
methodology: POISSON_BOOTSTRAP
}
confidence_interval {
lower_bound { value: 500 }
upper_bound { value: 500 }
t_distribution_value {
sample_mean { value: 500 }
sample_standard_deviation { value: 0 }
sample_degrees_of_freedom { value: 9 }
unsampled_value { value: 500}
}
}
}
}
""", metrics_for_slice_pb2.MetricsForSlice())
]
self.assertCountEqual(
auto_slicing_util.find_significant_slices(
metrics, metric_key='accuracy', comparison_type='LOWER'), [
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[1.0, 6.0)'),),
num_examples=500.0,
slice_metric=0.4,
base_metric=0.8,
p_value=0.0,
effect_size=4.0,
raw_slice_metrics=metrics[1])
])
self.assertCountEqual(
auto_slicing_util.find_significant_slices(
metrics, metric_key='accuracy', comparison_type='HIGHER'), [
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'),),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=7.356017854191938e-70,
effect_size=0.9999999999999996,
raw_slice_metrics=metrics[3]),
auto_slicing_util.SliceComparisonResult(
slice_key=(('country', 'USA'),),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=7.356017854191938e-70,
effect_size=0.9999999999999996,
raw_slice_metrics=metrics[4]),
auto_slicing_util.SliceComparisonResult(
slice_key=(('age', '[12.0, 18.0)'), ('country', 'USA')),
num_examples=500.0,
slice_metric=0.9,
base_metric=0.8,
p_value=7.356017854191938e-70,
effect_size=0.9999999999999996,
raw_slice_metrics=metrics[5])
])