def testFairessIndicatorsMetricsWithNanValue(self):
computations = fairness_indicators.FairnessIndicators(
thresholds=[0.5]).computations()
histogram = computations[0]
matrices = computations[1]
metrics = computations[2]
examples = [{
'labels': np.array([0.0]),
'predictions': np.array([0.1]),
'example_weights': np.array([1.0]),
}, {
'labels': np.array([0.0]),
'predictions': np.array([0.7]),
'example_weights': np.array([1.0]),
}]
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(examples)
| 'Process' >> beam.Map(metric_util.to_standard_metric_inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner)
| 'ComputeMatrices' >>
beam.Map(lambda x:
(x[0], matrices.result(x[1]))) # pyformat: ignore
| 'ComputeMetrics' >> beam.Map(lambda x:
(x[0], metrics.result(x[1])))
) # pyformat: ignore
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
self.assertLen(got_metrics, 6) # 1 threshold * 6 metrics
self.assertTrue(
math.isnan(got_metrics[metric_types.MetricKey(
name=
'fairness_indicators_metrics/[email protected]',
model_name='',
output_name='',
sub_key=None)]))
self.assertTrue(
math.isnan(got_metrics[metric_types.MetricKey(
name=
'fairness_indicators_metrics/[email protected]',
model_name='',
output_name='',
sub_key=None)]))
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsWithThresholds(self, kwargs,
expected_metrics_nums,
expected_metrics_keys):
# This is a parameterized test with following parameters.
# - metric parameters like thresholds.
# - expected number of metrics computed
# - expected list of metrics keys
computations = fairness_indicators.FairnessIndicators(
**kwargs).computations(example_weighted=True)
histogram = computations[0]
matrices = computations[1]
metrics = computations[2]
examples = [{
'labels': np.array([0.0]),
'predictions': np.array([0.1]),
'example_weights': np.array([1.0]),
}, {
'labels': np.array([0.0]),
'predictions': np.array([0.7]),
'example_weights': np.array([3.0]),
}]
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(examples)
| 'Process' >> beam.Map(metric_util.to_standard_metric_inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner)
| 'ComputeMatrices' >>
beam.Map(lambda x:
(x[0], matrices.result(x[1]))) # pyformat: ignore
| 'ComputeMetrics' >> beam.Map(lambda x:
(x[0], metrics.result(x[1])))
) # pyformat: ignore
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
self.assertLen(got_metrics, expected_metrics_nums)
for metrics_key in expected_metrics_keys:
self.assertIn(metrics_key, got_metrics)
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsWithInput(self, input_examples,
computations_kwargs,
expected_result):
# This is a parameterized test with following parameters.
# - input examples to be used in the test
# - parameters like model name etc.
# - expected result to assert on
computations = fairness_indicators.FairnessIndicators(
thresholds=[0.5]).computations(**computations_kwargs)
histogram = computations[0]
matrices = computations[1]
metrics = computations[2]
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(input_examples)
| 'Process' >> beam.Map(metric_util.to_standard_metric_inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner)
| 'ComputeMatrices' >>
beam.Map(lambda x:
(x[0], matrices.result(x[1]))) # pyformat: ignore
| 'ComputeMetrics' >> beam.Map(lambda x:
(x[0], metrics.result(x[1])))
) # pyformat: ignore
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
self.assertLen(got_metrics, 8) # 1 threshold * 8 metrics
for metrics_key in expected_result:
self.assertEqual(got_metrics[metrics_key],
expected_result[metrics_key])
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsGeneral(self):
computations = fairness_indicators.FairnessIndicators(
thresholds=[0.3, 0.7]).computations()
histogram = computations[0]
matrices = computations[1]
metrics = computations[2]
examples = [{
'labels': np.array([0.0]),
'predictions': np.array([0.1]),
'example_weights': np.array([1.0]),
}, {
'labels': np.array([0.0]),
'predictions': np.array([0.5]),
'example_weights': np.array([1.0]),
}, {
'labels': np.array([1.0]),
'predictions': np.array([0.5]),
'example_weights': np.array([1.0]),
}, {
'labels': np.array([1.0]),
'predictions': np.array([0.9]),
'example_weights': np.array([1.0]),
}]
with beam.Pipeline() as pipeline:
# pylint: disable=no-value-for-parameter
result = (
pipeline
| 'Create' >> beam.Create(examples)
| 'Process' >> beam.Map(metric_util.to_standard_metric_inputs)
| 'AddSlice' >> beam.Map(lambda x: ((), x))
| 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner)
| 'ComputeMatrices' >>
beam.Map(lambda x:
(x[0], matrices.result(x[1]))) # pyformat: ignore
| 'ComputeMetrics' >> beam.Map(lambda x:
(x[0], metrics.result(x[1])))
) # pyformat: ignore
# pylint: enable=no-value-for-parameter
def check_result(got):
try:
self.assertLen(got, 1)
got_slice_key, got_metrics = got[0]
self.assertEqual(got_slice_key, ())
self.assertLen(got_metrics, 16) # 2 thresholds * 8 metrics
self.assertDictElementsAlmostEqual(
got_metrics, {
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.5,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
1.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.5,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.75,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.25,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
1.0 / 3.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.5,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.5,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
1.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.25,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.75,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
0.0,
metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'):
1.0 / 3.0
})
except AssertionError as err:
raise util.BeamAssertException(err)
util.assert_that(result, check_result, label='result')
