def testFairessIndicatorsMetricsWithNanValue(self): computations = fairness_indicators.FairnessIndicators( thresholds=[0.5]).computations() histogram = computations[0] matrices = computations[1] metrics = computations[2] examples = [{ 'labels': np.array([0.0]), 'predictions': np.array([0.1]), 'example_weights': np.array([1.0]), }, { 'labels': np.array([0.0]), 'predictions': np.array([0.7]), 'example_weights': np.array([1.0]), }] with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter result = ( pipeline | 'Create' >> beam.Create(examples) | 'Process' >> beam.Map(metric_util.to_standard_metric_inputs) | 'AddSlice' >> beam.Map(lambda x: ((), x)) | 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner) | 'ComputeMatrices' >> beam.Map(lambda x: (x[0], matrices.result(x[1]))) # pyformat: ignore | 'ComputeMetrics' >> beam.Map(lambda x: (x[0], metrics.result(x[1]))) ) # pyformat: ignore # pylint: enable=no-value-for-parameter def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) self.assertLen(got_metrics, 6) # 1 threshold * 6 metrics self.assertTrue( math.isnan(got_metrics[metric_types.MetricKey( name= 'fairness_indicators_metrics/[email protected]', model_name='', output_name='', sub_key=None)])) self.assertTrue( math.isnan(got_metrics[metric_types.MetricKey( name= 'fairness_indicators_metrics/[email protected]', model_name='', output_name='', sub_key=None)])) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsWithThresholds(self, kwargs, expected_metrics_nums, expected_metrics_keys): # This is a parameterized test with following parameters. # - metric parameters like thresholds. # - expected number of metrics computed # - expected list of metrics keys computations = fairness_indicators.FairnessIndicators( **kwargs).computations(example_weighted=True) histogram = computations[0] matrices = computations[1] metrics = computations[2] examples = [{ 'labels': np.array([0.0]), 'predictions': np.array([0.1]), 'example_weights': np.array([1.0]), }, { 'labels': np.array([0.0]), 'predictions': np.array([0.7]), 'example_weights': np.array([3.0]), }] with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter result = ( pipeline | 'Create' >> beam.Create(examples) | 'Process' >> beam.Map(metric_util.to_standard_metric_inputs) | 'AddSlice' >> beam.Map(lambda x: ((), x)) | 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner) | 'ComputeMatrices' >> beam.Map(lambda x: (x[0], matrices.result(x[1]))) # pyformat: ignore | 'ComputeMetrics' >> beam.Map(lambda x: (x[0], metrics.result(x[1]))) ) # pyformat: ignore # pylint: enable=no-value-for-parameter def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) self.assertLen(got_metrics, expected_metrics_nums) for metrics_key in expected_metrics_keys: self.assertIn(metrics_key, got_metrics) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsWithInput(self, input_examples, computations_kwargs, expected_result): # This is a parameterized test with following parameters. # - input examples to be used in the test # - parameters like model name etc. # - expected result to assert on computations = fairness_indicators.FairnessIndicators( thresholds=[0.5]).computations(**computations_kwargs) histogram = computations[0] matrices = computations[1] metrics = computations[2] with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter result = ( pipeline | 'Create' >> beam.Create(input_examples) | 'Process' >> beam.Map(metric_util.to_standard_metric_inputs) | 'AddSlice' >> beam.Map(lambda x: ((), x)) | 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner) | 'ComputeMatrices' >> beam.Map(lambda x: (x[0], matrices.result(x[1]))) # pyformat: ignore | 'ComputeMetrics' >> beam.Map(lambda x: (x[0], metrics.result(x[1]))) ) # pyformat: ignore # pylint: enable=no-value-for-parameter def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) self.assertLen(got_metrics, 8) # 1 threshold * 8 metrics for metrics_key in expected_result: self.assertEqual(got_metrics[metrics_key], expected_result[metrics_key]) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(result, check_result, label='result')
def testFairessIndicatorsMetricsGeneral(self): computations = fairness_indicators.FairnessIndicators( thresholds=[0.3, 0.7]).computations() histogram = computations[0] matrices = computations[1] metrics = computations[2] examples = [{ 'labels': np.array([0.0]), 'predictions': np.array([0.1]), 'example_weights': np.array([1.0]), }, { 'labels': np.array([0.0]), 'predictions': np.array([0.5]), 'example_weights': np.array([1.0]), }, { 'labels': np.array([1.0]), 'predictions': np.array([0.5]), 'example_weights': np.array([1.0]), }, { 'labels': np.array([1.0]), 'predictions': np.array([0.9]), 'example_weights': np.array([1.0]), }] with beam.Pipeline() as pipeline: # pylint: disable=no-value-for-parameter result = ( pipeline | 'Create' >> beam.Create(examples) | 'Process' >> beam.Map(metric_util.to_standard_metric_inputs) | 'AddSlice' >> beam.Map(lambda x: ((), x)) | 'ComputeHistogram' >> beam.CombinePerKey(histogram.combiner) | 'ComputeMatrices' >> beam.Map(lambda x: (x[0], matrices.result(x[1]))) # pyformat: ignore | 'ComputeMetrics' >> beam.Map(lambda x: (x[0], metrics.result(x[1]))) ) # pyformat: ignore # pylint: enable=no-value-for-parameter def check_result(got): try: self.assertLen(got, 1) got_slice_key, got_metrics = got[0] self.assertEqual(got_slice_key, ()) self.assertLen(got_metrics, 16) # 2 thresholds * 8 metrics self.assertDictElementsAlmostEqual( got_metrics, { metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.5, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 1.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.5, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.75, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.25, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 1.0 / 3.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.5, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.5, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 1.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.25, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.75, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 0.0, metric_types.MetricKey(name='fairness_indicators_metrics/[email protected]'): 1.0 / 3.0 }) except AssertionError as err: raise util.BeamAssertException(err) util.assert_that(result, check_result, label='result')