def test_all_scores_valid(self): # test standard pandas table creation y_true = np.array([1, 1, 1, 0, 1, 0, 1, 1, 1, 1]) y_pred = np.array([0, 0, 0, 1, 0, 0, 0, 0, 0, 1]) is_member = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) df = BinaryFairnessMetrics.get_all_scores(y_true, y_pred, is_member) assert type(df) == pd.DataFrame assert self.extract_metric_from_df('Statistical Parity', df) == 0. assert self.extract_metric_from_df('Average Odds', df) == 0.375 assert self.extract_metric_from_df('Disparate Impact', df) == 1. assert self.extract_metric_from_df('FNR difference', df) == 0.25 assert self.extract_metric_from_df('Predictive Equality', df) == 1.00 assert self.extract_metric_from_df('Generalized Entropy Index', df) == 1.375 assert np.isclose(self.extract_metric_from_df('Theil Index', df), 1.263, atol=0.01) attr = 'Ideal Value' assert self.extract_metric_from_df('Average Odds', df, attr) == 0 assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == 0 assert self.extract_metric_from_df('FNR difference', df, attr) == 0 assert self.extract_metric_from_df('Generalized Entropy Index', df, attr) == 0 assert self.extract_metric_from_df('Predictive Equality', df, attr) == 0 assert self.extract_metric_from_df('Statistical Parity', df, attr) == 0 assert self.extract_metric_from_df('Theil Index', df, attr) == 0 attr = 'Lower Bound' assert self.extract_metric_from_df('Average Odds', df, attr) == -.2 assert self.extract_metric_from_df('Disparate Impact', df, attr) == .8 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == -.2 assert self.extract_metric_from_df('FNR difference', df, attr) == -.2 assert self.extract_metric_from_df('Generalized Entropy Index', df, attr) == 0. assert self.extract_metric_from_df('Predictive Equality', df, attr) == -.2 assert self.extract_metric_from_df('Statistical Parity', df, attr) == -.2 assert self.extract_metric_from_df('Theil Index', df, attr) == 0. attr = 'Upper Bound' assert self.extract_metric_from_df('Average Odds', df, attr) == .2 assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1.2 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == .2 assert self.extract_metric_from_df('FNR difference', df, attr) == .2 assert not np.isfinite( self.extract_metric_from_df('Generalized Entropy Index', df, attr)) assert self.extract_metric_from_df('Predictive Equality', df, attr) == .2 assert self.extract_metric_from_df('Statistical Parity', df, attr) == .2 assert not np.isfinite( self.extract_metric_from_df('Theil Index', df, attr))
def test_all_scores_invalid(self): # test conversion of None y_true = np.array([1, 1, 1, 0, 0, 1, 1, 1, 1, 1]) y_pred = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) is_member = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) with self.assertWarns(UserWarning): df = BinaryFairnessMetrics.get_all_scores(y_true, y_pred, is_member) assert type(df) == pd.DataFrame assert self.extract_metric_from_df('Statistical Parity', df) == 0. assert not np.isfinite(self.extract_metric_from_df('Average Odds', df)) assert not np.isfinite( self.extract_metric_from_df('Disparate Impact', df)) assert self.extract_metric_from_df('FNR difference', df) == 0. assert not np.isfinite( self.extract_metric_from_df('Predictive Equality', df)) assert self.extract_metric_from_df('Generalized Entropy Index', df) == 2. assert np.isclose(self.extract_metric_from_df('Theil Index', df), 1.609, atol=0.01) attr = 'Ideal Value' assert self.extract_metric_from_df('Average Odds', df, attr) == 0 assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == 0 assert self.extract_metric_from_df('FNR difference', df, attr) == 0 assert self.extract_metric_from_df('Generalized Entropy Index', df, attr) == 0 assert self.extract_metric_from_df('Predictive Equality', df, attr) == 0 assert self.extract_metric_from_df('Statistical Parity', df, attr) == 0 assert self.extract_metric_from_df('Theil Index', df, attr) == 0 attr = 'Lower Bound' assert self.extract_metric_from_df('Average Odds', df, attr) == -.2 assert self.extract_metric_from_df('Disparate Impact', df, attr) == .8 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == -.2 assert self.extract_metric_from_df('FNR difference', df, attr) == -.2 assert self.extract_metric_from_df('Generalized Entropy Index', df, attr) == 0. assert self.extract_metric_from_df('Predictive Equality', df, attr) == -.2 assert self.extract_metric_from_df('Statistical Parity', df, attr) == -.2 assert self.extract_metric_from_df('Theil Index', df, attr) == 0. attr = 'Upper Bound' assert self.extract_metric_from_df('Average Odds', df, attr) == .2 assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1.2 assert self.extract_metric_from_df('Equal Opportunity', df, attr) == .2 assert self.extract_metric_from_df('FNR difference', df, attr) == .2 assert not np.isfinite( self.extract_metric_from_df('Generalized Entropy Index', df, attr)) assert self.extract_metric_from_df('Predictive Equality', df, attr) == .2 assert self.extract_metric_from_df('Statistical Parity', df, attr) == .2 assert not np.isfinite( self.extract_metric_from_df('Theil Index', df, attr))