def test_all_scores_valid(self):
# test standard pandas table creation
y_true = np.array([1, 1, 1, 0, 1, 0, 1, 1, 1, 1])
y_pred = np.array([0, 0, 0, 1, 0, 0, 0, 0, 0, 1])
is_member = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
df = BinaryFairnessMetrics.get_all_scores(y_true, y_pred, is_member)
assert type(df) == pd.DataFrame
assert self.extract_metric_from_df('Statistical Parity', df) == 0.
assert self.extract_metric_from_df('Average Odds', df) == 0.375
assert self.extract_metric_from_df('Disparate Impact', df) == 1.
assert self.extract_metric_from_df('FNR difference', df) == 0.25
assert self.extract_metric_from_df('Predictive Equality', df) == 1.00
assert self.extract_metric_from_df('Generalized Entropy Index',
df) == 1.375
assert np.isclose(self.extract_metric_from_df('Theil Index', df),
1.263,
atol=0.01)
attr = 'Ideal Value'
assert self.extract_metric_from_df('Average Odds', df, attr) == 0
assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1
assert self.extract_metric_from_df('Equal Opportunity', df, attr) == 0
assert self.extract_metric_from_df('FNR difference', df, attr) == 0
assert self.extract_metric_from_df('Generalized Entropy Index', df,
attr) == 0
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == 0
assert self.extract_metric_from_df('Statistical Parity', df, attr) == 0
assert self.extract_metric_from_df('Theil Index', df, attr) == 0
attr = 'Lower Bound'
assert self.extract_metric_from_df('Average Odds', df, attr) == -.2
assert self.extract_metric_from_df('Disparate Impact', df, attr) == .8
assert self.extract_metric_from_df('Equal Opportunity', df,
attr) == -.2
assert self.extract_metric_from_df('FNR difference', df, attr) == -.2
assert self.extract_metric_from_df('Generalized Entropy Index', df,
attr) == 0.
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == -.2
assert self.extract_metric_from_df('Statistical Parity', df,
attr) == -.2
assert self.extract_metric_from_df('Theil Index', df, attr) == 0.
attr = 'Upper Bound'
assert self.extract_metric_from_df('Average Odds', df, attr) == .2
assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1.2
assert self.extract_metric_from_df('Equal Opportunity', df, attr) == .2
assert self.extract_metric_from_df('FNR difference', df, attr) == .2
assert not np.isfinite(
self.extract_metric_from_df('Generalized Entropy Index', df, attr))
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == .2
assert self.extract_metric_from_df('Statistical Parity', df,
attr) == .2
assert not np.isfinite(
self.extract_metric_from_df('Theil Index', df, attr))
def test_all_scores_invalid(self):
# test conversion of None
y_true = np.array([1, 1, 1, 0, 0, 1, 1, 1, 1, 1])
y_pred = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
is_member = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
with self.assertWarns(UserWarning):
df = BinaryFairnessMetrics.get_all_scores(y_true, y_pred,
is_member)
assert type(df) == pd.DataFrame
assert self.extract_metric_from_df('Statistical Parity', df) == 0.
assert not np.isfinite(self.extract_metric_from_df('Average Odds', df))
assert not np.isfinite(
self.extract_metric_from_df('Disparate Impact', df))
assert self.extract_metric_from_df('FNR difference', df) == 0.
assert not np.isfinite(
self.extract_metric_from_df('Predictive Equality', df))
assert self.extract_metric_from_df('Generalized Entropy Index',
df) == 2.
assert np.isclose(self.extract_metric_from_df('Theil Index', df),
1.609,
atol=0.01)
attr = 'Ideal Value'
assert self.extract_metric_from_df('Average Odds', df, attr) == 0
assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1
assert self.extract_metric_from_df('Equal Opportunity', df, attr) == 0
assert self.extract_metric_from_df('FNR difference', df, attr) == 0
assert self.extract_metric_from_df('Generalized Entropy Index', df,
attr) == 0
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == 0
assert self.extract_metric_from_df('Statistical Parity', df, attr) == 0
assert self.extract_metric_from_df('Theil Index', df, attr) == 0
attr = 'Lower Bound'
assert self.extract_metric_from_df('Average Odds', df, attr) == -.2
assert self.extract_metric_from_df('Disparate Impact', df, attr) == .8
assert self.extract_metric_from_df('Equal Opportunity', df,
attr) == -.2
assert self.extract_metric_from_df('FNR difference', df, attr) == -.2
assert self.extract_metric_from_df('Generalized Entropy Index', df,
attr) == 0.
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == -.2
assert self.extract_metric_from_df('Statistical Parity', df,
attr) == -.2
assert self.extract_metric_from_df('Theil Index', df, attr) == 0.
attr = 'Upper Bound'
assert self.extract_metric_from_df('Average Odds', df, attr) == .2
assert self.extract_metric_from_df('Disparate Impact', df, attr) == 1.2
assert self.extract_metric_from_df('Equal Opportunity', df, attr) == .2
assert self.extract_metric_from_df('FNR difference', df, attr) == .2
assert not np.isfinite(
self.extract_metric_from_df('Generalized Entropy Index', df, attr))
assert self.extract_metric_from_df('Predictive Equality', df,
attr) == .2
assert self.extract_metric_from_df('Statistical Parity', df,
attr) == .2
assert not np.isfinite(
self.extract_metric_from_df('Theil Index', df, attr))
