def demographic_parity(df_test_encoded, predictions, print_=False):
dpd_sex = demographic_parity_difference(df_test_encoded.earnings, predictions, sensitive_features=df_test_encoded.sex)
dpr_sex = demographic_parity_ratio(df_test_encoded.earnings, predictions, sensitive_features=df_test_encoded.sex)
if (print_):
print(f"Demographic parity difference sex:", dpd_sex)
print(f"Demographic parity ratio sex:", dpr_sex)
def fair_metrics(bst, data, column, thresh):
tr = list(data.get_label())
best_iteration = bst.best_ntree_limit
pred = bst.predict(data, ntree_limit=best_iteration)
pred = [1 if p > thresh else 0 for p in pred]
na0 = 0
na1 = 0
nd0 = 0
nd1 = 0
for p, c in zip(pred, column):
if (p == 1 and c == 0):
nd1 += 1
if (p == 1 and c == 1):
na1 += 1
if (p == 0 and c == 0):
nd0 += 1
if (p == 0 and c == 1):
na0 += 1
Pa1, Pd1, Pa0, Pd0 = na1 / (na1 + na0), nd1 / (nd1 + nd0), na0 / (
na1 + na0), nd0 / (nd1 + nd0)
dsp_metric = np.abs(Pd1 - Pa1)
#dsp_metric = np.abs((first-second)/(first+second))
sr_metric = selection_rate(tr, pred, pos_label=1)
dpd_metric = demographic_parity_difference(tr,
pred,
sensitive_features=column)
dpr_metric = demographic_parity_ratio(tr, pred, sensitive_features=column)
eod_metric = equalized_odds_difference(tr, pred, sensitive_features=column)
return dsp_metric, sr_metric, dpd_metric, dpr_metric, eod_metric
def test_against_demographic_parity_ratio(method):
expected = metrics.demographic_parity_ratio(
y_true, y_pred, sensitive_features=sf_binary, method=method
)
actual = metrics.selection_rate_ratio(
y_true, y_pred, sensitive_features=sf_binary, method=method
)
assert expected == actual
def test_demographic_parity_ratio(agg_method):
actual = demographic_parity_ratio(y_t,
y_p,
sensitive_features=g_1,
method=agg_method)
gm = MetricFrame(selection_rate, y_t, y_p, sensitive_features=g_1)
assert actual == gm.ratio(method=agg_method)
def demographic_parity_ratio(y, c, y_hat):
c = c.reshape(-1)
assert y_hat.shape[1] == 2
assert y.shape[0] == c.shape[0]
y_pred = y_hat[:, 1] > 0.5
return fairlearn_metrics.demographic_parity_ratio(y,
y_pred,
sensitive_features=c)
def __binary_group_fairness_measures(X,
prtc_attr,
y_true,
y_pred,
y_prob=None,
priv_grp=1):
"""[summary]
Args:
X (pandas DataFrame): Sample features
prtc_attr (named array-like): values for the protected attribute
(note: protected attribute may also be present in X)
y_true (pandas DataFrame): Sample targets
y_pred (pandas DataFrame): Sample target predictions
y_prob (pandas DataFrame, optional): Sample target probabilities. Defaults
to None.
Returns:
[type]: [description]
"""
pa_names = prtc_attr.columns.tolist()
gf_vals = {}
gf_key = 'Group Fairness'
gf_vals['Statistical Parity Difference'] = \
aif_mtrc.statistical_parity_difference(y_true, y_pred, prot_attr=pa_names)
gf_vals['Disparate Impact Ratio'] = \
aif_mtrc.disparate_impact_ratio(y_true, y_pred, prot_attr=pa_names)
if not helper.is_tutorial_running() and not len(pa_names) > 1:
gf_vals['Demographic Parity Difference'] = \
fl_mtrc.demographic_parity_difference(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Demographic Parity Ratio'] = \
fl_mtrc.demographic_parity_ratio(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Average Odds Difference'] = \
aif_mtrc.average_odds_difference(y_true, y_pred, prot_attr=pa_names)
gf_vals['Equal Opportunity Difference'] = \
aif_mtrc.equal_opportunity_difference(y_true, y_pred, prot_attr=pa_names)
if not helper.is_tutorial_running() and not len(pa_names) > 1:
gf_vals['Equalized Odds Difference'] = \
fl_mtrc.equalized_odds_difference(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Equalized Odds Ratio'] = \
fl_mtrc.equalized_odds_ratio(y_true, y_pred,
sensitive_features=prtc_attr)
gf_vals['Positive Predictive Parity Difference'] = \
aif_mtrc.difference(sk_metric.precision_score, y_true,
y_pred, prot_attr=pa_names, priv_group=priv_grp)
gf_vals['Balanced Accuracy Difference'] = \
aif_mtrc.difference(sk_metric.balanced_accuracy_score, y_true,
y_pred, prot_attr=pa_names, priv_group=priv_grp)
if y_prob is not None:
gf_vals['AUC Difference'] = \
aif_mtrc.difference(sk_metric.roc_auc_score, y_true, y_prob,
prot_attr=pa_names, priv_group=priv_grp)
return (gf_key, gf_vals)
def test_demographic_parity_ratio_weighted(agg_method):
actual = demographic_parity_ratio(y_t,
y_p,
sensitive_features=g_1,
sample_weight=s_w,
method=agg_method)
gm = MetricFrame(selection_rate,
y_t,
y_p,
sensitive_features=g_1,
sample_params={'sample_weight': s_w})
assert actual == gm.ratio(method=agg_method)
def conditional_demographic_parity_ratio(labels, pred, attr, groups):
"""
Calculate conditional demographic parity by calculating the average
demographic parity ratio across bins defined by `groups`.
"""
ratios = []
for group in set(groups):
mask = groups == group
ratios.append(
demographic_parity_ratio(labels[mask],
pred[mask],
sensitive_features=attr[mask]))
return np.mean(ratios)
def evaluate_model(model, device, criterion, data_loader):
model.eval()
y_true = []
y_pred = []
y_out = []
sensitives = []
for i, data in enumerate(data_loader):
x, y, sensitive_features = data
x = x.to(device)
y = y.to(device)
sensitive_features = sensitive_features.to(device)
with torch.no_grad():
logit = model(x)
# logit : binary prediction size=(b, 1)
bina = (torch.sigmoid(logit) > 0.5).float()
y_true += y.cpu().tolist()
y_pred += bina.cpu().tolist()
y_out += torch.sigmoid(logit).tolist()
sensitives += sensitive_features.cpu().tolist()
result = {}
result["acc"] = skm.accuracy_score(y_true, y_pred)
result["f1score"] = skm.f1_score(y_true, y_pred)
result["AUC"] = skm.roc_auc_score(y_true, y_out)
result['DP'] = {
"diff":
flm.demographic_parity_difference(
y_true, y_pred, sensitive_features=sensitive_features),
"ratio":
flm.demographic_parity_ratio(y_true,
y_pred,
sensitive_features=sensitive_features),
}
result["EO"] = {
"diff":
flm.equalized_odds_difference(y_true,
y_pred,
sensitive_features=sensitive_features),
"ratio":
flm.equalized_odds_ratio(y_true,
y_pred,
sensitive_features=sensitive_features),
}
return result