def test_error_non_binary_protected_class():
with pytest.raises(ValueError):
metrics.mean_difference(
[0, 0, 0, 0, 1, 1, 1, 1],
[1, 0, 0, 1, 2, 3, 1, 0])
with pytest.raises(ValueError):
metrics.normalized_mean_difference(
[0, 0, 0, 0, 1, 1, 1, 1],
[1, 0, 0, 1, 2, 3, 1, 0])
def test_error_non_numeric_target_and_protected_class():
# mean difference
with pytest.raises(ValueError):
metrics.mean_difference(
["a", "b", "c", "d"],
["a", "b", "c", "d"])
with pytest.raises(ValueError):
metrics.mean_difference(
["a", "b", "c", "d"],
[1, 0, 0, 1])
with pytest.raises(ValueError):
metrics.mean_difference(
[1, 0, 0, 1],
["a", "b", "c", "d"])
# normalized mean difference
with pytest.raises(ValueError):
metrics.normalized_mean_difference(
["a", "b", "c", "d"],
["a", "b", "c", "d"])
with pytest.raises(ValueError):
metrics.normalized_mean_difference(
["a", "b", "c", "d"],
[1, 0, 0, 1])
with pytest.raises(ValueError):
metrics.normalized_mean_difference(
[1, 0, 0, 1],
["a", "b", "c", "d"])
def run_experiment_iteration_themis_ml_ACF(X, X_no_sex, y, s_sex, train, test):
metrics = []
# define our model.
logistic_clf = LogisticRegression(penalty='l2',
C=0.001,
class_weight='balanced')
baseline_clf = logistic_clf
rpa_clf = logistic_clf
roc_clf = SingleROClassifier(estimator=logistic_clf)
acf_clf = LinearACFClassifier(target_estimator=logistic_clf,
binary_residual_type='absolute')
# train baseline model
baseline_clf.fit(X[train], y[train])
baseline_preds = baseline_clf.predict(X[test])
baseline_auc = roc_auc_score(y[test], baseline_preds)
metrics.append(
['B',
mean_difference(baseline_preds, s_sex[test])[0], baseline_auc])
# train 'remove protected attributes' model. Here we have to train two
# seperate ones for sex and foreign status.
# model trained with no explicitly sex-related variables
rpa_preds_no_sex = rpa_clf.fit(X_no_sex[train],
y[train]).predict(X_no_sex[test])
metrics.append([
'RPA',
mean_difference(rpa_preds_no_sex, s_sex[test])[0],
roc_auc_score(y[test], rpa_preds_no_sex)
])
# train reject-option classification model.
roc_clf.fit(X[train], y[train])
roc_preds_sex = roc_clf.predict(X[test], s_sex[test])
metrics.append([
'ROC',
mean_difference(roc_preds_sex, s_sex[test])[0],
roc_auc_score(y[test], roc_preds_sex)
])
# train additive counterfactually fair model.
acf_preds_sex = acf_clf.fit(X[train], y[train],
s_sex[train]).predict(X[test], s_sex[test])
metrics.append([
'ACF',
mean_difference(acf_preds_sex, s_sex[test])[0],
roc_auc_score(y[test], acf_preds_sex)
])
probs = acf_clf.predict_proba(X, s_sex)
accuracy = roc_auc_score(y[test], acf_preds_sex)
probs_would_not_default = [prob[0] for prob in probs]
# convert metrics list of lists into dataframe
return {
'prob': probs_would_not_default,
'accuracy': accuracy,
'acf_fit': acf_clf
}
# - "male_divorced/separated"
# - "female_divorced/separated/married"
# - "male_single"
# - "male_married/widowed"
# - "female_single
s_map = {"male": 0, "female": 1}
sex = df["personal_status_and_sex"].map(lambda x: s_map[x.split("_")[0]])
# get foreign worker status
# 1 = yes, 0 = no
foreign = df["foreign_worker"]
# The mean difference scores below suggest that men and non-foreign workers
# are more likely to have low credit risks compared to women and foreign
# workers respectively.
print("Mean difference scores:")
print("protected class = sex: %s" % mean_difference(credit_risk, sex)[0])
# 0.0748013090229
print("protected class = foreign: %s" %
mean_difference(credit_risk, foreign)[0])
# 0.199264685246
print("\nNormalized mean difference scores:")
# normalized mean difference
print("protected class = sex: %s" %
normalized_mean_difference(credit_risk, sex)[0])
# 0.0772946859903
print("protected class = foreign: %s" %
normalized_mean_difference(credit_risk, foreign)[0])
# 0.63963963964
def test_mean_difference_full_reverse_discrimination():
"""Binary case: advantaged group is fully discriminated against."""
y = [0, 0, 0, 0, 1, 1, 1, 1]
s = [0, 0, 0, 0, 1, 1, 1, 1]
assert _get_point_est(metrics.mean_difference(y, s)) == -1
assert _get_point_est(metrics.normalized_mean_difference(y, s)) == -1
def test_mean_difference_no_discrimination():
"""Binary case: proportion in each group with +ve outcome are equal."""
y = [0, 0, 1, 1, 0, 0, 1, 1]
s = [0, 0, 0, 0, 1, 1, 1, 1]
assert _get_point_est(metrics.mean_difference(y, s)) == 0
assert _get_point_est(metrics.normalized_mean_difference(y, s)) == 0
def test_mean_difference_partial_discrimination():
"""Binary case: disadvantaged group is partially discriminated against."""
y = [1, 1, 1, 1, 0, 0, 1, 1]
s = [0, 0, 0, 0, 1, 1, 1, 1]
assert _get_point_est(metrics.mean_difference(y, s)) == 0.5
assert _get_point_est(metrics.normalized_mean_difference(y, s)) == 1