def test_input_X_unchanged(self, transformA, transformY, transformX,
mocker):
# The purpose of this test is to ensure that X is passed to the underlying estimator
# unchanged. For y and sample_weight ExponentiatedGradient makes certain transformations
# which are required. They are expected as pandas.Series.
X, y, A = _get_data()
transformed_X = transformX(X)
transformed_y = transformY(y)
transformed_A = transformA(A)
# Using a mocked estimator here since we don't actually want to fit one, but rather care
# about having that object's fit method called exactly twice through the best_h calls.
estimator = LeastSquaresBinaryClassifierLearner()
estimator.predict = mocker.Mock(return_value=y)
estimator.fit = mocker.MagicMock()
# restrict ExponentiatedGradient to a single iteration
expgrad = ExponentiatedGradient(estimator,
constraints=DemographicParity(),
max_iter=1)
mocker.patch('copy.deepcopy', return_value=estimator)
expgrad.fit(transformed_X,
transformed_y,
sensitive_features=transformed_A)
# ensure that the input data wasn't changed by our mitigator before being passed to the
# underlying estimator
assert estimator.fit.call_count == 2
args, kwargs = estimator.fit.call_args
assert len(args) == 2 # X and y
assert len(kwargs) == 1 # sample_weight
assert isinstance(args[0], type(transformed_X))
assert isinstance(args[1], pd.Series)
assert isinstance(kwargs['sample_weight'], pd.Series)
def test_argument_types(self, transformX, transformY, transformA):
# This is an expanded-out version of one of the smoke tests
expgrad = ExponentiatedGradient(self.learner,
constraints=DemographicParity(),
eps=0.1)
expgrad.fit(transformX(self.X),
transformY(self.y),
sensitive_features=transformA(self.A))
res = expgrad._expgrad_result._as_dict()
Q = res["best_classifier"]
res["n_classifiers"] = len(res["classifiers"])
disp = DemographicParity()
disp.load_data(self.X, self.y, sensitive_features=self.A)
error = ErrorRate()
error.load_data(self.X, self.y, sensitive_features=self.A)
res["disp"] = disp.gamma(Q).max()
res["error"] = error.gamma(Q)[0]
assert res["best_gap"] == pytest.approx(0.0000, abs=self._PRECISION)
assert res["last_t"] == 5
assert res["best_t"] == 5
assert res["disp"] == pytest.approx(0.1, abs=self._PRECISION)
assert res["error"] == pytest.approx(0.25, abs=self._PRECISION)
assert res["n_oracle_calls"] == 32
assert res["n_classifiers"] == 3
def run_smoke_test_binary_classification(self, data, flipped=False):
learner = LeastSquaresBinaryClassifierLearner()
if "ratio" in data.keys():
disparity_moment = data["constraint_class"](
ratio_bound_slack=data["eps"], ratio_bound=data["ratio"])
else:
disparity_moment = data["constraint_class"](
difference_bound=data["eps"])
# Create Exponentiated Gradient object with a copy of the constraint.
# The original disparity_moment object is used for validation, so the
# assumption is that the moment logic is correct in these tests.
expgrad = ExponentiatedGradient(learner,
constraints=deepcopy(disparity_moment),
eps=data["eps"])
X, y, A = _get_data(A_two_dim=False, flip_y=flipped)
expgrad.fit(X, y, sensitive_features=A)
self._assert_expgrad_state(expgrad, data)
# select probability of predicting 1
def Q(X):
return expgrad._pmf_predict(X)[:, 1]
default_objective = ErrorRate()
disparity_moment.load_data(X, y, sensitive_features=A)
default_objective.load_data(X, y, sensitive_features=A)
disparity = disparity_moment.gamma(Q).max()
error = default_objective.gamma(Q)[0]
assert disparity == pytest.approx(data["disp"], abs=_PRECISION)
assert error == pytest.approx(data["error"], abs=_PRECISION)
def test_argument_types(self, transformX, transformY, transformA,
A_two_dim):
# This is an expanded-out version of one of the smoke tests
X, y, A = _get_data(A_two_dim)
merged_A = _map_into_single_column(A)
expgrad = ExponentiatedGradient(LeastSquaresBinaryClassifierLearner(),
constraints=DemographicParity(),
eps=0.1)
expgrad.fit(transformX(X),
transformY(y),
sensitive_features=transformA(A))
res = expgrad._expgrad_result._as_dict()
Q = res["best_classifier"]
res["n_classifiers"] = len(res["classifiers"])
disp = DemographicParity()
disp.load_data(X, y, sensitive_features=merged_A)
error = ErrorRate()
error.load_data(X, y, sensitive_features=merged_A)
res["disp"] = disp.gamma(Q).max()
res["error"] = error.gamma(Q)[0]
assert res["best_gap"] == pytest.approx(0.0000, abs=_PRECISION)
assert res["last_t"] == 5
assert res["best_t"] == 5
assert res["disp"] == pytest.approx(0.1, abs=_PRECISION)
assert res["error"] == pytest.approx(0.25, abs=_PRECISION)
assert res["n_oracle_calls"] == 32
assert res["n_classifiers"] == 3
def test_smoke_regression(self, data):
learner = LeastSquaresRegressor()
disparity_moment = data["constraint_class"](
loss=data["loss"],
upper_bound=data["upper_bound"])
# Create Exponentiated Gradient object with a copy of the constraint.
# The original disparity_moment object is used for validation, so the
# assumption is that the moment logic is correct in these tests.
expgrad = ExponentiatedGradient(
learner,
constraints=deepcopy(disparity_moment),
eps=data["eps"],
nu=data.get('nu'),
max_iter=data.get("max_iter", 50))
X, y, A = _get_data(A_two_dim=False, y_as_scores=True)
expgrad.fit(X, y, sensitive_features=A)
self._assert_expgrad_state(expgrad, data)
# check all predictors
disparity_moment.load_data(X, y, sensitive_features=A)
for i in range(len(expgrad.predictors_)):
def Q(X): return expgrad._pmf_predict(X)[i]
default_objective = MeanLoss(data["loss"])
default_objective.load_data(X, y, sensitive_features=A)
disparity = disparity_moment.gamma(Q).max()
error = default_objective.gamma(Q)[0]
assert disparity == pytest.approx(data["disp"][i], abs=_PRECISION)
assert error == pytest.approx(data["error"][i], abs=_PRECISION)
assert expgrad.weights_[i] == pytest.approx(data['weights'][i], abs=_PRECISION)
assert sum(expgrad.weights_) == pytest.approx(1, abs=_PRECISION)
def run_smoke_test(self, data, flipped=False):
ratio = 1.0
if "ratio" in data.keys():
ratio = data["ratio"]
expgrad = ExponentiatedGradient(
self.learner,
constraints=data["cons_class"](ratio=ratio),
eps=data["eps"])
expgrad.fit(self.X, (self.flipped_y if flipped else self.y),
sensitive_features=self.A)
def Q(X):
return expgrad._pmf_predict(X)[:, 1]
n_predictors = len(expgrad._predictors)
disparity_moment = data["cons_class"](ratio=ratio)
disparity_moment.load_data(self.X,
(self.flipped_y if flipped else self.y),
sensitive_features=self.A)
error = ErrorRate()
error.load_data(self.X, (self.flipped_y if flipped else self.y),
sensitive_features=self.A)
disparity = disparity_moment.gamma(Q).max()
error = error.gamma(Q)[0]
assert expgrad._best_gap == pytest.approx(data["best_gap"],
abs=self._PRECISION)
assert expgrad._last_t == data["last_t"]
assert expgrad._best_t == data["best_t"]
assert disparity == pytest.approx(data["disp"], abs=self._PRECISION)
assert error == pytest.approx(data["error"], abs=self._PRECISION)
assert expgrad._n_oracle_calls == data["n_oracle_calls"]
assert n_predictors == data["n_predictors"]
def test_single_y_value(self):
# Setup with data designed to result in "all single class"
# at some point in the grid
X_dict = {"c": [10, 50, 10]}
X = pd.DataFrame(X_dict)
y = [1, 1, 1]
A = ['a', 'b', 'b']
estimator = LogisticRegression(solver='liblinear',
fit_intercept=True,
random_state=97)
expgrad = ExponentiatedGradient(estimator, DemographicParity())
# Following line should not throw an exception
expgrad.fit(X, y, sensitive_features=A)
# Check the predictors for a ConstantPredictor
test_X_dict = {"c": [1, 2, 3, 4, 5, 6]}
test_X = pd.DataFrame(test_X_dict)
assert expgrad.n_oracle_calls_dummy_returned_ > 0
assert len(expgrad.oracle_execution_times_) == expgrad.n_oracle_calls_
for p in expgrad.predictors_:
assert isinstance(p, DummyClassifier)
assert np.array_equal(p.predict(test_X), [1, 1, 1, 1, 1, 1])
def run_smoke_test(self, data):
expgrad = ExponentiatedGradient(self.learner, constraints=data["cons_class"](),
eps=data["eps"])
expgrad.fit(self.X, self.y, sensitive_features=self.A)
res = expgrad._expgrad_result._as_dict()
Q = res["best_classifier"]
res["n_classifiers"] = len(res["classifiers"])
disp = data["cons_class"]()
disp.load_data(self.X, self.y, sensitive_features=self.A)
error = moments.MisclassificationError()
error.load_data(self.X, self.y, sensitive_features=self.A)
res["disp"] = disp.gamma(Q).max()
res["error"] = error.gamma(Q)[0]
assert res["best_gap"] == pytest.approx(
data["best_gap"], abs=self._PRECISION)
assert res["last_t"] == data["last_t"]
assert res["best_t"] == data["best_t"]
assert res["disp"] == pytest.approx(data["disp"], abs=self._PRECISION)
assert res["error"] == pytest.approx(
data["error"], abs=self._PRECISION)
assert res["n_oracle_calls"] == data["n_oracle_calls"]
assert res["n_classifiers"] == data["n_classifiers"]
def test_argument_types(self, transformX, transformY, transformA,
A_two_dim):
# This is an expanded-out version of one of the smoke tests
X, y, A = _get_data(A_two_dim)
merged_A = _map_into_single_column(A)
expgrad = ExponentiatedGradient(LeastSquaresBinaryClassifierLearner(),
constraints=DemographicParity(),
eps=0.1)
expgrad.fit(transformX(X),
transformY(y),
sensitive_features=transformA(A))
Q = expgrad._best_classifier
n_classifiers = len(expgrad._classifiers)
disparity_moment = DemographicParity()
disparity_moment.load_data(X, y, sensitive_features=merged_A)
error = ErrorRate()
error.load_data(X, y, sensitive_features=merged_A)
disparity = disparity_moment.gamma(Q).max()
error = error.gamma(Q)[0]
assert expgrad._best_gap == pytest.approx(0.0000, abs=_PRECISION)
assert expgrad._last_t == 5
assert expgrad._best_t == 5
assert disparity == pytest.approx(0.1, abs=_PRECISION)
assert error == pytest.approx(0.25, abs=_PRECISION)
assert expgrad._n_oracle_calls == 32
assert n_classifiers == 3
def test_equalized_odds():
# Have to do this one longhand, since it combines tpr and fpr
X, y = loan_scenario_generator(n, f, sfs, ibs, seed=632753)
X_dummy = pd.get_dummies(X)
metrics = {"tpr": true_positive_rate, "fpr": false_positive_rate}
unmitigated = LogisticRegression()
unmitigated.fit(X_dummy, y)
y_pred = unmitigated.predict(X_dummy)
mf_unmitigated = MetricFrame(
metrics=metrics,
y_true=y,
y_pred=y_pred,
sensitive_features=X["sens"],
control_features=X["ctrl"],
)
expgrad_basic = ExponentiatedGradient(
LogisticRegression(),
constraints=EqualizedOdds(difference_bound=0.01),
eps=0.01)
expgrad_basic.fit(X_dummy, y, sensitive_features=X["sens"])
y_pred_basic = expgrad_basic.predict(X_dummy, random_state=9235)
mf_basic = MetricFrame(
metrics=metrics,
y_true=y,
y_pred=y_pred_basic,
sensitive_features=X["sens"],
control_features=X["ctrl"],
)
expgrad_control = ExponentiatedGradient(
LogisticRegression(),
constraints=EqualizedOdds(difference_bound=0.01),
eps=0.01)
expgrad_control.fit(X_dummy,
y,
sensitive_features=X["sens"],
control_features=X["ctrl"])
y_pred_control = expgrad_control.predict(X_dummy, random_state=8152)
mf_control = MetricFrame(
metrics=metrics,
y_true=y,
y_pred=y_pred_control,
sensitive_features=X["sens"],
control_features=X["ctrl"],
)
compare_unmitigated = mf_control.difference(
method="to_overall") <= mf_unmitigated.difference(method="to_overall")
print(compare_unmitigated)
compare_basic = mf_control.difference(
method="to_overall") <= mf_basic.difference(method="to_overall")
print(compare_basic)
assert compare_basic.values.reshape(6).all()
assert compare_unmitigated.values.reshape(6).all()
def test_binary_classifier_0_1_required(self):
X, y, A = _get_data()
y = 2 * y
expgrad = ExponentiatedGradient(LogisticRegression(),
constraints=DemographicParity(),
max_iter=1)
with pytest.raises(ValueError) as execInfo:
expgrad.fit(X, y, sensitive_features=(A))
assert _LABELS_NOT_0_1_ERROR_MESSAGE == execInfo.value.args[0]
def run_comparisons(moment, metric_fn):
X, y = loan_scenario_generator(n, f, sfs, ibs, seed=163)
X_dummy = pd.get_dummies(X)
mf_input = MetricFrame(metric_fn, y, y,
sensitive_features=X['sens'],
control_features=X['ctrl'])
print("Metric for input:\n", mf_input.by_group)
print("Input Metric differences:\n", mf_input.difference(method='to_overall'), "\n")
unmitigated = LogisticRegression()
unmitigated.fit(X_dummy, y)
y_pred = unmitigated.predict(X_dummy)
mf_unmitigated = MetricFrame(metric_fn,
y, y_pred,
sensitive_features=X['sens'],
control_features=X['ctrl'])
print("Unmitigated metric:\n", mf_unmitigated.by_group)
print("Unmitigated metric differences:\n",
mf_unmitigated.difference(method='to_overall'), "\n")
expgrad_basic = ExponentiatedGradient(
LogisticRegression(),
constraints=moment(),
eps=0.005)
expgrad_basic.fit(X_dummy, y, sensitive_features=X['sens'])
y_pred_basic = expgrad_basic.predict(X_dummy, random_state=8235)
mf_basic = MetricFrame(metric_fn, y, y_pred_basic,
sensitive_features=X['sens'],
control_features=X['ctrl'])
print("Basic expgrad metric:\n", mf_basic.by_group)
print("Basic expgrad metric differences:\n",
mf_basic.difference(method='to_overall'), "\n")
expgrad_control = ExponentiatedGradient(
LogisticRegression(),
constraints=moment(),
eps=0.005)
expgrad_control.fit(X_dummy, y,
sensitive_features=X['sens'],
control_features=X['ctrl'])
y_pred_control = expgrad_control.predict(X_dummy, random_state=852)
mf_control = MetricFrame(metric_fn, y, y_pred_control,
sensitive_features=X['sens'],
control_features=X['ctrl'])
print("expgrad_control metric:\n", mf_control.by_group)
print("expgrad_control metric differences:\n",
mf_control.difference(method='to_overall'))
assert (mf_control.difference(method='to_overall') <=
mf_unmitigated.difference(method='to_overall')).all()
assert (mf_control.difference(method='to_overall') <=
mf_basic.difference(method='to_overall')).all()
def test_error_rate_consistency(self, eps, ratio, pos_copies):
learner = LeastSquaresBinaryClassifierLearner()
if ratio is None:
constraints_moment = EqualizedOdds(difference_bound=eps)
else:
constraints_moment = EqualizedOdds(ratio_bound=ratio, ratio_bound_slack=eps)
results = {}
for method in ["costs", "sampling"]:
X, y, A = _get_data()
if method == "sampling":
select = y == 1
X = pd.concat((X,) + (X.loc[select, :],) * pos_copies).values
y = pd.concat((y,) + (y[select],) * pos_copies).values
A = pd.concat((A,) + (A[select],) * pos_copies).values
objective_moment = ErrorRate()
else:
objective_moment = ErrorRate(costs={"fn": 1.0 + pos_copies, "fp": 1.0})
expgrad = ExponentiatedGradient(
learner,
constraints=deepcopy(constraints_moment),
objective=deepcopy(objective_moment),
eps=eps,
nu=1e-3,
)
expgrad.fit(X, y, sensitive_features=A)
# select probability of predicting 1
def Q(X):
return expgrad._pmf_predict(X)[:, 1]
constraints_eval = deepcopy(constraints_moment)
constraints_eval.load_data(X, y, sensitive_features=A)
disparity = constraints_eval.gamma(Q).max()
objective_eval = deepcopy(objective_moment)
objective_eval.load_data(X, y, sensitive_features=A)
total_error = objective_eval.gamma(Q)[0] * len(y)
results[method] = {
"error": objective_eval.gamma(Q)[0],
"total_error": total_error,
"disp": disparity,
"n_predictors": len(expgrad.predictors_),
"best_gap": expgrad.best_gap_,
"last_iter": expgrad.last_iter_,
"best_iter": expgrad.best_iter_,
"n_oracle_calls": expgrad.n_oracle_calls_,
"n_oracle_calls_dummy_returned": expgrad.n_oracle_calls_dummy_returned_,
}
self._assert_expgrad_two_states(results["costs"], results["sampling"])
def run_expgrad_classification(estimator, moment):
"""Run classification test with ExponentiatedGradient."""
X, Y, A = fetch_adult()
expgrad = ExponentiatedGradient(
estimator,
constraints=moment)
expgrad.fit(X, Y, sensitive_features=A)
assert expgrad.n_oracle_calls_ > 1
assert len(expgrad.predictors_) > 1
def run(fairness_constraints, use_proxy=False):
print(f"Start running experiment with Proxy: {use_proxy}.")
all_results = {}
all_results['eps'] = fairness_constraints
all_results['accuracy'] = {'train': [], 'test': []}
all_results['violation'] = {'train': [], 'test': []}
all_results['violation_male'] = {'train': [], 'test': []}
all_results['violation_female'] = {'train': [], 'test': []}
for eps in fairness_constraints:
begin = time.time()
if use_proxy:
sweep = ExponentiatedGradient(
LogisticRegression(solver='liblinear', fit_intercept=True),
constraints=ProxyEqualizedOdds(error_rate=error_rate),
eps=eps)
else:
sweep = ExponentiatedGradient(LogisticRegression(
solver='liblinear', fit_intercept=True),
constraints=EqualizedOdds(),
eps=eps)
try:
sweep.fit(X_train, Y_noised, sensitive_features=A_train)
prediction_train = sweep.predict(X_train)
prediction_test = sweep.predict(X_test)
except:
print(f"Fairlearn can't fit at fairness constraint {eps}")
pass
all_results['accuracy']['train'].append(
accuracy(prediction_train, Y_train))
all_results['accuracy']['test'].append(
accuracy(prediction_test, Y_test))
all_results['violation']['train'].append(
violation(prediction_train, Y_train, A_train))
all_results['violation']['test'].append(
violation(prediction_test, Y_test, A_test))
all_results['violation_male']['train'].append(
violation(prediction_train, Y_train, A_train, grp=1))
all_results['violation_male']['test'].append(
violation(prediction_test, Y_test, A_test, grp=1))
all_results['violation_female']['train'].append(
violation(prediction_train, Y_train, A_train, grp=0))
all_results['violation_female']['test'].append(
violation(prediction_test, Y_test, A_test, grp=0))
print(
f"Running fairness constraint: {eps}, Training Accuracy: {all_results['accuracy']['train']}, Test Accuracy: {all_results['accuracy']['test']}, Training Violation: {all_results['violation']['train']}, Test Violation: {all_results['violation']['test']}, Time cost: {time.time() - begin}"
)
return all_results
def run_estimation(fairness_constraints, proxy=False, lnl=False):
print(
f"Start running experiment with Proxy: {proxy}, Learning with Noisy Labels: {lnl}."
)
all_results_train, all_results_test = [], []
for eps in fairness_constraints:
begin = time.time()
if proxy and lnl:
clf = ExponentiatedGradient(
LogisticRegression(solver='liblinear', fit_intercept=True),
constraints=ProxyEqualizedOdds2(delta=delta),
eps=eps)
sweep = LearningWithNoisyLabels(clf=clf)
elif proxy:
sweep = ExponentiatedGradient(
LogisticRegression(solver='liblinear', fit_intercept=True),
constraints=ProxyEqualizedOdds2(delta=delta),
eps=eps)
elif lnl:
clf = ExponentiatedGradient(LogisticRegression(solver='liblinear',
fit_intercept=True),
constraints=EqualizedOdds(),
eps=eps)
sweep = LearningWithNoisyLabels(clf=clf)
else:
sweep = ExponentiatedGradient(LogisticRegression(
solver='liblinear', fit_intercept=True),
constraints=EqualizedOdds(),
eps=eps)
sweep.fit(X_train, Y_noised, sensitive_features=A_train)
prediction_train = sweep.predict(X_train)
prediction_test = sweep.predict(X_test)
accuracy_train = accuracy(prediction_train, Y_train)
accuracy_test = accuracy(prediction_test, Y_test)
all_results_train.append(accuracy_train)
all_results_test.append(accuracy_test)
print(
f"Running fairness constraint: {eps}, Training Accuracy: {accuracy_train}, Test Accuracy: {accuracy_test}, Training Violation: {violation(prediction_train, Y_train, A_train)}, Test Violation: {violation(prediction_test, Y_test, A_test)}, Time cost: {time.time() - begin}"
)
return all_results_train, all_results_test
def test_argument_types_ratio_bound(self, transformX, transformY,
transformA, A_two_dim):
# This is an expanded-out version of one of the smoke tests
X, y, A = _get_data(A_two_dim)
merged_A = _map_into_single_column(A)
transformed_X = transformX(X)
transformed_y = transformY(y)
transformed_A = transformA(A)
eps = 0.1
ratio = 1.0
expgrad = ExponentiatedGradient(
LeastSquaresBinaryClassifierLearner(),
constraints=DemographicParity(ratio_bound_slack=eps,
ratio_bound=ratio),
eps=eps,
)
expgrad.fit(transformed_X,
transformed_y,
sensitive_features=transformed_A)
def Q(X):
return expgrad._pmf_predict(X)[:, 1]
n_predictors = len(expgrad.predictors_)
disparity_moment = DemographicParity(ratio_bound_slack=eps,
ratio_bound=ratio)
disparity_moment.load_data(X, y, sensitive_features=merged_A)
error = ErrorRate()
error.load_data(X, y, sensitive_features=merged_A)
disparity = disparity_moment.gamma(Q).max()
disp = disparity_moment.gamma(Q)
disp_eps = disparity_moment.gamma(Q) - disparity_moment.bound()
error = error.gamma(Q)[0]
assert expgrad.best_gap_ == pytest.approx(0.0000, abs=_PRECISION)
assert expgrad.last_iter_ == 5
assert expgrad.best_iter_ == 5
assert disparity == pytest.approx(0.1, abs=_PRECISION)
assert np.all(np.isclose(disp - eps, disp_eps))
assert error == pytest.approx(0.25, abs=_PRECISION)
assert expgrad.n_oracle_calls_ == 32
assert n_predictors == 3
def evaluate(eps, X_train, y_train, X_test, y_test, sex_train, sex_test,
index):
estimator = GradientBoostingClassifier()
constraints = DemographicParity()
egsolver = ExponentiatedGradient(estimator, constraints, eps=eps)
egsolver.fit(X_train, y_train, sensitive_features=sex_train)
y_pred = egsolver.predict(X_test)
# print("y_pred",y_pred)
group_summary_adult = group_summary(accuracy_score,
y_test,
y_pred,
sensitive_features=sex_test)
selection_rate_summary = selection_rate_group_summary(
y_test, y_pred, sensitive_features=sex_test)
error = 1 - group_summary_adult["overall"]
dp = demographic(selection_rate_summary)
errorlist[index].append(error)
dplist[index].append(dp)
print("error:%f,dp:%f" % (error, dp))
def fit(self, X, y, sample_weight=None):
from fairlearn.reductions import ExponentiatedGradient, DemographicParity
_estimator = self._estimator
constraints = self.constraints
eps = float(self.eps)
nu = float(self.nu)
max_iter = int(self.max_iter)
eta0 = float(self.eta0) # renamed from eta_mul
# run_linprog_step = self.run_linprog_step # missing
# sample_weight_name = self.sample_weight_name #missing
# For now the sensitive feature is always the first one
sensitive_features = X[:,0]
constraints = DemographicParity(difference_bound = eps)
self.estimator = ExponentiatedGradient(
_estimator, constraints, eps = self.eps, T = max_iter, nu = nu, eta_mul = eta0)
self.estimator = self.estimator.fit(X, y, sensitive_features)
return self
def fit(train: DataTuple, args):
"""Fit a model."""
try:
from fairlearn.reductions import (
ConditionalSelectionRate,
DemographicParity,
EqualizedOdds,
ExponentiatedGradient,
)
except ImportError as e:
raise RuntimeError(
"In order to use Agarwal, install fairlearn==0.4.6.") from e
fairness_class: ConditionalSelectionRate
if args.fairness == "DP":
fairness_class = DemographicParity()
else:
fairness_class = EqualizedOdds()
if args.classifier == "SVM":
model = select_svm(C=args.C, kernel=args.kernel, seed=args.seed)
else:
random_state = np.random.RandomState(seed=args.seed)
model = LogisticRegression(solver="liblinear",
random_state=random_state,
max_iter=5000,
C=args.C)
data_x = train.x
data_y = train.y[train.y.columns[0]]
data_a = train.s[train.s.columns[0]]
exponentiated_gradient = ExponentiatedGradient(model,
constraints=fairness_class,
eps=args.eps,
T=args.iters)
exponentiated_gradient.fit(data_x, data_y, sensitive_features=data_a)
min_class_label = train.y[train.y.columns[0]].min()
exponentiated_gradient.min_class_label = min_class_label
return exponentiated_gradient
def test_simple_fit_predict(self):
estimator = LeastSquaresBinaryClassifierLearner()
constraints = DemographicParity()
expgrad = ExponentiatedGradient(estimator, constraints)
expgrad.fit(pd.DataFrame(X1), pd.Series(labels),
sensitive_features=pd.Series(sensitive_features))
expgrad.predict(pd.DataFrame(X1))
def run_corrupt(fairness_constraints):
all_results = {}
all_results['eps'] = fairness_constraints
all_results['accuracy'] = {
'train': [],
'test': []
}
all_results['violation'] = {
'train': [],
'test': []
}
all_results['violation_male'] = {
'train': [],
'test': []
}
all_results['violation_female'] = {
'train': [],
'test': []
}
for eps in fairness_constraints:
begin = time.time()
print(f"[INFO][RUN] Corrupt")
sweep = ExponentiatedGradient(LogisticRegression(solver='liblinear', fit_intercept=True),
constraints=EqualizedOdds(),
eps=eps)
try:
sweep.fit(X_train, Y_noised, sensitive_features=A_train)
prediction_train = sweep.predict(X_train)
prediction_test = sweep.predict(X_test)
except:
print(f"Fairlearn can't fit at fairness constraint {eps}")
pass
all_results['accuracy']['train'].append(accuracy(prediction_train, Y_train))
all_results['accuracy']['test'].append(accuracy(prediction_test, Y_test))
all_results['violation']['train'].append(violation(prediction_train, Y_train, A_train))
all_results['violation']['test'].append(violation(prediction_test, Y_test, A_test))
all_results['violation_male']['train'].append(violation(prediction_train, Y_train, A_train, grp=1))
all_results['violation_male']['test'].append(violation(prediction_test, Y_test, A_test, grp=1))
all_results['violation_female']['train'].append(violation(prediction_train, Y_train, A_train, grp=0))
all_results['violation_female']['test'].append(violation(prediction_test, Y_test, A_test, grp=0))
print(f"Running fairness constraint: {eps}, Training Accuracy: {all_results['accuracy']['train'][-1]}, Test Accuracy: {all_results['accuracy']['test'][-1]}, Training Violation: {all_results['violation']['train'][-1]}, Test Violation: {all_results['violation']['test'][-1]}, Time cost: {time.time() - begin}")
acc = np.array(all_results['accuracy']['test'])
v = np.array(all_results['violation']['test'])
all_results['accuracy']['mean'] = acc.mean()
all_results['accuracy']['std'] = acc.std()
all_results['violation']['mean'] = v.mean()
all_results['violation']['std'] = v.std()
return all_results
def reduction(name, fold, num_X=0):
# load data
train_data, test_data, cloumns, _, train_y_fair, train_y_proxy, test_y_fair, test_y_proxy = load_data(
name, fold, num_X=num_X, use_fair=False)
train_X = train_data.drop(columns=cloumns)
train_y_true = train_data[DATA2D[name]]
train_sex = train_data[DATA2S[name]]
# learn
learn = ExponentiatedGradient(LogisticRegression(solver='liblinear',
fit_intercept=True),
constraints=DemographicParity())
learn.fit(train_X, train_y_true, sensitive_features=train_sex)
# predict
test_X = test_data.drop(columns=cloumns)
prob_test = learn._pmf_predict(test_X)[:, 1]
prob_train = learn._pmf_predict(train_X)[:, 1]
s_train = train_sex.astype(bool)
s_test = np.array(test_data[DATA2S[name]]).astype(bool)
save_file(name, num_X, fold, "Reduction", prob_train, s_train,
train_y_fair, train_y_proxy, prob_test, s_test, test_y_fair,
test_y_proxy)
def test_input_X_unchanged(self, transformA, transformY, transformX,
mocker):
# The purpose of this test is to ensure that X is passed to the underlying estimator
# unchanged. For y and sample_weight ExponentiatedGradient makes certain transformations
# which are required. They are expected as pandas.Series.
X, y, A = _get_data()
transformed_X = transformX(X)
transformed_y = transformY(y)
transformed_A = transformA(A)
# Using a mocked estimator here since we don't actually want to fit one, but rather care
# about having that object's fit method called exactly twice through the best_h calls.
estimator = mocker.MagicMock()
estimator.predict = mocker.MagicMock(return_value=y)
# ExponentiatedGradient pickles and unpickles the estimator, which isn't possible for the
# mock object, so we patch import of pickle as well. It sets the result from pickle.loads
# as the estimator, so we can simply overwrite the return value to be our mocked estimator
# object.
mocker.patch('pickle.dumps')
mocker.patch('pickle.loads', return_value=estimator)
# restrict ExponentiatedGradient to a single iteration
expgrad = ExponentiatedGradient(estimator,
constraints=DemographicParity(),
max_iter=1)
expgrad.fit(transformed_X,
transformed_y,
sensitive_features=transformed_A)
# ensure that the input data wasn't changed by our mitigator before being passed to the
# underlying estimator
assert estimator.fit.call_count == 2
for name, args, kwargs in estimator.method_calls:
if name == 'fit':
assert len(args) == 2 # X and y
assert len(kwargs) == 1 # sample_weight
assert isinstance(args[0], type(transformed_X))
assert isinstance(args[1], pd.Series)
assert isinstance(kwargs['sample_weight'], pd.Series)
def run_smoke_test(self, data, flipped=False):
if flipped:
y = self.flipped_y
else:
y = self.y
ratio = 1.0
if "ratio" in data.keys():
ratio = data["ratio"]
expgrad = ExponentiatedGradient(
self.learner,
constraints=data["cons_class"](ratio=ratio),
eps=data["eps"])
expgrad.fit(self.X, y, sensitive_features=self.A)
def Q(X):
return expgrad._pmf_predict(X)[:, 1]
n_predictors = len(expgrad.predictors_)
disparity_moment = data["cons_class"](ratio=ratio)
disparity_moment.load_data(self.X, y, sensitive_features=self.A)
error = ErrorRate()
error.load_data(self.X, y, sensitive_features=self.A)
disparity = disparity_moment.gamma(Q).max()
error = error.gamma(Q)[0]
assert expgrad.best_gap_ == pytest.approx(data["best_gap"],
abs=self._PRECISION)
assert expgrad.last_iter_ == data["last_iter"]
assert expgrad.best_iter_ == data["best_iter"]
assert expgrad.last_iter_ >= _MIN_ITER
assert disparity == pytest.approx(data["disp"], abs=self._PRECISION)
assert error == pytest.approx(data["error"], abs=self._PRECISION)
assert expgrad.n_oracle_calls_ == data["n_oracle_calls"]
assert expgrad.n_oracle_calls_dummy_returned_ == data[
"n_oracle_calls_dummy_returned"]
assert n_predictors == data["n_predictors"]
assert len(expgrad.oracle_execution_times_) == expgrad.n_oracle_calls_
def run_expgrad_classification(estimator, moment):
"""Run classification test with ExponentiatedGradient."""
X_train, Y_train, A_train, X_test, Y_test, A_test = fetch_adult()
verification_moment = copy.deepcopy(moment)
unmitigated = copy.deepcopy(estimator)
unmitigated.fit(X_train, Y_train)
expgrad = ExponentiatedGradient(estimator, constraints=moment)
expgrad.fit(X_train, Y_train, sensitive_features=A_train)
assert expgrad.n_oracle_calls_ > 1
assert len(expgrad.predictors_) > 1
verification_moment.load_data(X_test, Y_test, sensitive_features=A_test)
gamma_unmitigated = verification_moment.gamma(
lambda x: unmitigated.predict(x))
gamma_mitigated = verification_moment.gamma(lambda x: expgrad.predict(x))
for idx in gamma_mitigated.index:
assert abs(gamma_mitigated[idx]) <= abs(
gamma_unmitigated[idx]), "Checking {0}".format(idx)
def train_and_predict(train: DataTuple, test: TestTuple, args: AgarwalArgs):
"""Train a logistic regression model and compute predictions on the given test data."""
random.seed(888)
np.random.seed(888)
fairness_class: ConditionalSelectionRate
if args.fairness == "DP":
fairness_class = DemographicParity()
else:
fairness_class = EqualizedOdds()
if args.classifier == "SVM":
model = select_svm(args.C, args.kernel)
else:
model = LogisticRegression(solver="liblinear",
random_state=888,
max_iter=5000,
C=args.C)
data_x = train.x
data_y = train.y[train.y.columns[0]]
data_a = train.s[train.s.columns[0]]
exponentiated_gradient = ExponentiatedGradient(model,
constraints=fairness_class,
eps=args.eps,
T=args.iters)
exponentiated_gradient.fit(data_x, data_y, sensitive_features=data_a)
randomized_predictions = exponentiated_gradient.predict(test.x)
preds = pd.DataFrame(randomized_predictions, columns=["preds"])
min_class_label = train.y[train.y.columns[0]].min()
if preds["preds"].min() != preds["preds"].max():
preds = preds.replace(preds["preds"].min(), min_class_label)
return preds
def run_smoke_test(self, data):
expgrad = ExponentiatedGradient(self.learner,
constraints=data["cons_class"](),
eps=data["eps"])
expgrad.fit(self.X, self.y, sensitive_features=self.A)
Q = expgrad._best_classifier
n_classifiers = len(expgrad._classifiers)
disparity_moment = data["cons_class"]()
disparity_moment.load_data(self.X, self.y, sensitive_features=self.A)
error = ErrorRate()
error.load_data(self.X, self.y, sensitive_features=self.A)
disparity = disparity_moment.gamma(Q).max()
error = error.gamma(Q)[0]
assert expgrad._best_gap == pytest.approx(data["best_gap"],
abs=self._PRECISION)
assert expgrad._last_t == data["last_t"]
assert expgrad._best_t == data["best_t"]
assert disparity == pytest.approx(data["disp"], abs=self._PRECISION)
assert error == pytest.approx(data["error"], abs=self._PRECISION)
assert expgrad._n_oracle_calls == data["n_oracle_calls"]
assert n_classifiers == data["n_classifiers"]
def test_sample_weights_argument(self):
estimator = Pipeline([('scaler', StandardScaler()),
('logistic',
LogisticRegression(solver='liblinear'))])
X, y, A = _get_data()
expgrad = ExponentiatedGradient(estimator,
constraints=DemographicParity(),
max_iter=1)
with pytest.raises(ValueError) as execInfo:
expgrad.fit(X, y, sensitive_features=(A))
assert 'Pipeline.fit does not accept the sample_weight parameter' in execInfo.value.args[
0]
expgrad = ExponentiatedGradient(
estimator,
constraints=DemographicParity(),
max_iter=1,
sample_weight_name='logistic__sample_weight')
expgrad.fit(X, y, sensitive_features=(A))
def test_random_state_exponentiated_gradient():
"""Test that the random_state argument works as expected.
This test case reproduces the problem reported in issue 588 if the
random_state does not work as intended within Exponentiated Gradient.
https://github.com/fairlearn/fairlearn/issues/588
"""
X_train, X_test, y_train, y_test, race_train, race_test = _get_test_data()
# Train a simple logistic regression model
lr = LogisticRegression(max_iter=1000, random_state=0)
lr.fit(X_train, y_train)
# Train threshold optimizer
expgrad = ExponentiatedGradient(estimator=lr, constraints=EqualizedOdds())
expgrad.fit(X_train, y_train, sensitive_features=race_train)
# score groups
y_pred_test = expgrad.predict(X_test, random_state=0)
for _ in range(100):
assert (y_pred_test == expgrad.predict(X_test, random_state=0)).all()
assert (y_pred_test != expgrad.predict(X_test, random_state=1)).any()
