def test_fit_error():
"""Test fit method errors out."""
# case: s array not the same length as y array
with pytest.raises(ValueError):
Relabeller().fit(create_linear_X(), create_y(), np.array([1, 0, 0, 1]))
# case: y targets are not a binary variable
with pytest.raises(TypeError):
targets = create_y()
targets[0] = 100
Relabeller().fit_transform(create_linear_X(), targets, create_s())
# case: s protected classes are not a binary variable
with pytest.raises(TypeError):
s_classes = create_y()
s_classes[0] = 100
Relabeller().fit_transform(create_linear_X(), targets, s_classes)
def test_multiple_ro_clf_fit_predict_demote_true():
"""Test fit and predict methods when demote is True.
The create_linear_X, create_y, and create_Y functions (see conftest.py)
are set up such a linear classifier can find a neat decision boundary to
perfectly classify examples.
"""
X = create_linear_X()
y = create_y()
s = create_s()
roc_clf = MultipleROClassifier(theta=0.2)
roc_clf.fit(X, y)
raw_pred_proba = roc_clf._raw_predict_proba(X, s)
raw_pred = (raw_pred_proba[:, 1] > DECISION_THRESHOLD).astype(int)
pred_proba = roc_clf.predict_proba(X, s)
pred = roc_clf.predict(X, s)
# all raw predictions should perfectly classify
assert (raw_pred == y).all()
assert (pred == (pred_proba[:, 1] > DECISION_THRESHOLD)).all()
# probabilities should be the weighted mean of probabilities from the
# specified estimators in the ensemble
expected_probs = np.zeros_like(y, dtype="float64")
for e, w in zip(roc_clf.estimators_, roc_clf.pred_weights_):
expected_probs += e.predict_proba(X)[:, 1] * w
expected_probs = expected_probs / roc_clf.pred_weights_.sum()
assert (raw_pred_proba[:, 1] == expected_probs).all()
def test_single_ro_clf_fit_predict_demote_false():
"""Test fit and predict methods when demote is False.
When demote=False, only disadvantage group examples should be promoted
(i.e. no advantaged group demotions).
"""
X = create_linear_X()
y = create_y()
s = create_s()
roc_clf = SingleROClassifier(theta=0.2, demote=False)
roc_clf.fit(X, y)
raw_pred_proba = roc_clf._raw_predict_proba(X, s)
raw_pred = (raw_pred_proba[:, 1] > DECISION_THRESHOLD).astype(int)
pred_proba = roc_clf.predict_proba(X, s)
pred = roc_clf.predict(X, s)
# all raw predictions should perfectly classify
assert (raw_pred == y).all()
assert (pred == (pred_proba[:, 1] > DECISION_THRESHOLD)).all()
# with the theta critical region threshold of 0.2, and demote=False,
# only the disadvantaged observation will be promoted.
midpoint = (X.shape[0] - 1) / 2.0
promote_index = int(math.floor(midpoint))
other_index = [i for i in range(X.shape[0]) if i not in [promote_index]]
# check correctly flipped predicted labels
assert pred[promote_index] == (1 - raw_pred[promote_index])
assert (pred[other_index] == raw_pred[other_index]).all()
# check correctly flipped predicted probability
assert pred_proba[promote_index, 1] == \
(1 - raw_pred_proba[promote_index, 1])
assert (pred[other_index] == raw_pred[other_index]).all()
def test_fit_transform_error():
"""Test fit_transform method errors out.
ValueError should occur when X input to transform method is not the same
as the X input to fit method.
"""
X_input = create_linear_X()
with pytest.raises(ValueError):
Relabeller().fit(X_input, create_y(), create_s()).transform(X_input.T)
def test_predict_value_error(random_X_data):
"""Raise ValueError if X doesn't have expected number of variables."""
X = create_random_X(random_X_data)
s = create_s()
lin_acf = counterfactually_fair_models.LinearACFClassifier()
lin_acf.fit(X, create_y(), s)
with pytest.raises(ValueError):
# pass in just a subset of the input variables
lin_acf.predict(X[:, 5], s)
lin_acf.predict_proba(X[:, 5], s)
def test_fit_predict(random_X_data):
"""Test happy path of LinearACFClassifier `fit` and `predict` methods."""
X = create_random_X(random_X_data)
y = create_y()
s = create_s()
for residual_type in ["pearson", "deviance", "absolute"]:
lin_acf = counterfactually_fair_models.LinearACFClassifier(
binary_residual_type=residual_type)
lin_acf.fit(X, y, s)
lin_acf_pred_proba = lin_acf.predict_proba(X, s)[:, 1]
assert (
lin_acf.fit_residuals_ == lin_acf._compute_residuals_on_predict(
X, s)).all()
assert is_binary(lin_acf.predict(X, s))
assert is_continuous(lin_acf_pred_proba)
assert max(lin_acf_pred_proba) < 1
assert min(lin_acf_pred_proba) > 0
def test_relabeller_fit():
"""Test that relabeller fitting """
relabeller = Relabeller()
X_input = create_linear_X()
targets = create_y()
protected_class = create_s()
# The formula to determine how many observations to promote/demote is
# the number needed to make the proportion of positive labels equal
# between the two groups. This proportion is rounded up.
expected_n = 3
# Given data specified in X function, the default LogisticRegression
# estimator should be able to draw a perfect decision boundary to seperate
# the y target.
relabeller.fit(X_input, targets, protected_class)
assert relabeller.n_relabels_ == expected_n
assert (relabeller.X_ == X_input).all()
assert (relabeller.y_ == targets).all()
assert (relabeller.s_ == protected_class).all()
def test_binary_single_class(random_X_data):
"""Linear ACF can handle training data with single-valued column."""
X = create_random_X(random_X_data)
X = np.concatenate([X, np.ones((X.shape[0], 1))], axis=1)
s = create_s()
y = create_y()
lin_acf = counterfactually_fair_models.LinearACFClassifier()
for residual_type in ["pearson", "deviance", "absolute"]:
lin_acf = counterfactually_fair_models.LinearACFClassifier(
binary_residual_type=residual_type)
lin_acf.fit(X, y, s)
lin_acf_pred_proba = lin_acf.predict_proba(X, s)[:, 1]
assert (
lin_acf.fit_residuals_ == lin_acf._compute_residuals_on_predict(
X, s)).all()
assert is_binary(lin_acf.predict(X, s))
assert is_continuous(lin_acf_pred_proba)
assert max(lin_acf_pred_proba) < 1
assert min(lin_acf_pred_proba) > 0
def test_single_ro_clf_fit_predict_demote_true():
"""Test fit and predict methods when demote is True.
The create_linear_X, create_y, and create_Y functions (see conftest.py)
are set up such a linear classifier can find a neat decision boundary to
perfectly classify examples.
"""
X = create_linear_X()
y = create_y()
s = create_s()
roc_clf = SingleROClassifier(theta=0.2)
roc_clf.fit(X, y)
raw_pred_proba = roc_clf._raw_predict_proba(X, s)
raw_pred = (raw_pred_proba[:, 1] > DECISION_THRESHOLD).astype(int)
pred_proba = roc_clf.predict_proba(X, s)
pred = roc_clf.predict(X, s)
# all raw predictions should perfectly classify
assert (raw_pred == y).all()
assert (pred == (pred_proba[:, 1] > DECISION_THRESHOLD)).all()
# with the theta critical region threshold of 0.2, the middle two
# observations in create_linear_X should be flipped in label.
midpoint = (X.shape[0] - 1) / 2.0
promote_index = int(math.floor(midpoint))
demote_index = int(math.ceil(midpoint))
other_index = [
i for i in range(X.shape[0]) if i not in [promote_index, demote_index]
]
# check correctly flipped predicted labels
assert pred[promote_index] == (1 - raw_pred[promote_index])
assert pred[demote_index] == (1 - raw_pred[demote_index])
assert (pred[other_index] == raw_pred[other_index]).all()
# check correctly flipped predicted probability
assert pred_proba[promote_index, 1] == \
(1 - raw_pred_proba[promote_index, 1])
assert pred_proba[demote_index, 1] == (1 - raw_pred_proba[demote_index, 1])
assert (pred[other_index] == raw_pred[other_index]).all()
def test_fairness_aware_meta_estimator():
X = create_linear_X()
y = create_y()
s = create_s()
# use sklearn estimator
lr = FairnessAwareMetaEstimator(LogisticRegression())
lr.fit(X, y)
lr.predict(X)
lr.predict_proba(X)
# use themis_ml LinearACFClassifier estimator
linear_acf = FairnessAwareMetaEstimator(
counterfactually_fair_models.LinearACFClassifier(
binary_residual_type="absolute"))
linear_acf.fit(X, y, s)
linear_acf.predict(X, s)
linear_acf.predict_proba(X, s)
# when fit/predict methods need s, raise ValueError
with pytest.raises(ValueError):
linear_acf.fit(X, y, s=None)
with pytest.raises(ValueError):
linear_acf.predict(X, s=None)
with pytest.raises(ValueError):
linear_acf.predict_proba(X, s=None)
# use themis_ml relabeller preprocessor
relabel_clf = FairnessAwareMetaEstimator(
estimator=LogisticRegression(), relabeller=relabelling.Relabeller())
relabel_clf.fit(X, y, s)
relabel_clf.predict(X)
relabel_clf.predict_proba(X)
# when estimator method does not need `s` on predict, raise ValueError if
# it is provided
with pytest.raises(ValueError):
relabel_clf.predict(X, s)
relabel_clf.predict_proba(X, s)
# use themis_ml RejectOption Classifier
single_reject_option_clf = FairnessAwareMetaEstimator(
reject_option_classification.SingleROClassifier())
single_reject_option_clf.fit(X, y)
single_reject_option_clf.predict(X, s)
single_reject_option_clf.predict_proba(X, s)
with pytest.raises(ValueError):
single_reject_option_clf.fit(X, y, s)
with pytest.raises(ValueError):
single_reject_option_clf.predict(X, s=None)
with pytest.raises(ValueError):
single_reject_option_clf.predict_proba(X, s=None)
multi_reject_option_clf = FairnessAwareMetaEstimator(
reject_option_classification.MultipleROClassifier())
multi_reject_option_clf.fit(X, y)
multi_reject_option_clf.predict(X, s)
multi_reject_option_clf.predict_proba(X, s)
with pytest.raises(ValueError):
multi_reject_option_clf.fit(X, y, s)
with pytest.raises(ValueError):
multi_reject_option_clf.predict(X, s=None)
with pytest.raises(ValueError):
multi_reject_option_clf.predict_proba(X, s=None)
def test_not_fitted_error():
"""Test raises not fitted error if predict before fit."""
with pytest.raises(NotFittedError):
SingleROClassifier().predict(create_linear_X(), create_y())
def test_relabeller_transform():
"""Test that relabeller correctly relabels targets."""
expected = np.array([[0, 0, 1, 1, 1, 0, 0, 0, 1, 1]])
assert (Relabeller().fit_transform(create_linear_X(),
create_y(),
s=create_s()) == expected).all()