def test_threshold_optimization_equalized_odds_e2e(
attributes, attribute_names, expected_positive_p0,
expected_positive_p1, expected_negative_p0, expected_negative_p1,
X_transform, y_transform, A_transform):
X = X_transform(_format_as_list_of_lists(attributes))
y = y_transform(example_labels)
A = A_transform(attributes)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=EQUALIZED_ODDS)
adjusted_model.fit(X, y, A)
predictions = adjusted_model.predict_proba(X, A)
# assert equalized odds
for a in attribute_names:
positive_indices = (np.array(attributes) == a) * \
(np.array(example_labels) == 1)
negative_indices = (np.array(attributes) == a) * \
(np.array(example_labels) == 0)
average_probs_positive_indices = np.average(
predictions[positive_indices], axis=0)
average_probs_negative_indices = np.average(
predictions[negative_indices], axis=0)
assert np.isclose(average_probs_positive_indices[0],
expected_positive_p0)
assert np.isclose(average_probs_positive_indices[1],
expected_positive_p1)
assert np.isclose(average_probs_negative_indices[0],
expected_negative_p0)
assert np.isclose(average_probs_negative_indices[1],
expected_negative_p1)
def test_predict_output_0_or_1(attributes, attribute_names, X_transform,
y_transform, A_transform, metric):
X = X_transform(_format_as_list_of_lists(attributes))
y = y_transform(example_labels)
A = A_transform(attributes)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
adjusted_model.fit(X, y, A)
predictions = adjusted_model.predict(X, A)
for prediction in predictions:
assert prediction in [0, 1]
def test_inconsistent_input_data_types(X, y, A, metric):
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
error_message = INPUT_DATA_FORMAT_ERROR_MESSAGE.format(
type(X).__name__,
type(y).__name__,
type(A).__name__)
if X is None or y is None and A is None:
with pytest.raises(TypeError) as exception:
adjusted_model.fit(X, y, A)
assert str(exception.value) == error_message
def test_threshold_optimization_different_input_lengths(
X_transform, y_transform, A_transform, metric):
n = len(example_attributes1)
for permutation in [(0, 1), (1, 0)]:
with pytest.raises(ValueError,
match=DIFFERENT_INPUT_LENGTH_ERROR_MESSAGE.format(
"X, sensitive_features, and y")):
X = X_transform(
_format_as_list_of_lists(example_attributes1)[:n -
permutation[0]])
y = y_transform(example_labels[:n - permutation[1]])
A = A_transform(example_attributes1)
adjusted_model = ThresholdOptimizer(
unconstrained_model=ExampleModel(), parity_criteria=metric)
adjusted_model.fit(X, y, A)
# try providing empty lists in all combinations
for permutation in [(0, n), (n, 0)]:
X = X_transform(
_format_as_list_of_lists(example_attributes1)[:n - permutation[0]])
y = y_transform(example_labels[:n - permutation[1]])
A = A_transform(example_attributes1)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
with pytest.raises(ValueError, match=EMPTY_INPUT_ERROR_MESSAGE):
adjusted_model.fit(X, y, A)
def test_threshold_optimization_non_binary_labels(X_transform, y_transform,
A_transform, metric):
non_binary_labels = copy.deepcopy(example_labels)
non_binary_labels[0] = 2
X = X_transform(_format_as_list_of_lists(example_attributes1))
y = y_transform(non_binary_labels)
A = A_transform(example_attributes1)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
with pytest.raises(ValueError, match=NON_BINARY_LABELS_ERROR_MESSAGE):
adjusted_model.fit(X, y, A)
def test_predict_before_fit_error(X_transform, A_transform,
predict_method_name, metric):
X = X_transform(_format_as_list_of_lists(example_attributes1))
A = A_transform(example_attributes1)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
with pytest.raises(ValueError, match=PREDICT_BEFORE_FIT_ERROR_MESSAGE):
getattr(adjusted_model, predict_method_name)(X, A)
def test_threshold_optimization_demographic_parity_e2e(
attributes, attribute_names, expected_p0, expected_p1, X_transform,
y_transform, A_transform):
X = X_transform(_format_as_list_of_lists(attributes))
y = y_transform(example_labels)
A = A_transform(attributes)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=DEMOGRAPHIC_PARITY)
adjusted_model.fit(X, y, A)
predictions = adjusted_model.predict_proba(X, A)
# assert demographic parity
for a in attribute_names:
average_probs = np.average(predictions[np.array(attributes) == a],
axis=0)
assert np.isclose(average_probs[0], expected_p0)
assert np.isclose(average_probs[1], expected_p1)
def test_predict_different_argument_lengths(attributes, attribute_names,
X_transform, y_transform,
A_transform, metric):
X = X_transform(_format_as_list_of_lists(attributes))
y = y_transform(example_labels)
A = A_transform(attributes)
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
adjusted_model.fit(X, y, A)
with pytest.raises(ValueError,
match=DIFFERENT_INPUT_LENGTH_ERROR_MESSAGE.format(
"X and sensitive_features")):
adjusted_model.predict(X, A_transform(attributes[:-1]))
with pytest.raises(ValueError,
match=DIFFERENT_INPUT_LENGTH_ERROR_MESSAGE.format(
"X and sensitive_features")):
adjusted_model.predict(
X_transform(_format_as_list_of_lists(attributes))[:-1], A)
def test_predict_multiple_attributes_columns_error(attributes, attribute_names,
X_transform, y_transform,
metric):
X = X_transform(_format_as_list_of_lists(attributes))
y = y_transform(example_labels)
A = pd.DataFrame({"A1": attributes, "A2": attributes})
adjusted_model = ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria=metric)
adjusted_model.fit(X, y, attributes)
with pytest.raises(ValueError,
match=MULTIPLE_DATA_COLUMNS_ERROR_MESSAGE.format(
"sensitive_features")):
adjusted_model.predict(X, A)
def test_not_model(metric):
with pytest.raises(ValueError, match=MISSING_PREDICT_ERROR_MESSAGE):
ThresholdOptimizer(unconstrained_model=ExampleNotModel(),
parity_criteria=metric)
def test_not_estimator(not_estimator, metric):
with pytest.raises(ValueError, match=MISSING_FIT_PREDICT_ERROR_MESSAGE):
ThresholdOptimizer(unconstrained_estimator=not_estimator,
parity_criteria=metric)
def test_metric_not_supported():
with pytest.raises(ValueError,
match=NOT_SUPPORTED_PARITY_CRITERIA_ERROR_MESSAGE):
ThresholdOptimizer(unconstrained_model=ExampleModel(),
parity_criteria="UnsupportedMetric")
def test_no_model_or_estimator_error(metric):
with pytest.raises(ValueError,
match=MODEL_OR_ESTIMATOR_REQUIRED_ERROR_MESSAGE):
ThresholdOptimizer(parity_criteria=metric)
def test_both_model_and_estimator_error(metric):
with pytest.raises(ValueError,
match=EITHER_MODEL_OR_ESTIMATOR_ERROR_MESSAGE):
ThresholdOptimizer(unconstrained_model=ExampleModel(),
unconstrained_estimator=ExampleEstimator(),
parity_criteria=metric)