def test_no_predict_before_fit(self): gs = GridSearch(self.estimator, self.disparity_criterion) X, _, _ = _quick_data() with pytest.raises(NotFittedError) as execInfo: gs.predict(X) assert not_fitted_error_msg.format(GridSearch.__name__) == execInfo.value.args[0]
def test_no_predict_before_fit(self): gs = GridSearch(self.estimator, self.disparity_criterion) X, _, _ = self._quick_data() with pytest.raises(NotFittedError) as execInfo: gs.predict(X) assert _NO_PREDICT_BEFORE_FIT == execInfo.value.args[0]
def test_no_predict_before_fit(self): gs = GridSearch(self.estimator, self.disparity_criterion) X, _, _ = self._quick_data() message = str("Must call fit before attempting to make predictions") with pytest.raises(NotFittedException) as execInfo: gs.predict(X) assert message == execInfo.value.args[0]
def test_demographicparity_fair_uneven_populations(A_two_dim): # Variant of test_demographicparity_already_fair, which has unequal # populations in the two classes # Also allow the threshold to be adjustable score_threshold = 0.625 number_a0 = 4 number_a1 = 4 a0_label = 17 a1_label = 37 X, Y, A = _simple_threshold_data(number_a0, number_a1, score_threshold, score_threshold, a0_label, a1_label, A_two_dim) target = GridSearch(LogisticRegression(solver='liblinear', fit_intercept=True), constraints=DemographicParity(), grid_size=11) target.fit(X, Y, sensitive_features=A) assert len(target.all_results) == 11 test_X = pd.DataFrame({"actual_feature": [0.2, 0.7], "sensitive_features": [a0_label, a1_label], "constant_ones_feature": [1, 1]}) sample_results = target.predict(test_X) sample_proba = target.predict_proba(test_X) assert np.allclose(sample_proba, [[0.53748641, 0.46251359], [0.46688736, 0.53311264]]) sample_results = target.all_results[0].predictor.predict(test_X) assert np.array_equal(sample_results, [1, 0])
def test_bgl_unfair(A_two_dim): a0_count = 5 a1_count = 7 a0_label = 2 a1_label = 3 a0_factor = 1 a1_factor = 16 grid_size = 7 X, Y, A = _simple_regression_data( a0_count, a1_count, a0_factor, a1_factor, a0_label, a1_label, A_two_dim ) bgl_square_loss = BoundedGroupLoss(SquareLoss(-np.inf, np.inf)) grid_search = GridSearch( LinearRegression(), constraints=bgl_square_loss, grid_size=grid_size ) grid_search.fit(X, Y, sensitive_features=A) assert_n_grid_search_results(grid_size, grid_search) test_X = pd.DataFrame( { "actual_feature": [0.2, 0.7], "sensitive_features": [a0_label, a1_label], "constant_ones_feature": [1, 1], } ) best_predict = grid_search.predict(test_X) assert np.allclose([-1.91764706, 9.61176471], best_predict) all_predict = [predictor.predict(test_X) for predictor in grid_search.predictors_] # TODO: investigate where the different outcomes for the first grid point are from, likely # due to some ignored data points at the edge resulting in another solution with the same # least squares loss (i.e. both solutions acceptable). # Reflects https://github.com/fairlearn/fairlearn/issues/265 assert logging_all_close([[3.2, 11.2]], [all_predict[0]]) or logging_all_close( [[3.03010885, 11.2]], [all_predict[0]] ) assert logging_all_close( [ [-3.47346939, 10.64897959], [-2.68, 10.12], [-1.91764706, 9.61176471], [-1.18461538, 9.12307692], [-0.47924528, 8.65283019], [0.2, 0.7], ], all_predict[1:], )
def test_demographicparity_fair_uneven_populations_with_grid_offset( A_two_dim, offset): # Grid of Lagrangian multipliers has some initial offset score_threshold = 0.625 number_a0 = 4 number_a1 = 4 a0_label = 17 a1_label = 37 grid_size = 11 iterables = [["+", "-"], ["all"], [a0_label, a1_label]] midx = pd.MultiIndex.from_product(iterables, names=["sign", "event", "group_id"]) grid_offset = pd.Series(offset, index=midx) X, Y, A = _simple_threshold_data( number_a0, number_a1, score_threshold, score_threshold, a0_label, a1_label, A_two_dim, ) grid_search = GridSearch( LogisticRegression(solver="liblinear", fit_intercept=True), constraints=DemographicParity(), grid_size=grid_size, grid_offset=grid_offset, ) grid_search.fit(X, Y, sensitive_features=A) assert_n_grid_search_results(grid_size, grid_search) test_X = pd.DataFrame({ "actual_feature": [0.2, 0.7], "sensitive_features": [a0_label, a1_label], "constant_ones_feature": [1, 1], }) sample_results = grid_search.predict(test_X) assert np.array_equal(sample_results, [0, 1]) sample_proba = grid_search.predict_proba(test_X) assert np.allclose(sample_proba, [[0.55069845, 0.44930155], [0.41546008, 0.58453992]]) sample_results = grid_search.predictors_[0].predict(test_X) assert np.array_equal(sample_results, [1, 0])
def lagrangian(constraint, model, constraint_weight, grid_size, X_train, Y_train, A_train, X_test): """ Conduct lagrangian algorithm and set the base classifier as the black-box estimator to train and predict. """ start_time = datetime.now() if constraint == 'DP': clf = GridSearch(models[model], constraints=DemographicParity(), constraint_weight=constraint_weight, grid_size=grid_size) elif constraint == 'EO': clf = GridSearch(models[model], constraints=EqualizedOdds(), constraint_weight=constraint_weight, grid_size=grid_size) clf.fit(X_train, Y_train, sensitive_features=A_train) Y_pred = clf.predict(X_test) end_time = datetime.now() return Y_pred, time_diff_in_microseconds(end_time - start_time)
def evaluate(weight, X_train, y_train, X_test, y_test, sex_train, sex_test, index): estimator = GradientBoostingClassifier() constraints = DemographicParity() gssolver = GridSearch(estimator, constraints, grid_size=10, constraint_weight=weight) gssolver.fit(X_train, y_train, sensitive_features=sex_train) y_pred = gssolver.predict(X_test) # print("y_pred",y_pred) group_summary_adult = group_summary(accuracy_score, y_test, y_pred, sensitive_features=sex_test) selection_rate_summary = selection_rate_group_summary( y_test, y_pred, sensitive_features=sex_test) error = 1 - group_summary_adult["overall"] dp = demographic(selection_rate_summary) errorlist[index].append(error) dplist[index].append(dp) print("error:%f,dp:%f" % (error, dp))
def run_gridsearch_classification(estimator, moment): """Run classification test with GridSearch.""" X_train, Y_train, A_train, X_test, Y_test, A_test = fetch_adult() verification_moment = copy.deepcopy(moment) unmitigated = copy.deepcopy(estimator) unmitigated.fit(X_train, Y_train) num_predictors = 11 gs = GridSearch(estimator, constraints=moment, grid_size=num_predictors) gs.fit(X_train, Y_train, sensitive_features=A_train) assert len(gs.predictors_) == num_predictors verification_moment.load_data(X_test, Y_test, sensitive_features=A_test) gamma_unmitigated = verification_moment.gamma( lambda x: unmitigated.predict(x)) gamma_mitigated = verification_moment.gamma(lambda x: gs.predict(x)) for idx in gamma_mitigated.index: assert abs(gamma_mitigated[idx]) <= abs( gamma_unmitigated[idx]), "Checking {0}".format(idx)
def test_bgl_unfair(A_two_dim): a0_count = 5 a1_count = 7 a0_label = 2 a1_label = 3 a0_factor = 1 a1_factor = 16 X, Y, A = _simple_regression_data(a0_count, a1_count, a0_factor, a1_factor, a0_label, a1_label, A_two_dim) bgl_square_loss = GroupLossMoment(SquareLoss(-np.inf, np.inf)) target = GridSearch(LinearRegression(), constraints=bgl_square_loss, grid_size=7) target.fit(X, Y, sensitive_features=A) assert len(target.all_results) == 7 test_X = pd.DataFrame({ "actual_feature": [0.2, 0.7], "sensitive_features": [a0_label, a1_label], "constant_ones_feature": [1, 1] }) best_predict = target.predict(test_X) assert np.allclose([-1.91764706, 9.61176471], best_predict) all_predict = [r.predictor.predict(test_X) for r in target.all_results] assert logging_all_close( [[3.2, 11.2], [-3.47346939, 10.64897959], [-2.68, 10.12], [-1.91764706, 9.61176471], [-1.18461538, 9.12307692], [-0.47924528, 8.65283019], [0.2, 0.7]], all_predict)
def test_bgl_unfair(): a0_count = 5 a1_count = 7 a0_label = 2 a1_label = 3 a0_factor = 1 a1_factor = 16 X, Y, A = _simple_regression_data(a0_count, a1_count, a0_factor, a1_factor, a0_label, a1_label) target = GridSearch(LinearRegression(), disparity_metric=moments.GroupLossMoment( moments.ZeroOneLoss()), quality_metric=SimpleRegressionQualityMetric(), grid_size=7) target.fit(X, Y, sensitive_features=A) assert len(target.all_results) == 7 test_X = pd.DataFrame({ "actual_feature": [0.2, 0.7], "sensitive_features": [a0_label, a1_label], "constant_ones_feature": [1, 1] }) best_predict = target.predict(test_X) assert np.allclose([-1.91764706, 9.61176471], best_predict) all_predict = target.posterior_predict(test_X) assert np.allclose( [[3.2, 11.2], [-3.47346939, 10.64897959], [-2.68, 10.12], [-1.91764706, 9.61176471], [-1.18461538, 9.12307692], [-0.47924528, 8.65283019], [0.2, 0.7]], all_predict)