def test_p_percent_pandas(sensitive_classification_dataset): X, y = sensitive_classification_dataset mod_unfair = LogisticRegression().fit(X, y) assert p_percent_score("x2")(mod_unfair, X) == 0 mod_fair = make_pipeline(ColumnSelector("x1"), LogisticRegression()).fit(X, y) assert p_percent_score("x2")(mod_fair, X) == 0.9
def objective(self, trial): C = trial.suggest_loguniform('C', 1e-10, 1e10) model = LogisticRegression(C=C, max_iter=10**3, tol=10**-6) model.fit(self.X_train, self.y_train, sample_weight=self.sample_weight) y_pred = model.predict(self.X_val) if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)==0 or p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val))==0: return float('inf') if self.metric=='accuracy': perf = sklearn.metrics.accuracy_score(self.y_val, y_pred) elif self.metric=='equal_opportunity': perf = equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val) elif self.metric=='p_percent': perf = p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val) elif self.metric=='c_variation': perf = 1/coefficient_of_variation(model, self.X_val, self.y_val) if perf>self.best_perf: self.best_perf = perf self.best_model = model return 1/perf if perf!=0 else float('inf')
def tune(self, metric=None): self.best_perf = 0 self.best_model = None if metric is not None: self.metric = metric if self.moo_ is None: self.moo_ = monise(weightedScalar=self.scalarization, singleScalar=self.scalarization, nodeTimeLimit=2, targetSize=150, targetGap=0, nodeGap=0.01, norm=False) self.moo_.optimize() for solution in self.moo_.solutionsList: y_pred = solution.x.predict(self.X_val) if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or equal_opportunity_score(sensitive_column=self.fair_feat)(solution.x, self.X_val, self.y_val)==0 or p_percent_score(sensitive_column=self.fair_feat)(solution.x, self.X_val))==0: continue if self.metric=='accuracy': perf = sklearn.metrics.accuracy_score(self.y_val, y_pred) elif self.metric=='equal_opportunity': perf = equal_opportunity_score(sensitive_column=self.fair_feat)(solution.x, self.X_val, self.y_val) elif self.metric=='p_percent': perf = p_percent_score(sensitive_column=self.fair_feat)(solution.x, self.X_val) elif self.metric=='c_variation': perf = 1/coefficient_of_variation(solution.x, self.X_val, self.y_val) if perf>self.best_perf: self.best_perf = perf self.best_model = solution.x return self.best_model
def test_warning_is_logged(sensitive_classification_dataset): X, y = sensitive_classification_dataset mod_fair = make_pipeline(ColumnSelector("x1"), LogisticRegression()).fit(X, y) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # Trigger a warning. p_percent_score("x2", positive_target=2)(mod_fair, X) assert issubclass(w[-1].category, RuntimeWarning)
def test_p_percent_pandas_multiclass(sensitive_multiclass_classification_dataset): X, y = sensitive_multiclass_classification_dataset mod_unfair = LogisticRegression(multi_class="ovr").fit(X, y) assert p_percent_score("x2")(mod_unfair, X) == 0 assert p_percent_score("x2", positive_target=2)(mod_unfair, X) == 0 mod_fair = make_pipeline(ColumnSelector("x1"), LogisticRegression()).fit(X, y) assert p_percent_score("x2")(mod_fair, X) == pytest.approx(0.9333333) assert p_percent_score("x2", positive_target=2)(mod_fair, X) == 0
def test_fairness(sensitive_classification_dataset): """tests whether fairness (measured by p percent score) increases as we decrease the covariance threshold""" X, y = sensitive_classification_dataset scorer = p_percent_score("x1") prev_fairness = -np.inf for cov_threshold in [None, 10, 0.5, 0.1]: fair = FairClassifier( covariance_threshold=cov_threshold, sensitive_cols=["x1"], penalty="none", train_sensitive_cols=False, ).fit(X, y) fairness = scorer(fair, X, y) assert fairness >= prev_fairness prev_fairness = fairness
def evaluate_model_test(model__, fair_feature, X_test, y_test): y_pred = model__.predict(X_test) metrics = { "Acc": accuracy_score(y_test, y_pred), #"BalancedAcc": balanced_accuracy_score(y_test, y_pred), #"F-score": f1_score(y_test, y_pred), "EO": equal_opportunity_score(sensitive_column=fair_feature)(model__, X_test, y_test), "DP": p_percent_score(sensitive_column=fair_feature)(model__, X_test), "CV": coefficient_of_variation(model__, X_test, y_test) } metrics["SingleClass"] = False if len(np.unique(y_pred)) == 1: #raise Exception("Model classifies every point to the same class") metrics["SingleClass"] = True return metrics
def test_p_percent_numpy(sensitive_classification_dataset): X, y = sensitive_classification_dataset X = X.values mod = LogisticRegression().fit(X, y) assert p_percent_score(1)(mod, X) == 0
def objective(self, trial): C = trial.suggest_loguniform('C', 1e-5, 1e5) c = trial.suggest_loguniform('c', 1e-5, 1e5) #print(c, C) try: #if 1==1: if self.base_model=='equal': model = EqualOpportunityClassifier(sensitive_cols=self.fair_feat, positive_target=True, covariance_threshold=c, C=C, max_iter=10**3) model.fit(self.X_train, self.y_train) elif self.base_model=='demographic': model = DemographicParityClassifier(sensitive_cols=self.fair_feat, covariance_threshold=c, C=C, max_iter=10**3) model.fit(self.X_train, self.y_train) elif self.base_model=='minimax': a_train = self.X_train[self.fair_feat].copy().astype('int') a_val = self.X_val[self.fair_feat].copy().astype('int') a_train[a_train==0] = -1 a_val[a_val==0] = -1 model = SKLearn_Weighted_LLR(self.X_train.values, self.y_train.values, a_train.values, self.X_val.values, self.y_val.values, a_val.values, C_reg=C) mua_ini = np.ones(a_val.max() + 1) mua_ini /= mua_ini.sum() results = APSTAR(model, mua_ini, niter=200, max_patience=200, Kini=1, Kmin=20, alpha=0.5, verbose=False) mu_best_list = results['mu_best_list'] mu_best = mu_best_list[-1] model.weighted_fit(self.X_train.values, self.y_train.values, a_train.values, mu_best) else: raise('Incorrect base_model.') y_pred = model.predict(self.X_val) except: return float('inf') if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)==0 or p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val))==0: return float('inf') if self.metric=='accuracy': perf = sklearn.metrics.accuracy_score(self.y_val, y_pred) elif self.metric=='equal_opportunity': perf = equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val) elif self.metric=='p_percent': perf = p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val) elif self.metric=='c_variation': perf = 1/coefficient_of_variation(model, self.X_val, self.y_val) if perf>self.best_perf: self.best_perf = perf self.best_model = model return 1/perf if perf!=0 else float('inf')