def test_p_percent_pandas_multiclass():
sensitive_classification_dataset = pd.DataFrame({
"x1": [1, 0, 1, 0, 1, 0, 1, 1],
"x2": [0, 0, 0, 0, 0, 1, 1, 1],
"y": [1, 1, 1, 0, 1, 0, 0, 2],
})
X, y = (
sensitive_classification_dataset.drop(columns="y"),
sensitive_classification_dataset["y"],
)
mod_1 = types.SimpleNamespace()
mod_1.predict = lambda X: np.array([2, 0, 1, 0, 1, 0, 1, 2])
assert (equal_opportunity_score(sensitive_column="x2", positive_target=2)(
mod_1, X, np.array([2, 0, 1, 0, 1, 0, 1, 2])) == 1)
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 1])
assert (equal_opportunity_score(sensitive_column="x2",
positive_target=2)(mod_1, X, y) == 0)
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 0])
assert (equal_opportunity_score(sensitive_column="x2",
positive_target=2)(mod_1, X, y) == 0)
def objective(self, trial):
C = trial.suggest_loguniform('C', 1e-10, 1e10)
model = LogisticRegression(C=C, max_iter=10**3, tol=10**-6)
model.fit(self.X_train, self.y_train, sample_weight=self.sample_weight)
y_pred = model.predict(self.X_val)
if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or
equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)==0 or
p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val))==0:
return float('inf')
if self.metric=='accuracy':
perf = sklearn.metrics.accuracy_score(self.y_val, y_pred)
elif self.metric=='equal_opportunity':
perf = equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)
elif self.metric=='p_percent':
perf = p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val)
elif self.metric=='c_variation':
perf = 1/coefficient_of_variation(model, self.X_val, self.y_val)
if perf>self.best_perf:
self.best_perf = perf
self.best_model = model
return 1/perf if perf!=0 else float('inf')
def tune(self, metric=None):
self.best_perf = 0
self.best_model = None
if metric is not None:
self.metric = metric
if self.moo_ is None:
self.moo_ = monise(weightedScalar=self.scalarization, singleScalar=self.scalarization,
nodeTimeLimit=2, targetSize=150,
targetGap=0, nodeGap=0.01, norm=False)
self.moo_.optimize()
for solution in self.moo_.solutionsList:
y_pred = solution.x.predict(self.X_val)
if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or
equal_opportunity_score(sensitive_column=self.fair_feat)(solution.x, self.X_val, self.y_val)==0 or
p_percent_score(sensitive_column=self.fair_feat)(solution.x, self.X_val))==0:
continue
if self.metric=='accuracy':
perf = sklearn.metrics.accuracy_score(self.y_val, y_pred)
elif self.metric=='equal_opportunity':
perf = equal_opportunity_score(sensitive_column=self.fair_feat)(solution.x, self.X_val, self.y_val)
elif self.metric=='p_percent':
perf = p_percent_score(sensitive_column=self.fair_feat)(solution.x, self.X_val)
elif self.metric=='c_variation':
perf = 1/coefficient_of_variation(solution.x, self.X_val, self.y_val)
if perf>self.best_perf:
self.best_perf = perf
self.best_model = solution.x
return self.best_model
def test_warning_is_logged(sensitive_classification_dataset):
X, y = sensitive_classification_dataset
mod_fair = make_pipeline(ColumnSelector("x1"),
LogisticRegression()).fit(X, y)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# Trigger a warning.
equal_opportunity_score("x2", positive_target=2)(mod_fair, X, y)
assert issubclass(w[-1].category, RuntimeWarning)
def simulate_eo(data, columnNames, label, model, numOfSimulation=100):
"""
description:
runs simulation based on the number of simulations
args:
data (list of list): list of attributes and values
label (string): classification label / ground truth
model (classification model): sckit classification model
numOfSimulation (int): number of simulations to run
returns:
equalOpportunities (list): list of equal opportunities
"""
numOfSimulation = 1000
equalOpportunities = list()
sensitiveVariableCount = list()
for time in tqdm(range(numOfSimulation)):
windowData = create_window(data, columnNames)
X, y = create_Xy(windowData, label)
sensitiveVarCount = compute_sensitive_ratio(X['sex'])
X, y = preprocess_columns(X, y)
eqTemp = equal_opportunity_score(sensitive_column="sex")(model, X, y)
equalOpportunities.append(eqTemp)
sensitiveVariableCount.append(sensitiveVarCount)
return equalOpportunities, sensitiveVariableCount
def test_equal_opportunity_pandas():
sensitive_classification_dataset = pd.DataFrame({
"x1": [1, 0, 1, 0, 1, 0, 1, 1],
"x2": [0, 0, 0, 0, 0, 1, 1, 1],
"y": [1, 1, 1, 0, 1, 0, 0, 1]
})
X, y = sensitive_classification_dataset.drop(
columns='y'), sensitive_classification_dataset['y']
mod_1 = types.SimpleNamespace()
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 1, 1])
assert equal_opportunity_score(sensitive_column="x2")(mod_1, X, y) == 0.75
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 1])
assert equal_opportunity_score(sensitive_column="x2")(mod_1, X, y) == 0.75
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 0])
assert equal_opportunity_score(sensitive_column="x2")(mod_1, X, y) == 0
def test_fairness(sensitive_classification_dataset):
"""tests whether fairness (measured by p percent score) increases as we decrease the covariance threshold"""
X, y = sensitive_classification_dataset
scorer = equal_opportunity_score("x1")
prev_fairness = -np.inf
for cov_threshold in [None, 10, 0.5, 0.1]:
fair = EqualOpportunityClassifier(
covariance_threshold=cov_threshold,
positive_target=True,
sensitive_cols=["x1"],
penalty="none",
train_sensitive_cols=False,
).fit(X, y)
fairness = scorer(fair, X, y)
assert fairness >= prev_fairness
prev_fairness = fairness
def evaluate_model_test(model__, fair_feature, X_test, y_test):
y_pred = model__.predict(X_test)
metrics = {
"Acc":
accuracy_score(y_test, y_pred),
#"BalancedAcc": balanced_accuracy_score(y_test, y_pred),
#"F-score": f1_score(y_test, y_pred),
"EO":
equal_opportunity_score(sensitive_column=fair_feature)(model__, X_test,
y_test),
"DP":
p_percent_score(sensitive_column=fair_feature)(model__, X_test),
"CV":
coefficient_of_variation(model__, X_test, y_test)
}
metrics["SingleClass"] = False
if len(np.unique(y_pred)) == 1:
#raise Exception("Model classifies every point to the same class")
metrics["SingleClass"] = True
return metrics
def test_p_percent_numpy(sensitive_classification_dataset):
X, y = sensitive_classification_dataset
X = X.values
mod = LogisticRegression().fit(X, y)
assert equal_opportunity_score(1)(mod, X, y) == 0
def objective(self, trial):
C = trial.suggest_loguniform('C', 1e-5, 1e5)
c = trial.suggest_loguniform('c', 1e-5, 1e5)
#print(c, C)
try:
#if 1==1:
if self.base_model=='equal':
model = EqualOpportunityClassifier(sensitive_cols=self.fair_feat, positive_target=True, covariance_threshold=c, C=C, max_iter=10**3)
model.fit(self.X_train, self.y_train)
elif self.base_model=='demographic':
model = DemographicParityClassifier(sensitive_cols=self.fair_feat, covariance_threshold=c, C=C, max_iter=10**3)
model.fit(self.X_train, self.y_train)
elif self.base_model=='minimax':
a_train = self.X_train[self.fair_feat].copy().astype('int')
a_val = self.X_val[self.fair_feat].copy().astype('int')
a_train[a_train==0] = -1
a_val[a_val==0] = -1
model = SKLearn_Weighted_LLR(self.X_train.values, self.y_train.values,
a_train.values, self.X_val.values,
self.y_val.values, a_val.values,
C_reg=C)
mua_ini = np.ones(a_val.max() + 1)
mua_ini /= mua_ini.sum()
results = APSTAR(model, mua_ini, niter=200, max_patience=200, Kini=1,
Kmin=20, alpha=0.5, verbose=False)
mu_best_list = results['mu_best_list']
mu_best = mu_best_list[-1]
model.weighted_fit(self.X_train.values, self.y_train.values, a_train.values, mu_best)
else:
raise('Incorrect base_model.')
y_pred = model.predict(self.X_val)
except:
return float('inf')
if (sklearn.metrics.accuracy_score(self.y_val, y_pred)==0 or
equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)==0 or
p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val))==0:
return float('inf')
if self.metric=='accuracy':
perf = sklearn.metrics.accuracy_score(self.y_val, y_pred)
elif self.metric=='equal_opportunity':
perf = equal_opportunity_score(sensitive_column=self.fair_feat)(model, self.X_val, self.y_val)
elif self.metric=='p_percent':
perf = p_percent_score(sensitive_column=self.fair_feat)(model, self.X_val)
elif self.metric=='c_variation':
perf = 1/coefficient_of_variation(model, self.X_val, self.y_val)
if perf>self.best_perf:
self.best_perf = perf
self.best_model = model
return 1/perf if perf!=0 else float('inf')
#mod_unfair = LogisticRegression(solver='lbfgs').fit(X, y)
from sklego.metrics import equal_opportunity_score
from sklearn.linear_model import LogisticRegression
import types
sensitive_classification_dataset = pd.DataFrame({
"x1": [1, 0, 1, 0, 1, 0, 1, 1],
"x2": [0, 0, 0, 0, 0, 1, 1, 1],
"y": [1, 1, 1, 0, 1, 0, 0, 1]
})
X, y = sensitive_classification_dataset.drop(
columns='y'), sensitive_classification_dataset['y']
mod_1 = types.SimpleNamespace()
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 1, 1])
print(mod_1)
print('equal_opportunity_score:',
equal_opportunity_score(sensitive_column="x2")(mod_1, X, y))
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 1])
print('equal_opportunity_score:',
equal_opportunity_score(sensitive_column="x2")(mod_1, X, y))
mod_1.predict = lambda X: np.array([1, 0, 1, 0, 1, 0, 0, 0])
print('equal_opportunity_score:',
equal_opportunity_score(sensitive_column="x2")(mod_1, X, y))
