def base_train_test(dataset,
size,
base_solver,
loss="square",
random_seed=DATA_SPLIT_SEED):
"""
Given a baseline method, train and test on a dataset.
Input:
- dataset name
- size parameter for data parser
- base_solver: the solver for baseline benchmark
- loss: loss function name
- random_seed for data splitting
Output: Results for
- baseline output
"""
if dataset == 'law_school':
x, a, y = parser.clean_lawschool_full()
sens_attr = 'race'
elif dataset == 'communities':
x, a, y = parser.clean_communities_full()
sens_attr = 'race'
elif dataset == 'adult':
x, a, y = parser.clean_adult_full()
sens_attr = 'sex'
## the difference is that sensitive features are clarified here.
else:
raise Exception('DATA SET NOT FOUND!')
if _SMALL:
x, a, y = subsample(x, a, y, size)
x_train, a_train, y_train, x_test, a_test, y_test = train_test_split_groups(
x, a, y, random_seed=DATA_SPLIT_SEED)
## until
if base_solver.name == "SEO":
# Evaluate SEO method
base_solver.fit(x_train, y_train, sens_attr)
h_base = lambda X: base_solver.predict(X, sens_attr)
else:
base_solver.fit(x_train, y_train)
h_base = lambda X: base_solver.predict(X)
base_train_eval = evaluate.eval_BenchmarkModel(x_train, a_train, y_train,
h_base, loss)
base_test_eval = evaluate.eval_BenchmarkModel(x_test, a_test, y_test,
h_base, loss)
result = {}
result['base_train_eval'] = base_train_eval
result['base_test_eval'] = base_test_eval
result['loss'] = loss
result['learner'] = base_solver.name
result['dataset'] = dataset
return result
def grid_train_test(lambda_list, learner):
"""
Take the adult dataset and get logistic regression learner from
grid search method.
"""
x, a, y = parser.clean_adult_full()
if _SMALL:
x, a, y = run_exp.subsample(x, a, y, 2000)
x_train, a_train, y_train, x_test, a_test, y_test = run_exp.train_test_split_groups(
x, a, y, random_seed=DATA_SPLIT_SEED)
models = {}
train_evaluation = {}
test_evaluation = {}
learners = {}
for lamb in lambda_list:
models[lamb], learners[lamb] = lambda_response(x_train, a_train,
y_train, learner, lamb)
train_evaluation[lamb] = evaluate.eval_BenchmarkModel(
x_train, a_train, y_train, models[lamb], "logistic")
print(lamb, train_evaluation[lamb]['average_loss'])
test_evaluation[lamb] = evaluate.eval_BenchmarkModel(
x_test, a_test, y_test, models[lamb], "logistic")
result = {}
result['learner'] = learner.name
result['loss'] = "logistic"
result['constraint'] = 'DP'
result['train_eval'] = train_evaluation
result['test_eval'] = test_evaluation
result['learners'] = learners
return result
def fair_train_test(dataset,
size,
eps_list,
learner,
constraint="DP",
loss="square",
random_seed=DATA_SPLIT_SEED,
init_cache=[]):
"""
Input:
- dataset name
- size parameter for data parser
- eps_list: list of epsilons for exp_grad
- learner: the solver for CSC
- constraint: fairness constraint name
- loss: loss function name
- random_seed
Output: Results for
- exp_grad: (eps, loss) for training and test sets
- benchmark method: (eps, loss) for training and test sets
"""
if dataset == 'law_school':
x, a, y = parser.clean_lawschool_full()
elif dataset == 'communities':
x, a, y = parser.clean_communities_full()
elif dataset == 'adult':
x, a, y = parser.clean_adult_full()
else:
raise Exception('DATA SET NOT FOUND!')
if _SMALL:
x, a, y = subsample(x, a, y, size)
x_train, a_train, y_train, x_test, a_test, y_test = train_test_split_groups(
x, a, y, random_seed=DATA_SPLIT_SEED)
fair_model = {}
train_evaluation = {}
test_evaluation = {}
for eps in eps_list:
fair_model[eps] = fairlearn.train_FairRegression(x_train,
a_train,
y_train,
eps,
Theta,
learner,
constraint,
loss,
init_cache=init_cache)
## return a model info (see exp_grad.py for more details)
train_evaluation[eps] = evaluate.evaluate_FairModel(
x_train, a_train, y_train, loss,
fair_model[eps]['exp_grad_result'], Theta)
## return a dict named evaluation (see eval.py for more details)
test_evaluation[eps] = evaluate.evaluate_FairModel(
x_test, a_test, y_test, loss, fair_model[eps]['exp_grad_result'],
Theta)
result = {}
result['dataset'] = dataset
result['learner'] = learner.name
result['loss'] = loss
result['constraint'] = constraint
result['train_eval'] = train_evaluation
result['test_eval'] = test_evaluation
return result
def disp_grid_search_test(result, base_list, full=True, paired_test=True):
"""
This function is specific to plotting grid search method
"""
# initialize
constraint = "DP"
err_bar = False
# First calcuate the baseline loss vector
base_res = base_list[0]
x, a, y = parser.clean_adult_full()
if not full:
x, a, y = run_exp.subsample(x, a, y, 2000)
_, _, _, x_test, a_test, y_test = run_exp.train_test_split_groups(
x, a, y, random_seed=DATA_SPLIT_SEED)
n = len(y_test)
loss = base_res['loss']
if paired_test:
base_pred = base_res['base_test_eval']['pred']
base_loss_vec = evaluate.loss_vec2(base_pred, y_test, loss)
base_mean_loss = np.mean(base_loss_vec)
train_eval = result['train_eval']
test_eval = result['test_eval']
constraint = result['constraint']
learner = result['learner']
eps_vals = train_eval.keys()
train_disp_dic = {}
test_disp_dic = {}
train_err_dic = {}
test_err_dic = {}
test_loss_std_dic = {}
test_disp_dev_dic = {}
if learner[:2] == "LR":
linestyle = '-'
else:
linestyle = '--'
for eps in eps_vals:
if constraint == "DP":
train_disp = train_eval[eps]["DP_disp"]
test_disp = test_eval[eps]["DP_disp"]
else:
raise Exception('Constraint not supported: ', str(constraint))
train_disp_dic[eps] = train_disp
test_disp_dic[eps] = test_disp
if paired_test:
test_total_pred = test_eval[eps]['pred']
loss_vec = evaluate.loss_vec2(test_total_pred, y_test, loss)
diff_vec = loss_vec - base_loss_vec
loss_mean, loss_std = norm.fit(diff_vec)
test_loss_std_dic[eps] = loss_std / np.sqrt(n)
else:
test_loss_std_dic[eps] = test_eval[eps]['loss_std']
test_disp_dev_dic[eps] = test_eval[eps]['disp_std']
train_err_dic[eps] = (train_eval[eps]['average_loss'])
test_err_dic[eps] = (test_eval[eps]['average_loss'] - base_mean_loss)
if _PARETO:
pareto_epsilons_train = convex_env_train(train_disp_dic, train_err_dic)
pareto_epsilons_test = convex_env_test(pareto_epsilons_train,
test_disp_dic, test_err_dic)
else:
pareto_epsilons_train = eps_vals
pareto_epsilons_test = eps_vals
# taking the pareto frontier
train_disp_list = [train_disp_dic[k] for k in pareto_epsilons_train]
test_disp_list = [test_disp_dic[k] for k in pareto_epsilons_test]
train_err_list = [train_err_dic[k] for k in pareto_epsilons_train]
test_err_list = [test_err_dic[k] for k in pareto_epsilons_test]
err_upperconf = [2 * test_loss_std_dic[k] for k in pareto_epsilons_test]
err_lowerconf = [2 * test_loss_std_dic[k] for k in pareto_epsilons_test]
disp_conf = [test_disp_dev_dic[k] for k in pareto_epsilons_test]
print(pareto_epsilons_test)
plt.fill_between(np.array(test_disp_list),
np.array(test_err_list) - np.array(err_lowerconf),
np.array(test_err_list) + np.array(err_upperconf),
alpha=0.2,
facecolor='black',
antialiased=True)
plt.errorbar(test_disp_list,
test_err_list,
color='black',
capthick=1,
markersize=5,
capsize=2,
linewidth=2,
linestyle=linestyle)
def disp_test_res(result_list, base_list, full=True, paired_test=True):
"""
Read in a result list and trace out the disp_curve_list
for different methods
"""
# initialize
constraint = "DP"
err_bar = False
# First calcuate the baseline loss vector
base_res = base_list[0]
dataset = base_res['dataset']
if dataset == 'law_school':
x, a, y = parser.clean_lawschool_full()
elif dataset == 'communities':
x, a, y = parser.clean_communities_full()
elif dataset == 'adult':
x, a, y = parser.clean_adult_full()
if not full:
x, a, y = run_exp.subsample(x, a, y, 2000)
_, _, _, x_test, a_test, y_test = run_exp.train_test_split_groups(
x, a, y, random_seed=DATA_SPLIT_SEED)
n = len(y_test)
loss = base_res['loss']
if paired_test:
base_pred = base_res['base_test_eval']['pred']
base_loss_vec = evaluate.loss_vec2(base_pred, y_test, loss)
base_mean_loss = np.mean(base_loss_vec)
print(base_mean_loss)
for result in result_list:
train_eval = result['train_eval']
test_eval = result['test_eval']
constraint = result['constraint']
learner = result['learner']
dataset = result['dataset']
eps_vals = train_eval.keys()
train_disp_dic = {}
test_disp_dic = {}
train_err_dic = {}
test_err_dic = {}
test_loss_std_dic = {}
test_disp_dev_dic = {}
linestyle = '-'
if learner == 'SVM_LP':
color = 'orange'
err_bar = True
elif learner == 'OLS':
color = 'red'
err_bar = True
elif learner[:2] == "RF":
color = 'brown'
err_bar = True
elif learner == "XGB Classifier":
color = 'blue'
err_bar = True
linestyle = '--'
elif learner == "XGB Regression":
color = 'red'
err_bar = True
linestyle = '--'
elif learner == "LR":
color = "blue"
err_bar = True
else:
color = 'tan'
err_bar = True
for eps in eps_vals:
if constraint == "DP":
train_disp = train_eval[eps]["DP_disp"]
test_disp = test_eval[eps]["DP_disp"]
elif constraint == "QEO":
train_disp = train_eval[eps]["QEO_disp"]
test_disp = test_eval[eps]["QEO_disp"]
else:
raise Exception('Constraint not supported: ', str(constraint))
train_disp_dic[eps] = train_disp
test_disp_dic[eps] = test_disp
if paired_test:
test_total_pred = test_eval[eps]['pred']
test_res_weights = test_eval[eps]['classifier_weights']
weighted_loss_vec = evaluate.loss_vec(test_total_pred, y_test,
test_res_weights, loss)
diff_vec = weighted_loss_vec - base_loss_vec
loss_mean, loss_std = norm.fit(diff_vec)
test_loss_std_dic[eps] = loss_std / np.sqrt(n)
else:
test_loss_std_dic[eps] = test_eval[eps]['loss_std']
test_disp_dev_dic[eps] = test_eval[eps]['disp_std']
if loss == "square":
# taking the RMSE
train_err_dic[eps] = np.sqrt(train_eval[eps]['weighted_loss'])
test_err_dic[eps] = np.sqrt(
test_eval[eps]['weighted_loss']) - np.sqrt(base_mean_loss)
else:
train_err_dic[eps] = (train_eval[eps]['weighted_loss'])
test_err_dic[eps] = (test_eval[eps]['weighted_loss'] -
base_mean_loss)
if _PARETO:
pareto_epsilons_train = convex_env_train(train_disp_dic,
train_err_dic)
pareto_epsilons_test = convex_env_test(pareto_epsilons_train,
test_disp_dic, test_err_dic)
else:
pareto_epsilons_train = eps_vals
pareto_epsilons_test = eps_vals
# taking the pareto frontier
train_disp_list = [train_disp_dic[k] for k in pareto_epsilons_train]
test_disp_list = [test_disp_dic[k] for k in pareto_epsilons_test]
train_err_list = [train_err_dic[k] for k in pareto_epsilons_train]
test_err_list = [test_err_dic[k] for k in pareto_epsilons_test]
# Getting error bars
if loss == "square":
err_upperconf = [
np.sqrt(test_eval[k]['weighted_loss'] +
2 * test_loss_std_dic[k]) -
np.sqrt(test_eval[k]['weighted_loss'])
for k in pareto_epsilons_test
]
err_lowerconf = [
np.sqrt(test_eval[k]['weighted_loss']) -
np.sqrt(test_eval[k]['weighted_loss'] -
2 * test_loss_std_dic[k]) for k in pareto_epsilons_test
]
else:
err_upperconf = [
2 * test_loss_std_dic[k] for k in pareto_epsilons_test
]
err_lowerconf = [
2 * test_loss_std_dic[k] for k in pareto_epsilons_test
]
disp_conf = [test_disp_dev_dic[k] for k in pareto_epsilons_test]
plt.fill_between(np.array(test_disp_list),
np.array(test_err_list) - np.array(err_lowerconf),
np.array(test_err_list) + np.array(err_upperconf),
alpha=0.2,
facecolor=color,
antialiased=True)
plt.errorbar(test_disp_list,
test_err_list,
color=color,
capthick=1,
markersize=5,
capsize=2,
linewidth=2,
linestyle=linestyle)
# Plotting benchmark
for base_res in base_list:
base_train_eval = base_res['base_train_eval']
base_test_eval = base_res['base_test_eval']
loss = base_res['loss']
learner = base_res['learner']
base_test_disp_conf = base_test_eval['disp_std']
base_test_loss_std = base_test_eval['loss_std']
dataset = base_res['dataset']
marker = '^'
label = 'unconstrained'
if learner == 'OLS':
# color = 'red'
color = 'darksalmon'
err_bar = True
elif learner == "SEO":
marker = 'v'
color = 'deepskyblue'
err_bar = True
label = 'SEO'
elif learner[:2] == "RF":
color = 'brown'
err_bar = True
elif learner[:3] == "XGB":
color = "green"
err_bar = True
elif learner == "LR":
color = "darksalmon"
err_bar = True
else:
color = 'tan'
err_bar = False
# Getting error bars
if loss == "square":
err_upperconf = np.sqrt(base_test_eval['average_loss'] +
2 * base_test_loss_std) - np.sqrt(
base_test_eval['average_loss'])
err_lowerconf = np.sqrt(base_test_eval['average_loss']) - np.sqrt(
base_test_eval['average_loss'] - 2 * base_test_loss_std)
else:
err_upperconf = 2 * base_test_loss_std
err_lowerconf = 2 * base_test_loss_std
if loss == "square":
if err_bar:
plt.errorbar([base_test_eval[constraint + '_disp']], [
np.sqrt(base_test_eval['average_loss']) -
np.sqrt(base_mean_loss)
],
xerr=base_test_disp_conf,
marker=marker,
markeredgecolor='black',
color='black',
markerfacecolor=color,
ecolor='black',
capthick=1,
markersize=11,
capsize=2)
else:
plt.scatter([base_test_eval[constraint + '_disp']], [
np.sqrt(base_test_eval['average_loss']) -
np.sqrt(base_mean_loss)
],
marker=marker,
edgecolors='black',
s=95,
label=label)
else:
if err_bar:
plt.errorbar([base_test_eval[constraint + '_disp']],
[base_test_eval['average_loss']] - base_mean_loss,
xerr=base_test_disp_conf,
marker=marker,
markeredgecolor='black',
markerfacecolor=color,
color='black',
ecolor='black',
capthick=1,
markersize=11,
capsize=2)
else:
plt.scatter([base_test_eval[constraint + '_disp']],
[(base_test_eval['average_loss'])] -
base_mean_loss,
marker=marker,
label=label)