def f_to_min1(hps):
pipeline, hps = f_clf1(hps)
if np.sum(pipeline.named_steps['selection'].mask) == 0:
return {
'loss': 4,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': 0.0,
'cv_acc': 0.0,
'cv_robust': 0.0,
'cv_number_features': 1.0
}
pipeline.fit(X_train, pd.DataFrame(y_train))
validation_number_features = float(
np.sum(pipeline.named_steps['selection']._get_support_mask())
) / float(X_train.shape[1])
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
current_time = time.time() - start_time
return {
'loss': loss,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time,
'updated_parameters': hps
}
def execute_feature_combo(feature_combo, number_of_evaluations):
hps = {}
for f_i in range(X_train.shape[1]):
if f_i in feature_combo:
hps['f_' + str(f_i)] = 1
else:
hps['f_' + str(f_i)] = 0
result = f_to_min1(hps)
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model = result['model']
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
pickle.dump(my_result, f_log)
return result, {'success': success}
return result, {}
def f_to_min1():
mask = np.array([True for f_i in range(X_train.shape[1])])
pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)])
pipeline.fit(X_train, y_train)
validation_number_features = 1.0
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
print(loss)
current_time = time.time() - start_time
return {
'loss': loss,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time
}
def weighted_ranking(X_train,
X_test,
y_train,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features: float = 1.0,
max_search_time=np.inf,
cv_splitter=None):
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score,
greater_is_better=True,
needs_threshold=True)
fair_train = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_train[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
#calculate rankings
rankings = []
for ranking_function_i in range(len(ranking_functions)):
rankings.append(ranking_functions[ranking_function_i](X_train,
y_train))
def f_clf1(hps):
weights = []
for i in range(len(rankings)):
weights.append(hps['weight' + str(i)])
model = Pipeline([('selection',
WeightedRankingSelection(scores=rankings,
weights=weights,
k=hps['k'] + 1,
names=np.array(names))),
('clf', clf)])
return model
def f_to_min1(hps):
print(hps)
model = f_clf1(hps)
robust_scorer = make_scorer(
robust_score,
greater_is_better=True,
X=X_train,
y=y_train,
model=clf,
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
cv = GridSearchCV(model,
param_grid={'clf__C': [1.0]},
cv=cv_splitter,
scoring={
'AUC': auc_scorer,
'Fairness': fair_train,
'Robustness': robust_scorer
},
refit=False)
cv.fit(X_train, pd.DataFrame(y_train))
cv_acc = cv.cv_results_['mean_test_AUC'][0]
cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0]
cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0]
cv_number_features = float(
np.sum(
model.named_steps['selection']._get_support_mask())) / float(
len(model.named_steps['selection']._get_support_mask()))
loss = 0.0
if cv_acc >= min_accuracy and \
cv_fair >= min_fairness and \
cv_robust >= min_robustness and \
cv_number_features <= max_number_features:
if min_fairness > 0.0:
loss += (min_fairness - cv_fair)
if min_accuracy > 0.0:
loss += (min_accuracy - cv_acc)
if min_robustness > 0.0:
loss += (min_robustness - cv_robust)
if max_number_features < 1.0:
loss += (cv_number_features - max_number_features)
else:
if min_fairness > 0.0 and cv_fair < min_fairness:
loss += (min_fairness - cv_fair)**2
if min_accuracy > 0.0 and cv_acc < min_accuracy:
loss += (min_accuracy - cv_acc)**2
if min_robustness > 0.0 and cv_robust < min_robustness:
loss += (min_robustness - cv_robust)**2
if max_number_features < 1.0 and cv_number_features > max_number_features:
loss += (cv_number_features - max_number_features)**2
return {
'loss': loss,
'status': STATUS_OK,
'model': model,
'cv_fair': cv_fair,
'cv_acc': cv_acc,
'cv_robust': cv_robust,
'cv_number_features': cv_number_features
}
max_k = max(int(max_number_features * X_train.shape[1]), 1)
space = {'k': hp.randint('k', max_k)}
if len(rankings) > 1:
for i in range(len(rankings)):
#space['weight' + str(i)] = hp.lognormal('weight' + str(i), 0, 1)
space['weight' + str(i)] = hp.choice(
'weight' + str(i) + 'choice',
[(0.0),
hp.lognormal('weight' + str(i) + 'specified', 0, 1)])
else:
space['weight' + str(0)] = 1.0
cv_fair = 0
cv_acc = 0
cv_robust = 0
cv_number_features = 1.0
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
while True:
if time.time() - start_time > max_search_time:
break
fmin(f_to_min1,
space=space,
algo=tpe.suggest,
max_evals=i,
trials=trials)
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
cv_acc = trials.trials[-1]['result']['cv_acc']
cv_robust = trials.trials[-1]['result']['cv_robust']
cv_number_features = trials.trials[-1]['result']['cv_number_features']
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model = trials.trials[-1]['result']['model']
model.fit(X_train, pd.DataFrame(y_train))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' +
str(min(cv_acc, test_acc)) + ' robust: ' +
str(min(test_robust, cv_robust)) + ' k: ' +
str(cv_number_features))
success = True
break
i += 1
if not success:
try:
cv_fair = trials.best_trial['result']['cv_fair']
cv_acc = trials.best_trial['result']['cv_acc']
cv_robust = trials.best_trial['result']['cv_robust']
cv_number_features = trials.best_trial['result'][
'cv_number_features']
except:
pass
runtime = time.time() - start_time
return {
'time': runtime,
'success': success,
'cv_acc': cv_acc,
'cv_robust': cv_robust,
'cv_fair': cv_fair,
'cv_number_features': cv_number_features,
'cv_number_evaluations': number_of_evaluations
}
def objective(features):
if 'time' in cheating_global.successfull_result[hash]:
return [0.0, 0.0, 0.0, 0.0]
if np.sum(features) == 0:
return [0.0, 0.0, 0.0, 0.0]
model = f_clf1(features)
robust_scorer = make_scorer(
robust_score,
greater_is_better=True,
X=X_train,
y=y_train,
model=clf,
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
cv = GridSearchCV(model,
param_grid={'clf__C': [1.0]},
cv=cv_splitter,
scoring={
'AUC': auc_scorer,
'Fairness': fair_train,
'Robustness': robust_scorer
},
refit=False)
cv.fit(X_train, pd.DataFrame(y_train))
cv_acc = cv.cv_results_['mean_test_AUC'][0]
cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0]
cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0]
cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1
cv_number_features = float(
np.sum(
model.named_steps['selection']._get_support_mask())) / float(
len(model.named_steps['selection']._get_support_mask()))
cv_simplicity = 1.0 - cv_number_features
if not 'cv_acc' in cheating_global.successfull_result[hash] or \
calculate_loss(cheating_global.successfull_result[hash]['cv_acc'],
cheating_global.successfull_result[hash]['cv_fair'],
cheating_global.successfull_result[hash]['cv_robust'],
cheating_global.successfull_result[hash]['cv_number_features']
) > calculate_loss(cv_acc, cv_fair, cv_robust, cv_number_features):
cheating_global.successfull_result[hash]['cv_acc'] = cv_acc
cheating_global.successfull_result[hash]['cv_robust'] = cv_robust
cheating_global.successfull_result[hash]['cv_fair'] = cv_fair
cheating_global.successfull_result[hash][
'cv_number_features'] = cv_number_features
# check constraints for test set
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model.fit(X_train, pd.DataFrame(y_train))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' +
str(min(cv_acc, test_acc)) + ' robust: ' +
str(min(test_robust, cv_robust)) + ' k: ' +
str(cv_number_features))
cheating_global.successfull_result[hash]['time'] = time.time(
) - start_time
cheating_global.successfull_result[hash]['cv_acc'] = cv_acc
cheating_global.successfull_result[hash][
'cv_robust'] = cv_robust
cheating_global.successfull_result[hash]['cv_fair'] = cv_fair
cheating_global.successfull_result[hash][
'cv_number_features'] = cv_number_features
return [cv_acc, cv_fair, cv_robust, cv_simplicity]
def hyperparameter_optimization(X_train,
X_test,
y_train,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features=None,
max_search_time=np.inf,
cv_splitter=None):
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score,
greater_is_better=True,
needs_threshold=True)
fair_train = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_train[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
def f_clf1(hps):
mask = np.zeros(len(hps), dtype=bool)
for k, v in hps.items():
mask[int(k.split('_')[1])] = v
#repair number of features if neccessary
max_k = max(int(max_number_features * X_train.shape[1]), 1)
if np.sum(mask) > max_k:
id_features_used = np.nonzero(mask)[
0] # indices where features are used
np.random.shuffle(id_features_used) # shuffle ids
ids_tb_deactived = id_features_used[max_k:] # deactivate features
for item_to_remove in ids_tb_deactived:
mask[item_to_remove] = False
for mask_i in range(len(mask)):
hps['f_' + str(mask_i)] = mask[mask_i]
model = Pipeline([('selection', MaskSelection(mask)),
('clf', LogisticRegression())])
return model, hps
def f_to_min1(hps):
model, hps = f_clf1(hps)
print("hyperopt: " + str(hps))
if np.sum(model.named_steps['selection'].mask) == 0:
return {
'loss': 4,
'status': STATUS_OK,
'model': model,
'cv_fair': 0.0,
'cv_acc': 0.0,
'cv_robust': 0.0,
'cv_number_features': 1.0
}
robust_scorer = make_scorer(
robust_score,
greater_is_better=True,
X=X_train,
y=y_train,
model=clf,
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
cv = GridSearchCV(model,
param_grid={'clf__C': [1.0]},
cv=cv_splitter,
scoring={
'AUC': auc_scorer,
'Fairness': fair_train,
'Robustness': robust_scorer
},
refit=False)
cv.fit(X_train, pd.DataFrame(y_train))
cv_acc = cv.cv_results_['mean_test_AUC'][0]
cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0]
cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0]
cv_number_features = float(
np.sum(
model.named_steps['selection']._get_support_mask())) / float(
len(model.named_steps['selection']._get_support_mask()))
loss = 0.0
if cv_acc >= min_accuracy and \
cv_fair >= min_fairness and \
cv_robust >= min_robustness:
if min_fairness > 0.0:
loss += (min_fairness - cv_fair)
if min_accuracy > 0.0:
loss += (min_accuracy - cv_acc)
if min_robustness > 0.0:
loss += (min_robustness - cv_robust)
else:
if min_fairness > 0.0 and cv_fair < min_fairness:
loss += (min_fairness - cv_fair)**2
if min_accuracy > 0.0 and cv_acc < min_accuracy:
loss += (min_accuracy - cv_acc)**2
if min_robustness > 0.0 and cv_robust < min_robustness:
loss += (min_robustness - cv_robust)**2
return {
'loss': loss,
'status': STATUS_OK,
'model': model,
'cv_fair': cv_fair,
'cv_acc': cv_acc,
'cv_robust': cv_robust,
'cv_number_features': cv_number_features,
'updated_parameters': hps
}
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
cv_fair = 0
cv_acc = 0
cv_robust = 0
cv_number_features = 1.0
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
while True:
if time.time() - start_time > max_search_time:
break
fmin(f_to_min1,
space=space,
algo=tpe.suggest,
max_evals=i,
trials=trials)
#update repair in database
try:
current_trial = trials.trials[-1]
if type(current_trial['result']['updated_parameters']) != type(
None):
trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals(
current_trial['result']['updated_parameters'])
except:
print("found an error in repair")
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
cv_acc = trials.trials[-1]['result']['cv_acc']
cv_robust = trials.trials[-1]['result']['cv_robust']
cv_number_features = trials.trials[-1]['result']['cv_number_features']
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model = trials.trials[-1]['result']['model']
model.fit(X_train, pd.DataFrame(y_train))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' +
str(min(cv_acc, test_acc)) + ' robust: ' +
str(min(test_robust, cv_robust)) + ' k: ' +
str(cv_number_features))
success = True
break
i += 1
if not success:
try:
cv_fair = trials.best_trial['result']['cv_fair']
cv_acc = trials.best_trial['result']['cv_acc']
cv_robust = trials.best_trial['result']['cv_robust']
cv_number_features = trials.best_trial['result'][
'cv_number_features']
except:
pass
runtime = time.time() - start_time
return {
'time': runtime,
'success': success,
'cv_acc': cv_acc,
'cv_robust': cv_robust,
'cv_fair': cv_fair,
'cv_number_features': cv_number_features,
'cv_number_evaluations': number_of_evaluations
}
def transfer_to_other_model(self,
model,
X_train,
X_validation,
X_test,
y_train,
y_validation,
y_test,
sensitive_ids=None,
hyperparameters=None):
exp_results = self.read_file()
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
min_loss = np.inf
best_run = None
for min_r in range(len(exp_results)):
if 'loss' in exp_results[
min_r] and exp_results[min_r]['loss'] < min_loss:
min_loss = exp_results[min_r]['loss']
best_run = min_r
feature_selection = exp_results[best_run]['selected_features']
start_time = time.time()
pipeline = Pipeline([('selection', feature_selection), ('clf', model)])
accuracy_scorer = make_scorer(f1_score)
grid_result = run_grid_search(pipeline,
X_train,
y_train,
X_validation,
y_validation,
accuracy_scorer,
sensitive_ids,
1.0,
1.0,
1.0,
0.0,
model_hyperparameters=hyperparameters,
start_time=start_time)
model = grid_result['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
fair_test = None
if type(sensitive_ids) != type(None):
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
test_number_features = exp_results[best_run]['cv_number_features']
categories = ['Accuracy', 'Fairness', 'Simplicity', 'Safety']
fig = go.Figure()
fig.add_trace(
go.Scatterpolar(r=[
test_acc, test_fair, 1.0 - test_number_features, test_robust
],
theta=categories,
fill='toself',
name='Best Result on Test'))
fig.update_layout(
title='Best Result on Test',
polar=dict(radialaxis=dict(visible=True, range=[0, 1])),
showlegend=False)
return fig
def hyperparameter_optimization(X_train,
X_validation,
X_train_val,
X_test,
y_train,
y_validation,
y_train_val,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features=None,
max_search_time=np.inf,
log_file=None,
algo=tpe.suggest,
accuracy_scorer=make_scorer(
roc_auc_score,
greater_is_better=True,
needs_threshold=True),
avoid_robustness=False,
model_hyperparameters=None):
min_loss = np.inf
start_time = time.time()
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(
true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
stored_results = {} #avoid duplicate hyperparameters
def f_clf1(hps):
mask = np.zeros(len(hps), dtype=bool)
for k, v in hps.items():
mask[int(k.split('_')[1])] = v
#repair number of features if neccessary
max_k = max(int(max_number_features * X_train.shape[1]), 1)
if np.sum(mask) > max_k:
id_features_used = np.nonzero(mask)[
0] # indices where features are used
np.random.shuffle(id_features_used) # shuffle ids
ids_tb_deactived = id_features_used[max_k:] # deactivate features
for item_to_remove in ids_tb_deactived:
mask[item_to_remove] = False
for mask_i in range(len(mask)):
hps['f_' + str(mask_i)] = mask[mask_i]
pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)])
return pipeline, hps
def f_to_min1(hps):
if str(hps) in stored_results:
return stored_results[str(hps)]
pipeline, hps = f_clf1(hps)
if np.sum(pipeline.named_steps['selection'].mask) == 0:
stored_results[str(hps)] = {
'loss': 4,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': 0.0,
'cv_acc': 0.0,
'cv_robust': 0.0,
'cv_number_features': 1.0
}
return stored_results[str(hps)]
stored_results[str(hps)] = run_grid_search(
pipeline,
X_train,
y_train,
X_validation,
y_validation,
accuracy_scorer,
sensitive_ids,
min_fairness,
min_accuracy,
min_robustness,
max_number_features,
model_hyperparameters=model_hyperparameters,
start_time=start_time,
avoid_robustness=avoid_robustness)
stored_results[str(hps)]['updated_parameters'] = hps
return stored_results[str(hps)]
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
fail_counter = [0]
while True:
if time.time() - start_time > max_search_time:
break
if fail_counter[0] >= 10:
fail_counter[0] += 1
break
fail_counter[0] = 0
fmin(f_to_min1, space=space, algo=algo, max_evals=i, trials=trials)
#update repair in database
try:
current_trial = trials.trials[-1]
if type(current_trial['result']['updated_parameters']) != type(
None):
trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals(
current_trial['result']['updated_parameters'])
except:
print("found an error in repair")
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
validation_acc = trials.trials[-1]['result']['cv_acc']
validation_robust = trials.trials[-1]['result']['cv_robust']
cv_number_features = trials.trials[-1]['result']['cv_number_features']
my_result = trials.trials[-1]['result']
my_result['number_evaluations'] = number_of_evaluations
model = trials.trials[-1]['result']['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 0.0
if not avoid_robustness:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(
min_fairness, min_accuracy, min_robustness,
max_number_features):
my_result['Finished'] = True
my_result['Validation_Satisfied'] = True
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(
my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new,
f_log,
protocol=pickle.HIGHEST_PROTOCOL)
return {'success': success}
if min_loss > trials.trials[-1]['result']['loss']:
min_loss = trials.trials[-1]['result']['loss']
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(
my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new,
f_log,
protocol=pickle.HIGHEST_PROTOCOL)
i += 1
my_result = {
'number_evaluations': number_of_evaluations,
'success_test': False,
'final_time': time.time() - start_time,
'Finished': True
}
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': False}
def exhaustive(X_train, X_validation, X_test, y_train, y_validation, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None):
f_log = open(log_file, 'wb+')
min_loss = np.inf
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True)
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
def f_clf1(hps):
mask = np.zeros(len(hps), dtype=bool)
for k, v in hps.items():
mask[int(k.split('_')[1])] = v
#repair number of features if neccessary
max_k = max(int(max_number_features * X_train.shape[1]), 1)
if np.sum(mask) > max_k:
id_features_used = np.nonzero(mask)[0] # indices where features are used
np.random.shuffle(id_features_used) # shuffle ids
ids_tb_deactived = id_features_used[max_k:] # deactivate features
for item_to_remove in ids_tb_deactived:
mask[item_to_remove] = False
for mask_i in range(len(mask)):
hps['f_' + str(mask_i)] = mask[mask_i]
model = Pipeline([
('selection', MaskSelection(mask)),
('clf', clf)
])
return model, hps
def f_to_min1(hps):
pipeline, hps = f_clf1(hps)
if np.sum(pipeline.named_steps['selection'].mask) == 0:
return {'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0}
pipeline.fit(X_train, y_train)
validation_number_features = float(np.sum(pipeline.named_steps['selection']._get_support_mask())) / float(X_train.shape[1])
validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_validation, y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair) ** 2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc) ** 2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust) ** 2
print(loss)
current_time = time.time() - start_time
return {'loss': loss,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time,
'updated_parameters': hps}
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
cv_fair = 0
cv_acc = 0
cv_robust = 0
cv_number_features = 1.0
number_of_evaluations = 0
max_k = max(int(max_number_features * X_train.shape[1]), 1)
for l in range(1, max_k + 1):
for feature_combo in itertools.combinations(range(X_train.shape[1]), l):
hps = {}
for f_i in range(X_train.shape[1]):
if f_i in feature_combo:
hps['f_' + str(f_i)] = 1
else:
hps['f_' + str(f_i)] = 0
result = f_to_min1(hps)
number_of_evaluations += 1
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model = result['model']
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
pickle.dump(my_result, f_log)
f_log.close()
return {'success': success}
if min_loss > my_result['loss']:
min_loss = my_result['loss']
pickle.dump(my_result, f_log)
my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'time': time.time() - start_time,
'Finished': True}
pickle.dump(my_result, f_log)
f_log.close()
return {'success': False}
def fullfeatures(X_train,
X_validation,
X_test,
y_train,
y_validation,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features=None,
max_search_time=np.inf,
log_file=None):
f_log = open(log_file, 'wb+')
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score,
greater_is_better=True,
needs_threshold=True)
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(
true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
def f_to_min1():
mask = np.array([True for f_i in range(X_train.shape[1])])
pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)])
pipeline.fit(X_train, y_train)
validation_number_features = 1.0
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
print(loss)
current_time = time.time() - start_time
return {
'loss': loss,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time
}
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
number_of_evaluations = 0
result = f_to_min1()
number_of_evaluations += 1
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features:
model = result['model']
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
pickle.dump(my_result, f_log)
f_log.close()
return {'success': success}
pickle.dump(my_result, f_log)
my_result = {
'number_evaluations': number_of_evaluations,
'success_test': False,
'time': time.time() - start_time,
'Finished': True
}
pickle.dump(my_result, f_log)
f_log.close()
return {'success': False}
def f_to_min1(x, s):
opt_start_time = time.time()
mask = x > 0.5
model = f_clf1(mask)
if np.sum(model.named_steps['selection'].mask) == 0:
return 4, (time.time() - opt_start_time)
s_max = y_train_fab.shape[0]
shuffle = np.random.permutation(np.arange(s_max))
train_subset = X_train_fab[shuffle[:s]]
train_targets_subset = y_train_fab[shuffle[:s]]
model.fit(train_subset, pd.DataFrame(train_targets_subset))
cv_acc = auc_scorer(model, X_val, pd.DataFrame(y_val))
cv_fair = 1.0 - fair_val(model, X_val, pd.DataFrame(y_val))
cv_robust = 1.0 - robust_score_test(
X_test=X_val,
y_test=y_val,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
cv_number_features = float(
np.sum(
model.named_steps['selection']._get_support_mask())) / float(
len(model.named_steps['selection']._get_support_mask()))
print("accuracy: " + str(cv_acc) + ' fair: ' + str(cv_fair) + ' k: ' +
str(cv_number_features))
loss = 0.0
if cv_acc >= min_accuracy and \
cv_fair >= min_fairness and \
cv_robust >= min_robustness and \
cv_number_features <= max_number_features:
if min_fairness > 0.0:
loss += (min_fairness - cv_fair)
if min_accuracy > 0.0:
loss += (min_accuracy - cv_acc)
if min_robustness > 0.0:
loss += (min_robustness - cv_robust)
if max_number_features < 1.0:
loss += (cv_number_features - max_number_features)
else:
if min_fairness > 0.0 and cv_fair < min_fairness:
loss += (min_fairness - cv_fair)**2
if min_accuracy > 0.0 and cv_acc < min_accuracy:
loss += (min_accuracy - cv_acc)**2
if min_robustness > 0.0 and cv_robust < min_robustness:
loss += (min_robustness - cv_robust)**2
if max_number_features < 1.0 and cv_number_features > max_number_features:
loss += (cv_number_features - max_number_features)**2
print("loss: " + str(loss))
return loss, (time.time() - opt_start_time)
def hyperparameter_optimization(X_train,
X_validation,
X_test,
y_train,
y_validation,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features=None,
max_search_time=np.inf,
log_file=None,
algo=tpe.suggest):
f_log = open(log_file, 'wb+')
min_loss = np.inf
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score,
greater_is_better=True,
needs_threshold=True)
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(
true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
stored_results = {} #avoid duplicate hyperparameters
def f_clf1(hps):
mask = np.zeros(len(hps), dtype=bool)
for k, v in hps.items():
mask[int(k.split('_')[1])] = v
#repair number of features if neccessary
max_k = max(int(max_number_features * X_train.shape[1]), 1)
if np.sum(mask) > max_k:
id_features_used = np.nonzero(mask)[
0] # indices where features are used
np.random.shuffle(id_features_used) # shuffle ids
ids_tb_deactived = id_features_used[max_k:] # deactivate features
for item_to_remove in ids_tb_deactived:
mask[item_to_remove] = False
for mask_i in range(len(mask)):
hps['f_' + str(mask_i)] = mask[mask_i]
pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)])
return pipeline, hps
def f_to_min1(hps):
if str(hps) in stored_results:
return stored_results[str(hps)]
pipeline, hps = f_clf1(hps)
if np.sum(pipeline.named_steps['selection'].mask) == 0:
stored_results[str(hps)] = {
'loss': 4,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': 0.0,
'cv_acc': 0.0,
'cv_robust': 0.0,
'cv_number_features': 1.0
}
return stored_results[str(hps)]
pipeline.fit(X_train, pd.DataFrame(y_train))
validation_number_features = float(
np.sum(pipeline.named_steps['selection']._get_support_mask())
) / float(X_train.shape[1])
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
current_time = time.time() - start_time
stored_results[str(hps)] = {
'loss': loss,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time,
'updated_parameters': hps
}
return stored_results[str(hps)]
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
fail_counter = [0]
while True:
if time.time() - start_time > max_search_time:
break
if fail_counter[0] >= 10:
fail_counter[0] += 1
break
fail_counter[0] = 0
fmin(f_to_min1, space=space, algo=algo, max_evals=i, trials=trials)
#update repair in database
try:
current_trial = trials.trials[-1]
if type(current_trial['result']['updated_parameters']) != type(
None):
trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals(
current_trial['result']['updated_parameters'])
except:
print("found an error in repair")
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
validation_acc = trials.trials[-1]['result']['cv_acc']
validation_robust = trials.trials[-1]['result']['cv_robust']
cv_number_features = trials.trials[-1]['result']['cv_number_features']
my_result = trials.trials[-1]['result']
my_result['number_evaluations'] = number_of_evaluations
if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features:
model = trials.trials[-1]['result']['model']
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
pickle.dump(my_result, f_log)
f_log.close()
return {'success': success}
if min_loss > trials.trials[-1]['result']['loss']:
min_loss = trials.trials[-1]['result']['loss']
pickle.dump(my_result, f_log)
i += 1
my_result = {
'number_evaluations': number_of_evaluations,
'success_test': False,
'time': time.time() - start_time,
'Finished': True
}
pickle.dump(my_result, f_log)
f_log.close()
return {'success': False}
def exhaustive(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None):
min_loss = np.inf
start_time = time.time()
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
def f_clf1(hps):
mask = np.zeros(len(hps), dtype=bool)
for k, v in hps.items():
mask[int(k.split('_')[1])] = v
#repair number of features if neccessary
max_k = max(int(max_number_features * X_train.shape[1]), 1)
if np.sum(mask) > max_k:
id_features_used = np.nonzero(mask)[0] # indices where features are used
np.random.shuffle(id_features_used) # shuffle ids
ids_tb_deactived = id_features_used[max_k:] # deactivate features
for item_to_remove in ids_tb_deactived:
mask[item_to_remove] = False
for mask_i in range(len(mask)):
hps['f_' + str(mask_i)] = mask[mask_i]
model = Pipeline([
('selection', MaskSelection(mask)),
('clf', clf)
])
return model, hps
def f_to_min1(hps):
pipeline, hps = f_clf1(hps)
if np.sum(pipeline.named_steps['selection'].mask) == 0:
return {'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0}
grid_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation,
accuracy_scorer, sensitive_ids,
min_fairness, min_accuracy, min_robustness, max_number_features,
model_hyperparameters=model_hyperparameters, start_time=start_time)
grid_result['updated_parameters'] = hps
return grid_result
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
number_of_evaluations = 0
max_k = max(int(max_number_features * X_train.shape[1]), 1)
for l in range(1, max_k + 1):
for feature_combo in itertools.combinations(range(X_train.shape[1]), l):
hps = {}
for f_i in range(X_train.shape[1]):
if f_i in feature_combo:
hps['f_' + str(f_i)] = 1
else:
hps['f_' + str(f_i)] = 0
result = f_to_min1(hps)
number_of_evaluations += 1
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
##check on test
model = result['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features):
my_result['Finished'] = True
my_result['Validation_Satisfied'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': success}
if min_loss > my_result['loss']:
min_loss = my_result['loss']
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time,
'Finished': True}
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': False}
def objective(features, min_loss): if 'success' in cheating_global.successfull_result[hash]: return [0.0, 0.0, 0.0, 0.0, min_loss] if np.sum(features) == 0: return [0.0, 0.0, 0.0, 0.0, min_loss] pipeline = f_clf1(features) my_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time) new_pipeline = copy.deepcopy(my_result['model']) validation_acc = my_result['cv_acc'] validation_fair = my_result['cv_fair'] validation_robust = my_result['cv_robust'] loss = my_result['loss'] cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1 validation_simplicity = 1.0 - my_result['cv_number_features'] #check constraints for test set my_result['number_evaluations'] = cheating_global.successfull_result[hash]['cv_number_evaluations'] new_pipeline.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(new_pipeline, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(new_pipeline, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=new_pipeline.named_steps['clf'], feature_selector=new_pipeline.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if validation_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True cheating_global.successfull_result[hash]['success'] = success my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return [validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss] if min_loss > loss: min_loss = loss with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return [validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss]
def fullfeatures(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None):
start_time = time.time()
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
def f_to_min1():
mask = np.array([True for f_i in range(X_train.shape[1])])
pipeline = Pipeline([
('selection', MaskSelection(mask)),
('clf', clf)
])
grid_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation,
accuracy_scorer, sensitive_ids,
min_fairness, min_accuracy, min_robustness, max_number_features,
model_hyperparameters=model_hyperparameters, start_time=start_time)
return grid_result
space = {}
for f_i in range(X_train.shape[1]):
space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2)
number_of_evaluations = 0
result = f_to_min1()
number_of_evaluations += 1
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
model = result['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features):
my_result['Finished'] = True
my_result['Validation_Satisfied'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': success}
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time,
'Finished': True}
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': False}
def weighted_ranking(X_train,
X_validation,
X_test,
y_train,
y_validation,
y_test,
names,
sensitive_ids,
ranking_functions=[],
clf=None,
min_accuracy=0.0,
min_fairness=0.0,
min_robustness=0.0,
max_number_features: float = 1.0,
max_search_time=np.inf,
log_file=None):
min_loss = np.inf
f_log = open(log_file, 'wb+')
start_time = time.time()
auc_scorer = make_scorer(roc_auc_score,
greater_is_better=True,
needs_threshold=True)
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(
true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score,
greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
#calculate rankings
rankings = []
for ranking_function_i in range(len(ranking_functions)):
rankings.append(ranking_functions[ranking_function_i](X_train,
y_train))
def f_clf1(hps):
weights = []
for i in range(len(rankings)):
weights.append(hps['weight' + str(i)])
pipeline = Pipeline([('selection',
WeightedRankingSelection(scores=rankings,
weights=weights,
k=hps['k'],
names=np.array(names))),
('clf', clf)])
return pipeline
stored_results = {}
fail_counter = [0]
def f_to_min1(hps):
if str(hps) in stored_results:
fail_counter[0] += 1
return stored_results[str(hps)]
fail_counter[0] = 0
pipeline = f_clf1(hps)
pipeline.fit(X_train, pd.DataFrame(y_train))
validation_number_features = float(
np.sum(pipeline.named_steps['selection']._get_support_mask())
) / float(X_train.shape[1])
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
current_time = time.time() - start_time
stored_results[str(hps)] = {
'loss': loss,
'status': STATUS_OK,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time,
'updated_parameters': hps
}
return stored_results[str(hps)]
max_k = max(int(max_number_features * X_train.shape[1]), 1)
space = {'k': scope.int(hp.quniform('k', 1, max_k, 1))}
if len(rankings) > 1:
for i in range(len(rankings)):
space['weight' + str(i)] = hp.choice(
'weight' + str(i) + 'choice',
[(0.0),
hp.lognormal('weight' + str(i) + 'specified', 0, 1)])
else:
space['weight' + str(0)] = 1.0
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
while True:
if time.time() - start_time > max_search_time:
break
if len(stored_results) >= max_k:
break
if fail_counter[0] >= 10:
break
fmin(f_to_min1,
space=space,
algo=tpe.suggest,
max_evals=i,
trials=trials)
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
validation_acc = trials.trials[-1]['result']['cv_acc']
validation_robust = trials.trials[-1]['result']['cv_robust']
cv_number_features = trials.trials[-1]['result']['cv_number_features']
my_result = trials.trials[-1]['result']
my_result['number_evaluations'] = number_of_evaluations
if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features:
model = trials.trials[-1]['result']['model']
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(model, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
pickle.dump(my_result, f_log)
f_log.close()
return {'success': success}
if min_loss > trials.trials[-1]['result']['loss']:
min_loss = trials.trials[-1]['result']['loss']
pickle.dump(my_result, f_log)
i += 1
my_result = {
'number_evaluations': number_of_evaluations,
'success_test': False,
'time': time.time() - start_time,
'Finished': True
}
pickle.dump(my_result, f_log)
f_log.close()
return {'success': False}
def run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters, start_time, avoid_robustness=False):
fair_validation = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]])
search_configs = [{}]
if type(model_hyperparameters) != type(None):
new_model_hyperparameters = {}
for k, v in model_hyperparameters.items():
new_model_hyperparameters['clf__' + k] = v
#print(new_model_hyperparameters)
search_configs = [dict(zip(new_model_hyperparameters, v)) for v in product(*new_model_hyperparameters.values())]
grid_results = {}
for configuration in search_configs:
new_pipeline = copy.deepcopy(pipeline)
if len(configuration) > 0:
new_pipeline.set_params(**configuration)
new_pipeline.fit(X_train, pd.DataFrame(y_train))
validation_number_features = float(np.sum(new_pipeline.named_steps['selection']._get_support_mask())) / float(X_train.shape[1])
validation_acc = accuracy_scorer(new_pipeline, X_validation, pd.DataFrame(y_validation))
#print("accuracy: " + str(validation_acc))
validation_fair = 0.0
if type(sensitive_ids) != type(None):
validation_fair = 1.0 - fair_validation(new_pipeline, X_validation, pd.DataFrame(y_validation))
validation_robust = 0.0
if not avoid_robustness:
validation_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_validation, y_test=y_validation,
model=new_pipeline.named_steps['clf'],
feature_selector=new_pipeline.named_steps['selection'],
scorer=accuracy_scorer)
loss = 0.0
if not np.isinf(min_fairness) and min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair) ** 2
if not np.isinf(min_accuracy) and min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc) ** 2
if not np.isinf(min_robustness) and min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust) ** 2
if not np.isinf(max_number_features) and max_number_features < 1.0 and validation_number_features > max_number_features:
loss += (validation_number_features - max_number_features) ** 2
#if constraints are satisfied, we can start to optimize utility
if loss == 0.0:
if min_fairness == np.inf: #fairness is utility
loss -= validation_fair
if min_accuracy == np.inf: #accuracy is utility
loss -= validation_acc
if min_robustness == np.inf: #robustness is utility
loss -= validation_robust
if max_number_features == -np.inf: #complexity is utility
loss -= 1.0 - validation_number_features
grid_results[immutabledict(configuration)] = {'loss': loss,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features}
# here, we optimize model parameters for accuracy
max_acc = -1
best_parameter_configuration = {}
for k, v in grid_results.items():
if max_acc < v['cv_acc']:
max_acc = v['cv_acc']
best_parameter_configuration = k
#print(best_parameter_configuration)
best_pipeline = copy.deepcopy(pipeline)
best_pipeline.set_params(**best_parameter_configuration)
results = grid_results[best_parameter_configuration]
results['model'] = best_pipeline
results['time'] = time.time() - start_time
results['status'] = STATUS_OK
return results
def execute_feature_combo(feature_combo, number_of_evaluations):
hps = {}
for f_i in range(X_train.shape[1]):
if f_i in feature_combo:
hps['f_' + str(f_i)] = 1
else:
hps['f_' + str(f_i)] = 0
result = f_to_min1(hps)
cv_fair = result['cv_fair']
cv_acc = result['cv_acc']
cv_robust = result['cv_robust']
cv_number_features = result['cv_number_features']
my_result = result
my_result['number_evaluations'] = number_of_evaluations
model = result['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(
min_fairness, min_accuracy, min_robustness,
max_number_features):
my_result['Finished'] = True
my_result['Validation_Satisfied'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(
my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new,
f_log,
protocol=pickle.HIGHEST_PROTOCOL)
return my_result, {'success': success}
return my_result, {}
def objective(features, min_loss):
if 'success' in cheating_global.successfull_result[hash]:
return [0.0, 0.0, 0.0, 0.0, min_loss]
if np.sum(features) == 0:
return [0.0, 0.0, 0.0, 0.0, min_loss]
pipeline = f_clf1(features)
pipeline.fit(X_train, pd.DataFrame(y_train))
cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1
validation_number_features = float(
np.sum(pipeline.named_steps['selection']._get_support_mask())
) / float(X_train.shape[1])
validation_acc = auc_scorer(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_fair = 0.0
if type(sensitive_ids) != type(None) and min_fairness > 0.0:
validation_fair = 1.0 - fair_validation(pipeline, X_validation,
pd.DataFrame(y_validation))
validation_robust = 0.0
if min_robustness > 0.0:
validation_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_validation,
y_test=y_validation,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
loss = 0.0
if min_fairness > 0.0 and validation_fair < min_fairness:
loss += (min_fairness - validation_fair)**2
if min_accuracy > 0.0 and validation_acc < min_accuracy:
loss += (min_accuracy - validation_acc)**2
if min_robustness > 0.0 and validation_robust < min_robustness:
loss += (min_robustness - validation_robust)**2
print(loss)
validation_simplicity = 1.0 - validation_number_features
#check constraints for test set
current_time = time.time() - start_time
my_result = {
'loss': loss,
'model': pipeline,
'cv_fair': validation_fair,
'cv_acc': validation_acc,
'cv_robust': validation_robust,
'cv_number_features': validation_number_features,
'time': current_time
}
my_result['number_evaluations'] = cheating_global.successfull_result[
hash]['cv_number_evaluations']
if validation_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and validation_number_features <= max_number_features:
X_train_val = np.vstack((X_train, X_validation))
y_train_val = np.append(y_train, y_validation)
pipeline.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = 0.0
if min_accuracy > 0.0:
test_acc = auc_scorer(pipeline, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if min_fairness > 0.0:
test_fair = 1.0 - fair_test(pipeline, X_test,
pd.DataFrame(y_test))
test_robust = 0.0
if min_robustness > 0.0:
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=pipeline.named_steps['clf'],
feature_selector=pipeline.named_steps['selection'],
scorer=auc_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
my_result['Finished'] = True
success = False
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
cheating_global.successfull_result[hash]['success'] = success
my_result['success_test'] = success
pickle.dump(my_result, f_log)
f_log.close()
return [
validation_acc, validation_fair, validation_robust,
validation_simplicity, min_loss
]
if min_loss > loss:
min_loss = loss
pickle.dump(my_result, f_log)
return [
validation_acc, validation_fair, validation_robust,
validation_simplicity, min_loss
]
def weighted_ranking(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features: float = 1.0, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None):
min_loss = np.inf
start_time = time.time()
fair_validation = None
fair_test = None
if type(sensitive_ids) != type(None):
fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_validation[:, sensitive_ids[0]])
fair_test = make_scorer(true_positive_rate_score, greater_is_better=True,
sensitive_data=X_test[:, sensitive_ids[0]])
#calculate rankings
try:
rankings = []
for ranking_function_i in range(len(ranking_functions)):
rankings.append(ranking_functions[ranking_function_i](X_train, y_train))
except Exception as e:
my_result = {'error': e}
#with open(log_file, 'ab') as f_log:
# pickle.dump(my_result, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': False}
def f_clf1(hps):
weights = []
for i in range(len(rankings)):
weights.append(hps['weight' + str(i)])
pipeline = Pipeline([
('selection', WeightedRankingSelection(scores=rankings, weights=weights, k=hps['k'], names=np.array(names))),
('clf', clf)
])
return pipeline
stored_results = {}
fail_counter = [0]
def f_to_min1(hps):
if str(hps) in stored_results:
fail_counter[0] += 1
return stored_results[str(hps)]
fail_counter[0] = 0
pipeline = f_clf1(hps)
stored_results[str(hps)] = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation,
accuracy_scorer, sensitive_ids,
min_fairness, min_accuracy, min_robustness, max_number_features,
model_hyperparameters=model_hyperparameters, start_time=start_time)
stored_results[str(hps)]['updated_parameters'] = hps
return stored_results[str(hps)]
max_k = max(int(max_number_features * X_train.shape[1]), 1)
space = {'k': scope.int(hp.quniform('k', 1, max_k, 1))}
if len(rankings) > 1:
for i in range(len(rankings)):
space['weight' + str(i)] = hp.choice('weight' + str(i) + 'choice',
[
(0.0),
hp.lognormal('weight' + str(i) + 'specified', 0, 1)
])
else:
space['weight' + str(0)] = 1.0
number_of_evaluations = 0
trials = Trials()
i = 1
success = False
while True:
if time.time() - start_time > max_search_time:
break
if len(stored_results) >= max_k:
break
if fail_counter[0] >= 10:
break
fmin(f_to_min1, space=space, algo=tpe.suggest, max_evals=i, trials=trials)
number_of_evaluations += 1
cv_fair = trials.trials[-1]['result']['cv_fair']
validation_acc = trials.trials[-1]['result']['cv_acc']
validation_robust = trials.trials[-1]['result']['cv_robust']
my_result = trials.trials[-1]['result']
my_result['number_evaluations'] = number_of_evaluations
#check test results
model = trials.trials[-1]['result']['model']
model.fit(X_train_val, pd.DataFrame(y_train_val))
test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 0.0
if type(sensitive_ids) != type(None):
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=accuracy_scorer)
my_result['test_fair'] = test_fair
my_result['test_acc'] = test_acc
my_result['test_robust'] = test_robust
my_result['final_time'] = time.time() - start_time
if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features):
my_result['Finished'] = True
my_result['Validation_Satisfied'] = True
if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness:
success = True
my_result['success_test'] = success
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': success}
if min_loss > trials.trials[-1]['result']['loss']:
min_loss = trials.trials[-1]['result']['loss']
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection'])
my_result_new['model'] = None
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
i += 1
my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time, 'Finished': True}
with open(log_file, 'ab') as f_log:
my_result_new = copy.deepcopy(my_result)
pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL)
return {'success': False}
robust_scorer_test = make_scorer(
robust_score_test,
greater_is_better=True,
needs_threshold=True,
X_test=X_test,
model=model,
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
model.fit(X_train, pd.DataFrame(y_train))
test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test))
test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test))
test_robust = 1.0 - robust_score_test(
eps=0.1,
X_test=X_test,
y_test=y_test,
model=model.named_steps['clf'],
feature_selector=model.named_steps['selection'],
scorer=auc_scorer)
#print('acc: ' + str(test_acc) + ' fair: ' + str(test_fair) + ' robust: ' + str(test_robust))
cv_fair = trials.trials[-1]['result']['cv_fair']
cv_acc = trials.trials[-1]['result']['cv_acc']
print('cv-fair: ' + str(cv_fair) + ' test fair: ' + str(test_fair) +
' cv-acc: ' + str(cv_acc) + ' test acc: ' + str(test_acc))
satisfied_constraints.append(
[min(test_acc, cv_acc), min(test_fair, cv_fair)])
times.append(time.time() - start_time)
iterations.append(i)
