def f_to_min1(hps): pipeline, hps = f_clf1(hps) if np.sum(pipeline.named_steps['selection'].mask) == 0: return { 'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0 } pipeline.fit(X_train, pd.DataFrame(y_train)) validation_number_features = float( np.sum(pipeline.named_steps['selection']._get_support_mask()) ) / float(X_train.shape[1]) validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 current_time = time.time() - start_time return { 'loss': loss, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time, 'updated_parameters': hps }
def execute_feature_combo(feature_combo, number_of_evaluations): hps = {} for f_i in range(X_train.shape[1]): if f_i in feature_combo: hps['f_' + str(f_i)] = 1 else: hps['f_' + str(f_i)] = 0 result = f_to_min1(hps) cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model = result['model'] X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success pickle.dump(my_result, f_log) return result, {'success': success} return result, {}
def f_to_min1(): mask = np.array([True for f_i in range(X_train.shape[1])]) pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)]) pipeline.fit(X_train, y_train) validation_number_features = 1.0 validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 print(loss) current_time = time.time() - start_time return { 'loss': loss, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time }
def weighted_ranking(X_train, X_test, y_train, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features: float = 1.0, max_search_time=np.inf, cv_splitter=None): start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_train = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_train[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) #calculate rankings rankings = [] for ranking_function_i in range(len(ranking_functions)): rankings.append(ranking_functions[ranking_function_i](X_train, y_train)) def f_clf1(hps): weights = [] for i in range(len(rankings)): weights.append(hps['weight' + str(i)]) model = Pipeline([('selection', WeightedRankingSelection(scores=rankings, weights=weights, k=hps['k'] + 1, names=np.array(names))), ('clf', clf)]) return model def f_to_min1(hps): print(hps) model = f_clf1(hps) robust_scorer = make_scorer( robust_score, greater_is_better=True, X=X_train, y=y_train, model=clf, feature_selector=model.named_steps['selection'], scorer=auc_scorer) cv = GridSearchCV(model, param_grid={'clf__C': [1.0]}, cv=cv_splitter, scoring={ 'AUC': auc_scorer, 'Fairness': fair_train, 'Robustness': robust_scorer }, refit=False) cv.fit(X_train, pd.DataFrame(y_train)) cv_acc = cv.cv_results_['mean_test_AUC'][0] cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0] cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0] cv_number_features = float( np.sum( model.named_steps['selection']._get_support_mask())) / float( len(model.named_steps['selection']._get_support_mask())) loss = 0.0 if cv_acc >= min_accuracy and \ cv_fair >= min_fairness and \ cv_robust >= min_robustness and \ cv_number_features <= max_number_features: if min_fairness > 0.0: loss += (min_fairness - cv_fair) if min_accuracy > 0.0: loss += (min_accuracy - cv_acc) if min_robustness > 0.0: loss += (min_robustness - cv_robust) if max_number_features < 1.0: loss += (cv_number_features - max_number_features) else: if min_fairness > 0.0 and cv_fair < min_fairness: loss += (min_fairness - cv_fair)**2 if min_accuracy > 0.0 and cv_acc < min_accuracy: loss += (min_accuracy - cv_acc)**2 if min_robustness > 0.0 and cv_robust < min_robustness: loss += (min_robustness - cv_robust)**2 if max_number_features < 1.0 and cv_number_features > max_number_features: loss += (cv_number_features - max_number_features)**2 return { 'loss': loss, 'status': STATUS_OK, 'model': model, 'cv_fair': cv_fair, 'cv_acc': cv_acc, 'cv_robust': cv_robust, 'cv_number_features': cv_number_features } max_k = max(int(max_number_features * X_train.shape[1]), 1) space = {'k': hp.randint('k', max_k)} if len(rankings) > 1: for i in range(len(rankings)): #space['weight' + str(i)] = hp.lognormal('weight' + str(i), 0, 1) space['weight' + str(i)] = hp.choice( 'weight' + str(i) + 'choice', [(0.0), hp.lognormal('weight' + str(i) + 'specified', 0, 1)]) else: space['weight' + str(0)] = 1.0 cv_fair = 0 cv_acc = 0 cv_robust = 0 cv_number_features = 1.0 number_of_evaluations = 0 trials = Trials() i = 1 success = False while True: if time.time() - start_time > max_search_time: break fmin(f_to_min1, space=space, algo=tpe.suggest, max_evals=i, trials=trials) number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] cv_acc = trials.trials[-1]['result']['cv_acc'] cv_robust = trials.trials[-1]['result']['cv_robust'] cv_number_features = trials.trials[-1]['result']['cv_number_features'] if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model = trials.trials[-1]['result']['model'] model.fit(X_train, pd.DataFrame(y_train)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' + str(min(cv_acc, test_acc)) + ' robust: ' + str(min(test_robust, cv_robust)) + ' k: ' + str(cv_number_features)) success = True break i += 1 if not success: try: cv_fair = trials.best_trial['result']['cv_fair'] cv_acc = trials.best_trial['result']['cv_acc'] cv_robust = trials.best_trial['result']['cv_robust'] cv_number_features = trials.best_trial['result'][ 'cv_number_features'] except: pass runtime = time.time() - start_time return { 'time': runtime, 'success': success, 'cv_acc': cv_acc, 'cv_robust': cv_robust, 'cv_fair': cv_fair, 'cv_number_features': cv_number_features, 'cv_number_evaluations': number_of_evaluations }
def objective(features): if 'time' in cheating_global.successfull_result[hash]: return [0.0, 0.0, 0.0, 0.0] if np.sum(features) == 0: return [0.0, 0.0, 0.0, 0.0] model = f_clf1(features) robust_scorer = make_scorer( robust_score, greater_is_better=True, X=X_train, y=y_train, model=clf, feature_selector=model.named_steps['selection'], scorer=auc_scorer) cv = GridSearchCV(model, param_grid={'clf__C': [1.0]}, cv=cv_splitter, scoring={ 'AUC': auc_scorer, 'Fairness': fair_train, 'Robustness': robust_scorer }, refit=False) cv.fit(X_train, pd.DataFrame(y_train)) cv_acc = cv.cv_results_['mean_test_AUC'][0] cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0] cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0] cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1 cv_number_features = float( np.sum( model.named_steps['selection']._get_support_mask())) / float( len(model.named_steps['selection']._get_support_mask())) cv_simplicity = 1.0 - cv_number_features if not 'cv_acc' in cheating_global.successfull_result[hash] or \ calculate_loss(cheating_global.successfull_result[hash]['cv_acc'], cheating_global.successfull_result[hash]['cv_fair'], cheating_global.successfull_result[hash]['cv_robust'], cheating_global.successfull_result[hash]['cv_number_features'] ) > calculate_loss(cv_acc, cv_fair, cv_robust, cv_number_features): cheating_global.successfull_result[hash]['cv_acc'] = cv_acc cheating_global.successfull_result[hash]['cv_robust'] = cv_robust cheating_global.successfull_result[hash]['cv_fair'] = cv_fair cheating_global.successfull_result[hash][ 'cv_number_features'] = cv_number_features # check constraints for test set if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model.fit(X_train, pd.DataFrame(y_train)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' + str(min(cv_acc, test_acc)) + ' robust: ' + str(min(test_robust, cv_robust)) + ' k: ' + str(cv_number_features)) cheating_global.successfull_result[hash]['time'] = time.time( ) - start_time cheating_global.successfull_result[hash]['cv_acc'] = cv_acc cheating_global.successfull_result[hash][ 'cv_robust'] = cv_robust cheating_global.successfull_result[hash]['cv_fair'] = cv_fair cheating_global.successfull_result[hash][ 'cv_number_features'] = cv_number_features return [cv_acc, cv_fair, cv_robust, cv_simplicity]
def hyperparameter_optimization(X_train, X_test, y_train, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features=None, max_search_time=np.inf, cv_splitter=None): start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_train = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_train[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) def f_clf1(hps): mask = np.zeros(len(hps), dtype=bool) for k, v in hps.items(): mask[int(k.split('_')[1])] = v #repair number of features if neccessary max_k = max(int(max_number_features * X_train.shape[1]), 1) if np.sum(mask) > max_k: id_features_used = np.nonzero(mask)[ 0] # indices where features are used np.random.shuffle(id_features_used) # shuffle ids ids_tb_deactived = id_features_used[max_k:] # deactivate features for item_to_remove in ids_tb_deactived: mask[item_to_remove] = False for mask_i in range(len(mask)): hps['f_' + str(mask_i)] = mask[mask_i] model = Pipeline([('selection', MaskSelection(mask)), ('clf', LogisticRegression())]) return model, hps def f_to_min1(hps): model, hps = f_clf1(hps) print("hyperopt: " + str(hps)) if np.sum(model.named_steps['selection'].mask) == 0: return { 'loss': 4, 'status': STATUS_OK, 'model': model, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0 } robust_scorer = make_scorer( robust_score, greater_is_better=True, X=X_train, y=y_train, model=clf, feature_selector=model.named_steps['selection'], scorer=auc_scorer) cv = GridSearchCV(model, param_grid={'clf__C': [1.0]}, cv=cv_splitter, scoring={ 'AUC': auc_scorer, 'Fairness': fair_train, 'Robustness': robust_scorer }, refit=False) cv.fit(X_train, pd.DataFrame(y_train)) cv_acc = cv.cv_results_['mean_test_AUC'][0] cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0] cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0] cv_number_features = float( np.sum( model.named_steps['selection']._get_support_mask())) / float( len(model.named_steps['selection']._get_support_mask())) loss = 0.0 if cv_acc >= min_accuracy and \ cv_fair >= min_fairness and \ cv_robust >= min_robustness: if min_fairness > 0.0: loss += (min_fairness - cv_fair) if min_accuracy > 0.0: loss += (min_accuracy - cv_acc) if min_robustness > 0.0: loss += (min_robustness - cv_robust) else: if min_fairness > 0.0 and cv_fair < min_fairness: loss += (min_fairness - cv_fair)**2 if min_accuracy > 0.0 and cv_acc < min_accuracy: loss += (min_accuracy - cv_acc)**2 if min_robustness > 0.0 and cv_robust < min_robustness: loss += (min_robustness - cv_robust)**2 return { 'loss': loss, 'status': STATUS_OK, 'model': model, 'cv_fair': cv_fair, 'cv_acc': cv_acc, 'cv_robust': cv_robust, 'cv_number_features': cv_number_features, 'updated_parameters': hps } space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) cv_fair = 0 cv_acc = 0 cv_robust = 0 cv_number_features = 1.0 number_of_evaluations = 0 trials = Trials() i = 1 success = False while True: if time.time() - start_time > max_search_time: break fmin(f_to_min1, space=space, algo=tpe.suggest, max_evals=i, trials=trials) #update repair in database try: current_trial = trials.trials[-1] if type(current_trial['result']['updated_parameters']) != type( None): trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals( current_trial['result']['updated_parameters']) except: print("found an error in repair") number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] cv_acc = trials.trials[-1]['result']['cv_acc'] cv_robust = trials.trials[-1]['result']['cv_robust'] cv_number_features = trials.trials[-1]['result']['cv_number_features'] if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model = trials.trials[-1]['result']['model'] model.fit(X_train, pd.DataFrame(y_train)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: print('fair: ' + str(min(cv_fair, test_fair)) + ' acc: ' + str(min(cv_acc, test_acc)) + ' robust: ' + str(min(test_robust, cv_robust)) + ' k: ' + str(cv_number_features)) success = True break i += 1 if not success: try: cv_fair = trials.best_trial['result']['cv_fair'] cv_acc = trials.best_trial['result']['cv_acc'] cv_robust = trials.best_trial['result']['cv_robust'] cv_number_features = trials.best_trial['result'][ 'cv_number_features'] except: pass runtime = time.time() - start_time return { 'time': runtime, 'success': success, 'cv_acc': cv_acc, 'cv_robust': cv_robust, 'cv_fair': cv_fair, 'cv_number_features': cv_number_features, 'cv_number_evaluations': number_of_evaluations }
def transfer_to_other_model(self, model, X_train, X_validation, X_test, y_train, y_validation, y_test, sensitive_ids=None, hyperparameters=None): exp_results = self.read_file() X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) min_loss = np.inf best_run = None for min_r in range(len(exp_results)): if 'loss' in exp_results[ min_r] and exp_results[min_r]['loss'] < min_loss: min_loss = exp_results[min_r]['loss'] best_run = min_r feature_selection = exp_results[best_run]['selected_features'] start_time = time.time() pipeline = Pipeline([('selection', feature_selection), ('clf', model)]) accuracy_scorer = make_scorer(f1_score) grid_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, 1.0, 1.0, 1.0, 0.0, model_hyperparameters=hyperparameters, start_time=start_time) model = grid_result['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) fair_test = None if type(sensitive_ids) != type(None): fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) test_number_features = exp_results[best_run]['cv_number_features'] categories = ['Accuracy', 'Fairness', 'Simplicity', 'Safety'] fig = go.Figure() fig.add_trace( go.Scatterpolar(r=[ test_acc, test_fair, 1.0 - test_number_features, test_robust ], theta=categories, fill='toself', name='Best Result on Test')) fig.update_layout( title='Best Result on Test', polar=dict(radialaxis=dict(visible=True, range=[0, 1])), showlegend=False) return fig
def hyperparameter_optimization(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features=None, max_search_time=np.inf, log_file=None, algo=tpe.suggest, accuracy_scorer=make_scorer( roc_auc_score, greater_is_better=True, needs_threshold=True), avoid_robustness=False, model_hyperparameters=None): min_loss = np.inf start_time = time.time() fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer( true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) stored_results = {} #avoid duplicate hyperparameters def f_clf1(hps): mask = np.zeros(len(hps), dtype=bool) for k, v in hps.items(): mask[int(k.split('_')[1])] = v #repair number of features if neccessary max_k = max(int(max_number_features * X_train.shape[1]), 1) if np.sum(mask) > max_k: id_features_used = np.nonzero(mask)[ 0] # indices where features are used np.random.shuffle(id_features_used) # shuffle ids ids_tb_deactived = id_features_used[max_k:] # deactivate features for item_to_remove in ids_tb_deactived: mask[item_to_remove] = False for mask_i in range(len(mask)): hps['f_' + str(mask_i)] = mask[mask_i] pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)]) return pipeline, hps def f_to_min1(hps): if str(hps) in stored_results: return stored_results[str(hps)] pipeline, hps = f_clf1(hps) if np.sum(pipeline.named_steps['selection'].mask) == 0: stored_results[str(hps)] = { 'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0 } return stored_results[str(hps)] stored_results[str(hps)] = run_grid_search( pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time, avoid_robustness=avoid_robustness) stored_results[str(hps)]['updated_parameters'] = hps return stored_results[str(hps)] space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) number_of_evaluations = 0 trials = Trials() i = 1 success = False fail_counter = [0] while True: if time.time() - start_time > max_search_time: break if fail_counter[0] >= 10: fail_counter[0] += 1 break fail_counter[0] = 0 fmin(f_to_min1, space=space, algo=algo, max_evals=i, trials=trials) #update repair in database try: current_trial = trials.trials[-1] if type(current_trial['result']['updated_parameters']) != type( None): trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals( current_trial['result']['updated_parameters']) except: print("found an error in repair") number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] validation_acc = trials.trials[-1]['result']['cv_acc'] validation_robust = trials.trials[-1]['result']['cv_robust'] cv_number_features = trials.trials[-1]['result']['cv_number_features'] my_result = trials.trials[-1]['result'] my_result['number_evaluations'] = number_of_evaluations model = trials.trials[-1]['result']['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if not avoid_robustness: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined( min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy( my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': success} if min_loss > trials.trials[-1]['result']['loss']: min_loss = trials.trials[-1]['result']['loss'] with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy( my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) i += 1 my_result = { 'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time, 'Finished': True } with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': False}
def exhaustive(X_train, X_validation, X_test, y_train, y_validation, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None): f_log = open(log_file, 'wb+') min_loss = np.inf start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) def f_clf1(hps): mask = np.zeros(len(hps), dtype=bool) for k, v in hps.items(): mask[int(k.split('_')[1])] = v #repair number of features if neccessary max_k = max(int(max_number_features * X_train.shape[1]), 1) if np.sum(mask) > max_k: id_features_used = np.nonzero(mask)[0] # indices where features are used np.random.shuffle(id_features_used) # shuffle ids ids_tb_deactived = id_features_used[max_k:] # deactivate features for item_to_remove in ids_tb_deactived: mask[item_to_remove] = False for mask_i in range(len(mask)): hps['f_' + str(mask_i)] = mask[mask_i] model = Pipeline([ ('selection', MaskSelection(mask)), ('clf', clf) ]) return model, hps def f_to_min1(hps): pipeline, hps = f_clf1(hps) if np.sum(pipeline.named_steps['selection'].mask) == 0: return {'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0} pipeline.fit(X_train, y_train) validation_number_features = float(np.sum(pipeline.named_steps['selection']._get_support_mask())) / float(X_train.shape[1]) validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair) ** 2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc) ** 2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust) ** 2 print(loss) current_time = time.time() - start_time return {'loss': loss, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time, 'updated_parameters': hps} space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) cv_fair = 0 cv_acc = 0 cv_robust = 0 cv_number_features = 1.0 number_of_evaluations = 0 max_k = max(int(max_number_features * X_train.shape[1]), 1) for l in range(1, max_k + 1): for feature_combo in itertools.combinations(range(X_train.shape[1]), l): hps = {} for f_i in range(X_train.shape[1]): if f_i in feature_combo: hps['f_' + str(f_i)] = 1 else: hps['f_' + str(f_i)] = 0 result = f_to_min1(hps) number_of_evaluations += 1 cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model = result['model'] X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success pickle.dump(my_result, f_log) f_log.close() return {'success': success} if min_loss > my_result['loss']: min_loss = my_result['loss'] pickle.dump(my_result, f_log) my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'time': time.time() - start_time, 'Finished': True} pickle.dump(my_result, f_log) f_log.close() return {'success': False}
def fullfeatures(X_train, X_validation, X_test, y_train, y_validation, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features=None, max_search_time=np.inf, log_file=None): f_log = open(log_file, 'wb+') start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer( true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) def f_to_min1(): mask = np.array([True for f_i in range(X_train.shape[1])]) pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)]) pipeline.fit(X_train, y_train) validation_number_features = 1.0 validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 print(loss) current_time = time.time() - start_time return { 'loss': loss, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time } space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) number_of_evaluations = 0 result = f_to_min1() number_of_evaluations += 1 cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features: model = result['model'] X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success pickle.dump(my_result, f_log) f_log.close() return {'success': success} pickle.dump(my_result, f_log) my_result = { 'number_evaluations': number_of_evaluations, 'success_test': False, 'time': time.time() - start_time, 'Finished': True } pickle.dump(my_result, f_log) f_log.close() return {'success': False}
def f_to_min1(x, s): opt_start_time = time.time() mask = x > 0.5 model = f_clf1(mask) if np.sum(model.named_steps['selection'].mask) == 0: return 4, (time.time() - opt_start_time) s_max = y_train_fab.shape[0] shuffle = np.random.permutation(np.arange(s_max)) train_subset = X_train_fab[shuffle[:s]] train_targets_subset = y_train_fab[shuffle[:s]] model.fit(train_subset, pd.DataFrame(train_targets_subset)) cv_acc = auc_scorer(model, X_val, pd.DataFrame(y_val)) cv_fair = 1.0 - fair_val(model, X_val, pd.DataFrame(y_val)) cv_robust = 1.0 - robust_score_test( X_test=X_val, y_test=y_val, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) cv_number_features = float( np.sum( model.named_steps['selection']._get_support_mask())) / float( len(model.named_steps['selection']._get_support_mask())) print("accuracy: " + str(cv_acc) + ' fair: ' + str(cv_fair) + ' k: ' + str(cv_number_features)) loss = 0.0 if cv_acc >= min_accuracy and \ cv_fair >= min_fairness and \ cv_robust >= min_robustness and \ cv_number_features <= max_number_features: if min_fairness > 0.0: loss += (min_fairness - cv_fair) if min_accuracy > 0.0: loss += (min_accuracy - cv_acc) if min_robustness > 0.0: loss += (min_robustness - cv_robust) if max_number_features < 1.0: loss += (cv_number_features - max_number_features) else: if min_fairness > 0.0 and cv_fair < min_fairness: loss += (min_fairness - cv_fair)**2 if min_accuracy > 0.0 and cv_acc < min_accuracy: loss += (min_accuracy - cv_acc)**2 if min_robustness > 0.0 and cv_robust < min_robustness: loss += (min_robustness - cv_robust)**2 if max_number_features < 1.0 and cv_number_features > max_number_features: loss += (cv_number_features - max_number_features)**2 print("loss: " + str(loss)) return loss, (time.time() - opt_start_time)
def hyperparameter_optimization(X_train, X_validation, X_test, y_train, y_validation, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features=None, max_search_time=np.inf, log_file=None, algo=tpe.suggest): f_log = open(log_file, 'wb+') min_loss = np.inf start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer( true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) stored_results = {} #avoid duplicate hyperparameters def f_clf1(hps): mask = np.zeros(len(hps), dtype=bool) for k, v in hps.items(): mask[int(k.split('_')[1])] = v #repair number of features if neccessary max_k = max(int(max_number_features * X_train.shape[1]), 1) if np.sum(mask) > max_k: id_features_used = np.nonzero(mask)[ 0] # indices where features are used np.random.shuffle(id_features_used) # shuffle ids ids_tb_deactived = id_features_used[max_k:] # deactivate features for item_to_remove in ids_tb_deactived: mask[item_to_remove] = False for mask_i in range(len(mask)): hps['f_' + str(mask_i)] = mask[mask_i] pipeline = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)]) return pipeline, hps def f_to_min1(hps): if str(hps) in stored_results: return stored_results[str(hps)] pipeline, hps = f_clf1(hps) if np.sum(pipeline.named_steps['selection'].mask) == 0: stored_results[str(hps)] = { 'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0 } return stored_results[str(hps)] pipeline.fit(X_train, pd.DataFrame(y_train)) validation_number_features = float( np.sum(pipeline.named_steps['selection']._get_support_mask()) ) / float(X_train.shape[1]) validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 current_time = time.time() - start_time stored_results[str(hps)] = { 'loss': loss, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time, 'updated_parameters': hps } return stored_results[str(hps)] space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) number_of_evaluations = 0 trials = Trials() i = 1 success = False fail_counter = [0] while True: if time.time() - start_time > max_search_time: break if fail_counter[0] >= 10: fail_counter[0] += 1 break fail_counter[0] = 0 fmin(f_to_min1, space=space, algo=algo, max_evals=i, trials=trials) #update repair in database try: current_trial = trials.trials[-1] if type(current_trial['result']['updated_parameters']) != type( None): trials._dynamic_trials[-1]['misc']['vals'] = map_hyper2vals( current_trial['result']['updated_parameters']) except: print("found an error in repair") number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] validation_acc = trials.trials[-1]['result']['cv_acc'] validation_robust = trials.trials[-1]['result']['cv_robust'] cv_number_features = trials.trials[-1]['result']['cv_number_features'] my_result = trials.trials[-1]['result'] my_result['number_evaluations'] = number_of_evaluations if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features: model = trials.trials[-1]['result']['model'] X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success pickle.dump(my_result, f_log) f_log.close() return {'success': success} if min_loss > trials.trials[-1]['result']['loss']: min_loss = trials.trials[-1]['result']['loss'] pickle.dump(my_result, f_log) i += 1 my_result = { 'number_evaluations': number_of_evaluations, 'success_test': False, 'time': time.time() - start_time, 'Finished': True } pickle.dump(my_result, f_log) f_log.close() return {'success': False}
def exhaustive(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None): min_loss = np.inf start_time = time.time() fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) def f_clf1(hps): mask = np.zeros(len(hps), dtype=bool) for k, v in hps.items(): mask[int(k.split('_')[1])] = v #repair number of features if neccessary max_k = max(int(max_number_features * X_train.shape[1]), 1) if np.sum(mask) > max_k: id_features_used = np.nonzero(mask)[0] # indices where features are used np.random.shuffle(id_features_used) # shuffle ids ids_tb_deactived = id_features_used[max_k:] # deactivate features for item_to_remove in ids_tb_deactived: mask[item_to_remove] = False for mask_i in range(len(mask)): hps['f_' + str(mask_i)] = mask[mask_i] model = Pipeline([ ('selection', MaskSelection(mask)), ('clf', clf) ]) return model, hps def f_to_min1(hps): pipeline, hps = f_clf1(hps) if np.sum(pipeline.named_steps['selection'].mask) == 0: return {'loss': 4, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': 0.0, 'cv_acc': 0.0, 'cv_robust': 0.0, 'cv_number_features': 1.0} grid_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time) grid_result['updated_parameters'] = hps return grid_result space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) number_of_evaluations = 0 max_k = max(int(max_number_features * X_train.shape[1]), 1) for l in range(1, max_k + 1): for feature_combo in itertools.combinations(range(X_train.shape[1]), l): hps = {} for f_i in range(X_train.shape[1]): if f_i in feature_combo: hps['f_' + str(f_i)] = 1 else: hps['f_' + str(f_i)] = 0 result = f_to_min1(hps) number_of_evaluations += 1 cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations ##check on test model = result['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': success} if min_loss > my_result['loss']: min_loss = my_result['loss'] with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time, 'Finished': True} with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': False}
def objective(features, min_loss): if 'success' in cheating_global.successfull_result[hash]: return [0.0, 0.0, 0.0, 0.0, min_loss] if np.sum(features) == 0: return [0.0, 0.0, 0.0, 0.0, min_loss] pipeline = f_clf1(features) my_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time) new_pipeline = copy.deepcopy(my_result['model']) validation_acc = my_result['cv_acc'] validation_fair = my_result['cv_fair'] validation_robust = my_result['cv_robust'] loss = my_result['loss'] cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1 validation_simplicity = 1.0 - my_result['cv_number_features'] #check constraints for test set my_result['number_evaluations'] = cheating_global.successfull_result[hash]['cv_number_evaluations'] new_pipeline.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(new_pipeline, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(new_pipeline, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=new_pipeline.named_steps['clf'], feature_selector=new_pipeline.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if validation_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True cheating_global.successfull_result[hash]['success'] = success my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return [validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss] if min_loss > loss: min_loss = loss with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return [validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss]
def fullfeatures(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features = None, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None): start_time = time.time() fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) def f_to_min1(): mask = np.array([True for f_i in range(X_train.shape[1])]) pipeline = Pipeline([ ('selection', MaskSelection(mask)), ('clf', clf) ]) grid_result = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time) return grid_result space = {} for f_i in range(X_train.shape[1]): space['f_' + str(f_i)] = hp.randint('f_' + str(f_i), 2) number_of_evaluations = 0 result = f_to_min1() number_of_evaluations += 1 cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations model = result['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': success} with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time, 'Finished': True} with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': False}
def weighted_ranking(X_train, X_validation, X_test, y_train, y_validation, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features: float = 1.0, max_search_time=np.inf, log_file=None): min_loss = np.inf f_log = open(log_file, 'wb+') start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer( true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) #calculate rankings rankings = [] for ranking_function_i in range(len(ranking_functions)): rankings.append(ranking_functions[ranking_function_i](X_train, y_train)) def f_clf1(hps): weights = [] for i in range(len(rankings)): weights.append(hps['weight' + str(i)]) pipeline = Pipeline([('selection', WeightedRankingSelection(scores=rankings, weights=weights, k=hps['k'], names=np.array(names))), ('clf', clf)]) return pipeline stored_results = {} fail_counter = [0] def f_to_min1(hps): if str(hps) in stored_results: fail_counter[0] += 1 return stored_results[str(hps)] fail_counter[0] = 0 pipeline = f_clf1(hps) pipeline.fit(X_train, pd.DataFrame(y_train)) validation_number_features = float( np.sum(pipeline.named_steps['selection']._get_support_mask()) ) / float(X_train.shape[1]) validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 current_time = time.time() - start_time stored_results[str(hps)] = { 'loss': loss, 'status': STATUS_OK, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time, 'updated_parameters': hps } return stored_results[str(hps)] max_k = max(int(max_number_features * X_train.shape[1]), 1) space = {'k': scope.int(hp.quniform('k', 1, max_k, 1))} if len(rankings) > 1: for i in range(len(rankings)): space['weight' + str(i)] = hp.choice( 'weight' + str(i) + 'choice', [(0.0), hp.lognormal('weight' + str(i) + 'specified', 0, 1)]) else: space['weight' + str(0)] = 1.0 number_of_evaluations = 0 trials = Trials() i = 1 success = False while True: if time.time() - start_time > max_search_time: break if len(stored_results) >= max_k: break if fail_counter[0] >= 10: break fmin(f_to_min1, space=space, algo=tpe.suggest, max_evals=i, trials=trials) number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] validation_acc = trials.trials[-1]['result']['cv_acc'] validation_robust = trials.trials[-1]['result']['cv_robust'] cv_number_features = trials.trials[-1]['result']['cv_number_features'] my_result = trials.trials[-1]['result'] my_result['number_evaluations'] = number_of_evaluations if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and cv_number_features <= max_number_features: model = trials.trials[-1]['result']['model'] X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success pickle.dump(my_result, f_log) f_log.close() return {'success': success} if min_loss > trials.trials[-1]['result']['loss']: min_loss = trials.trials[-1]['result']['loss'] pickle.dump(my_result, f_log) i += 1 my_result = { 'number_evaluations': number_of_evaluations, 'success_test': False, 'time': time.time() - start_time, 'Finished': True } pickle.dump(my_result, f_log) f_log.close() return {'success': False}
def run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters, start_time, avoid_robustness=False): fair_validation = None if type(sensitive_ids) != type(None): fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) search_configs = [{}] if type(model_hyperparameters) != type(None): new_model_hyperparameters = {} for k, v in model_hyperparameters.items(): new_model_hyperparameters['clf__' + k] = v #print(new_model_hyperparameters) search_configs = [dict(zip(new_model_hyperparameters, v)) for v in product(*new_model_hyperparameters.values())] grid_results = {} for configuration in search_configs: new_pipeline = copy.deepcopy(pipeline) if len(configuration) > 0: new_pipeline.set_params(**configuration) new_pipeline.fit(X_train, pd.DataFrame(y_train)) validation_number_features = float(np.sum(new_pipeline.named_steps['selection']._get_support_mask())) / float(X_train.shape[1]) validation_acc = accuracy_scorer(new_pipeline, X_validation, pd.DataFrame(y_validation)) #print("accuracy: " + str(validation_acc)) validation_fair = 0.0 if type(sensitive_ids) != type(None): validation_fair = 1.0 - fair_validation(new_pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if not avoid_robustness: validation_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_validation, y_test=y_validation, model=new_pipeline.named_steps['clf'], feature_selector=new_pipeline.named_steps['selection'], scorer=accuracy_scorer) loss = 0.0 if not np.isinf(min_fairness) and min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair) ** 2 if not np.isinf(min_accuracy) and min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc) ** 2 if not np.isinf(min_robustness) and min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust) ** 2 if not np.isinf(max_number_features) and max_number_features < 1.0 and validation_number_features > max_number_features: loss += (validation_number_features - max_number_features) ** 2 #if constraints are satisfied, we can start to optimize utility if loss == 0.0: if min_fairness == np.inf: #fairness is utility loss -= validation_fair if min_accuracy == np.inf: #accuracy is utility loss -= validation_acc if min_robustness == np.inf: #robustness is utility loss -= validation_robust if max_number_features == -np.inf: #complexity is utility loss -= 1.0 - validation_number_features grid_results[immutabledict(configuration)] = {'loss': loss, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features} # here, we optimize model parameters for accuracy max_acc = -1 best_parameter_configuration = {} for k, v in grid_results.items(): if max_acc < v['cv_acc']: max_acc = v['cv_acc'] best_parameter_configuration = k #print(best_parameter_configuration) best_pipeline = copy.deepcopy(pipeline) best_pipeline.set_params(**best_parameter_configuration) results = grid_results[best_parameter_configuration] results['model'] = best_pipeline results['time'] = time.time() - start_time results['status'] = STATUS_OK return results
def execute_feature_combo(feature_combo, number_of_evaluations): hps = {} for f_i in range(X_train.shape[1]): if f_i in feature_combo: hps['f_' + str(f_i)] = 1 else: hps['f_' + str(f_i)] = 0 result = f_to_min1(hps) cv_fair = result['cv_fair'] cv_acc = result['cv_acc'] cv_robust = result['cv_robust'] cv_number_features = result['cv_number_features'] my_result = result my_result['number_evaluations'] = number_of_evaluations model = result['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if cv_fair >= min_fairness and cv_acc >= min_accuracy and cv_robust >= min_robustness and cv_number_features <= max_number_features and not is_utility_defined( min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy( my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return my_result, {'success': success} return my_result, {}
def objective(features, min_loss): if 'success' in cheating_global.successfull_result[hash]: return [0.0, 0.0, 0.0, 0.0, min_loss] if np.sum(features) == 0: return [0.0, 0.0, 0.0, 0.0, min_loss] pipeline = f_clf1(features) pipeline.fit(X_train, pd.DataFrame(y_train)) cheating_global.successfull_result[hash]['cv_number_evaluations'] += 1 validation_number_features = float( np.sum(pipeline.named_steps['selection']._get_support_mask()) ) / float(X_train.shape[1]) validation_acc = auc_scorer(pipeline, X_validation, pd.DataFrame(y_validation)) validation_fair = 0.0 if type(sensitive_ids) != type(None) and min_fairness > 0.0: validation_fair = 1.0 - fair_validation(pipeline, X_validation, pd.DataFrame(y_validation)) validation_robust = 0.0 if min_robustness > 0.0: validation_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_validation, y_test=y_validation, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) loss = 0.0 if min_fairness > 0.0 and validation_fair < min_fairness: loss += (min_fairness - validation_fair)**2 if min_accuracy > 0.0 and validation_acc < min_accuracy: loss += (min_accuracy - validation_acc)**2 if min_robustness > 0.0 and validation_robust < min_robustness: loss += (min_robustness - validation_robust)**2 print(loss) validation_simplicity = 1.0 - validation_number_features #check constraints for test set current_time = time.time() - start_time my_result = { 'loss': loss, 'model': pipeline, 'cv_fair': validation_fair, 'cv_acc': validation_acc, 'cv_robust': validation_robust, 'cv_number_features': validation_number_features, 'time': current_time } my_result['number_evaluations'] = cheating_global.successfull_result[ hash]['cv_number_evaluations'] if validation_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and validation_number_features <= max_number_features: X_train_val = np.vstack((X_train, X_validation)) y_train_val = np.append(y_train, y_validation) pipeline.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = 0.0 if min_accuracy > 0.0: test_acc = auc_scorer(pipeline, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if min_fairness > 0.0: test_fair = 1.0 - fair_test(pipeline, X_test, pd.DataFrame(y_test)) test_robust = 0.0 if min_robustness > 0.0: test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=pipeline.named_steps['clf'], feature_selector=pipeline.named_steps['selection'], scorer=auc_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time my_result['Finished'] = True success = False if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True cheating_global.successfull_result[hash]['success'] = success my_result['success_test'] = success pickle.dump(my_result, f_log) f_log.close() return [ validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss ] if min_loss > loss: min_loss = loss pickle.dump(my_result, f_log) return [ validation_acc, validation_fair, validation_robust, validation_simplicity, min_loss ]
def weighted_ranking(X_train, X_validation, X_train_val, X_test, y_train, y_validation, y_train_val, y_test, names, sensitive_ids, ranking_functions= [], clf=None, min_accuracy = 0.0, min_fairness = 0.0, min_robustness = 0.0, max_number_features: float = 1.0, max_search_time=np.inf, log_file=None, accuracy_scorer=make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True), model_hyperparameters=None): min_loss = np.inf start_time = time.time() fair_validation = None fair_test = None if type(sensitive_ids) != type(None): fair_validation = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_validation[:, sensitive_ids[0]]) fair_test = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_test[:, sensitive_ids[0]]) #calculate rankings try: rankings = [] for ranking_function_i in range(len(ranking_functions)): rankings.append(ranking_functions[ranking_function_i](X_train, y_train)) except Exception as e: my_result = {'error': e} #with open(log_file, 'ab') as f_log: # pickle.dump(my_result, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': False} def f_clf1(hps): weights = [] for i in range(len(rankings)): weights.append(hps['weight' + str(i)]) pipeline = Pipeline([ ('selection', WeightedRankingSelection(scores=rankings, weights=weights, k=hps['k'], names=np.array(names))), ('clf', clf) ]) return pipeline stored_results = {} fail_counter = [0] def f_to_min1(hps): if str(hps) in stored_results: fail_counter[0] += 1 return stored_results[str(hps)] fail_counter[0] = 0 pipeline = f_clf1(hps) stored_results[str(hps)] = run_grid_search(pipeline, X_train, y_train, X_validation, y_validation, accuracy_scorer, sensitive_ids, min_fairness, min_accuracy, min_robustness, max_number_features, model_hyperparameters=model_hyperparameters, start_time=start_time) stored_results[str(hps)]['updated_parameters'] = hps return stored_results[str(hps)] max_k = max(int(max_number_features * X_train.shape[1]), 1) space = {'k': scope.int(hp.quniform('k', 1, max_k, 1))} if len(rankings) > 1: for i in range(len(rankings)): space['weight' + str(i)] = hp.choice('weight' + str(i) + 'choice', [ (0.0), hp.lognormal('weight' + str(i) + 'specified', 0, 1) ]) else: space['weight' + str(0)] = 1.0 number_of_evaluations = 0 trials = Trials() i = 1 success = False while True: if time.time() - start_time > max_search_time: break if len(stored_results) >= max_k: break if fail_counter[0] >= 10: break fmin(f_to_min1, space=space, algo=tpe.suggest, max_evals=i, trials=trials) number_of_evaluations += 1 cv_fair = trials.trials[-1]['result']['cv_fair'] validation_acc = trials.trials[-1]['result']['cv_acc'] validation_robust = trials.trials[-1]['result']['cv_robust'] my_result = trials.trials[-1]['result'] my_result['number_evaluations'] = number_of_evaluations #check test results model = trials.trials[-1]['result']['model'] model.fit(X_train_val, pd.DataFrame(y_train_val)) test_acc = accuracy_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 0.0 if type(sensitive_ids) != type(None): test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test(eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=accuracy_scorer) my_result['test_fair'] = test_fair my_result['test_acc'] = test_acc my_result['test_robust'] = test_robust my_result['final_time'] = time.time() - start_time if cv_fair >= min_fairness and validation_acc >= min_accuracy and validation_robust >= min_robustness and not is_utility_defined(min_fairness, min_accuracy, min_robustness, max_number_features): my_result['Finished'] = True my_result['Validation_Satisfied'] = True if test_fair >= min_fairness and test_acc >= min_accuracy and test_robust >= min_robustness: success = True my_result['success_test'] = success with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': success} if min_loss > trials.trials[-1]['result']['loss']: min_loss = trials.trials[-1]['result']['loss'] with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) my_result_new['selected_features'] = copy.deepcopy(my_result_new['model'].named_steps['selection']) my_result_new['model'] = None pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) i += 1 my_result = {'number_evaluations': number_of_evaluations, 'success_test': False, 'final_time': time.time() - start_time, 'Finished': True} with open(log_file, 'ab') as f_log: my_result_new = copy.deepcopy(my_result) pickle.dump(my_result_new, f_log, protocol=pickle.HIGHEST_PROTOCOL) return {'success': False}
robust_scorer_test = make_scorer( robust_score_test, greater_is_better=True, needs_threshold=True, X_test=X_test, model=model, feature_selector=model.named_steps['selection'], scorer=auc_scorer) model.fit(X_train, pd.DataFrame(y_train)) test_acc = auc_scorer(model, X_test, pd.DataFrame(y_test)) test_fair = 1.0 - fair_test(model, X_test, pd.DataFrame(y_test)) test_robust = 1.0 - robust_score_test( eps=0.1, X_test=X_test, y_test=y_test, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) #print('acc: ' + str(test_acc) + ' fair: ' + str(test_fair) + ' robust: ' + str(test_robust)) cv_fair = trials.trials[-1]['result']['cv_fair'] cv_acc = trials.trials[-1]['result']['cv_acc'] print('cv-fair: ' + str(cv_fair) + ' test fair: ' + str(test_fair) + ' cv-acc: ' + str(cv_acc) + ' test acc: ' + str(test_acc)) satisfied_constraints.append( [min(test_acc, cv_acc), min(test_fair, cv_fair)]) times.append(time.time() - start_time) iterations.append(i)