def run(self): # We optimize s on a log scale, as we expect that the performance varies # logarithmically across s s_max = self.x_train.shape[0] s_min = s_max // 27 subsets = [27] * 8 subsets.extend([9] * 4) subsets.extend([3] * 2) subsets.extend([1] * 1) # subsets = [64, 32, 16] # Defining the bounds and dimensions of the # input space (configuration space + environment space) # We also optimize the hyperparameters of the svm on a log scale lower = np.array([1e-3, 1e-5, 1e-5]) upper = np.array([1e5, 10, 1e-1]) # Start Fabolas to optimize the objective function try: results = fabolas(objective_function=self.objective_function, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=len(subsets), num_iterations=1000, n_hypers=30, subsets=subsets, output_path=self.output_path, inc_estimation="last_seen") except ValueError as e: print(e)
def test_bayesian_optimization(self): res = fabolas(objective_function=objective, lower=self.lower, upper=self.upper, subsets=[10, 20], s_min=10, s_max=10000, n_init=1, num_iterations=3) assert len(res["x_opt"]) == self.lower.shape[0] assert np.all(np.array(res["x_opt"]) >= self.lower) assert np.all(np.array(res["x_opt"]) <= self.upper)
def test_bayesian_optimization(self): res = fabolas(objective_function=objective, lower=self.lower, upper=self.upper, subsets=[10, 20], s_min=10, s_max=10000, n_init=2, num_iterations=3) assert len(res["x_opt"]) == self.lower.shape[0] assert np.all(np.array(res["x_opt"]) >= self.lower) assert np.all(np.array(res["x_opt"]) <= self.upper)
# Initial data setup ################################################# version = 1 n_init_num = 5 budget_iter = 80 initial = 0 x_init_dir = home_dir + '/IBO_master/experiments_IBO/' + exp_name + '/initial_data/' x_init_name = 'x_init_{}_v{}.pkl'.format(exp_name, version) ################################################# # IBO main function ################################################# for it in range(n_runs): results_over_runs[it] = fabolas(objective_function, lower=lower, upper=upper, s_min=s_min, s_max=s_max, init_data=x_init_dir + x_init_name, subsets=[1], num_iterations=budget_iter) ################################################# # Saving the results ################################################# output_main_dir = home_dir + '/IBO_master/experiments_IBO/' + exp_name + '/output_main/' pickle.dump( results_over_runs, open( output_main_dir + "results_{}_{}_init{}_budget{}_v{}.pkl".format( exp_name, method, n_init_num, budget_iter, version), "wb"))
os.makedirs(output_path, exist_ok=True) def objective(x, s): dataset_fraction = s / s_max res = f.objective_function(x, dataset_fraction=dataset_fraction) return res["function_value"], res["cost"] info = f.get_meta_information() bounds = np.array(info['bounds']) lower = bounds[:, 0] upper = bounds[:, 1] results = fabolas(objective_function=objective, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=10, num_iterations=num_iterations, n_hypers=30, subsets=subsets, rng=rng, output_path=output_path) results["run_id"] = run_id results['X'] = results['X'].tolist() results['y'] = results['y'].tolist() results['c'] = results['c'].tolist() test_error = [] current_inc = None current_inc_val = None key = "incumbents" for inc in results["incumbents"]: print(inc)
subsets.extend([32] * 2) subsets.extend([4] * 1) def objective(x, s): dataset_fraction = s / s_max res = f.objective_function(x, dataset_fraction=dataset_fraction) return res["function_value"], res["cost"] info = f.get_meta_information() bounds = np.array(info['bounds']) lower = bounds[:, 0] upper = bounds[:, 1] results = fabolas(objective_function=objective, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=len(subsets), num_iterations=num_iterations, n_hypers=30, subsets=subsets, inc_estimation="mean", rng=rng) results["run_id"] = run_id results['X'] = results['X'].tolist() results['y'] = results['y'].tolist() results['c'] = results['c'].tolist() test_error = [] cum_cost = 0 for i, inc in enumerate(results["incumbents"]): y = f.objective_function_test(np.array(inc))["function_value"] test_error.append(y) # Compute the time it would have taken to evaluate this configuration
def main(): parser = argparse.ArgumentParser() parser.add_argument("run_id") parser.add_argument("dataset") parser.add_argument("num_iterations", type=int) parser.add_argument("s_min") parser.add_argument("s_max") # parser.add_argument( # "subsets" # ) # parser.add_argument( # "seed" # ) args = parser.parse_args() benchmark_function = { "LeNet": lenet_function, }[args.dataset] s_min, s_max = int(args.s_min), int(args.s_max) config_space = ImageAugmentation.get_config_space() hyperparameters = config_space.get_hyperparameters() lower = [] upper = [] for hyperparameter in hyperparameters: if hasattr(hyperparameter, "lower"): lower_bound = hyperparameter.lower upper_bound = hyperparameter.upper else: domain = hyperparameter.choices lower_bound, upper_bound = min(domain), max(domain) lower.append(lower_bound) upper.append(upper_bound) lower = np.array(lower) upper = np.array(upper) # Start Fabolas to optimize the objective function results = fabolas(lambda x, s: benchmark_function( x=x, s=int(s), config_space=config_space), lower=lower, upper=upper, s_min=s_min, s_max=s_max, num_iterations=50) x_best = results["x_opt"] print("best configuration", x_best) print(benchmark_function(x_best[:, :-1], s=x_best[:, None, -1])) path = path_join( abspath("."), "Workspace/MastersThesis/AutoDA/experiments/results/mnist/fabolas") with open(os.path.join(path, "fabolas_optimized_%d.json" % args.run_id), "w") as fh: json.dump(x_best, fh)
def optimize(self) -> TuningResult: """ Method performs a hyperparameter optimization run according to the selected HPO-method. :return: result: TuningResult TuningResult-object that contains the results of this optimization run. :return: """ # Convert the skopt hyperparameter space into a continuous space for RoBO hp_space_lower = np.zeros(shape=(len(self.hp_space), )) hp_space_upper = np.zeros(shape=(len(self.hp_space), )) for i in range(len(self.hp_space)): if type(self.hp_space[i]) == skopt.space.space.Integer: hp_space_lower[i, ] = self.hp_space[i].low hp_space_upper[i, ] = self.hp_space[i].high elif type(self.hp_space[i]) == skopt.space.space.Categorical: n_choices = len(list(self.hp_space[i].categories)) hp_space_lower[i, ] = 0 hp_space_upper[i, ] = n_choices - 1 elif type(self.hp_space[i]) == skopt.space.space.Real: hp_space_lower[i, ] = self.hp_space[i].low hp_space_upper[i, ] = self.hp_space[i].high else: raise Exception( 'The skopt HP-space could not be converted correctly!') # Set the random seed of the random number generator rand_num_generator = np.random.RandomState(seed=self.random_seed) # Optimize on the predefined n_func_evals and measure the wall clock times start_time = time.time() self.times = [] # Initialize a list for saving the wall clock times # Use a warmstart configuration (only possible for BOHAMIANN, not FABOLAS) if self.do_warmstart == 'Yes': # Initialize numpy arrays for saving the warmstart configuration and the warmstart loss warmstart_config = np.zeros(shape=(1, len(self.hp_space))) warmstart_loss = np.zeros(shape=(1, 1)) # Retrieve the default hyperparameters and the default loss for the ML-algorithm default_params = self.get_warmstart_configuration() try: # Dictionary for saving the warmstart HP-configuration (only contains the HPs, which are part of the # 'tuned' HP-space warmstart_dict = {} # Iterate over all HPs of this ML-algorithm's tuned HP-space and append the default values to # the numpy array for i in range(len(self.hp_space)): this_param = self.hp_space[i].name # Categorical HPs need to be encoded as integer values for RoBO if type(self.hp_space[i]) == skopt.space.space.Categorical: choices = self.hp_space[i].categories this_warmstart_value_cat = default_params[this_param] dict_value = this_warmstart_value_cat # Find the index of the default / warmstart HP in the list of possible choices for j in range(len(choices)): if this_warmstart_value_cat == choices[j]: this_warmstart_value = j # For all non-categorical HPs else: this_warmstart_value = default_params[this_param] dict_value = this_warmstart_value # For some HPs (e.g. max_depth of RF) the default value is None, although their typical dtype is # different (e.g. int) if this_warmstart_value is None: # Try to impute these values by the mean value this_warmstart_value = int( 0.5 * (self.hp_space[i].low + self.hp_space[i].high)) dict_value = this_warmstart_value # Pass the warmstart value to the according numpy array warmstart_config[0, i] = this_warmstart_value warmstart_dict[this_param] = dict_value # Pass the default loss to the according numpy array warmstart_loss[0, 0] = self.get_warmstart_loss( warmstart_dict=warmstart_dict) # Pass the warmstart configuration as a kwargs dict kwargs = {'X_init': warmstart_config, 'Y_init': warmstart_loss} # Set flag to indicate that a warmstart took place did_warmstart = True except: print('Warmstarting RoBO failed!') kwargs = {} # Set flag to indicate that NO warmstart took place did_warmstart = False # No warmstart requested else: kwargs = {} # Set flag to indicate that NO warmstart took place did_warmstart = False # Select the specified HPO-tuning method try: if self.hpo_method == 'Fabolas': # Budget correct? // Set further parameters? s_max = len( self.x_train ) # Maximum number of data points for the training data set s_min = int( 0.05 * s_max ) # Maximum number of data points for the training data set n_init = int(self.n_func_evals / 3) # Requirement of the fabolas implementation result_dict = fabolas( objective_function=self.objective_fabolas, s_min=s_min, s_max=s_max, lower=hp_space_lower, upper=hp_space_upper, num_iterations=self.n_func_evals, rng=rand_num_generator, n_init=n_init) run_successful = True elif self.hpo_method == 'Bohamiann': if did_warmstart: # A single initial design point (warm start hyperparameter configuration) kwargs['n_init'] = 1 # Budget correct? // Set further parameters? result_dict = bayesian_optimization( objective_function=self.objective_bohamiann, lower=hp_space_lower, upper=hp_space_upper, model_type='bohamiann', num_iterations=self.n_func_evals, rng=rand_num_generator, **kwargs) run_successful = True else: raise Exception('Unknown HPO-method!') # Algorithm crashed except: # Add a warning here run_successful = False # If the optimization run was successful, determine the optimization results if run_successful: for i in range(len(self.times)): # Subtract the start time to receive the wall clock time of each function evaluation self.times[i] = self.times[i] - start_time wall_clock_time = max(self.times) # Insert timestamp of 0.0 for the warm start hyperparameter configuration if did_warmstart: self.times.insert(0, 0.0) # Timestamps timestamps = self.times # Losses (not incumbent losses) losses = result_dict['y'] evaluation_ids = list(range(1, len(losses) + 1)) best_loss = min(losses) configurations = () for config in result_dict['X']: # Cut off the unused Fabolas budget value at the end config = config[:len(self.hp_space)] config_dict = {} for i in range(len(config)): if type(self.hp_space[i]) == skopt.space.space.Integer: config_dict[self.hp_space[i].name] = int( round(config[i])) elif type( self.hp_space[i]) == skopt.space.space.Categorical: config_dict[self.hp_space[i].name] = list( self.hp_space[i].categories)[int(round(config[i]))] elif type(self.hp_space[i]) == skopt.space.space.Real: config_dict[self.hp_space[i].name] = config[i] else: raise Exception( 'The continuous HP-space could not be converted correctly!' ) configurations = configurations + (config_dict, ) # Find the best hyperparameter configuration (incumbent) best_configuration = {} x_opt = result_dict['x_opt'] for i in range(len(x_opt)): if type(self.hp_space[i]) == skopt.space.space.Integer: best_configuration[self.hp_space[i].name] = int( round(x_opt[i])) elif type(self.hp_space[i]) == skopt.space.space.Categorical: best_configuration[self.hp_space[i].name] = list( self.hp_space[i].categories)[int(round(x_opt[i]))] elif type(self.hp_space[i]) == skopt.space.space.Real: best_configuration[self.hp_space[i].name] = x_opt[i] else: raise Exception( 'The continuous HP-space could not be converted correctly!' ) # Run not successful (algorithm crashed) else: evaluation_ids, timestamps, losses, configurations, best_loss, best_configuration, wall_clock_time = \ self.impute_results_for_crash() # Pass the results to a TuningResult-Object result = TuningResult(evaluation_ids=evaluation_ids, timestamps=timestamps, losses=losses, configurations=configurations, best_loss=best_loss, best_configuration=best_configuration, wall_clock_time=wall_clock_time, successful=run_successful, did_warmstart=did_warmstart) return result
# Validate this hyperparameter configuration on the full validation data y = 1 - clf.score(X_val, y_val) c = time.time() - start_time return y, c # Load the data X_train, y_train, X_val, y_val, X_test, y_test = load_dataset() # We optimize s on a log scale, as we expect that the performance varies # logarithmically across s s_min = 100 s_max = 50000 # Defining the bounds and dimensions of the # input space (configuration space + environment space) # We also optimize the hyperparameters of the svm on a log scale lower = np.array([-10, -10]) upper = np.array([10, 10]) # Start Fabolas to optimize the objective function res = fabolas(objective_function, lower=lower, upper=upper, s_min=s_min, s_max=s_max, num_iterations=100) x_best = res["x_opt"] print(x_best) print(objective_function(x_best[:, :-1], s=x_best[:, None, -1]))
dataset_fraction = s / s_max res = f.objective_function(x, dataset_fraction=dataset_fraction) return res["function_value"], res["cost"] info = f.get_meta_information() bounds = np.array(info['bounds']) lower = bounds[:, 0] upper = bounds[:, 1] results = fabolas(objective_function=objective, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=len(subsets), num_iterations=num_iterations, n_hypers=30, subsets=subsets, inc_estimation="mean", rng=rng) results["run_id"] = run_id results['X'] = results['X'].tolist() results['y'] = results['y'].tolist() results['c'] = results['c'].tolist() test_error = [] cum_cost = 0 for i, inc in enumerate(results["incumbents"]):
dataset_fraction = s / s_max res = f.objective_function(x, dataset_fraction=dataset_fraction) return res["function_value"], res["cost"] info = f.get_meta_information() bounds = np.array(info['bounds']) lower = bounds[:, 0] upper = bounds[:, 1] results = fabolas(objective_function=objective, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=len(subsets), num_iterations=num_iterations, n_hypers=30, subsets=subsets, rng=rng, output_path=output_path, inc_estimation="last_seen") results["run_id"] = run_id results['X'] = results['X'].tolist() results['y'] = results['y'].tolist() results['c'] = results['c'].tolist() test_error = [] current_inc = None current_inc_val = None
os.makedirs(output_path, exist_ok=True) def objective(x, s): dataset_fraction = s / s_max res = f.objective_function(x, dataset_fraction=dataset_fraction) return res["function_value"], res["cost"] info = f.get_meta_information() bounds = np.array(info['bounds']) lower = bounds[:, 0] upper = bounds[:, 1] results = fabolas(objective_function=objective, lower=lower, upper=upper, s_min=s_min, s_max=s_max, n_init=len(subsets), num_iterations=num_iterations, n_hypers=30, subsets=subsets, rng=rng, output_path=output_path, inc_estimation="last_seen") results["run_id"] = run_id results['X'] = results['X'].tolist() results['y'] = results['y'].tolist() results['c'] = results['c'].tolist() test_error = [] current_inc = None current_inc_val = None key = "incumbents" for inc in results["incumbents"]: print(inc)
def run_fabolas(X_train, X_test, y_train, y_test, names, sensitive_ids, ranking_functions=[], clf=None, min_accuracy=0.0, min_fairness=0.0, min_robustness=0.0, max_number_features=None, max_search_time=np.inf, cv_splitter=None): X_train_fab, X_val, y_train_fab, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train) start_time = time.time() auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) #fair_train = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_train[:, sensitive_ids[0]]) fair_val = make_scorer(true_positive_rate_score, greater_is_better=True, sensitive_data=X_val[:, sensitive_ids[0]]) def f_clf1(mask): model = Pipeline([('selection', MaskSelection(mask)), ('clf', clf)]) return model def f_to_min1(x, s): opt_start_time = time.time() mask = x > 0.5 model = f_clf1(mask) if np.sum(model.named_steps['selection'].mask) == 0: return 4, (time.time() - opt_start_time) s_max = y_train_fab.shape[0] shuffle = np.random.permutation(np.arange(s_max)) train_subset = X_train_fab[shuffle[:s]] train_targets_subset = y_train_fab[shuffle[:s]] model.fit(train_subset, pd.DataFrame(train_targets_subset)) cv_acc = auc_scorer(model, X_val, pd.DataFrame(y_val)) cv_fair = 1.0 - fair_val(model, X_val, pd.DataFrame(y_val)) cv_robust = 1.0 - robust_score_test( X_test=X_val, y_test=y_val, model=model.named_steps['clf'], feature_selector=model.named_steps['selection'], scorer=auc_scorer) cv_number_features = float( np.sum( model.named_steps['selection']._get_support_mask())) / float( len(model.named_steps['selection']._get_support_mask())) print("accuracy: " + str(cv_acc) + ' fair: ' + str(cv_fair) + ' k: ' + str(cv_number_features)) loss = 0.0 if cv_acc >= min_accuracy and \ cv_fair >= min_fairness and \ cv_robust >= min_robustness and \ cv_number_features <= max_number_features: if min_fairness > 0.0: loss += (min_fairness - cv_fair) if min_accuracy > 0.0: loss += (min_accuracy - cv_acc) if min_robustness > 0.0: loss += (min_robustness - cv_robust) if max_number_features < 1.0: loss += (cv_number_features - max_number_features) else: if min_fairness > 0.0 and cv_fair < min_fairness: loss += (min_fairness - cv_fair)**2 if min_accuracy > 0.0 and cv_acc < min_accuracy: loss += (min_accuracy - cv_acc)**2 if min_robustness > 0.0 and cv_robust < min_robustness: loss += (min_robustness - cv_robust)**2 if max_number_features < 1.0 and cv_number_features > max_number_features: loss += (cv_number_features - max_number_features)**2 print("loss: " + str(loss)) return loss, (time.time() - opt_start_time) cv_fair = 0 cv_acc = 0 cv_robust = 0 cv_number_features = 1.0 number_of_evaluations = 0 ''' start fabolas ''' # Defining the bounds and dimensions of the # input space (configuration space + environment space) # We also optimize the hyperparameters of the svm on a log scale lower = np.zeros(X_train.shape[1]) upper = np.ones(X_train.shape[1]) # Start Fabolas to optimize the objective function print("xshape: " + str(X_train.shape)) res = fabolas(f_to_min1, lower=lower, upper=upper, s_min=100, s_max=X_train.shape[0], num_iterations=1000, n_init=10, chain_length=5, burnin=5, n_hypers=10) '''