def _fit(self): """ Run the hyper-parameter optimization process Returns ------- tested_parameters_ : ndarray, the parameters tested during the process cv_scores_ : if score_format == 'cv', list of all the CV results of the parameters tested; if score_format == 'avg', array of the mean CV results of the parameters tested """ n_tested_parameters = 0 tested_parameters = np.zeros((self.n_iter, self.n_parameters)) cv_scores = [] if (self.detailed_res == 2): search_path = np.zeros((self.n_iter, self.n_parameters)) ### Initialize with random candidates ### init_candidates = utils.sample_candidates(self.n_init, self.param_bounds, self.param_isInt) self.n_init = init_candidates.shape[0] if (self.verbose): print('Start random init') for i in range(self.n_init): dict_candidate = self.vector_to_dict(init_candidates[i, :]) cv_score = self.score(dict_candidate) if (self.verbose): print('Step ' + str(i) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in, idx = utils.is_in_ndarray( init_candidates[i, :], tested_parameters[:n_tested_parameters, :]) if not is_in: tested_parameters[n_tested_parameters, :] = init_candidates[ i, :] cv_scores.append(cv_score) n_tested_parameters += 1 else: if (self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if (self.detailed_res == 2): search_path[i, :] = init_candidates[i, :] ### Smart Search ### if (self.verbose): print('Start smart search') i_mod_10 = 0 for i in range(self.n_iter - self.n_init - self.n_final_iter): if (i == 20 and cluster_evol == 'step'): self.GCP_args[1] = n_clusters if (i / 10 > (i_mod_10 + 2) and self.cluster_evol == 'variable'): self.GCP_args[0] = self.GCP_args[0] self.GCP_args[1] = min(self.GCP_args[1] + 1, self.n_clusters_max) i_mod_10 += 3 # Sample candidates and predict their corresponding acquisition values candidates = utils.sample_candidates(self.n_candidates, self.param_bounds, self.param_isInt) # Model and retrieve the candidate that maximezes the acquisiton function best_candidate = utils.find_best_candidate( self.model, tested_parameters[:n_tested_parameters, :], cv_scores, self.GCP_args, candidates, self.verbose, self.acquisition_function) dict_candidate = self.vector_to_dict(best_candidate) cv_score = self.score(dict_candidate) if (self.verbose): print('Step ' + str(i + self.n_init) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in, idx = utils.is_in_ndarray( best_candidate, tested_parameters[:n_tested_parameters, :]) if not is_in: tested_parameters[n_tested_parameters, :] = best_candidate cv_scores.append(cv_score) n_tested_parameters += 1 else: if (self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if (self.detailed_res == 2): search_path[i + self.n_init, :] = best_candidate ### Final steps ### self.acquisition_function = 'Simple' for i in range(self.n_final_iter): # Sample candidates and predict their corresponding acquisition values candidates = utils.sample_candidates(self.n_candidates, self.param_bounds, self.param_isInt) # Model and retrieve the candidate that maximezes the acquisiton function best_candidate = utils.find_best_candidate( self.model, tested_parameters[:n_tested_parameters, :], cv_scores, self.GCP_args, candidates, self.verbose, self.acquisition_function) dict_candidate = self.vector_to_dict(best_candidate) cv_score = self.score(dict_candidate) if (self.verbose): print('Step ' + str(i + self.n_iter - self.n_final_iter) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in, idx = utils.is_in_ndarray( best_candidate, tested_parameters[:n_tested_parameters, :]) if not is_in: tested_parameters[n_tested_parameters, :] = best_candidate cv_scores.append(cv_score) n_tested_parameters += 1 else: if (self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if (self.detailed_res == 2): search_path[i + self.n_iter - self.n_final_iter, :] = best_candidate # compute the averages of CV results mean_scores = [] for o in cv_scores: mean_scores.append(np.mean(o)) # find the max best_idx = np.argmax(mean_scores) vector_best_param = tested_parameters[best_idx] best_parameter = self.vector_to_dict(vector_best_param) # store self.best_parameter_ = best_parameter self.tested_parameters_ = tested_parameters[:n_tested_parameters, :] if (self.verbose): print('\nTested ' + str(n_tested_parameters) + ' parameters') print('Max cv score ' + str(mean_scores[best_idx])) print('Best parameter ' + str(tested_parameters[best_idx])) print(best_parameter) if (self.detailed_res == 1): self.cv_scores_ = list(cv_scores) return tested_parameters[:n_tested_parameters, :], cv_scores elif (self.detailed_res == 2): self.cv_scores_ = list(cv_scores) return tested_parameters[: n_tested_parameters, :], search_path, cv_scores, mean_scores else: self.cv_scores_ = mean_scores return tested_parameters[:n_tested_parameters, :], mean_scores
def _fit(self): """ Run the hyper-parameter optimization process Returns ------- `tested_parameters_` : ndarray, the parameters tested during the process `cv_scores_` : if score_format == 'cv', list of all the CV results of the parameters tested; if score_format == 'avg', array of the mean CV results of the parameters tested """ n_tested_parameters = 0 tested_parameters = np.zeros((self.n_iter,self.n_parameters)) cv_scores = [] if(self.detailed_res ==2): search_path = np.zeros((self.n_iter,self.n_parameters)) ### Initialize with random candidates ### init_candidates = utils.sample_candidates(self.n_init,self.param_bounds,self.param_isInt) self.n_init = init_candidates.shape[0] if(self.verbose): print('Start random init') for i in range(self.n_init): dict_candidate = self.vector_to_dict(init_candidates[i,:]) cv_score = self.score(dict_candidate) if(self.verbose): print ('Step ' + str(i) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in,idx = utils.is_in_ndarray(init_candidates[i,:],tested_parameters[:n_tested_parameters,:]) if not is_in: tested_parameters[n_tested_parameters,:] = init_candidates[i,:] cv_scores.append(cv_score) n_tested_parameters += 1 else: if(self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if(self.detailed_res ==2): search_path[i,:] = init_candidates[i,:] ### Smart Search ### if(self.verbose): print('Start smart search') i_mod_10 = 0 for i in range(self.n_iter - self.n_init - self.n_final_iter): if(i==20 and self.cluster_evol=='step'): self.GCP_args[1] = n_clusters if(i/10 > (i_mod_10+2) and self.cluster_evol=='variable'): self.GCP_args[0] = self.GCP_args[0] self.GCP_args[1] = min(self.GCP_args[1]+1,self.n_clusters_max) i_mod_10 += 3 # Sample candidates and predict their corresponding acquisition values candidates = utils.sample_candidates(self.n_candidates,self.param_bounds,self.param_isInt) # Model and retrieve the candidate that maximezes the acquisiton function best_candidate = utils.find_best_candidate(self.model, tested_parameters[:n_tested_parameters,:], cv_scores, self.GCP_args, candidates, self.verbose, self.acquisition_function) dict_candidate = self.vector_to_dict(best_candidate) cv_score = self.score(dict_candidate) if(self.verbose): print ('Step ' + str(i+self.n_init) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in,idx = utils.is_in_ndarray(best_candidate,tested_parameters[:n_tested_parameters,:]) if not is_in: tested_parameters[n_tested_parameters,:] = best_candidate cv_scores.append(cv_score) n_tested_parameters += 1 else: if(self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if(self.detailed_res ==2): search_path[i + self.n_init,:] = best_candidate ### Final steps ### self.acquisition_function = 'Simple' for i in range(self.n_final_iter): # Sample candidates and predict their corresponding acquisition values candidates = utils.sample_candidates(self.n_candidates,self.param_bounds,self.param_isInt) # Model and retrieve the candidate that maximezes the acquisiton function best_candidate = utils.find_best_candidate(self.model, tested_parameters[:n_tested_parameters,:], cv_scores, self.GCP_args, candidates, self.verbose, self.acquisition_function) dict_candidate = self.vector_to_dict(best_candidate) cv_score = self.score(dict_candidate) if(self.verbose): print ('Step ' + str(i+self.n_iter - self.n_final_iter) + ' - Hyperparameter ' + str(dict_candidate) + ' ' + str(np.mean(cv_score))) is_in,idx = utils.is_in_ndarray(best_candidate,tested_parameters[:n_tested_parameters,:]) if not is_in: tested_parameters[n_tested_parameters,:] = best_candidate cv_scores.append(cv_score) n_tested_parameters += 1 else: if(self.verbose): print('Hyperparameter already tesed') cv_scores[idx] += cv_score if(self.detailed_res ==2): search_path[i + self.n_iter - self.n_final_iter,:] = best_candidate # compute the averages of CV results mean_scores = [] for o in cv_scores: mean_scores.append(np.mean(o)) # find the max best_idx = np.argmax(mean_scores) vector_best_param = tested_parameters[best_idx] best_parameter = self.vector_to_dict(vector_best_param) # store self.best_parameter_ = best_parameter self.tested_parameters_ = tested_parameters[:n_tested_parameters,:] if(self.verbose): print ('\nTested ' + str(n_tested_parameters) + ' parameters') print ('Max cv score ' + str(mean_scores[best_idx])) print ('Best parameter ' + str(tested_parameters[best_idx])) print(best_parameter) if(self.detailed_res == 1): self.cv_scores_ = list(cv_scores) return tested_parameters[:n_tested_parameters,:], cv_scores elif(self.detailed_res == 2): self.cv_scores_ = list(cv_scores) return tested_parameters[:n_tested_parameters,:], search_path, cv_scores, mean_scores else: self.cv_scores_ = mean_scores return tested_parameters[:n_tested_parameters,:], mean_scores
fig = plt.figure() abs = range(0, 400) f_plot = [scoring_function(i) for i in abs] n_rows = nb_GCP_steps / 3 if not (nb_GCP_steps % 3 == 0): n_rows += 1 if (save_plots): save_data = np.asarray([np.asarray(abs), np.asarray(f_plot)[:, 0]]).T np.savetxt('data_plots/data_plot.csv', save_data, delimiter=',') #-------------------- Random initialization --------------------# # sample n_random_init random parameters to initialize the process init_rand_candidates = utils.sample_candidates(n_random_init, parameter_bounds, isInt) for i in range(init_rand_candidates.shape[0]): print i rand_candidate = init_rand_candidates[i] new_output = scoring_function(rand_candidate) if (verbose): print('Random try ' + str(rand_candidate) + ', score : ' + str(np.mean(new_output))) if (parameters is None): parameters = np.asarray([rand_candidate]) raw_outputs = [new_output] mean_outputs = [np.mean(new_output)] std_outputs = [np.std(new_output)] else:
fig = plt.figure() abs = range(0,400) f_plot = [scoring_function(i) for i in abs] n_rows = nb_GCP_steps/3 if not(nb_GCP_steps% 3 == 0): n_rows += 1 if(save_plots): save_data = np.asarray([np.asarray(abs),np.asarray(f_plot)[:,0]]).T np.savetxt('data_plots/data_plot.csv',save_data,delimiter=',') #-------------------- Random initialization --------------------# # sample n_random_init random parameters to initialize the process init_rand_candidates = utils.sample_candidates(n_random_init,parameter_bounds,isInt) for i in range(init_rand_candidates.shape[0]): print i rand_candidate = init_rand_candidates[i] new_output = scoring_function(rand_candidate) if(verbose): print('Random try '+str(rand_candidate)+', score : '+str(np.mean(new_output))) if(parameters is None): parameters = np.asarray([rand_candidate]) raw_outputs = [new_output] mean_outputs = [np.mean(new_output)] std_outputs = [np.std(new_output)] else: parameters,raw_outputs,mean_outputs,std_outputs = \