def post_process_skopt_results(skopt_results, results, opt_path): mpl.rcParams.update(mpl.rcParamsDefault) skopt_plots(skopt_results, pref=opt_path) fname = os.path.join(opt_path, 'gp_parameters') sr_res = SerializeSKOptResults(skopt_results) if 'folder' in list(results.items())[0]: clear_weights(results=results, opt_dir=opt_path) else: clear_weights(results=results, opt_dir=opt_path) clear_weights(opt_dir=opt_path) try: dump(skopt_results, os.path.join(opt_path, os.path.basename(opt_path))) except PicklingError: print("could not pickle results") try: with open(fname + '.json', 'w') as fp: json.dump(sr_res.serialized_results, fp, sort_keys=True, indent=4) except TypeError: with open(fname + '.json', 'w') as fp: json.dump(str(sr_res.serialized_results), fp, sort_keys=True, indent=4) return
def __call__(self, res): """ Parameters ---------- * `res` [`OptimizeResult`, scipy object]: The optimization as a OptimizeResult object. """ dump(res, self.checkpoint_path, **self.dump_options)
def _export_results_object(results): from io import BytesIO results.specs['args'].pop('callback', None) buffer = BytesIO() dump(results, buffer, store_objective=False) buffer.seek(0) return buffer
def run(n_calls=50, n_runs=5, acq_optimizer="lbfgs"): bounds = [(-5.0, 10.0), (0.0, 15.0)] optimizers = [("gp_minimize", gp_minimize), ("forest_minimize", forest_minimize), ("gbrt_minimize", gbrt_minimize)] for name, optimizer in optimizers: print(name) results = [] min_func_calls = [] time_ = 0.0 for random_state in range(n_runs): if name == "gp_minimize": res = optimizer(branin, bounds, random_state=random_state, n_calls=n_calls, noise=1e-10, verbose=True, acq_optimizer=acq_optimizer, n_jobs=-1) else: res = optimizer(branin, bounds, random_state=random_state, n_calls=n_calls, acq_optimizer=acq_optimizer) results.append(res) print("Dumping results of run %d" % random_state) dump(res, "%d_run" % random_state) min_func_calls.append(np.argmin(res.func_vals) + 1) optimal_values = [result.fun for result in results] mean_optimum = np.mean(optimal_values) std = np.std(optimal_values) best = np.min(optimal_values) print("Mean optimum: " + str(mean_optimum)) print("Std of optimal values" + str(std)) print("Best optima:" + str(best)) mean_fcalls = np.mean(min_func_calls) std_fcalls = np.std(min_func_calls) best_fcalls = np.min(min_func_calls) print("Mean func_calls to reach min: " + str(mean_fcalls)) print("Std func_calls to reach min: " + str(std_fcalls)) print("Fastest no of func_calls to reach min: " + str(best_fcalls))
def fit(self, X, Y, total_duration=6e7, n_iter=100, cv_iter=None, optimizer=None, acq_func='gp_hedge', **kwargs): start = datetime.now() def splitter(itr): for train_idx, test_idx in itr: yield X[train_idx], Y[train_idx], X[test_idx], Y[test_idx] def splitter_dict(itr_dict): n_splits = len(list(itr_dict.values())[0]) for i in range(n_splits): X_train = dict() Y_train = dict() X_test = dict() Y_test = dict() for n_obj, itr in itr_dict.items(): train_idx = itr[i][0] test_idx = itr[i][1] X_train[n_obj] = np.copy(X[n_obj][train_idx]) X_test[n_obj] = np.copy(X[n_obj][test_idx]) Y_train[n_obj] = np.copy(Y[n_obj][train_idx]) Y_test[n_obj] = np.copy(Y[n_obj][test_idx]) yield X_train, Y_train, X_test, Y_test if cv_iter is None: cv_iter = ShuffleSplit(n_splits=3, test_size=0.1, random_state=self.random_state) if isinstance(X, dict): splits = dict() for n_obj, arr in X.items(): if arr.shape[0] == 1: splits[n_obj] = [([0], [0]) for i in range(cv_iter.n_splits)] else: splits[n_obj] = list(cv_iter.split(arr)) else: splits = list(cv_iter.split(X)) # Pre-compute splits for reuse # Here we fix a random seed for all simulations to correlate the random # streams: seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the ranking algorithm: {}'.format(seed)) opt_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug('Random seed for the optimizer: {}'.format(opt_seed)) gp_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the GP surrogate: {}'.format(gp_seed)) if optimizer is not None: opt = optimizer self.logger.debug('Setting the provided optimizer') self.log_best_params(opt) else: transformed = [] for param in self.parameter_ranges: transformed.append(check_dimension(param)) self.logger.info("Parameter Space: {}".format(transformed)) space = normalize_dimensions(transformed) self.logger.info( "Parameter Space after transformation: {}".format(space)) # Todo: Make this passable base_estimator = cook_estimator("GP", space=space, random_state=gp_seed, noise="gaussian") opt = Optimizer(dimensions=self.parameter_ranges, random_state=opt_seed, base_estimator=base_estimator, acq_func=acq_func, **kwargs) self._callbacks_set_optimizer(opt) self._callbacks_on_optimization_begin() time_taken = duration_tillnow(start) total_duration -= time_taken max_fit_duration = -10000 self.logger.info('Time left for {} iterations is {}'.format( n_iter, microsec_to_time(total_duration))) try: for t in range(n_iter): start = datetime.now() self._callbacks_on_iteration_begin(t) self.logger.info( 'Starting optimization iteration: {}'.format(t)) if t > 0: self.log_best_params(opt) next_point = opt.ask() self.logger.info('Next parameters:\n{}'.format(next_point)) results = [] running_times = [] if isinstance(X, dict): for X_train, Y_train, X_test, Y_test in splitter_dict( splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) else: for X_train, Y_train, X_test, Y_test in splitter(splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) results = np.array(results) running_times = np.array(running_times) mean_result = np.mean(results) mean_fitting_duration = np.mean(running_times) # Storing the maximum time to run the splitting model and adding the time for out of sample evaluation if max_fit_duration < np.sum(running_times): max_fit_duration = np.sum(running_times) self.logger.info( 'Validation error for the parameters is {:.4f}'.format( mean_result)) self.logger.info('Time taken for the parameters is {}'.format( microsec_to_time(np.sum(running_times)))) if "ps" in opt.acq_func: opt.tell(next_point, [mean_result, mean_fitting_duration]) else: opt.tell(next_point, mean_result) self._callbacks_on_iteration_end(t) self.logger.info( "Main optimizer iterations done {} and saving the model". format(np.array(opt.yi).shape[0])) dump(opt, self.optimizer_path) time_taken = duration_tillnow(start) total_duration -= time_taken self.logger.info('Time left for simulations is {} '.format( microsec_to_time(total_duration))) if (total_duration - max_fit_duration) < 0: self.logger.info( 'At iteration {} maximum time required by model to validate a parameter values' .format(microsec_to_time(max_fit_duration))) self.logger.info( 'At iteration {} simulation stops, due to time deficiency' .format(t)) break except KeyboardInterrupt: self.logger.debug( 'Optimizer interrupted saving the model at {}'.format( self.optimizer_path)) self.log_best_params(opt) else: self.logger.debug( 'Finally, fit a model on the complete training set and storing the model at {}' .format(self.optimizer_path)) self._fit_params["epochs"] = self._fit_params.get("epochs", 1000) if "ps" in opt.acq_func: best_point = opt.Xi[np.argmin(np.array(opt.yi)[:, 0])] else: best_point = opt.Xi[np.argmin(opt.yi)] self._set_new_parameters(best_point) self.model = copy.copy(self.ranker) self.model.fit(X, Y, **self._fit_params) finally: self._callbacks_on_optimization_end() self.optimizer = opt if np.array(opt.yi).shape[0] != 0: dump(opt, self.optimizer_path)
import json import adaptive from skopt import gp_minimize from skopt.utils import dump from openmc_model import objective # Optimisation for 2D EXAMPLE # Uses adaptive sampling methods from task 8 to obtain starting points for the optimiser learner = adaptive.Learner2D(objective, bounds=[(0, 100), (0, 100)]) runner = adaptive.Runner(learner, ntasks=1, goal=lambda l: l.npoints > 30) runner.ioloop.run_until_complete(runner.task) # Gaussian Processes based optimisation that returns an SciPy optimisation object res = gp_minimize( objective, # the function to minimize dimensions=[(0., 100.), (0., 100.)], # the bounds on each dimension of x n_calls=40, # the number of evaluations of f n_random_starts=0, # the number of random initialization points verbose=True, x0=[i for i in list(learner.data.keys()) ], # initial data from the adaptive sampling method y0=list(learner.data.values() ) # initial data from the adaptive sampling method ) # Saves the optimisation simulation reults to a file dump(res, 'saved_optimisation_2d.dat')
def fit(self, X, Y, total_duration=600, n_iter=100, cv_iter=None, acq_func='gp_hedge', **kwargs): start = datetime.now() def splitter(itr): for train_idx, test_idx in itr: yield X[train_idx], Y[train_idx], X[test_idx], Y[test_idx] def splitter_dict(itr_dict): n_splits = len(list(itr_dict.values())[0]) for i in range(n_splits): X_train = dict() Y_train = dict() X_test = dict() Y_test = dict() for n_obj, itr in itr_dict.items(): train_idx = itr[i][0] test_idx = itr[i][1] X_train[n_obj] = np.copy(X[n_obj][train_idx]) X_test[n_obj] = np.copy(X[n_obj][test_idx]) Y_train[n_obj] = np.copy(Y[n_obj][train_idx]) Y_test[n_obj] = np.copy(Y[n_obj][test_idx]) yield X_train, Y_train, X_test, Y_test if cv_iter is None: cv_iter = ShuffleSplit(n_splits=3, test_size=0.1, random_state=self.random_state) if isinstance(X, dict): splits = dict() for n_obj, arr in X.items(): if arr.shape[0] == 1: splits[n_obj] = [([0], [0]) for i in range(cv_iter.n_splits)] else: splits[n_obj] = list(cv_iter.split(arr)) else: splits = list(cv_iter.split(X)) # Pre-compute splits for reuse # Here we fix a random seed for all simulations to correlate the random # streams: seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the ranking algorithm: {}'.format(seed)) opt_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug('Random seed for the optimizer: {}'.format(opt_seed)) gp_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the GP surrogate: {}'.format(gp_seed)) n_iter = self.set_optimizer(n_iter, opt_seed, acq_func, gp_seed, **kwargs) self._callbacks_set_optimizer(self.opt) self._callbacks_on_optimization_begin() time_taken = duration_till_now(start) total_duration -= time_taken max_fit_duration = -np.inf self.logger.info('Time left for {} iterations is {}'.format( n_iter, seconds_to_time(total_duration))) try: for t in range(n_iter): if total_duration <= 0: break start = datetime.now() self._callbacks_on_iteration_begin(t) self.logger.info( 'Starting optimization iteration: {}'.format(t)) if t > 0: self.log_best_params() next_point = self.opt.ask() self.logger.info('Next parameters:\n{}'.format(next_point)) results = [] running_times = [] if isinstance(X, dict): for X_train, Y_train, X_test, Y_test in splitter_dict( splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) else: for X_train, Y_train, X_test, Y_test in splitter(splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) results = np.array(results) running_times = np.array(running_times) mean_result = np.mean(results) mean_fitting_duration = np.mean(running_times) # Storing the maximum time to run the splitting model and adding the time for out of sample evaluation if max_fit_duration < np.sum(running_times): max_fit_duration = np.sum(running_times) self.logger.info( 'Validation error for the parameters is {:.4f}'.format( mean_result)) self.logger.info('Time taken for the parameters is {}'.format( seconds_to_time(np.sum(running_times)))) if "ps" in self.opt.acq_func: self.opt.tell(next_point, [mean_result, mean_fitting_duration]) else: self.opt.tell(next_point, mean_result) self._callbacks_on_iteration_end(t) self.logger.info( "Main optimizer iterations done {} and saving the model". format(np.array(self.opt.yi).shape[0])) dump(self.opt, self.optimizer_path) time_taken = duration_till_now(start) total_duration -= time_taken self.logger.info('Time left for simulations is {} '.format( seconds_to_time(total_duration))) # Delete Tensorflow graph, to prevent memory leaks: K.clear_session() sess = tf.Session() K.set_session(sess) if (total_duration - max_fit_duration) < 0: self.logger.info( 'Maximum time required by model to validate parameter values {}' .format(seconds_to_time(max_fit_duration))) self.logger.info( 'At iteration {} simulation stops, due to time deficiency' .format(t)) break except KeyboardInterrupt: self.logger.debug( 'Optimizer interrupted saving the model at {}'.format( self.optimizer_path)) self.log_best_params() else: self.logger.debug( 'Finally, fit a model on the complete training set and storing the model at {}' .format(self.optimizer_path)) finally: K.clear_session() sess = tf.Session() K.set_session(sess) self._callbacks_on_optimization_end() # self._fit_params["epochs"] = np.min([self._fit_params.get("epochs", 500) * 2, 1000]) if "ps" in self.opt.acq_func: best_point = self.opt.Xi[np.argmin( np.array(self.opt.yi)[:, 0])] else: best_point = self.opt.Xi[np.argmin(self.opt.yi)] self._set_new_parameters(best_point) self.model = copy.copy(self.learner) self.model.fit(X, Y, **self._fit_params) if np.array(self.opt.yi).shape[0] != 0: dump(self.opt, self.optimizer_path)
'optimizer': torch.optim.Adam, 'distance_metric': 'cosine' } mini_func = gp_minimize optimize_types = ['subspace_dim', 'beta', 'lr', 'mini_batch_size'] minimizer = Minimizer(base_workspace, optimize_types, mini_func) dims = [4, 8, 16, 32, 64, 128] for dim in dims: # order should be the same as the "optimize_types" space = [ Categorical([dim]), Integer(1, 30), Real(10**-5, 10**0, "log-uniform"), Categorical([32, 64, 128, 256, 512]) ] x0 = [dim, 6, 0.005, 512] checkpoint_fname = '_'.join([emb_fname, str(dim)]) + '_checkpoint.pkl' checkpoint_callback = CheckpointSaver(checkpoint_fname, store_objective=False) res = minimizer.minimize(space, n_calls=30, verbose=True, x0=x0, callback=[checkpoint_callback]) results_fname = '_'.join(['results', emb_fname, str(dim)]) dump(res, results_fname + '.pkl', store_objective=False)
# this decorator allows your objective function to receive a the parameters as # keyword arguments. This is particularly convenient when you want to set scikit-learn # estimator parameters @use_named_args(space) def objective(**params): reg.set_params(**params) return -np.mean( cross_val_score( reg, X, y, cv=5, n_jobs=-1, scoring="neg_mean_absolute_error")) from skopt import gp_minimize # checkpoint_saver = CheckpointSaver("test_checkpoint.pkl", store_objective=False) # res_gp = gp_minimize(objective, space, n_calls=50, callback=[checkpoint_saver], random_state=0) # res_gp.specs['args']['func'] = None # dump(res_gp,'test_skopt.pkl') res = load('test_checkpoint.pkl') res_gp = gp_minimize(objective, space, x0=res.x_iters, y0=res.func_vals, n_calls=10, random_state=0, verbose=True) res_gp.specs['args']['func'] = None dump(res_gp, 'test_skopt2.pkl')
def hyper_parameter_optimization_process( self, train_x: 'np.array', train_y: 'np.array', val_x: 'np.array', val_y: 'np.array', model_epochs: int = 10, n_calls: int = 11, acq_func: str = 'gp_hedge', verbose: bool = True, kappa: float = 1.96, noise: float = 0.01, n_jobs: int = -1) -> 'Skopt HPO Object': """Conducts hyperparameter optimization process & return its results""" # Initializing concerned hyper-parameters dim_learning_rate = Real(low=1e-4, high=1e-2, prior='log-uniform', name='learning_rate') dim_num_lstm_nodes = Integer(low=20, high=300, name='num_lstm_nodes') dim_dropout = Real(low=0.1, high=0.9, prior='log-uniform', name='dropout') dim_batch_size = Integer(low=1, high=128, name='batch_size') # Setting default values for the concerned hyperparameters default_parameters = [1e-3, 50, 0.1, 64] dimensions = [ dim_learning_rate, dim_num_lstm_nodes, dim_dropout, dim_batch_size ] # Fitness function as objective function served to HPO process @use_named_args(dimensions=dimensions) def fitness(learning_rate, num_lstm_nodes, dropout, batch_size): # Setting seed and clearing model graphs in backend tf.random.set_seed(SimpleCNN.seed_num) K.clear_session() tf.compat.v1.reset_default_graph() # Initializing model, compiling & training it model = self.generate_model(use_optimised_hyperparameters=False, dropout=dropout, number_of_lstm_nodes=num_lstm_nodes) optimizer = SGD(learning_rate=learning_rate) model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(train_x, train_y, batch_size=batch_size, epochs=model_epochs, verbose=2, shuffle=False, validation_data=(val_x, val_y)) # Generating prediction on Validation data validation_data_prediction = model.predict(val_x) # Calculating MSE for the model trained with candidate Hyperparameters of this iteration mse_validation = mean_squared_error(val_y, validation_data_prediction) # Deleting created model del model return mse_validation hpo_result = gp_minimize(func=fitness, dimensions=dimensions, acq_func=acq_func, n_calls=n_calls, noise=noise, n_jobs=n_jobs, kappa=kappa, x0=default_parameters, verbose=verbose) self.hpo_result = hpo_result # Storing optimised Hyperparameters dump(hpo_result, self.path_to_save_hyperparameters, store_objective=False) # Generating Dataframe of all HPO process candidates and their MSE hpo_iterations_df = mu.generate_hyperparameter_optimization_iterations_df( hpo_result=hpo_result, columns=["Learning Rate", "# LSTM Nodes", "Dropout", "Batch Size"]) # Saving all HPO plots dvu.save_all_hpo_plots(hpo_result=hpo_result, hpo_iterations_df=hpo_iterations_df, path_map=self.path_to_save_hpo_plots) return hpo_result
import adaptive from skopt.utils import dump from openmc_model import objective # Optimisation for 1D example learner = adaptive.SKOptLearner( objective, dimensions=[(0., 100.)], base_estimator="GP", acq_func="gp_hedge", acq_optimizer="lbfgs", ) runner = adaptive.Runner(learner, ntasks=1, goal=lambda l: l.npoints > 40) runner.ioloop.run_until_complete(runner.task) dump(runner.learner, 'saved_optimisation_1d.dat')
#1. first load existed optimization model or build a new model # when we first time use model opt = None if os.path.exists(model_filename): opt = skopt_utils.load(model_filename) print("read model") else: # default setting: there are three workers for each application, cpu range is from 200m to 500m. restriction=[] for i in range(len(app_name)): for j in range(3): restriction += [(100, 3950)] opt = Optimizer(restriction, n_initial_points=5, acq_func="gp_hedge", base_estimator="GP") app_info, keys = read_container_info() ask_BO([], keys) skopt_utils.dump(opt, model_filename) sys.exit() #3. read input data from file' app_info, keys = read_container_info() cpu = [] measured = [] measured = read_measured_data() y, stop = bo_function(app_info) if stop == False: opt.tell(measured, y) print(y, stop) print("-----") ask_BO(measured, keys)