def plot_skopt_objective(opt_res, path): """Plots the objective plot from the skopt package. Args: opt_res (scipy.optimize.OptimizeResult): Optimization result object. path (str): Directory at which to save plot. """ if not os.path.exists(path): os.makedirs(path) skplot.plot_objective( result=opt_res, levels=64, n_points=64, n_samples=256, size=2, zscale="linear", dimensions=None, sample_source="random", minimum="result", n_minimum_search=None, plot_dims=None, show_points=True, cmap="viridis_r", ) fig = plt.gcf() fig.tight_layout() fig.savefig(path + "objective") fig.show()
def plot_objective(self, filename=None, n_points=25, n_samples=250, cutoff=0.0): # if `savefig(filename)`, don't `show()` if filename is not None: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import skopt from skopt.plots import plot_objective spaces = list(self.spaces.values()) pnames = list(self.spaces.keys()) opt = skopt.Optimizer(spaces, "ET", acq_optimizer="sampling") self.cutoff = cutoff hyperpars, objective = self.trials_and_results # skopt minimizes. Therefore use: acc = 1-acc objective = 1-objective hyperpars = [list(f) for f in hyperpars] objective = list(objective) opt.tell(hyperpars, objective) opt_result = opt.run(lambda x: 0, n_iter=0) plot_objective(opt_result, n_points=n_points, n_samples=n_samples, dimensions=pnames) plt.tight_layout() if filename is None: plt.show() else: plt.savefig(filename)
def test_hyper_search_example_trainer(): list_data = generate_toy_data(100000) data_split_hash = ExampleTrainer(list_data, seed=42).data_split_hash @use_named_args(space) def run_example_trainer(**kwargs): cfg = AttrDict(kwargs) cfg.batch_size = int(round(cfg.batch_size)) cfg.act_name = 'leaky_relu' cfg.act_params = (cfg.leaky_param,) del cfg['leaky_param'] cfg.max_epochs = 1024 trainer = ExampleTrainer(list_data, seed=42, nbr_readouts=0, **cfg) assert data_split_hash == trainer.data_split_hash makedirs('data/hyper', exist_ok=True) path = cfg.get_hashed_path('data/hyper') try: trainer.restore_best_state(path) except EnvironmentError: trainer.train(path) return trainer.best_validation_score res_gp = skopt.gp_minimize(run_example_trainer, space, n_calls=1, random_state=0, n_random_starts=1) print([d.name for d in res_gp.space.dimensions]) ts = get_time_stamp() plt.close() plot_objective(res_gp) plt.savefig('data/hyper/%s-objective.png' % ts) plt.close() plot_evaluations(res_gp) plt.savefig('data/hyper/%s-evaluations.png' % ts) skopt.dump(res_gp, 'data/hyper/%s-result.gz' % ts, store_objective=False)
def test_plots_work(): """Basic smoke tests to make sure plotting doesn't crash.""" SPACE = [ Integer(1, 20, name='max_depth'), Integer(2, 100, name='min_samples_split'), Integer(5, 30, name='min_samples_leaf'), Integer(1, 30, name='max_features'), Categorical(['gini', 'entropy'], name='criterion'), Categorical(list('abcdefghij'), name='dummy'), ] def objective(params): clf = DecisionTreeClassifier(random_state=3, **{ dim.name: val for dim, val in zip(SPACE, params) if dim.name != 'dummy' }) return -np.mean(cross_val_score(clf, *load_breast_cancer(True))) res = gp_minimize(objective, SPACE, n_calls=10, random_state=3) plots.plot_convergence(res) plots.plot_evaluations(res) plots.plot_objective(res) plots.plot_regret(res)
def _plot(self, names=['scores', 'evals', 'partial']): if 'scores' in names: plt.plot(self.result.func_vals) if 'evals' in names: plot_evaluations(self.result) if 'partial' in names: plot_objective(self.result)
def plot_evaluations(self): res = create_result(Xi=self.opt.Xi, yi=self.opt.yi, space=self.opt.space, rng=self.opt.rng, models=self.opt.models) plot_objective(res, dimensions=self.parameter_names) plt.show()
def find_best_hyperparameters(model, X, y, dynamic_params_space, scoring, plot, nfold, **HPO_params): # filter these warnings - they are not consistent, arise even for float features from warnings import filterwarnings # simplefilter("ignore", UserWarning) filterwarnings("ignore", message="The objective has been evaluated at this point before", category=UserWarning) # Get model name model_name = model.__class__.__name__ # Get dynamic parameters names: @use_named_args(dynamic_params_space) def get_params_names(**dynamic_params): return list(dynamic_params.keys()) param_names = get_params_names(dynamic_params_space) # Define an objective function @use_named_args(dynamic_params_space) def objective(**dynamic_params): #model.set_params(**static_params) model.set_params(**dynamic_params) cv = StratifiedKFold(n_splits=nfold, random_state=seed, shuffle=True) scores = cross_validate(model, X, y, cv=cv, scoring = scoring, n_jobs=-1) val_score = np.mean(scores['test_score']) return -val_score print(model_name, 'model training...') # Load previously trained results and get starting point (x0) as best model from previous run try: res = load(r'output/models/'+model_name) x0 = res.x # If not trained before -> no initial point provided except: x0 = None res = forest_minimize(objective, dynamic_params_space, x0 = x0, **HPO_params) # add attribute - parameters names to the res res.param_names = param_names print('Optimized parameters: ', res.param_names) print('Previous best parameters:', x0) print('Current best parameters:', res.x) print('Best score:', -res.fun) # Saved optimization result dump(res, r'output/models/'+model_name, store_objective=False) if plot == True: plt.figure(figsize=(5,2)) plot_convergence(res) try: # plot_objective would not work if only one parameter was searched for plot_objective(res) except: pass plt.show()
def get_maximum(self, n_calls=10, n_random_starts=5, noise=0.01, verbose=True, plot_results=False): """Performs Bayesian optimization by iteratively evaluating the given function on points that are likely to be a global maximum. After the optimization, the evaluated values are stored in self.x_values and self.y_values and appended to the data file if provided. Args: n_calls (int): Number of iterations n_random_starts (int): Initial random evaluations if no previpus values are provided noise (float): Estimated noise in the data verbose (bool): Whether to print optimization details at each evaluation plot_results (bool): Whether to plot an analysis of the solution Returns: A tuple (x, y) with the argmax and max found of the evaluated function. """ x_values = [x for x in self.x_values] if len(self.x_values) > 0 else None # Negate y_values because skopt performs minimization instead of maximization y_values = [-y for y in self.y_values] if len(self.y_values) > 0 else None rand_starts = 2 if len(self.x_values) == 0 and n_random_starts == 0 else n_random_starts res = gp_minimize(func=GaussianProcessSearch.evaluate, dimensions=self.search_space, n_calls=n_calls, n_random_starts=rand_starts, acq_func='EI', acq_optimizer='lbfgs', x0=x_values, y0=y_values, noise=noise, n_jobs=-1, callback=self.__save_res, verbose=verbose) if plot_results: ax = plot_objective(res) plt.show() ax = plot_evaluations(res) plt.show() self.x_values = [[float(val) for val in point] for point in res.x_iters] self.y_values = [-val for val in res.func_vals] if self.output_file is not None: self.save_values() try: ax = plot_objective(res) plt.savefig( self.output_file + "_objective_plot.png") except Exception as e: print(e) try: ax = plot_evaluations(res) plt.savefig( self.output_file + "_evaluations_plot.png") except Exception as e: print(e) return res.x, -res.fun
def save_skopt_plots(dirpath,search_result,prior_names): if not os.path.exists(dirpath): os.makedirs(dirpath) # ---- Evalution plot_evaluations(search_result, bins=20) plt.savefig( os.path.join(dirpath,'evaluation_plot.png') ) # ---- Convergence (previously looked better enquire what is going on) plot_convergence(search_result) plt.savefig( os.path.join(dirpath,'convergence_plot.png') ) # ---- Partial Dependence plots are only approximations of the modelled fitness function # - which in turn is only an approximation of the true fitness function in fitness plot_objective(result=search_result) plt.savefig( os.path.join(dirpath,'objective_plot.png') )
def plot_status(self, opt): result = opt.run(None, 0) output = self.output() plot_convergence(result) output["conv"].dump(plt.gcf(), bbox_inches="tight") plot_objective(result) output["obj"].dump(plt.gcf(), bbox_inches="tight") plt.close()
def test_plots_work_without_cat(): """Basic smoke tests to make sure plotting doesn't crash.""" SPACE = [ Integer(1, 20, name='max_depth'), Integer(2, 100, name='min_samples_split'), Integer(5, 30, name='min_samples_leaf'), Integer(1, 30, name='max_features'), ] def objective(params): clf = DecisionTreeClassifier(random_state=3, **{ dim.name: val for dim, val in zip(SPACE, params) if dim.name != 'dummy' }) return -np.mean(cross_val_score(clf, *load_breast_cancer(True))) res = gp_minimize(objective, SPACE, n_calls=10, random_state=3) plots.plot_convergence(res) plots.plot_evaluations(res) plots.plot_objective(res) plots.plot_objective(res, minimum='expected_minimum') plots.plot_objective(res, sample_source='expected_minimum', n_minimum_search=10) plots.plot_objective(res, sample_source='result') plots.plot_regret(res)
def run(self): result = self.input()["collection"].targets[0]["opt"].load().run( None, 0) output = self.output() plot_convergence(result) output["convergence"].dump(plt.gcf(), bbox_inches="tight") plt.close() plot_evaluations(result, bins=10) output["evaluations"].dump(plt.gcf(), bbox_inches="tight") plt.close() if self.plot_objective: plot_objective(result) output["objective"].dump(plt.gcf(), bbox_inches="tight") plt.close()
def make_plots(results, dir): import matplotlib import matplotlib.pyplot as plt # https://matplotlib.org/faq/usage_faq.html#non-interactive-example # https://matplotlib.org/api/_as_gen/matplotlib.pyplot.savefig.html if not dir.exists(): dir.mkdir(parents=True, exist_ok=True) # WIP: there is currently no support for plotting categorical variables... # You have to manually checkout this pull request: # git pr 675 # install https://github.com/tj/git-extras # git pull origin master # https://github.com/scikit-optimize/scikit-optimize/pull/675 from skopt.plots import plot_convergence _ = plot_convergence(results) plt.savefig(dir / 'plot_convergence.png') from skopt.plots import plot_objective _ = plot_objective(results) plt.savefig(dir / 'plot_objective.png') from skopt.plots import plot_regret _ = plot_regret(results) plt.savefig(dir / 'plot_regret.png') from skopt.plots import plot_evaluations _ = plot_evaluations(results) plt.savefig(dir / 'plot_evaluations.png') # compare convergence... # https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/strategy-comparison.ipynb # for all runs... # from skopt.plots import plot_convergence # plot = plot_convergence(("dummy_minimize", dummy_res), # ("gp_minimize", gp_res), # ("forest_minimize('rf')", rf_res), # ("forest_minimize('et)", et_res), # true_minimum=0.397887, yscale="log") # plot.legend(loc="best", prop={'size': 6}, numpoints=1); # parallel optimization # in the yaml: # parallel: X (if you have a small dataset.self.) # https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/parallel-optimization.ipynb # from sklearn.externals.joblib import Parallel, delayed # x = optimizer.ask(n_points=4) # x is a list of n_points points # y = Parallel()(delayed(branin)(v) for v in x) # evaluate points in parallel # optimizer.tell(x, y) # checkpoints? # https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/interruptible-optimization.ipynb # https://github.com/scikit-optimize/scikit-optimize/blob/master/examples/store-and-load-results.ipynb # poor man's solution: # import pickle # with open('my-optimizer.pkl', 'wb') as f: # pickle.dump(opt, f) # with open('my-optimizer.pkl', 'rb') as f: # opt_restored = pickle.load(f)
def compute_batch(self, trials, nb_eval, iteration, function): space = copy.deepcopy(self.searchspace) # curr_eval = getattr(trials, '_ids') # if curr_eval == set(): # curr_eval = 0 # else: # curr_eval = max(curr_eval) + 1 # space[self.string] = iteration #hp.quniform(self.string, -.5+iteration, .5+iteration, 1) space.append((0,iteration+1e-10)) # print(curr_eval) temp=0 if isinstance(trials[0],type(None)): temp =2 coucou = skopt.gp_minimize(function,space,n_calls=nb_eval,x0=trials[0],y0=trials[1],n_initial_points=temp) for i in coucou.x_iters: i[-1] = iteration trials[0] = coucou.x_iters trials[1] = coucou.func_vals # fmin( # function, # space, algo=self.algo, # max_evals=curr_eval+nb_eval, # trials=trials, # verbose=self.verbose # ) if(iteration==5): _ = plot_objective(coucou)
def _log_plot_objective(results, experiment, name='diagnostics'): try: fig = plt.figure(figsize=(16, 12)) fig = axes2fig(sk_plots.plot_objective(results), fig=fig) experiment.log_image(name, fig) except Exception as e: print('Could not create the objective chart due to error: {}'.format(e))
def show_optimized(optimized, output=None): """ Takes the dump of an optimizer and outputs relevant infos Both with visualisations, in the terminal and in the optional output file where it puts the best parameters in a JSON format """ print(optimized) if output != None: x = optimized.x data = dict() data["jnd_length"] = x[0] data["jnd_angle"] = x[1] data["jnd_direction"] = x[2] data["angle_baseline_a"] = x[3] data["angle_baseline_d"] = x[4] if len(x) > 5: data["coef_l"] = x[5] data["coef_a"] = x[6] data["coef_d"] = x[7] else: data["coef_l"] = 1. data["coef_a"] = 1. data["coef_d"] = 1. json.dump(data, open(output, "w"), indent=4) p1 = plot_objective(optimized) p2 = plot_evaluations(optimized) plt.set_cmap("viridis") plt.grid() plt.legend() plt.show()
def test_names_dimensions(): # Define objective def objective(x, noise_level=0.1): return np.sin(5 * x[0]) * (1 - np.tanh(x[0] ** 2)) +\ np.random.randn() * noise_level # Initialize Optimizer opt = Optimizer([(-2.0, 2.0)], n_initial_points=1) # Optimize for i in range(2): next_x = opt.ask() f_val = objective(next_x) res = opt.tell(next_x, f_val) # Plot results plots.plot_objective(res)
def plot_objective(self): results = SkoptResults() results.space = self.optimizer.space results.x_iters = self.optimizer.Xi results = self._convert_categorical_hyperparameters(results) results.x = results.x_iters[np.argmin(self.optimizer.yi)] results.models = self.optimizer.models plt.figure(figsize=(10, 10)) return plot_objective(results)
def save_objective_figure(save_data_folder, result, all_param): print("saving objective_figure") plt.clf() _ = plot_objective(result, n_points=len(all_param) - num_of_solution_samples, minimum='expected_minimum') plt.savefig( save_data_folder ) # without white boxes savefig('foo.png', bbox_inches='tight')
def run(self): from skopt.plots import plot_objective, plot_evaluations, plot_convergence import matplotlib.pyplot as plt result = self.input().load().run(None, 0) output = self.output() with output.targets["convergence"].localize("w") as tmp: plot_convergence(result) tmp.dump(plt.gcf(), bbox_inches="tight") plt.close() with output.targets["evaluations"].localize("w") as tmp: plot_evaluations(result, bins=10) tmp.dump(plt.gcf(), bbox_inches="tight") plt.close() if self.has_fitted_model(): plot_objective(result) with output.targets["objective"].localize("w") as tmp: tmp.dump(plt.gcf(), bbox_inches="tight") plt.close()
def plotting(results): fig1 = plt.figure(1) ax1 = plt.axes() plots.plot_convergence(results) plt.savefig(logdir + 'convergence.png', dpi=500) plt.close() fig2 = plt.figure(1) ax2 = plt.axes() plots.plot_evaluations(results) plt.savefig(logdir + 'eval.png', dpi=500) plt.close() # plot_objective can only be called if the parameters are being calculated instead of selected randomly if len(results.models) > 1: fig3 = plt.figure(3) ax3 = plt.axes() plots.plot_objective(results) plt.savefig(logdir + 'objective.png', dpi=500) plt.close() filename = logdir + 'gp_min.sav' pickle.dump(results, open(filename, 'wb'))
def plotting(results): fig1 = plt.figure(1) ax1 = plt.axes() plots.plot_convergence(results) plt.savefig(logdir + 'convergence.png', dpi=500) plt.close() fig2 = plt.figure(1) ax2 = plt.axes() plots.plot_evaluations(results) plt.savefig(logdir + 'eval.png', dpi=500) plt.close() #plot_objective kann nur aufgerufen werden, sobald die Parameter berechnet und nicht mehr zufällig gewählt sind if len(results.models) > 1: fig3 = plt.figure(3) ax3 = plt.axes() plots.plot_objective(results) plt.savefig(logdir + 'objective.png', dpi=500) plt.close() filename = logdir + 'gp_min.sav' pickle.dump(results, open(filename, 'wb'))
def save_objective(search_result, folder_path): """ :param search_result: :param folder_path: :return: """ ax = plot_objective(result=search_result) folder_path.mkdir(exist_ok=True, parents=True) plt.savefig(folder_path / 'objective.png') plt.close() print(f'Objective saved at {folder_path / "objective.png"}')
def main(): from functools import partial from skopt.plots import plot_evaluations, plot_objective from skopt import gp_minimize, forest_minimize, dummy_minimize bounds = [ (0., 1.), ] * 8 n_calls = 200 forest_res = gp_minimize(Main_Loop, bounds) _ = plot_evaluations(forest_res) _ = plot_objective(forest_res)
def plot_bayesian_hparam_opt(model_name, hparam_names, search_results, save_fig=False): ''' Plot all 2D hyperparameter comparisons from the logs of a Bayesian hyperparameter optimization. :param model_name: Name of the model :param hparam_names: List of hyperparameter identifiers :param search_results: The object resulting from a Bayesian hyperparameter optimization with the skopt package :param save_fig: :return: ''' # Abbreviate hyperparameters to improve plot readability axis_labels = hparam_names.copy() for i in range(len(axis_labels)): if len(axis_labels[i]) >= 12: axis_labels[i] = axis_labels[i][:4] + '...' + axis_labels[i][-4:] # Plot axes = plot_objective(result=search_results, dimensions=axis_labels) # Create a title fig = plt.gcf() fig.suptitle('Bayesian Hyperparameter\n Optimization for ' + model_name, fontsize=15, x=0.65, y=0.97) # Indicate which hyperparameter abbreviations correspond with which hyperparameter hparam_abbrs_text = '' for i in range(len(hparam_names)): hparam_abbrs_text += axis_labels[i] + ':\n' fig.text(0.50, 0.8, hparam_abbrs_text, fontsize=10, style='italic', color='mediumblue') hparam_names_text = '' for i in range(len(hparam_names)): hparam_names_text += hparam_names[i] + '\n' fig.text(0.65, 0.8, hparam_names_text, fontsize=10, color='darkblue') fig.tight_layout() if save_fig: plt.savefig(cfg['PATHS']['EXPERIMENT_VISUALIZATIONS'] + 'Bayesian_opt_' + model_name + '_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '.png')
def skopt_plots(search_result): from skopt.plots import plot_evaluations, plot_objective, plot_convergence _ = plot_evaluations(search_result) plt.savefig('evaluations', dpi=400, bbox_inches='tight') plt.show() _ = plot_objective(search_result) plt.savefig('objective', dpi=400, bbox_inches='tight') plt.show() _ = plot_convergence(search_result) plt.savefig('convergence', dpi=400, bbox_inches='tight') plt.show()
def plot_objective(self, *args, **kwargs): """Calls skopt.plots.plot_objective.""" def _coconut_mock_12(self, *args, **kwargs): return self, args, kwargs while True: from skopt.plots import plot_objective try: _coconut_tre_check_2 = plot_objective is _coconut_recursive_func_31 except _coconut.NameError: _coconut_tre_check_2 = False if _coconut_tre_check_2: self, args, kwargs = _coconut_mock_12(self.get_skopt_result(), *args, **kwargs) continue else: return plot_objective(self.get_skopt_result(), *args, **kwargs) return None
def skopt_plots(search_result, pref=os.getcwd()): from skopt.plots import plot_evaluations, plot_objective, plot_convergence plt.close('all') _ = plot_evaluations(search_result) plt.savefig(os.path.join(pref, 'evaluations'), dpi=400, bbox_inches='tight') plt.close('all') _ = plot_objective(search_result) plt.savefig(os.path.join(pref, 'objective'), dpi=400, bbox_inches='tight') plt.close('all') _ = plot_convergence(search_result) plt.savefig(os.path.join(pref, 'convergence'), dpi=400, bbox_inches='tight')
def send_plot_objective(results, experiment=None, channel_name='objective'): """Logs skopt plot_objective figure to neptune. Image channel `objective` is created and the output of the plot_objective function is first covented to `neptune.Image` and then sent to neptune. Args: results('scipy.optimize.OptimizeResult'): Results object that is typically an output of the function like `skopt.forest_minimize(...)` experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. Examples: Run skopt training:: ... results = skopt.forest_minimize(objective, space, base_estimator='ET', n_calls=100, n_random_starts=10) Send skopt plot_objective figure to neptune:: import neptune import neptunecontrib.monitoring.skopt as sk_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') sk_monitor.send_plot_objective(results) """ _exp = experiment if experiment else neptune fig = plt.figure(figsize=(16, 12)) try: fig = axes2fig(sk_plots.plot_objective(results), fig=fig) with tempfile.NamedTemporaryFile(suffix='.png') as f: fig.savefig(f.name) _exp.send_image(channel_name, f.name) except Exception as e: print( 'Could not create ans objective chart due to error: {}'.format(e))
def plot_objective_(OptimizeResult, dimensions, fig_savepath, figsize=(12, 12), format='PNG', dpi=300): """ Pairwise partial dependence plot of the objective function. The diagonal shows the partial dependence for dimension i with respect to the objective function. The off-diagonal shows the partial dependence for dimensions i and j with respect to the objective function. The objective function is approximated by result.model. Pairwise scatter plots of the points at which the objective function was directly evaluated are shown on the off-diagonal. A red point indicates the found minimum. Note: search spaces that contain Categorical dimensions are currently not supported by this function. Parameters result [OptimizeResult] The result for which to create the scatter plot matrix. levels [int, default=10] Number of levels to draw on the contour plot, passed directly to plt.contour(). n_points [int, default=40] Number of points at which to evaluate the partial dependence along each dimension. n_samples [int, default=250] Number of random samples to use for averaging the model function at each of the n_points. size [float, default=2] Height (in inches) of each facet. zscale [str, default='linear'] Scale to use for the z axis of the contour plots. Either 'linear' or 'log'. dimensions [list of str, default=None] Labels of the dimension variables. None defaults to space.dimensions[i].name, or if also None to ['X_0', 'X_1', ..]. fig_savepath [string]: The path to save a convergence figure Returns ax: [Axes]: The matplotlib axes. """ fig = plt.figure(num=1, figsize=figsize) ax0 = plt.gca() _ = plot_objective(OptimizeResult, dimensions=dimensions) plt.tight_layout() # plt.subplots_adjust(left=0.08, bottom=0.12, right=0.98, top=0.98, hspace=0.1, wspace=0.2) plt.savefig(fig_savepath, format=format, dpi=dpi)
# test run # _fitness(x=default_parameters) # run model search_result = gp_minimize(func=_fitness, dimensions=dimensions, acq_func='EI', # Expected Improvement. n_calls=50, x0=default_parameters) # print results print("############RESULTS############") space = search_result.space print("RESULTS: ", space.point_to_dict(search_result.x)) print("Fitness value: ", search_result.fun) print("Nodes searched: ") print(sorted(zip(search_result.func_vals, search_result.x_iters))) print("##############################") # plot convergence plot_convergence(search_result) plt.show() # plot dependence # dim_names = ['learning_rate', 'batch_size', 'train_steps'] dim_names = ['learning_rate', 'batch_size'] fig, ax = plot_objective(result=search_result, dimension_names=dim_names) plt.show() fig, ax = plot_evaluations(result=search_result, dimension_names=dim_names) plt.show()