def _evaluate_min_params(result, params='result', n_minimum_search=None, random_state=None): """Returns the minimum based on `params`""" x_vals = None space = result.space if isinstance(params, str): if params == 'result': # Using the best observed result x_vals = result.x elif params == 'expected_minimum': if result.space.is_partly_categorical: # space is also categorical raise ValueError('expected_minimum does not support any' 'categorical values') # Do a gradient based minimum search using scipys own minimizer if n_minimum_search: # If a value for # expected_minimum_samples has been parsed x_vals, _ = expected_minimum(result, n_random_starts=n_minimum_search, random_state=random_state) else: # Use standard of 20 random starting points x_vals, _ = expected_minimum(result, n_random_starts=20, random_state=random_state) elif params == 'expected_minimum_random': # Do a minimum search by evaluating the function with # n_samples sample values if n_minimum_search: # If a value for # n_minimum_samples has been parsed x_vals, _ = expected_minimum_random_sampling( result, n_random_starts=n_minimum_search, random_state=random_state) else: # Use standard of 10^n_parameters. Note this # becomes very slow for many parameters x_vals, _ = expected_minimum_random_sampling( result, n_random_starts=10**len(result.x), random_state=random_state) else: raise ValueError('Argument ´eval_min_params´ must be a valid' 'string (´result´)') elif isinstance(params, list): assert len(params) == len(result.x), 'Argument' \ '´eval_min_params´ of type list must have same length as' \ 'number of features' # Using defined x_values x_vals = params else: raise ValueError('Argument ´eval_min_params´ must' 'be a string or a list') return x_vals
def test_expected_minimum(): res = gp_minimize(bench3, [(-2.0, 2.0)], x0=[0.], noise=1e-8, n_calls=20, random_state=1) x_min, f_min = expected_minimum(res, random_state=1) x_min2, f_min2 = expected_minimum(res, random_state=1) assert f_min <= res.fun # true since noise ~= 0.0 assert x_min == x_min2 assert f_min == f_min2
def test_expected_minimum(): res = gp_minimize(bench3, [(-2.0, 2.0)], x0=[0.], noise=1e-8, n_calls=20, random_state=1) x_min, f_min = expected_minimum(res, random_state=1) x_min2, f_min2 = expected_minimum(res, random_state=1) assert f_min <= res.fun # true since noise ~= 0.0 assert x_min == x_min2 assert f_min == f_min2
def plot_results(self): """ Visualize Hyperparameter Optimization Results """ super(SkoptOptimizer, self).plot_results() import skopt.plots try: skopt.plots.plot_convergence(self.skopt_result) except: pass try: skopt.plots.plot_objective(self.skopt_result) except: pass try: skopt.plots.plot_evaluations(self.skopt_result) except: pass try: from skopt import expected_minimum print(expected_minimum(self.skopt_result)) except: pass
def test_evaluate_min_params(): res = gp_minimize(bench3, [(-2.0, 2.0)], x0=[0.], noise=1e-8, n_calls=8, n_random_starts=3, random_state=1) x_min, f_min = expected_minimum(res, random_state=1) x_min2, f_min2 = expected_minimum_random_sampling(res, n_random_starts=1000, random_state=1) plots.plot_gaussian_process(res) assert _evaluate_min_params(res, params='result') == res.x assert _evaluate_min_params(res, params=[1.]) == [1.] assert _evaluate_min_params(res, params='expected_minimum', random_state=1) == x_min assert _evaluate_min_params(res, params='expected_minimum', n_minimum_search=20, random_state=1) == x_min assert _evaluate_min_params(res, params='expected_minimum_random', n_minimum_search=1000, random_state=1) == x_min2
def test_plots_work(): """Basic smoke tests to make sure plotting doesn't crash.""" SPACE = [ Integer(1, 20, name='max_depth'), Integer(2, 100, name='min_samples_split'), Integer(5, 30, name='min_samples_leaf'), Integer(1, 30, name='max_features'), Categorical(['gini', 'entropy'], name='criterion'), Categorical(list('abcdefghij'), name='dummy'), ] def objective(params): clf = DecisionTreeClassifier(random_state=3, **{ dim.name: val for dim, val in zip(SPACE, params) if dim.name != 'dummy' }) return -np.mean(cross_val_score(clf, *load_breast_cancer(True))) res = gp_minimize(objective, SPACE, n_calls=10, random_state=3) x_min, f_min = expected_minimum_random_sampling(res, random_state=1) x_min2, f_min2 = expected_minimum(res, random_state=1) assert x_min == x_min2 assert f_min == f_min2 plots.plot_convergence(res) plots.plot_evaluations(res) plots.plot_objective(res) plots.plot_objective(res, minimum='expected_minimum_random') plots.plot_objective(res, sample_source='expected_minimum_random', n_minimum_search=10000) plots.plot_objective(res, sample_source='result') plots.plot_regret(res)
def get_expected_minimum(result: OptimizeResult, **kwargs) -> Optional[List[float]]: if len(result.models) == 0: return None return expected_minimum(result, **kwargs)
def test_plots_work(): """Basic smoke tests to make sure plotting doesn't crash.""" SPACE = [ Integer(1, 20, name='max_depth'), Integer(2, 100, name='min_samples_split'), Integer(5, 30, name='min_samples_leaf'), Integer(1, 30, name='max_features'), Categorical(['gini', 'entropy'], name='criterion'), Categorical(list('abcdefghij'), name='dummy'), ] def objective(params): clf = DecisionTreeClassifier(random_state=3, **{ dim.name: val for dim, val in zip(SPACE, params) if dim.name != 'dummy' }) return -np.mean(cross_val_score(clf, *load_breast_cancer(True))) res = gp_minimize(objective, SPACE, n_calls=10, random_state=3) x = [[11, 52, 8, 14, 'entropy', 'f'], [14, 90, 10, 2, 'gini', 'a'], [7, 90, 6, 14, 'entropy', 'f']] samples = res.space.transform(x) xi_ = [1., 10.5, 20.] yi_ = [-0.9240883492576596, -0.9240745890422687, -0.9240586402439884] xi, yi = partial_dependence_1D(res.space, res.models[-1], 0, samples, n_points=3) assert_array_almost_equal(xi, xi_) assert_array_almost_equal(yi, yi_, 1e-3) xi_ = [0, 1] yi_ = [-0.9241087603770617, -0.9240188905968352] xi, yi = partial_dependence_1D(res.space, res.models[-1], 4, samples, n_points=3) assert_array_almost_equal(xi, xi_) assert_array_almost_equal(yi, yi_, 1e-3) xi_ = [0, 1] yi_ = [1., 10.5, 20.] zi_ = [[-0.92412562, -0.92403575], [-0.92411186, -0.92402199], [-0.92409591, -0.92400604]] xi, yi, zi = partial_dependence_2D(res.space, res.models[-1], 0, 4, samples, n_points=3) assert_array_almost_equal(xi, xi_) assert_array_almost_equal(yi, yi_) assert_array_almost_equal(zi, zi_, 1e-3) x_min, f_min = expected_minimum_random_sampling(res, random_state=1) x_min2, f_min2 = expected_minimum(res, random_state=1) x_min, f_min = expected_minimum_random_sampling(res, random_state=1) x_min2, f_min2 = expected_minimum(res, random_state=1) assert x_min == x_min2 assert f_min == f_min2 plots.plot_convergence(res) plots.plot_evaluations(res) plots.plot_objective(res) plots.plot_objective(res, dimensions=["a", "b", "c", "d", "e", "f"]) plots.plot_objective(res, minimum='expected_minimum_random') plots.plot_objective(res, sample_source='expected_minimum_random', n_minimum_search=10000) plots.plot_objective(res, sample_source='result') plots.plot_regret(res) plots.plot_objective_2D(res, 0, 4) plots.plot_histogram(res, 0, 4)
def make_figure(datasets, opt, figure_path, n_levels=32): fig = plt.figure(figsize=(5.1, 1.9)) gs = gridspec.GridSpec(2, 4, width_ratios=[8, 8, 10, 1]) exp_axis = plt.subplot(gs[0, 0]) err_axis = plt.subplot(gs[0, 1]) com_axis = plt.subplot(gs[1, 0]) cst_axis = plt.subplot(gs[1, 1]) fit_axis = plt.subplot(gs[0:2, 2]) leg_axis = plt.subplot(gs[0:2, 3]) (dataset, label, color, color_conf, linestyle) = datasets for measure, axis in [('expressivity', exp_axis), ('error', err_axis), ('complexity', com_axis), ('cost', cst_axis)]: mean, conf_pos, conf_neg, start_gen = dataset[measure] xvals = list(range(start_gen, start_gen + len(mean))) axis.plot(xvals, mean, c=color, label=label, linestyle=linestyle, dash_capstyle="round", linewidth=2) axis.fill_between(xvals, conf_neg, conf_pos, facecolor=color_conf, alpha=0.5) ylim = measure_bounds[measure] ypad = (ylim[1] - ylim[0]) * 0.05 axis.set_ylim(ylim[0] - ypad, ylim[1] + ypad) axis.set_ylabel(measures_names[measure], fontsize=8) axis.set_yticklabels([]) axis.set_yticks([]) for tick in axis.xaxis.get_major_ticks(): tick.label.set_fontsize(7) axis.set_xlim(0, 10) if measure == 'expressivity' or measure == 'error': axis.set_xticks([]) else: axis.set_xticks([0, 2, 4, 6, 8, 10]) axis.set_xlabel('Generation', fontsize=8) (w_star, e_star), neg_log_p_sim = expected_minimum(opt) print('Simplicity:', len(opt.func_vals), 'iterations)') print('w* =', w_star) print('e* =', e_star) print('log(P) =', -neg_log_p_sim) space = opt.space samples = np.asarray(opt.x_iters) rvs_transformed = space.transform(space.rvs(n_samples=250)) xi, yi, zi = plots.partial_dependence(space, opt.models[-1], 1, 0, rvs_transformed, 250) zmin = zi.min() zmax = zi.max() levels = np.geomspace(zmin, zmax, n_levels + 1) cs = fit_axis.contourf(xi, yi, zi, levels, cmap='viridis_r') fit_axis.plot([0, w_star, w_star], [e_star, e_star, 0], c='k', linestyle=':') fit_axis.scatter(w_star, e_star, c='k', s=100, lw=0, marker='*') fit_axis.set_xlim(0, 2) fit_axis.set_ylim(0, 1) fit_axis.set_xticks([0, 0.5, 1.0, 1.5, 2.0]) for tick in fit_axis.xaxis.get_major_ticks(): tick.label.set_fontsize(7) fit_axis.set_yticklabels([]) fit_axis.set_yticks([]) fit_axis.set_xlabel('Weight (w)', fontsize=8) fit_axis.set_ylabel('Noise (ε)', fontsize=8) cb = plt.colorbar(cs, cax=leg_axis) cb.set_ticks([]) leg_axis.yaxis.set_label_position("left") leg_axis.set_ylabel('log likelihood', fontsize=8) leg_axis.invert_yaxis() fig.tight_layout(pad=0.1, h_pad=0.5, w_pad=0.5, rect=(0.01, 0, 1, 1)) fig.savefig(figure_path, format='svg') tools.format_svg_labels(figure_path) if not figure_path.endswith('.svg'): tools.convert_svg(figure_path, figure_path)