def test_conversion_from_distribution_to_dimension() -> None: sampler = optuna.integration.SkoptSampler() study = optuna.create_study(sampler=sampler) with patch("skopt.Optimizer") as mock_object: study.optimize(_objective, n_trials=2, catch=()) dimensions = [ # Original: trial.suggest_float('p0', -3.3, 5.2) space.Real(-3.3, 5.2), # Original: trial.suggest_float('p1', 2.0, 2.0) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_float('p9', 2.2, 2.2, step=0.5) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_categorical('p10', ['9', '3', '0', '8']) space.Categorical(("9", "3", "0", "8")), # Original: trial.suggest_float('p2', 0.0001, 0.3, log=True) space.Real(0.0001, 0.3, prior="log-uniform"), # Original: trial.suggest_float('p3', 1.1, 1.1, log=True) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_int('p4', -100, 8) space.Integer(0, 108), # Original: trial.suggest_int('p5', -20, -20) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_int('p6', 1, 8, log=True) space.Real(0.5, 8.5, prior="log-uniform"), # Original: trial.suggest_float('p7', 10, 20, step=2) space.Integer(0, 5), # Original: trial.suggest_float('p8', 0.1, 1.0, step=0.1) space.Integer(0, 8), ] assert mock_object.mock_calls[0] == call(dimensions)
def __init__(self, search_space, skopt_kwargs): # type: (Dict[str, BaseDistribution], Dict[str, Any]) -> None self._search_space = search_space dimensions = [] for name, distribution in sorted(self._search_space.items()): # TODO(nzw0301) support IntLogUniform if isinstance(distribution, distributions.UniformDistribution): # Convert the upper bound from exclusive (optuna) to inclusive (skopt). high = np.nextafter(distribution.high, float("-inf")) dimension = space.Real(distribution.low, high) elif isinstance(distribution, distributions.LogUniformDistribution): # Convert the upper bound from exclusive (optuna) to inclusive (skopt). high = np.nextafter(distribution.high, float("-inf")) dimension = space.Real(distribution.low, high, prior="log-uniform") elif isinstance(distribution, distributions.IntUniformDistribution): count = (distribution.high - distribution.low) // distribution.step dimension = space.Integer(0, count) elif isinstance(distribution, distributions.DiscreteUniformDistribution): count = int((distribution.high - distribution.low) // distribution.q) dimension = space.Integer(0, count) elif isinstance(distribution, distributions.CategoricalDistribution): dimension = space.Categorical(distribution.choices) else: raise NotImplementedError( "The distribution {} is not implemented.".format(distribution) ) dimensions.append(dimension) self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def test_skopt_kwargs() -> None: sampler = optuna.integration.SkoptSampler(skopt_kwargs={"base_estimator": "GBRT"}) study = optuna.create_study(sampler=sampler) with patch("skopt.Optimizer") as mock_object: study.optimize(lambda t: t.suggest_int("x", -10, 10), n_trials=2) dimensions = [space.Integer(0, 20)] assert mock_object.mock_calls[0] == call(dimensions, base_estimator="GBRT")
def test_skopt_kwargs_dimensions() -> None: # User specified `dimensions` argument will be ignored in `SkoptSampler`. sampler = optuna.integration.SkoptSampler(skopt_kwargs={"dimensions": []}) study = optuna.create_study(sampler=sampler) with patch("skopt.Optimizer") as mock_object: study.optimize(lambda t: t.suggest_int("x", -10, 10), n_trials=2) expected_dimensions = [space.Integer(0, 20)] assert mock_object.mock_calls[0] == call(expected_dimensions)
def __init__(self, search_space: Dict[str, distributions.BaseDistribution], skopt_kwargs: Dict[str, Any]) -> None: self._search_space = search_space dimensions = [] for name, distribution in sorted(self._search_space.items()): if isinstance(distribution, distributions.IntDistribution): if distribution.log: low = distribution.low - 0.5 high = distribution.high + 0.5 dimension = space.Real(low, high, prior="log-uniform") else: count = (distribution.high - distribution.low) // distribution.step dimension = space.Integer(0, count) elif isinstance(distribution, distributions.CategoricalDistribution): dimension = space.Categorical(distribution.choices) elif isinstance(distribution, distributions.FloatDistribution): # Convert the upper bound from exclusive (optuna) to inclusive (skopt). if distribution.log: high = np.nextafter(distribution.high, float("-inf")) dimension = space.Real(distribution.low, high, prior="log-uniform") elif distribution.step is not None: count = int((distribution.high - distribution.low) // distribution.step) dimension = space.Integer(0, count) else: high = np.nextafter(distribution.high, float("-inf")) dimension = space.Real(distribution.low, high) else: raise NotImplementedError( "The distribution {} is not implemented.".format( distribution)) dimensions.append(dimension) self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def test_conversion_from_distribution_to_dimenstion(): # type: () -> None sampler = optuna.integration.SkoptSampler() study = optuna.create_study(sampler=sampler) with patch('skopt.Optimizer') as mock_object: study.optimize(_objective, n_trials=2, catch=()) dimensions = [ # Original: trial.suggest_uniform('p0', -3.3, 5.2) space.Real(-3.3, 5.2), # Original: trial.suggest_uniform('p1', 2.0, 2.0) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_loguniform('p2', 0.0001, 0.3) space.Real(0.0001, 0.3, prior='log-uniform'), # Original: trial.suggest_loguniform('p3', 1.1, 1.1) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_int('p4', -100, 8) space.Integer(-100, 8), # Original: trial.suggest_int('p5', -20, -20) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_discrete_uniform('p6', 10, 20, 2) space.Integer(0, 5), # Original: trial.suggest_discrete_uniform('p7', 0.1, 1.0, 0.1) space.Integer(0, 8), # Original: trial.suggest_discrete_uniform('p8', 2.2, 2.2, 0.5) # => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension. # Original: trial.suggest_categorical('p9', ['9', '3', '0', '8']) space.Categorical(('9', '3', '0', '8')) ] assert mock_object.mock_calls[0] == call(dimensions)
def __init__(self, hyper_param_conf, command, expdir, exp_recipe_dir, recipe, computing, exp_proposal_watch_dir=None): base_estimator = 'GP' self.hyper_param_conf = hyper_param_conf self.command = command self.expdir = expdir self.exp_recipe_dir = exp_recipe_dir self.recipe = recipe self.computing = computing # read the hyper parameter file hyper_param_cfg = configparser.ConfigParser() hyper_param_cfg.read(hyper_param_conf) hyper_info = dict(hyper_param_cfg.items('info')) self.hyper_param_names = hyper_info['hyper_params'].split(' ') self.num_iters = int(hyper_info['num_iters']) self.n_initial_points = int(hyper_info['n_initial_points']) self.n_initial_points_to_start = int( hyper_info['n_initial_points_to_start']) self.max_parallel_jobs = int(hyper_info['max_parallel_jobs']) self.selected_segment_length = hyper_info['segment_length'] self.selected_task = hyper_info['task'] if 'adapt_hyper_param' in hyper_info: self.adapt_param = { 'param_name': hyper_info['adapt_hyper_param'], 'param_thr': int(hyper_info['param_thr']), 'par_cnt_scheme': hyper_info['par_cnt_scheme'] } else: self.adapt_param = None hyper_param_dict = dict() skopt_dims = [] for par_name in self.hyper_param_names: par_dict = dict(hyper_param_cfg.items(par_name)) par_type = par_dict['type'] if par_type == 'Integer': skopt_dim = skopt_space.Integer(low=int(par_dict['min']), high=int(par_dict['max']), name=par_name) elif par_type == 'Real': skopt_dim = skopt_space.Real(low=float(par_dict['min']), high=float(par_dict['max']), name=par_name) elif par_type == 'Categorical': skopt_dim = skopt_space.Categorical( categories=par_dict['categories'].split(' '), name=par_name) else: raise ValueError('Type %s is not a valid parameter type' % par_type) hyper_param_dict[par_name] = par_dict skopt_dims.append(skopt_dim) self.hyper_param_dict = hyper_param_dict self.skopt_dims = skopt_dims self.last_result = None # self.all_results = [] self.start_new_run_flag = True self.iter_ind = 0 self.watch_list = dict() self.all_dim_values = [] self.all_losses = dict() self.n_job_running = 0 self.n_initial_points_started = 0 self.n_unsuitable_points_for_estimator = 0 self.max_n_unsuitable_points_for_estimator = 10000 self.unsuitable_runs = [] self.lost_runs = [] self.exp_proposal_watch_dir = exp_proposal_watch_dir self.use_proposal_run = False self.proposed_loss_runs = [] # only 0.25% of the point sample in the hyper space are wanted (since they lead to rougly the wanted amount of # trainable parameters) self.acq_optimizer_kwargs = {'n_points': 4000000} if 'debug' in expdir: self.acq_optimizer_kwargs = {'n_points': 40000} if base_estimator == 'boundedGP': # Make own estimator based on Gaussian Process Regressor. if skopt_dims is not None: space = Space(skopt_dims) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern(length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = BoundedGaussianProcessRegressor( space, self.hyper_param_names, self.adapt_param, kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) super(HyperParamOptimizer, self).__init__(skopt_dims, base_estimator=base_estimator, n_initial_points=self.n_initial_points, acq_optimizer_kwargs=self.acq_optimizer_kwargs)