def test_conversion_from_distribution_to_dimension() -> None:
sampler = optuna.integration.SkoptSampler()
study = optuna.create_study(sampler=sampler)
with patch("skopt.Optimizer") as mock_object:
study.optimize(_objective, n_trials=2, catch=())
dimensions = [
# Original: trial.suggest_float('p0', -3.3, 5.2)
space.Real(-3.3, 5.2),
# Original: trial.suggest_float('p1', 2.0, 2.0)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_float('p9', 2.2, 2.2, step=0.5)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_categorical('p10', ['9', '3', '0', '8'])
space.Categorical(("9", "3", "0", "8")),
# Original: trial.suggest_float('p2', 0.0001, 0.3, log=True)
space.Real(0.0001, 0.3, prior="log-uniform"),
# Original: trial.suggest_float('p3', 1.1, 1.1, log=True)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_int('p4', -100, 8)
space.Integer(0, 108),
# Original: trial.suggest_int('p5', -20, -20)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_int('p6', 1, 8, log=True)
space.Real(0.5, 8.5, prior="log-uniform"),
# Original: trial.suggest_float('p7', 10, 20, step=2)
space.Integer(0, 5),
# Original: trial.suggest_float('p8', 0.1, 1.0, step=0.1)
space.Integer(0, 8),
]
assert mock_object.mock_calls[0] == call(dimensions)
def __init__(self, search_space, skopt_kwargs):
# type: (Dict[str, BaseDistribution], Dict[str, Any]) -> None
self._search_space = search_space
dimensions = []
for name, distribution in sorted(self._search_space.items()):
# TODO(nzw0301) support IntLogUniform
if isinstance(distribution, distributions.UniformDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high)
elif isinstance(distribution, distributions.LogUniformDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high, prior="log-uniform")
elif isinstance(distribution, distributions.IntUniformDistribution):
count = (distribution.high - distribution.low) // distribution.step
dimension = space.Integer(0, count)
elif isinstance(distribution, distributions.DiscreteUniformDistribution):
count = int((distribution.high - distribution.low) // distribution.q)
dimension = space.Integer(0, count)
elif isinstance(distribution, distributions.CategoricalDistribution):
dimension = space.Categorical(distribution.choices)
else:
raise NotImplementedError(
"The distribution {} is not implemented.".format(distribution)
)
dimensions.append(dimension)
self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def test_skopt_kwargs() -> None:
sampler = optuna.integration.SkoptSampler(skopt_kwargs={"base_estimator": "GBRT"})
study = optuna.create_study(sampler=sampler)
with patch("skopt.Optimizer") as mock_object:
study.optimize(lambda t: t.suggest_int("x", -10, 10), n_trials=2)
dimensions = [space.Integer(0, 20)]
assert mock_object.mock_calls[0] == call(dimensions, base_estimator="GBRT")
def test_skopt_kwargs_dimensions() -> None:
# User specified `dimensions` argument will be ignored in `SkoptSampler`.
sampler = optuna.integration.SkoptSampler(skopt_kwargs={"dimensions": []})
study = optuna.create_study(sampler=sampler)
with patch("skopt.Optimizer") as mock_object:
study.optimize(lambda t: t.suggest_int("x", -10, 10), n_trials=2)
expected_dimensions = [space.Integer(0, 20)]
assert mock_object.mock_calls[0] == call(expected_dimensions)
def __init__(self, search_space: Dict[str, distributions.BaseDistribution],
skopt_kwargs: Dict[str, Any]) -> None:
self._search_space = search_space
dimensions = []
for name, distribution in sorted(self._search_space.items()):
if isinstance(distribution, distributions.IntDistribution):
if distribution.log:
low = distribution.low - 0.5
high = distribution.high + 0.5
dimension = space.Real(low, high, prior="log-uniform")
else:
count = (distribution.high -
distribution.low) // distribution.step
dimension = space.Integer(0, count)
elif isinstance(distribution,
distributions.CategoricalDistribution):
dimension = space.Categorical(distribution.choices)
elif isinstance(distribution, distributions.FloatDistribution):
# Convert the upper bound from exclusive (optuna) to inclusive (skopt).
if distribution.log:
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low,
high,
prior="log-uniform")
elif distribution.step is not None:
count = int((distribution.high - distribution.low) //
distribution.step)
dimension = space.Integer(0, count)
else:
high = np.nextafter(distribution.high, float("-inf"))
dimension = space.Real(distribution.low, high)
else:
raise NotImplementedError(
"The distribution {} is not implemented.".format(
distribution))
dimensions.append(dimension)
self._optimizer = skopt.Optimizer(dimensions, **skopt_kwargs)
def test_conversion_from_distribution_to_dimenstion():
# type: () -> None
sampler = optuna.integration.SkoptSampler()
study = optuna.create_study(sampler=sampler)
with patch('skopt.Optimizer') as mock_object:
study.optimize(_objective, n_trials=2, catch=())
dimensions = [
# Original: trial.suggest_uniform('p0', -3.3, 5.2)
space.Real(-3.3, 5.2),
# Original: trial.suggest_uniform('p1', 2.0, 2.0)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_loguniform('p2', 0.0001, 0.3)
space.Real(0.0001, 0.3, prior='log-uniform'),
# Original: trial.suggest_loguniform('p3', 1.1, 1.1)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_int('p4', -100, 8)
space.Integer(-100, 8),
# Original: trial.suggest_int('p5', -20, -20)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_discrete_uniform('p6', 10, 20, 2)
space.Integer(0, 5),
# Original: trial.suggest_discrete_uniform('p7', 0.1, 1.0, 0.1)
space.Integer(0, 8),
# Original: trial.suggest_discrete_uniform('p8', 2.2, 2.2, 0.5)
# => Skipped because `skopt.Optimizer` cannot handle an empty `Real` dimension.
# Original: trial.suggest_categorical('p9', ['9', '3', '0', '8'])
space.Categorical(('9', '3', '0', '8'))
]
assert mock_object.mock_calls[0] == call(dimensions)
def __init__(self,
hyper_param_conf,
command,
expdir,
exp_recipe_dir,
recipe,
computing,
exp_proposal_watch_dir=None):
base_estimator = 'GP'
self.hyper_param_conf = hyper_param_conf
self.command = command
self.expdir = expdir
self.exp_recipe_dir = exp_recipe_dir
self.recipe = recipe
self.computing = computing
# read the hyper parameter file
hyper_param_cfg = configparser.ConfigParser()
hyper_param_cfg.read(hyper_param_conf)
hyper_info = dict(hyper_param_cfg.items('info'))
self.hyper_param_names = hyper_info['hyper_params'].split(' ')
self.num_iters = int(hyper_info['num_iters'])
self.n_initial_points = int(hyper_info['n_initial_points'])
self.n_initial_points_to_start = int(
hyper_info['n_initial_points_to_start'])
self.max_parallel_jobs = int(hyper_info['max_parallel_jobs'])
self.selected_segment_length = hyper_info['segment_length']
self.selected_task = hyper_info['task']
if 'adapt_hyper_param' in hyper_info:
self.adapt_param = {
'param_name': hyper_info['adapt_hyper_param'],
'param_thr': int(hyper_info['param_thr']),
'par_cnt_scheme': hyper_info['par_cnt_scheme']
}
else:
self.adapt_param = None
hyper_param_dict = dict()
skopt_dims = []
for par_name in self.hyper_param_names:
par_dict = dict(hyper_param_cfg.items(par_name))
par_type = par_dict['type']
if par_type == 'Integer':
skopt_dim = skopt_space.Integer(low=int(par_dict['min']),
high=int(par_dict['max']),
name=par_name)
elif par_type == 'Real':
skopt_dim = skopt_space.Real(low=float(par_dict['min']),
high=float(par_dict['max']),
name=par_name)
elif par_type == 'Categorical':
skopt_dim = skopt_space.Categorical(
categories=par_dict['categories'].split(' '),
name=par_name)
else:
raise ValueError('Type %s is not a valid parameter type' %
par_type)
hyper_param_dict[par_name] = par_dict
skopt_dims.append(skopt_dim)
self.hyper_param_dict = hyper_param_dict
self.skopt_dims = skopt_dims
self.last_result = None
# self.all_results = []
self.start_new_run_flag = True
self.iter_ind = 0
self.watch_list = dict()
self.all_dim_values = []
self.all_losses = dict()
self.n_job_running = 0
self.n_initial_points_started = 0
self.n_unsuitable_points_for_estimator = 0
self.max_n_unsuitable_points_for_estimator = 10000
self.unsuitable_runs = []
self.lost_runs = []
self.exp_proposal_watch_dir = exp_proposal_watch_dir
self.use_proposal_run = False
self.proposed_loss_runs = []
# only 0.25% of the point sample in the hyper space are wanted (since they lead to rougly the wanted amount of
# trainable parameters)
self.acq_optimizer_kwargs = {'n_points': 4000000}
if 'debug' in expdir:
self.acq_optimizer_kwargs = {'n_points': 40000}
if base_estimator == 'boundedGP':
# Make own estimator based on Gaussian Process Regressor.
if skopt_dims is not None:
space = Space(skopt_dims)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] *
n_dims,
nu=2.5)
base_estimator = BoundedGaussianProcessRegressor(
space,
self.hyper_param_names,
self.adapt_param,
kernel=cov_amplitude * other_kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
super(HyperParamOptimizer,
self).__init__(skopt_dims,
base_estimator=base_estimator,
n_initial_points=self.n_initial_points,
acq_optimizer_kwargs=self.acq_optimizer_kwargs)
