def __init__(self, dim=2, bounds=None,
noise_std=0, random_state=None):
self.dim = dim
params = {'x%d' % i: (0, 10, 5) for i in range(1, 1+self.dim)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std, optimal_value=0, random_state=random_state)
def get_cs_bc():
cs_bc = ConfigurationSpace()
x0 = UniformFloatHyperparameter("x0", scale1[0], scale1[1])
# x0 = UniformIntegerHyperparameter("x0", scale1[0], scale1[1]) # test int
x1 = UniformFloatHyperparameter("x1", scale2[0], scale2[1])
cs_bc.add_hyperparameters([x0, x1])
return cs_bc
def impute_default_values(configuration_space: ConfigurationSpace,
configs_array: np.ndarray) -> np.ndarray:
"""Impute inactive hyperparameters in configuration array with their default.
Necessary to apply an EPM to the data.
Parameters
----------
configuration_space : ConfigurationSpace
configs_array : np.ndarray
Array of configurations.
Returns
-------
np.ndarray
Array with configuration hyperparameters. Inactive values are imputed
with their default value.
"""
for hp in configuration_space.get_hyperparameters():
default = hp.normalized_default_value
idx = configuration_space.get_idx_by_hyperparameter_name(hp.name)
nonfinite_mask = ~np.isfinite(configs_array[:, idx])
configs_array[nonfinite_mask, idx] = default
return configs_array
def get_configspace(self):
hp_num = self._hp_cnt + self._delta
if hp_num > self.hp_size:
hp_num = self.hp_size
hps = self.config_space.get_hyperparameters()
cs = ConfigurationSpace()
for _id in range(hp_num):
_hp_id = self.importance_list[_id]
for _hp in hps:
if _hp.name == _hp_id:
cs.add_hyperparameter(_hp)
history_list = list()
if len(self.history_dict.keys()) > 0 and self._hp_cnt < self.hp_size:
for _config in self.history_dict.keys():
# Impute the default value for new hyperparameter.
_config_dict = _config.get_dictionary().copy()
for _idx in range(self._hp_cnt, hp_num):
new_hp = self.importance_list[_idx]
# print('hp_num=', self._hp_cnt, 'new hp is', new_hp)
_config_dict[new_hp] = self.defaults[new_hp]
history_list.append((_config_dict, self.history_dict[_config]))
if len(history_list) == 0:
history_list = [(_config, self.history_dict[_config])
for _config in self.history_dict.keys()]
return cs, history_list
def __init__(self, noise_std=0, random_state=None):
lb, ub = -4.5, 4.5
dim = 2
params = {'x%d' % i: (lb, ub, (lb + ub)/2)
for i in range(1, dim+1)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std, optimal_value=0, random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
params = {'x1': (-10, 0, -5), 'x2': (-6.5, 0, -3.25)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
optimal_value=-106.7645367,
optimal_point=[(-3.1302468, -1.5821422)],
random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
params = {'x%d' % i: (-1.25, 1.25, 1) for i in [1, 2]}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
optimal_value=-0.072,
optimal_point=[(0.84852813, -0.84852813), (-0.84852813, 0.84852813)],
random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
params = {'x1': (-5, 10, 0),
'x2': (0, 15, 0)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
optimal_value=0.397887,
optimal_point=[(-np.pi, 12.275), (np.pi, 2.275), (9.42478, 2.475)],
random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
self.ref_point = [1864.72022, 11.81993945, 0.2903999384]
params = {'x%d' % i: (1.0, 3.0) for i in range(1, 6)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
num_objs=3,
random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
params = {'x1': (-15.0, -5.0, -10.0),
'x2': (-3.0, 3.0, 0)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
optimal_value=0,
optimal_point=[(-10.0, 1.0)],
random_state=random_state)
def __init__(self, dim: int, num_constraints=0, noise_std=0, random_state=None):
self.dim = dim
self.ref_point = [11.0, 11.0]
params = {'x%d' % i: (0, 1) for i in range(1, dim+1)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
num_objs=2, num_constraints=num_constraints,
random_state=random_state)
def __init__(self, dim=2, constrained=False, noise_std=0, random_state=None):
self.dim = dim
self.constrained = constrained
params = {'x%d' % i: (-5.0, 10.0, 2.5) for i in range(1, 1+self.dim)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
optimal_value=0,
optimal_point=[tuple(1.0 for _ in range(self.dim))],
random_state=random_state)
def __init__(self, noise_std=0, random_state=None):
self.ref_point = [10.0, 10.0]
params = {'x1': (0.1, 10.0),
'x2': (0.0, 5.0)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
num_objs=2, num_constraints=2,
random_state=random_state)
def get_xgboost_config_space(task_type='cls'):
if task_type == 'cls':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
100,
1000,
q=50,
default_value=500)
max_depth = UniformIntegerHyperparameter("max_depth", 1, 12)
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-3,
0.9,
log=True,
default_value=0.1)
min_child_weight = UniformFloatHyperparameter("min_child_weight",
0,
10,
q=0.1,
default_value=1)
subsample = UniformFloatHyperparameter("subsample",
0.1,
1,
q=0.1,
default_value=1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.1,
1,
q=0.1,
default_value=1)
gamma = UniformFloatHyperparameter("gamma",
0,
10,
q=0.1,
default_value=0)
reg_alpha = UniformFloatHyperparameter("reg_alpha",
0,
10,
q=0.1,
default_value=0)
reg_lambda = UniformFloatHyperparameter("reg_lambda",
1,
10,
q=0.1,
default_value=1)
cs.add_hyperparameters([
n_estimators, max_depth, learning_rate, min_child_weight,
subsample, colsample_bytree, gamma, reg_alpha, reg_lambda
])
return cs
elif task_type == 'rgs':
raise NotImplementedError
else:
raise ValueError('Unsupported task type: %s.' % (task_type, ))
def __init__(self, constrained=False, noise_std=0, random_state=None):
self.ref_point = [18.0, 6.0]
self.constrained = constrained
num_constraints = 1 if self.constrained else 0
params = {'x1': (0, 1, 0.5),
'x2': (0, 1, 0.5)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std,
num_objs=2,
num_constraints=num_constraints,
random_state=random_state)
def __init__(self, dim, num_objs=2, num_constraints=0, noise_std=0, random_state=None):
if dim <= num_objs:
raise ValueError(
"dim must be > num_objs, but got %s and %s" % (dim, num_objs)
)
self.dim = dim
self.k = self.dim - num_objs + 1
self.bounds = [(0.0, 1.0) for _ in range(self.dim)]
self.ref_point = [self._ref_val for _ in range(num_objs)]
params = {'x%d' % i: (0, 1, i/dim) for i in range(1, dim+1)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std, num_objs, num_constraints, random_state=random_state)
def string2config_space(space_desc: str):
line_list = space_desc.split('\n')
cur_line = 2
cs = ConfigurationSpace()
status = 'hp'
hp_list = list()
while cur_line != len(line_list) - 1:
line_content = line_list[cur_line]
if line_content == ' Conditions:':
hp_dict = {hp.name: hp for hp in hp_list}
status = 'cond'
elif line_content == ' Forbidden Clauses:':
status = 'bid'
else:
if status == 'hp':
hp = string2hyperparameter(line_content)
hp_list.append(hp)
cs.add_hyperparameter(hp)
elif status == 'cond':
cond = string2condition(line_content, hp_dict)
cs.add_condition(cond)
else:
forbid = string2forbidden(line_content, hp_dict)
cs.add_forbidden_clause(forbid)
cur_line += 1
return cs
def __init__(self, dim=2, bounds=None, constrained=False,
noise_std=0, random_state=None):
self.constrained = constrained
if bounds is None:
if constrained:
lb, ub = -5, 10
else:
lb, ub = -10, 15
else:
lb, ub = bounds
params = {'x%d' % i: (lb, ub, (lb + ub)/2)
for i in range(1, dim+1)}
config_space = ConfigurationSpace()
config_space.add_hyperparameters([UniformFloatHyperparameter(k, *v) for k, v in params.items()])
super().__init__(config_space, noise_std, optimal_value=0, random_state=random_state)
def get_hyperparameter_search_space(space_size='large'):
"""
['n_estimators', 'learning_rate', 'max_depth', 'colsample_bytree', 'gamma',
'min_child_weight', 'reg_alpha', 'reg_lambda', 'subsample']
"""
cs = ConfigurationSpace()
if space_size == 'large':
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, q=10, default_value=500)
learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.9, log=True, default_value=0.1)
max_depth = UniformIntegerHyperparameter("max_depth", 1, 12)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.1, 1, q=0.1, default_value=1)
gamma = UniformFloatHyperparameter("gamma", 0, 10, q=0.1, default_value=0)
min_child_weight = UniformFloatHyperparameter("min_child_weight", 0, 10, q=0.1, default_value=1)
reg_alpha = UniformFloatHyperparameter("reg_alpha", 0, 10, q=0.1, default_value=0)
reg_lambda = UniformFloatHyperparameter("reg_lambda", 1, 10, q=0.1, default_value=1)
subsample = UniformFloatHyperparameter("subsample", 0.1, 1, q=0.1, default_value=1)
elif space_size == 'medium':
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, q=10, default_value=500)
learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.9, log=True, default_value=0.1)
max_depth = UniformIntegerHyperparameter("max_depth", 1, 12)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.1, 1, q=0.1, default_value=1)
gamma = UniformFloatHyperparameter("gamma", 0, 10, q=0.1, default_value=0)
min_child_weight = UniformFloatHyperparameter("min_child_weight", 0, 10, q=0.1, default_value=1)
reg_alpha = UnParametrizedHyperparameter("reg_alpha", 0)
reg_lambda = UnParametrizedHyperparameter("reg_lambda", 1)
subsample = UnParametrizedHyperparameter("subsample", 1)
else:
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, q=10, default_value=500)
learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.9, log=True, default_value=0.1)
max_depth = UniformIntegerHyperparameter("max_depth", 1, 12)
colsample_bytree = UnParametrizedHyperparameter("colsample_bytree", 1)
gamma = UnParametrizedHyperparameter("gamma", 0)
min_child_weight = UnParametrizedHyperparameter("min_child_weight", 1)
reg_alpha = UnParametrizedHyperparameter("reg_alpha", 0)
reg_lambda = UnParametrizedHyperparameter("reg_lambda", 1)
subsample = UnParametrizedHyperparameter("subsample", 1)
cs.add_hyperparameters([n_estimators, max_depth, learning_rate, min_child_weight, subsample,
colsample_bytree, gamma, reg_alpha, reg_lambda])
return cs
def config_space2string(config_space: ConfigurationSpace):
pattern = r'[,|{}\'=<>&]'
for hp in config_space.get_hyperparameters():
if re.search(pattern, hp.name):
raise NameError('Invalid character in hyperparameter name!')
if hasattr(hp, 'choices'):
for value in hp.choices:
if re.search(pattern, value):
raise NameError('Invalid character in categorical hyperparameter value!')
return str(config_space)
def sample_configurations(configuration_space: ConfigurationSpace,
num: int) -> List[Configuration]:
result = []
cnt = 0
while cnt < num:
config = configuration_space.sample_configuration(1)
if config not in result:
result.append(config)
cnt += 1
return result
def expand_configurations(configs: List[Configuration],
configuration_space: ConfigurationSpace, num: int):
num_config = len(configs)
num_needed = num - num_config
config_cnt = 0
while config_cnt < num_needed:
config = configuration_space.sample_configuration(1)
if config not in configs:
configs.append(config)
config_cnt += 1
return configs
def get_lightgbm_config_space(task_type='cls'):
if task_type == 'cls':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
100,
1000,
default_value=500,
q=50)
num_leaves = UniformIntegerHyperparameter("num_leaves",
31,
2047,
default_value=128)
max_depth = Constant('max_depth', 15)
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-3,
0.3,
default_value=0.1,
log=True)
min_child_samples = UniformIntegerHyperparameter("min_child_samples",
5,
30,
default_value=20)
subsample = UniformFloatHyperparameter("subsample",
0.7,
1,
default_value=1,
q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.7,
1,
default_value=1,
q=0.1)
cs.add_hyperparameters([
n_estimators, num_leaves, max_depth, learning_rate,
min_child_samples, subsample, colsample_bytree
])
return cs
elif task_type == 'rgs':
raise NotImplementedError
else:
raise ValueError('Unsupported task type: %s.' % (task_type, ))
def sample_configurations(configuration_space: ConfigurationSpace, num: int,
excluded_configs: List[Configuration] = None) -> List[Configuration]:
if excluded_configs is None:
excluded_configs = []
result = []
cnt = 0
while cnt < num:
config = configuration_space.sample_configuration(1)
if config not in result and config not in excluded_configs:
result.append(config)
cnt += 1
return result
def expand_configurations(configs: List[Configuration], configuration_space: ConfigurationSpace, num: int,
excluded_configs: List[Configuration] = None):
if excluded_configs is None:
excluded_configs = []
num_config = len(configs)
num_needed = num - num_config
config_cnt = 0
while config_cnt < num_needed:
config = configuration_space.sample_configuration(1)
if config not in configs and config not in excluded_configs:
configs.append(config)
config_cnt += 1
return configs
def get_cs_lightgbm(): # todo q and int for compare?
cs = ConfigurationSpace()
n_estimators = UniformFloatHyperparameter("n_estimators",
100,
1000,
default_value=500,
q=50)
num_leaves = UniformIntegerHyperparameter("num_leaves",
31,
2047,
default_value=128)
# max_depth = Constant('max_depth', 15)
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-3,
0.3,
default_value=0.1,
log=True)
min_child_samples = UniformIntegerHyperparameter("min_child_samples",
5,
30,
default_value=20)
subsample = UniformFloatHyperparameter("subsample",
0.7,
1,
default_value=1,
q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.7,
1,
default_value=1,
q=0.1)
# cs.add_hyperparameters([n_estimators, num_leaves, max_depth, learning_rate, min_child_samples, subsample,
# colsample_bytree])
cs.add_hyperparameters([
n_estimators, num_leaves, learning_rate, min_child_samples,
subsample, colsample_bytree
])
return cs
def get_config_space_from_dict(space_dict: dict):
cs = ConfigurationSpace()
params_dict = space_dict['parameters']
for key in params_dict:
param_dict = params_dict[key]
param_type = param_dict['type']
if param_type in ['float', 'int']:
bound = param_dict['bound']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
if 'log' in param_dict:
optional_args['log'] = parse_bool(param_dict['log'])
if 'q' in param_dict:
optional_args['q'] = param_dict['q']
if param_type == 'float':
param = UniformFloatHyperparameter(key, bound[0], bound[1], **optional_args)
else:
param = UniformIntegerHyperparameter(key, bound[0], bound[1], **optional_args)
elif param_type == 'cat':
choices = param_dict['choice']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
param = CategoricalHyperparameter(key, choices, **optional_args)
elif param_type == 'const':
value = param_dict['value']
param = Constant(key, value)
else:
raise ValueError("Parameter type %s not supported!" % param_type)
cs.add_hyperparameter(param)
return cs
def sample_configuration(configuration_space: ConfigurationSpace, excluded_configs: List[Configuration] = None):
"""
sample one config not in excluded_configs
"""
if excluded_configs is None:
excluded_configs = []
sample_cnt = 0
max_sample_cnt = 1000
while True:
config = configuration_space.sample_configuration()
sample_cnt += 1
if config not in excluded_configs:
break
if sample_cnt >= max_sample_cnt:
raise ValueError('Cannot sample non duplicate configuration after %d iterations.' % max_sample_cnt)
return config
def __init__(self, bounds=None, noise_std=0, random_state=None):
if bounds is None:
bounds = [0, 20]
lb, ub = bounds
config_space = ConfigurationSpace()
config_space.add_hyperparameter(UniformFloatHyperparameter('x1', lb, ub, 1))
config_space.add_hyperparameter(UniformIntegerHyperparameter('x2', lb, ub, 1))
super().__init__(config_space, noise_std,
optimal_value=-31.9998,
optimal_point=[(5.333, 4)],
random_state=random_state)
def __init__(self,
config_space: ConfigurationSpace,
size,
lower_bounds=None,
upper_bounds=None,
random_state=None):
"""
Parameters
----------
config_space : ConfigurationSpace
ConfigurationSpace to do sampling.
size : int N
Number of samples.
lower_bounds : lower bounds in [0, 1] for continuous dimensions (optional)
upper_bounds : upper bounds in [0, 1] for continuous dimensions (optional)
"""
self.config_space = config_space
self.search_dims = []
for i, param in enumerate(config_space.get_hyperparameters()):
if isinstance(param, UniformFloatHyperparameter):
self.search_dims.append((0.0, 1.0))
elif isinstance(param, UniformIntegerHyperparameter):
self.search_dims.append((0.0, 1.0))
else:
raise NotImplementedError(
'Only Integer and Float are supported in %s.' %
self.__class__.__name__)
self.size = size
default_lb, default_ub = zip(*self.search_dims)
self.lower_bounds = np.array(
default_lb) if lower_bounds is None else np.clip(
lower_bounds, default_lb, default_ub)
self.upper_bounds = np.array(
default_ub) if upper_bounds is None else np.clip(
upper_bounds, default_lb, default_ub)
self.rng = check_random_state(random_state)
