def test_check_neighbouring_config_diamond_str(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', ['red', 'green'])
left = CategoricalHyperparameter('left', ['red', 'green'])
right = CategoricalHyperparameter('right', ['red', 'green', 'blue', 'yellow'])
bottom = CategoricalHyperparameter('bottom', ['red', 'green'])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 'red'))
diamond.add_condition(EqualsCondition(right, head, 'red'))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 'green'),
EqualsCondition(bottom, right, 'green')))
config = Configuration(diamond, {'bottom': 'red', 'head': 'red', 'left': 'green', 'right': 'green'})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = ConfigSpace.c_util.change_hp_value(
diamond,
config.get_array(),
hp_name,
neighbor_value,
index
)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(NormalFloatHyperparameter('b', 1, 2,
log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def test_add_hyperparameters_with_equal_names(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
self.assertRaisesRegex(ValueError,
"Hyperparameter 'name' is already in the "
"configuration space.",
cs.add_hyperparameter, hp)
def setUp(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("parent", [0, 1]))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
hp3 = cs.add_hyperparameter(
UniformIntegerHyperparameter("friend", 0, 5))
self.cs = cs
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def test_illegal_default_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default='l1')
hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default='l1')
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
forb1 = ForbiddenEqualsClause(hp1, "l1")
forb2 = ForbiddenEqualsClause(hp2, "l1")
forb3 = ForbiddenAndConjunction(forb1, forb2)
# cs.add_forbidden_clause(forb3)
self.assertRaisesRegexp(ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && "
"Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def deactivate_inactive_hyperparameters(configuration: dict,
configuration_space: ConfigurationSpace):
hyperparameters = configuration_space.get_hyperparameters()
configuration = Configuration(configuration_space=configuration_space,
values=configuration,
allow_inactive_with_values=True)
hps = deque()
unconditional_hyperparameters = configuration_space.get_all_unconditional_hyperparameters()
hyperparameters_with_children = list()
for uhp in unconditional_hyperparameters:
children = configuration_space._children_of[uhp]
if len(children) > 0:
hyperparameters_with_children.append(uhp)
hps.extendleft(hyperparameters_with_children)
inactive = set()
while len(hps) > 0:
hp = hps.pop()
children = configuration_space._children_of[hp]
for child in children:
conditions = configuration_space._parent_conditions_of[child.name]
for condition in conditions:
if not condition.evaluate_vector(configuration.get_array()):
dic = configuration.get_dictionary()
try:
del dic[child.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
inactive.add(child.name)
hps.appendleft(child.name)
for hp in hyperparameters:
if hp.name in inactive:
dic = configuration.get_dictionary()
try:
del dic[hp.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
return Configuration(configuration_space, values=configuration.get_dictionary())
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_add_forbidden_clause(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
forb = ForbiddenEqualsClause(hp1, 1)
# TODO add checking whether a forbidden clause makes sense at all
cs.add_forbidden_clause(forb)
# TODO add something to properly retrieve the forbidden clauses
self.assertEqual(str(cs), "Configuration space object:\n "
"Hyperparameters:\n input1, "
"Type: Categorical, Choices: {0, 1}, "
"Default: 0\n"
" Forbidden Clauses:\n"
" Forbidden: input1 == 1\n")
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10)
x2 = UniformFloatHyperparameter("x2", 0, 15)
cs.add_hyperparameters([x1, x2])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'x1': hp.uniform('hp_x1', -5, 10),
'x2': hp.uniform('hp_x2', 0, 15),
}
return space
elif optimizer == 'gpflowopt':
import gpflowopt
domain = (gpflowopt.domain.ContinuousParameter('x1', -5, 10) +
gpflowopt.domain.ContinuousParameter('x2', 0, 15))
return domain
else:
raise ValueError('Unknown optimizer %s when getting configspace' %
optimizer)
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def get_runhistory(self, num_success, num_capped, num_timeout):
cs = ConfigurationSpace()
cs.add_hyperparameter(
CategoricalHyperparameter(name="cat_a_b",
choices=["a", "b"],
default_value="a"))
cs.add_hyperparameter(
UniformFloatHyperparameter(name="float_0_1",
lower=0,
upper=1,
default_value=0.5))
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='integer_0_100',
lower=-10,
upper=10,
default_value=0))
rh = runhistory.RunHistory()
rs = numpy.random.RandomState(1)
successes = 0
capped = 0
timeouts = 0
while successes < num_success or capped < num_capped or timeouts < num_timeout:
config, seed, runtime, status, instance_id = \
generate_config(cs=cs, rs=rs)
if status == StatusType.SUCCESS and successes < num_success:
successes += 1
add = True
elif status == StatusType.TIMEOUT:
if runtime < 40 and capped < num_capped:
capped += 1
add = True
elif runtime == 40 and timeouts < num_timeout:
timeouts += 1
add = True
else:
add = False
else:
add = False
if add:
rh.add(config=config,
cost=runtime,
time=runtime,
status=status,
instance_id=instance_id,
seed=seed,
additional_info=None)
return cs, rh
def _get_configuration_space(self, builtin_transformers, trans_type=None):
config_dict = dict()
for tran_key in builtin_transformers:
tran = builtin_transformers[tran_key]
tran_id = tran().type
if trans_type is None or tran_id in trans_type:
try:
sub_configuration_space = builtin_transformers[
tran_key].get_hyperparameter_search_space()
config_dict[tran_key] = sub_configuration_space
except:
config_dict[tran_key] = ConfigurationSpace()
return config_dict
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter('n_estimators',
50,
1000,
default_value=100)
criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"],
default_value="gini")
max_features = UniformFloatHyperparameter("max_features",
0.,
1.,
default_value=0.5)
max_depth = UniformIntegerHyperparameter("max_depth",
4,
12,
default_value=5)
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
"min_weight_fraction_leaf", 0.)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"],
default_value="True")
cs.add_hyperparameters([
n_estimators, criterion, max_features, max_depth,
min_samples_split, min_samples_leaf, min_weight_fraction_leaf,
max_leaf_nodes, bootstrap, min_impurity_decrease
])
return cs
def test_random_neighborhood_int(self):
hp = UniformIntegerHyperparameter('a', 1, 10)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
self.assertAlmostEqual(5.8125, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(5.60234375, np.var(all_neighbors), places=2)
hp = UniformIntegerHyperparameter('a', 1, 10, log=True)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
# Default value is 3.16
self.assertAlmostEqual(3.9425, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(5.91, np.var(all_neighbors), places=2)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp)
for val in range(1, 11):
config = Configuration(cs, values={'a': val})
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, 1)
neighbors = [neighbor['a'] for neighbor in neighborhood]
self.assertEqual(len(neighbors), len(np.unique(neighbors)))
self.assertNotIn(val, neighbors)
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_check_configuration_input_checking(self):
cs = ConfigurationSpace()
self.assertRaisesRegexp(
TypeError, "The method check_configuration must be called "
"with an instance of %s. "
"Your input was of type %s" % (Configuration, type("String")),
cs.check_configuration, "String")
# For the check configuration method with vector representation
self.assertRaisesRegexp(
TypeError, "The method check_configuration must"
" be called with an instance of "
"np.ndarray Your input was of type %s" % (type("String")),
cs.check_configuration_vector_representation, "String")
def test_multi_sample_quantized_uihp(self):
# This unit test covers a problem with sampling multiple entries at a time from a configuration space with at
# least one UniformIntegerHyperparameter which is quantized.
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter("uihp", lower=1, upper=100, q=2, log=False))
self.assertIsNotNone(cs.sample_configuration(size=1))
self.assertEqual(10, len(cs.sample_configuration(size=10)))
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegex(ValueError, r"Hyperparameter configuration "
r"contains a cycle \[\['child', 'parent'\]\]",
cs.add_condition, cond2)
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def _check_and_cast_fidelity(fidelity: Union[dict, ConfigSpace.Configuration, None],
fidelity_space: ConfigSpace.ConfigurationSpace, **kwargs) \
-> ConfigSpace.Configuration:
""" Helper-function to evaluate the given fidelity object.
Similar to the checking and casting from above, we validate the fidelity object. To do so, we cast it to a
ConfigSpace.Configuration object.
If the fidelity is not specified (None), then we use the default fidelity of the benchmark.
If the benchmark is a multi-multi-fidelity benchmark and only a subset of the available fidelities is
specified, we fill the missing ones with their default values.
"""
# Make a check, that no fidelities are in the kwargs.
f_in_kwargs = []
for f in fidelity_space.get_hyperparameters():
if f.name in kwargs:
f_in_kwargs.append(f.name)
if len(f_in_kwargs) != 0:
raise ValueError(f'Fidelity parameters {", ".join(f_in_kwargs)} should not be part of kwargs\n'
f'Fidelity: {fidelity}\n Kwargs: {kwargs}')
default_fidelities = fidelity_space.get_default_configuration()
if fidelity is None:
fidelity = default_fidelities
if isinstance(fidelity, dict):
default_fidelities_cfg = default_fidelities.get_dictionary()
fidelity_copy = fidelity.copy()
fidelity = {k: fidelity_copy.pop(k, v) for k, v in default_fidelities_cfg.items()}
assert len(fidelity_copy) == 0, 'Provided fidelity dict contained unknown fidelity ' \
f'values: {fidelity_copy.keys()}'
fidelity = ConfigSpace.Configuration(fidelity_space, fidelity)
elif isinstance(fidelity, ConfigSpace.Configuration):
fidelity = fidelity
else:
raise TypeError(f'Fidelity has to be an instance of type None, dict, or '
f'ConfigSpace.Configuration but was {type(fidelity)}')
# Ensure that the extracted fidelity values play well with the defined fidelity space
fidelity_space.check_configuration(fidelity)
return fidelity
def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, default_value=500, q=50)
num_leaves = UniformIntegerHyperparameter("num_leaves", 31, 2047, default_value=128)
max_depth = Constant('max_depth', 15)
learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.3, default_value=0.1, log=True)
min_child_samples = UniformIntegerHyperparameter("min_child_samples", 5, 30, default_value=20)
subsample = UniformFloatHyperparameter("subsample", 0.7, 1, default_value=1, q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.7, 1, default_value=1, q=0.1)
cs.add_hyperparameters([n_estimators, num_leaves, max_depth, learning_rate, min_child_samples, subsample,
colsample_bytree])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': (hp.randint('lgb_n_estimators', 19) + 2) * 50,
'num_leaves': hp.randint('lgb_num_leaves', 2017) + 31,
'max_depth': 15,
'learning_rate': hp.loguniform('lgb_learning_rate', np.log(1e-3), np.log(0.3)),
'min_child_samples': hp.randint('lgb_min_child_samples', 26) + 5,
'subsample': (hp.randint('lgb_subsample', 4) + 7) * 0.1,
'colsample_bytree': (hp.randint('lgb_colsample_bytree', 4) + 7) * 0.1,
}
return space
elif optimizer == 'gpflowopt':
from gpflowopt.domain import ContinuousParameter
domain = (
ContinuousParameter('n_estimators', 100, 1000) +
ContinuousParameter('num_leaves', 31, 2047) +
ContinuousParameter('max_depth', 15, 16) +
ContinuousParameter("learning_rate", 1e-3, 0.3) +
ContinuousParameter("min_child_samples", 5, 30) +
ContinuousParameter("subsample", 0.7, 1) +
ContinuousParameter("colsample_bytree", 0.7, 1)
)
return domain
else:
raise ValueError('Unknown optimizer %s when getting configspace' % optimizer)
def get_configs(self, variables):
api_config = OrderedDict()
cs = ConfigurationSpace()
hp_list = []
for var in variables:
if isinstance(var, ContinuousVariable):
api_config[var.name] = {
'type': 'real',
'space': 'linear',
'range': var.domain
}
hp_list.append(
UniformFloatHyperparameter(name=var.name,
lower=var.domain[0],
upper=var.domain[1],
log=False))
else:
api_config[var.name] = {'type': 'cat', 'values': var.domain}
hp_list.append(
CategoricalHyperparameter(name=var.name,
choices=var.domain))
cs.add_hyperparameters(hp_list)
return api_config, cs
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace,
leader_model):
assert isinstance(value, list)
for item in value:
assert isinstance(item, dict)
clauses = []
for name, forbidden_values in item.items():
true_name = add_leader_model(name, leader_model,
SERIES_CONNECT_LEADER_TOKEN)
if isinstance(forbidden_values,
list) and len(forbidden_values) == 1:
forbidden_values = forbidden_values[0]
if isinstance(forbidden_values, list):
clauses.append(
ForbiddenInClause(
store[true_name],
list(map(smac_hdl._encode, forbidden_values))))
else:
clauses.append(
ForbiddenEqualsClause(
store[true_name],
smac_hdl._encode(forbidden_values)))
cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
for i in range(self.dim):
xi = UniformFloatHyperparameter("x%d" % i, self.lb, self.ub)
cs.add_hyperparameter(xi)
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'x%d' % i: hp.uniform('hp_x%d' % i, self.lb, self.ub)
for i in range(self.dim)
}
return space
elif optimizer == 'gpflowopt':
import gpflowopt
domain = (
gpflowopt.domain.ContinuousParameter('x0', self.lb, self.ub) +
gpflowopt.domain.ContinuousParameter('x1', self.lb, self.ub))
return domain
else:
raise ValueError('Unknown optimizer %s when getting configspace' %
optimizer)
def CS2HyperoptSpace(cs: ConfigurationSpace):
result = {}
for hyperparameter in cs.get_hyperparameters():
name = hyperparameter.name
if isinstance(hyperparameter, CategoricalHyperparameter):
result[name] = hp.choice(name, hyperparameter.choices)
elif isinstance(hyperparameter, UniformFloatHyperparameter):
lower = hyperparameter.lower
upper = hyperparameter.upper
result[name] = hp.uniform(name, lower, upper)
else:
raise ValueError
# todo: 考虑更多情况
return result
def main(argv):
if len(argv) < 2:
raise ValueError("You should input filename.")
if FLAGS.processes < 1 or FLAGS.processes > cpu_count():
raise ValueError("Please input a proper number of processes.")
# 读取输入文件
if argv[1].endswith(".csv"):
# 如果后缀名为csv则按csv读取
df = pd.read_csv(sys.argv[1])
X = df[df.columns[:-1]].values
y = df[df.columns[-1]].values
else:
# 否则按照libsvm读取
with open(argv[1], "r") as fp:
lines = fp.readlines()
X, y = libsvm2sparse(lines)
# 分割数据集
X_train, X_valid, y_train, y_valid = train_test_split(
X, y, test_size=0.33, random_state=1)
processes = FLAGS.processes
# 指定输入输出目录
dirs = ["tmpfile/smac3-output_%s" % (
datetime.datetime.fromtimestamp(time.time()).strftime(
'%Y-%m-%d_%H:%M:%S_%f')) for _ in range(processes)]
# 创建进程池
cs = ConfigurationSpace()
# 超参搜索空间，使用[1e-6, 1]
alpha = UniformFloatHyperparameter(
name="alpha", lower=1e-3, upper=1,
default_value=1, log=False)
cs.add_hyperparameters([alpha])
# 指定并行pSMAC目录和当前的输出目录
smac = OurSMAC(X_train, y_train, X_valid, y_valid,
dirs=dirs, smbo_id=0,
cs=cs, our_work=FLAGS.our_work)
smac.run()
def test_factorial(self):
def get_uniform_param(name: str):
return UniformFloatHyperparameter(name, 0, 1)
def get_constant_param(name: str):
return Constant(name, 0.)
def get_categorical_param(name: str):
return CategoricalHyperparameter(name, choices=["a", "b", "c"])
def get_ordinal_param(name: str):
return OrdinalHyperparameter(name, [8, 6, 4, 2])
get_params = [
get_uniform_param, get_constant_param, get_categorical_param,
get_ordinal_param
]
dims = np.arange(1, 5)
for n_dim in dims:
cs = ConfigurationSpace()
for i in range(n_dim):
for j, get_param in enumerate(get_params):
param_name = f"x{i+1}_{j}"
cs.add_hyperparameter(get_param(param_name))
factorial_kwargs = dict(
rng=np.random.RandomState(1),
traj_logger=unittest.mock.Mock(),
ta_run_limit=1000,
configs=None,
n_configs_x_params=None,
max_config_fracs=0.25,
init_budget=1,
)
FactorialInitialDesign(cs=cs,
**factorial_kwargs).select_configurations()
def get_discrete_configspace(configspace: CS.ConfigurationSpace,
grid_size: int = 10,
seed: Union[int, None] = None,
fidelity_space: bool = False):
""" Generates a new discretized ConfigurationSpace from a generally defined space
Given the discretization grid size for each dimension, the new ConfigurationSpace contains
each hyperparmater as an OrdinalParameter with the discretized values for that dimension as
the sequence of choices available for that hyperparameter.
Parameters
----------
configspace : ConfigSpace.ConfigurationSpace
grid_size : int
The number of steps to divide a parameter dimension into
seed : int
Returns
-------
ConfigSpace.ConfigurationSpace
"""
if fidelity_space:
grid_list = pd.DataFrame(get_fidelity_grid(configspace, grid_size))
else:
grid_list = pd.DataFrame(get_parameter_grid(configspace, grid_size))
cs = CS.ConfigurationSpace(seed=seed)
hp_names = np.sort(configspace.get_hyperparameter_names()).tolist()
for i, k in enumerate(hp_names):
choices = grid_list.iloc[:, i].unique()
if isinstance(configspace.get_hyperparameter(k),
CS.UniformIntegerHyperparameter):
choices = choices.astype(int)
elif isinstance(configspace.get_hyperparameter(k),
CS.UniformFloatHyperparameter):
choices = choices.astype(np.float32)
cs.add_hyperparameter(CS.OrdinalHyperparameter(str(k), choices))
return cs
def get_config_space():
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter(name='a',
lower=0,
upper=100))
cs.add_hyperparameter(UniformIntegerHyperparameter(name='b',
lower=0,
upper=100))
return cs
def test_fix_types(self):
# Test categorical and ordinal
for hyperparameter_type in [
CategoricalHyperparameter, OrdinalHyperparameter
]:
cs = ConfigurationSpace()
cs.add_hyperparameters([
hyperparameter_type('bools', [True, False]),
hyperparameter_type('ints', [1, 2, 3, 4, 5]),
hyperparameter_type('floats', [1.5, 2.5, 3.5, 4.5, 5.5]),
hyperparameter_type('str', ['string', 'ding', 'dong']),
hyperparameter_type('mixed',
[2, True, 1.5, 'string', False, 'False']),
])
c = cs.get_default_configuration().get_dictionary()
# Check bools
for b in [False, True]:
c['bools'] = b
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
# Check legal mixed values
for m in [2, True, 1.5, 'string']:
c['mixed'] = m
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
# Check error on cornercase that cannot be caught
for m in [False, 'False']:
c['mixed'] = m
c_str = {k: str(v) for k, v in c.items()}
self.assertRaises(ValueError, fix_types, c_str, cs)
# Test constant
for m in [2, 1.5, 'string']:
cs = ConfigurationSpace()
cs.add_hyperparameter(Constant('constant', m))
c = cs.get_default_configuration().get_dictionary()
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_create_scenario(self):
# check if the 'optimize_on' is not set to value other then 'runtime' or 'quality'
cs = ConfigurationSpace()
try:
BayesianOptimizationPipeline._create_scenario(cs=cs, cutoff_time=200, iteration=2, optimize_on='runtime')
except (UnboundLocalError, NameError, Exception):
self.fail()
try:
BayesianOptimizationPipeline._create_scenario(cs=cs, cutoff_time=200, iteration=2, optimize_on='quality')
except (UnboundLocalError, NameError, Exception):
self.fail()
with self.assertRaises(UnboundLocalError):
BayesianOptimizationPipeline._create_scenario(cs=cs, cutoff_time=200, iteration=2,
optimize_on='some_value_other_than_quality_or_runtime')
def get_pipeline_config_space(algorithm_candidates):
cs = ConfigurationSpace()
estimator_choice = CategoricalHyperparameter(
"estimator",
algorithm_candidates,
default_value=algorithm_candidates[0])
cs.add_hyperparameter(estimator_choice)
aug_space = get_aug_hyperparameter_space()
cs.add_hyperparameters(aug_space.get_hyperparameters())
cs.add_conditions(aug_space.get_conditions())
for estimator_id in algorithm_candidates:
sub_cs = get_model_config_space(estimator_id,
include_estimator=False,
include_aug=False)
parent_hyperparameter = {
'parent': estimator_choice,
'value': estimator_id
}
cs.add_configuration_space(estimator_id,
sub_cs,
parent_hyperparameter=parent_hyperparameter)
return cs
def get_hyperparameter_search_space():
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2"], default_value="l2")
loss = CategoricalHyperparameter(
"loss", ["hinge", "squared_hinge"], default_value="squared_hinge")
dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True')
# This is set ad-hoc
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
multi_class = UnParametrizedHyperparameter("multi_class", "ovr")
# These are set ad-hoc
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
intercept_scaling = UnParametrizedHyperparameter("intercept_scaling", 1)
cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class,
fit_intercept, intercept_scaling])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge")
)
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge")
)
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "True"),
ForbiddenEqualsClause(penalty, "l1")
)
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
class TestLHDesign(unittest.TestCase):
def setUp(self):
def get_uniform_param(name: str):
return UniformFloatHyperparameter(name, 0, 1)
def get_constant_param(name: str):
return Constant(name, 0.)
def get_categorical_param(name: str):
return CategoricalHyperparameter(name, choices=["a", "b", "c"])
def get_ordinal_param(name: str):
return OrdinalHyperparameter(name, [8, 6, 4, 2])
get_params = [
get_uniform_param, get_constant_param, get_categorical_param,
get_ordinal_param
]
self.cs = ConfigurationSpace()
for j, get_param in enumerate(get_params):
param_name = f"x{j}"
self.cs.add_hyperparameter(get_param(param_name))
param_constrained = CategoricalHyperparameter("constrained",
choices=["a", "b", "c"])
self.cs.add_hyperparameter(param_constrained)
self.cs.add_forbidden_clause(
ForbiddenEqualsClause(param_constrained, "b"))
for i in range(5):
self.cs.add_hyperparameter(
UniformFloatHyperparameter('x%d' % (i + len(get_params)), 0,
1))
def test_latin_hypercube_design(self):
kwargs = dict(
rng=np.random.RandomState(1),
traj_logger=unittest.mock.Mock(),
ta_run_limit=1000,
configs=None,
n_configs_x_params=None,
max_config_fracs=0.25,
init_budget=1000,
)
LHDesign(cs=self.cs, **kwargs).select_configurations()
def test_choose_next(self):
configspace = ConfigurationSpace()
configspace.add_hyperparameter(UniformFloatHyperparameter('a', 0, 1))
configspace.add_hyperparameter(UniformFloatHyperparameter('b', 0, 1))
dataset_name = 'foo'
func_eval_time_limit = 15
total_walltime_limit = 15
memory_limit = 3000
auto = AutoMLSMBO(None, dataset_name, None, func_eval_time_limit,
total_walltime_limit, memory_limit, None)
auto.config_space = configspace
scenario = Scenario({
'cs': configspace,
'cutoff-time': func_eval_time_limit,
'wallclock-limit': total_walltime_limit,
'memory-limit': memory_limit,
'run-obj': 'quality'
})
smac = SMAC(scenario)
self.assertRaisesRegex(
ValueError, 'Cannot use SMBO algorithm on '
'empty runhistory', auto.choose_next, smac)
config = Configuration(configspace, values={'a': 0.1, 'b': 0.2})
# TODO make sure the incumbent is always set?
smac.solver.incumbent = config
runhistory = smac.solver.runhistory
runhistory.add(config=config,
cost=0.5,
time=0.5,
status=StatusType.SUCCESS)
auto.choose_next(smac)
def _add_hierachical_configspace(self, cs, config, parent_name):
config_cand = list(config.keys())
config_cand.append("empty")
config_option = CategoricalHyperparameter(parent_name, config_cand,
default_value=config_cand[-1])
cs.add_hyperparameter(config_option)
for config_item in config_cand:
if config_item == 'empty':
sub_configuration_space = ConfigurationSpace()
else:
sub_configuration_space = config[config_item]
parent_hyperparameter = {'parent': config_option,
'value': config_item}
cs.add_configuration_space(config_item, sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def main():
for synthetic_function_cls in synthetic_functions:
meta_info = synthetic_function_cls.get_meta_information()
if "num_function_evals" in meta_info:
max_iter = meta_info["num_function_evals"]
else:
max_iter = 200
# 构造超参空间
config_space = ConfigurationSpace()
config_space.generate_all_continuous_from_bounds(
synthetic_function_cls.get_meta_information()['bounds'])
synthetic_function = synthetic_function_cls()
# 定义目标函数
def evaluation(config: dict):
config = Configuration(config_space, values=config)
return synthetic_function.objective_function(config)["function_value"] - \
synthetic_function.get_meta_information()["f_opt"]
# 对experiment_param的删除等操作放在存储后面
res = pd.DataFrame(columns=[f"trial-{i}" for i in range(10)],
index=range(max_iter))
for trial in range(10):
random_state = 50 + trial * 10
# 设置超参空间的随机种子（会影响后面的采样）
config_space.seed(random_state)
print("==========================")
print(f"= Trial -{trial:01d}- =")
print("==========================")
print('iter | loss | config origin')
print('----------------------------')
cg = SamplingSortOptimizer(config_space, [1],
min_points_in_model=25,
n_samples=2500)
loss = np.inf
for ix in range(max_iter):
config, config_info = cg.get_config(1)
cur_loss = evaluation(config)
loss = min(loss, cur_loss)
print(f" {ix:03d} {loss:.4f} {config_info.get('origin')}")
job = Job("")
job.result = {"loss": cur_loss}
job.kwargs = {
"budget": 1,
"config": config,
"config_info": config_info
}
cg.new_result(job)
res.loc[ix, f"trial-{trial}"] = cur_loss
res = raw2min(res)
m = res.mean(1)
s = res.std(1)
name = synthetic_function.get_meta_information()["name"]
final_result[name] = {"mean": m.tolist(), "std": s.tolist()}
Path("EETPE.json").write_text(json.dumps(final_result))
def _get_configuration_space(self, builtin_transformers, trans_type=None, optimizer='smac'):
config_dict = dict()
for tran_key in builtin_transformers:
tran = builtin_transformers[tran_key]
tran_id = tran().type
if trans_type is None or tran_id in trans_type:
try:
sub_configuration_space = builtin_transformers[tran_key].get_hyperparameter_search_space(
optimizer=optimizer)
config_dict[tran_key] = sub_configuration_space
except:
if optimizer == 'smac':
config_dict[tran_key] = ConfigurationSpace()
elif optimizer == 'tpe':
config_dict[tran_key] = {}
return config_dict
def _build_hierachical_configspace(self,
pre_config1,
pre_config2,
gen_config,
res_config,
sel_config,
bal_config=None):
cs = ConfigurationSpace()
self._add_hierachical_configspace(cs, pre_config1, "preprocessor1")
self._add_hierachical_configspace(cs, pre_config2, "preprocessor2")
if bal_config is not None:
self._add_hierachical_configspace(cs, bal_config, "balancer")
self._add_hierachical_configspace(cs, gen_config, "generator")
self._add_hierachical_configspace(cs, res_config, "rescaler")
self._add_hierachical_configspace(cs, sel_config, "selector")
return cs
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegex(ValueError,
r"Adding a second condition \(different\) for a "
r"hyperparameter is ambigouos and "
r"therefore forbidden. Add a conjunction "
r"instead!",
cs.add_condition, cond2)
def test_keys(self):
# A regression test to make sure issue #49 does no longer pop up. By
# iterating over the configuration in the for loop, it should not raise
# a KeyError if the child hyperparameter is inactive.
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None",
)
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5,
)
cs.add_hyperparameters([shrinkage, shrinkage_factor])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
for i in range(10):
config = cs.sample_configuration()
{hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None}
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegex(ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition,
cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegex(ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition,
cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegex(ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.",
cs2.add_condition, cond)
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter("classifier", ["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
# check backward compatibility with checking configurations instead of vectors
cs.check_configuration(configuration)
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_add_hyperparameter(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
def test_sample_no_configuration(self):
cs = ConfigurationSpace()
rval = cs.sample_configuration(size=0)
self.assertEqual(len(rval), 0)
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_setting_illegal_value(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('x', 0, 1))
configuration = {'x': 2}
self.assertRaises(ValueError, Configuration, cs, configuration)
