def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(
NormalFloatHyperparameter('b', 1, 2, log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def test_add_hyperparameters_with_equal_names(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
self.assertRaisesRegexp(
ValueError, "Hyperparameter 'name' is already in the "
"configuration space.", cs.add_hyperparameter, hp)
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.cs = ConfigurationSpace()
self.cs.add_hyperparameter(CategoricalHyperparameter(
name="cat_a_b", choices=["a", "b"], default_value="a"))
self.cs.add_hyperparameter(UniformFloatHyperparameter(
name="float_0_1", lower=0, upper=1, default_value=0.5))
self.cs.add_hyperparameter(UniformIntegerHyperparameter(
name='integer_0_100', lower=-10, upper=10, default_value=0))
self.rh = runhistory.RunHistory(aggregate_func=average_cost)
rs = numpy.random.RandomState(1)
to_count = 0
cn_count = 0
for i in range(500):
config, seed, runtime, status, instance_id = \
generate_config(cs=self.cs, rs=rs)
if runtime == 40:
to_count += 1
if runtime < 40 and status == StatusType.TIMEOUT:
cn_count += 1
self.rh.add(config=config, cost=runtime, time=runtime,
status=status, instance_id=instance_id,
seed=seed, additional_info=None)
print("%d TIMEOUTs, %d censored" % (to_count, cn_count))
self.scen = Scen()
self.scen.run_obj = "runtime"
self.scen.overall_obj = "par10"
self.scen.cutoff = 40
types, bounds = get_types(self.cs, None)
self.model = RandomForestWithInstances(
types=types, bounds=bounds,
instance_features=None, seed=1234567980)
def setUp(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter(
"parent", [0, 1]))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
hp3 = cs.add_hyperparameter(
UniformIntegerHyperparameter("friend", 0, 5))
self.cs = cs
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def testRandomImputation(self):
rs = numpy.random.RandomState(1)
for i in range(0, 150, 15):
# First random imputation sanity check
num_samples = max(1, i*10)
num_feat = max(1, i)
num_censored = int(num_samples*0.1)
X = rs.rand(num_samples, num_feat)
y = numpy.sin(X[:, 0:1])
cutoff = max(y) * 0.9
y[y > cutoff] = cutoff
# We have some cen data
cen_X = X[:num_censored, :]
cen_y = y[:num_censored]
uncen_X = X[num_censored:, :]
uncen_y = y[num_censored:]
cen_y /= 2
cs = ConfigurationSpace()
for i in range(num_feat):
cs.add_hyperparameter(UniformFloatHyperparameter(
name="a_%d" % i, lower=0, upper=1, default_value=0.5)
)
types, bounds = get_types(cs, None)
print(types)
print(bounds)
print('#'*120)
print(cen_X)
print(uncen_X)
print('~'*120)
self.model = RandomForestWithInstances(types=types, bounds=bounds,
instance_features=None,
seed=1234567980)
imputor = rfr_imputator.RFRImputator(rng=rs,
cutoff=cutoff,
threshold=cutoff*10,
change_threshold=0.01,
max_iter=5,
model=self.model)
imp_y = imputor.impute(censored_X=cen_X, censored_y=cen_y,
uncensored_X=uncen_X,
uncensored_y=uncen_y)
if imp_y is None:
continue
for idx in range(cen_y.shape[0]):
self.assertGreater(imp_y[idx], cen_y[idx])
self.assertTrue(numpy.isfinite(imp_y).all())
def deactivate_inactive_hyperparameters(
configuration: Dict,
configuration_space: ConfigurationSpace,
):
hyperparameters = configuration_space.get_hyperparameters()
configuration = Configuration(configuration_space=configuration_space,
values=configuration,
allow_inactive_with_values=True)
hps = deque()
unconditional_hyperparameters = configuration_space.get_all_unconditional_hyperparameters()
hyperparameters_with_children = list()
for uhp in unconditional_hyperparameters:
children = configuration_space._children_of[uhp]
if len(children) > 0:
hyperparameters_with_children.append(uhp)
hps.extendleft(hyperparameters_with_children)
inactive = set()
while len(hps) > 0:
hp = hps.pop()
children = configuration_space._children_of[hp]
for child in children:
conditions = configuration_space._parent_conditions_of[child.name]
for condition in conditions:
if not condition.evaluate_vector(configuration.get_array()):
dic = configuration.get_dictionary()
try:
del dic[child.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
inactive.add(child.name)
hps.appendleft(child.name)
for hp in hyperparameters:
if hp.name in inactive:
dic = configuration.get_dictionary()
try:
del dic[hp.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
return Configuration(configuration_space, values=configuration.get_dictionary())
def setUp(self):
logging.basicConfig()
self.logger = logging.getLogger(self.__module__ + '.' +
self.__class__.__name__)
self.logger.setLevel(logging.DEBUG)
base_directory = os.path.split(__file__)[0]
base_directory = os.path.abspath(
os.path.join(base_directory, '..', '..'))
self.current_dir = os.getcwd()
os.chdir(base_directory)
self.cs = ConfigurationSpace()
self.test_scenario_dict = {
'algo': 'echo Hello',
'paramfile': 'test/test_files/scenario_test/param.pcs',
'execdir': '.',
'deterministic': 0,
'run_obj': 'runtime',
'overall_obj': 'mean10',
'cutoff_time': 5,
'wallclock-limit': 18000,
'instance_file': 'test/test_files/scenario_test/training.txt',
'test_instance_file': 'test/test_files/scenario_test/test.txt',
'feature_file': 'test/test_files/scenario_test/features.txt',
'output_dir': 'test/test_files/scenario_test/tmp_output'
}
def test_illegal_default_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("loss", ["l1", "l2"],
default_value='l1')
hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"],
default_value='l1')
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
forb1 = ForbiddenEqualsClause(hp1, "l1")
forb2 = ForbiddenEqualsClause(hp2, "l1")
forb3 = ForbiddenAndConjunction(forb1, forb2)
# cs.add_forbidden_clause(forb3)
self.assertRaisesRegexp(
ValueError,
"Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && "
"Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def test_meta_data_stored(self):
meta_data = {
'additional': 'meta-data',
'useful': 'for integrations',
'input_id': 42
}
cs = ConfigurationSpace(meta=dict(meta_data))
self.assertEqual(cs.meta, meta_data)
def test_keys(self):
# A regression test to make sure issue #49 does no longer pop up. By
# iterating over the configuration in the for loop, it should not raise
# a KeyError if the child hyperparameter is inactive.
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage",
["None", "auto", "manual"],
default_value="None",
)
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor",
0.,
1.,
0.5,
)
cs.add_hyperparameters([shrinkage, shrinkage_factor])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage,
"manual"))
for i in range(10):
config = cs.sample_configuration()
d = {
hp_name: config[hp_name]
for hp_name in config if config[hp_name] is not None
}
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegexp(
ValueError, "Hyperparameter configuration "
"contains a cycle \[\['child', 'parent'\]\]", cs.add_condition,
cond2)
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter(
"input1", [0, 1]))
forb1 = cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
cond = cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(
str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def get_config_space():
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='a', lower=0, upper=100))
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='b', lower=0, upper=100))
return cs
def test_check_configuration_input_checking(self):
cs = ConfigurationSpace()
self.assertRaisesRegexp(
TypeError, "The method check_configuration must be called "
"with an instance of %s. "
"Your input was of type %s" % (Configuration, type("String")),
cs.check_configuration, "String")
# For the check configuration method with vector representation
self.assertRaisesRegexp(
TypeError, "The method check_configuration must"
" be called with an instance of "
"np.ndarray Your input was of type %s" % (type("String")),
cs.check_configuration_vector_representation, "String")
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegexp(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegexp(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegexp(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegexp(
ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.", cs2.add_condition, cond)
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegexp(
ValueError, "Adding a second condition \(different\) for a "
"hyperparameter is ambigouos and "
"therefore forbidden. Add a conjunction "
"instead!", cs.add_condition, cond2)
def test_add_forbidden_clause(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
forb = ForbiddenEqualsClause(hp1, 1)
# TODO add checking whether a forbidden clause makes sense at all
cs.add_forbidden_clause(forb)
# TODO add something to properly retrieve the forbidden clauses
self.assertEqual(
str(cs), "Configuration space object:\n "
"Hyperparameters:\n input1, "
"Type: Categorical, Choices: {0, 1}, "
"Default: 0\n"
" Forbidden Clauses:\n"
" Forbidden: input1 == 1\n")
def fix_types(configuration: dict,
configuration_space: ConfigurationSpace):
'''
iterates over all hyperparameters in the ConfigSpaceNNI
and fixes the types of the parameter values in configuration.
Arguments
---------
configuration: dict
param name -> param value
configuration_space: ConfigurationSpace
Configuration space which knows the types for all parameter values
Returns
-------
configuration: dict
with fixed types of parameter values
'''
for param in configuration_space.get_hyperparameters():
param_name = param.name
if configuration.get(param_name) is not None:
if isinstance(param, (CategoricalHyperparameter)):
# should be unnecessary, but to be on the safe param_name:
configuration[param_name] = str(configuration[param_name])
elif isinstance(param, (OrdinalHyperparameter)):
# should be unnecessary, but to be on the safe side:
configuration[param_name] = str(configuration[param_name])
elif isinstance(param, Constant):
# should be unnecessary, but to be on the safe side:
configuration[param_name] = str(configuration[param_name])
elif isinstance(param, UniformFloatHyperparameter):
configuration[param_name] = float(configuration[param_name])
elif isinstance(param, UniformIntegerHyperparameter):
configuration[param_name] = int(configuration[param_name])
else:
raise TypeError("Unknown hyperparameter type %s" % type(param))
return configuration
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def test_setting_illegal_value(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('x', 0, 1))
configuration = {'x': 2}
self.assertRaises(ValueError, Configuration, cs, configuration)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(
UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'],
default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError,
"Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space."
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_add_non_hyperparameter(self):
cs = ConfigurationSpace()
non_hp = unittest.TestSuite()
self.assertRaises(TypeError, cs.add_hyperparameter, non_hp)
def test_add_hyperparameter(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
