def test_build_new_GreaterThanIntCondition(self):
expected = "a real [0.0, 1.0] [0.5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 0.5"
cs = ConfigurationSpace()
a = UniformFloatHyperparameter("a", 0, 1, 0.5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 0.5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
expected = "a integer [0, 10] [5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 5"
cs = ConfigurationSpace()
a = UniformIntegerHyperparameter("a", 0, 10, 5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def get_config_space():
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='a', lower=0, upper=100))
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='b', lower=0, upper=100))
return cs
def test__sample_UniformIntegerHyperparameter(self):
hp = UniformIntegerHyperparameter("uihp", 0, 10)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertEqual(len(np.unique(values)), 11)
def test_write_forbidden(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 2)
hp3 = UniformIntegerHyperparameter("child2", 0, 2)
hp4 = UniformIntegerHyperparameter("child3", 0, 2)
hp5 = CategoricalHyperparameter("child4", [4, 5, 6, 7])
cs.add_hyperparameters([hp1, hp2, hp3, hp4, hp5])
forb2 = ForbiddenEqualsClause(hp1, 1)
forb3 = ForbiddenInClause(hp2, range(2, 3))
forb4 = ForbiddenInClause(hp3, range(2, 3))
forb5 = ForbiddenInClause(hp4, range(2, 3))
forb6 = ForbiddenInClause(hp5, [6, 7])
and1 = ForbiddenAndConjunction(forb2, forb3)
and2 = ForbiddenAndConjunction(forb2, forb4)
and3 = ForbiddenAndConjunction(forb2, forb5)
cs.add_forbidden_clauses(
[forb2, forb3, forb4, forb5, forb6, and1, and2, and3])
value = irace.write(cs) # generates file called forbidden.txt
def setUp(self):
current_dir = os.path.dirname(__file__)
self.test_files_dir = os.path.join(current_dir, '..', 'test_files')
seed = np.random.randint(1, 100000)
self.cs = ConfigurationSpace(seed=seed)
x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5)
self.cs.add_hyperparameter(x1)
x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5)
self.cs.add_hyperparameter(x2)
x3 = CategoricalHyperparameter("x3",
[5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4],
default_value=5)
self.cs.add_hyperparameter(x3)
x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5)
self.cs.add_hyperparameter(x4)
def test_sample_UniformIntegerHyperparameter(self):
# TODO: disentangle, actually test _sample and test sample on the
# base class
def sample(hp):
rs = np.random.RandomState(1)
counts_per_bin = [0 for i in range(21)]
values = []
for i in range(100000):
value = hp.sample(rs)
values.append(value)
index = int(
float(value - hp.lower) / (hp.upper - hp.lower) * 20)
counts_per_bin[index] += 1
self.assertIsInstance(value, int)
return counts_per_bin
# Quantized Uniform
hp = UniformIntegerHyperparameter("uihp", 0, 10)
counts_per_bin = sample(hp)
print(counts_per_bin)
for bin in counts_per_bin[::2]:
self.assertTrue(9302 > bin > 8700)
for bin in counts_per_bin[1::2]:
self.assertEqual(bin, 0)
self.assertEqual(sample(hp), sample(hp))
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.cs = ConfigurationSpace()
self.cs.add_hyperparameter(CategoricalHyperparameter(
name="cat_a_b", choices=["a", "b"], default_value="a"))
self.cs.add_hyperparameter(UniformFloatHyperparameter(
name="float_0_1", lower=0, upper=1, default_value=0.5))
self.cs.add_hyperparameter(UniformIntegerHyperparameter(
name='integer_0_100', lower=-10, upper=10, default_value=0))
self.rh = runhistory.RunHistory(aggregate_func=average_cost)
rs = numpy.random.RandomState(1)
to_count = 0
cn_count = 0
for i in range(500):
config, seed, runtime, status, instance_id = \
generate_config(cs=self.cs, rs=rs)
if runtime == 40:
to_count += 1
if runtime < 40 and status == StatusType.TIMEOUT:
cn_count += 1
self.rh.add(config=config, cost=runtime, time=runtime,
status=status, instance_id=instance_id,
seed=seed, additional_info=None)
print("%d TIMEOUTs, %d censored" % (to_count, cn_count))
self.scen = Scen()
self.scen.run_obj = "runtime"
self.scen.overall_obj = "par10"
self.scen.cutoff = 40
types, bounds = get_types(self.cs, None)
self.model = RandomForestWithInstances(
types=types, bounds=bounds,
instance_features=None, seed=1234567980)
def test_write_log_int(self):
expected = "int_log '--int_log ' i (2, 4)\n"
int_log = UniformIntegerHyperparameter("int_log", 10, 100, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(int_log)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_int(self):
expected = "Q16_int_a integer [16, 1024] [520]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("int_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def _construct_hyperparameter(hyperparameter: Dict) -> Hyperparameter:
hp_type = hyperparameter['type']
name = hyperparameter['name']
if hp_type == 'constant':
return Constant(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'unparametrized':
return UnParametrizedHyperparameter(
name=name,
value=hyperparameter['value'],
)
elif hp_type == 'uniform_float':
return UniformFloatHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_float':
return NormalFloatHyperparameter(
name=name,
log=hyperparameter['log'],
mu=hyperparameter['mu'],
sigma=hyperparameter['sigma'],
default_value=hyperparameter['default'],
)
elif hp_type == 'uniform_int':
return UniformIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'normal_int':
return NormalIntegerHyperparameter(
name=name,
log=hyperparameter['log'],
lower=hyperparameter['lower'],
upper=hyperparameter['upper'],
default_value=hyperparameter['default'],
)
elif hp_type == 'categorical':
return CategoricalHyperparameter(
name=name,
choices=hyperparameter['choices'],
default_value=hyperparameter['default'],
)
elif hp_type == 'ordinal':
return OrdinalHyperparameter(
name=name,
sequence=hyperparameter['sequence'],
default_value=hyperparameter['default'],
)
else:
raise ValueError(hp_type)
def test_and_conjunction(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 2)
hp3 = UniformIntegerHyperparameter("child2", 0, 2)
hp4 = UniformIntegerHyperparameter("child3", 0, 2)
forb2 = ForbiddenEqualsClause(hp1, 1)
forb3 = ForbiddenInClause(hp2, range(2, 3))
forb4 = ForbiddenInClause(hp3, range(2, 3))
forb5 = ForbiddenInClause(hp4, range(2, 3))
and1 = ForbiddenAndConjunction(forb2, forb3)
and2 = ForbiddenAndConjunction(forb2, forb4)
and3 = ForbiddenAndConjunction(forb2, forb5)
total_and = ForbiddenAndConjunction(and1, and2, and3)
self.assertEqual(
"((Forbidden: parent == 1 && Forbidden: child in {2}) "
"&& (Forbidden: parent == 1 && Forbidden: child2 in {2}) "
"&& (Forbidden: parent == 1 && Forbidden: child3 in "
"{2}))", str(total_and))
results = [
False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, False,
False, False, False, False, False, False, False, False, True
]
for i, values in enumerate(
product(range(2), range(3), range(3), range(3))):
is_forbidden = total_and.is_forbidden(
{
"parent": values[0],
"child": values[1],
"child2": values[2],
"child3": values[3]
},
True,
)
self.assertEqual(results[i], is_forbidden)
self.assertFalse(total_and.is_forbidden({}, strict=False))
def test_condition_from_cryptominisat(self):
parent = CategoricalHyperparameter('blkrest', ['0', '1'],
default_value='1')
child = UniformIntegerHyperparameter('blkrestlen',
2000,
10000,
log=True)
condition = EqualsCondition(child, parent, '1')
self.assertFalse(condition.evaluate(dict(blkrest='0')))
self.assertTrue(condition.evaluate(dict(blkrest='1')))
def test_write_equals_condition_numerical(self):
expected = "temp '--temp ' i (1, 2)\nls '--ls ' c {sa,ca,ny}| temp==2\n"
temp = UniformIntegerHyperparameter("temp", 1, 2)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(ls, temp, 2)
cs.add_condition(c1)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_not_equals_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = NotEqualsCondition(hp2, hp1, 0)
cond_ = NotEqualsCondition(hp2, hp1, 0)
self.assertEqual(cond, cond_)
# Test vector value:
self.assertEqual(cond.vector_value, hp1._inverse_transform(0))
self.assertEqual(cond.vector_value, cond_.vector_value)
cond_reverse = NotEqualsCondition(hp1, hp2, 0)
self.assertNotEqual(cond, cond_reverse)
self.assertNotEqual(cond, dict())
self.assertEqual("child | parent != 0", str(cond))
def test_get_parents(self):
# Necessary because we couldn't call cs.get_parents for
# clasp-sat-params-nat.pcs
counter = UniformIntegerHyperparameter('bump', 10, 4096, log=True)
_1_S_countercond = CategoricalHyperparameter('cony', ['yes', 'no'])
_1_0_restarts = CategoricalHyperparameter(
'restarts', ['F', 'L', 'D', 'x', '+', 'no'], default_value='x')
condition = EqualsCondition(counter, _1_S_countercond, 'yes')
# All conditions inherit get_parents from abstractcondition
self.assertEqual([_1_S_countercond], condition.get_parents())
condition2 = InCondition(counter, _1_0_restarts,
['F', 'D', 'L', 'x', '+'])
# All conjunctions inherit get_parents from abstractconjunction
conjunction = AndConjunction(condition, condition2)
self.assertEqual([_1_S_countercond, _1_0_restarts],
conjunction.get_parents())
def test_uniformint_legal_float_values(self):
n_iter = UniformIntegerHyperparameter("n_iter",
5.,
1000.,
default_value=20.0)
self.assertIsInstance(n_iter.default_value, int)
self.assertRaisesRegexp(ValueError,
"For the Integer parameter n_iter, "
"the value must be an Integer, too."
" Right now it is a <(type|class) "
"'float'>"
" with value 20.5.",
UniformIntegerHyperparameter,
"n_iter",
5.,
1000.,
default_value=20.5)
def test_with_ordinal(self):
cs = smac.configspace.ConfigurationSpace()
a = cs.add_hyperparameter(
CategoricalHyperparameter('a', [0, 1], default_value=0))
b = cs.add_hyperparameter(
OrdinalHyperparameter('b', [0, 1], default_value=1))
b = cs.add_hyperparameter(
UniformFloatHyperparameter('c',
lower=0.,
upper=1.,
default_value=1))
b = cs.add_hyperparameter(
UniformIntegerHyperparameter('d',
lower=0,
upper=10,
default_value=1))
cs.seed(1)
feat_array = np.array([0, 0, 0]).reshape(1, -1)
types, bounds = get_types(cs, feat_array)
model = RandomForestWithInstances(types=types,
bounds=bounds,
instance_features=feat_array,
seed=1,
ratio_features=1.0,
pca_components=9)
self.assertEqual(bounds[0][0], 2)
self.assertTrue(bounds[0][1] is np.nan)
self.assertEqual(bounds[1][0], 0)
self.assertEqual(bounds[1][1], 1)
self.assertEqual(bounds[2][0], 0.)
self.assertEqual(bounds[2][1], 1.)
self.assertEqual(bounds[3][0], 0.)
self.assertEqual(bounds[3][1], 1.)
X = np.array([[0., 0., 0., 0., 0, 0, 0], [0., 0., 1., 0., 0, 0, 0],
[0., 1., 0., 9., 0, 0, 0], [0., 1., 1., 4., 0, 0, 0]],
dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
model.train(np.vstack((X, X, X, X, X, X, X, X, X, X)),
np.vstack((y, y, y, y, y, y, y, y, y, y)))
mean, _ = model.predict(X)
for idx, m in enumerate(mean):
self.assertAlmostEqual(y[idx], m, 0.05)
def test_in_condition(self):
hp1 = CategoricalHyperparameter("parent", list(range(0, 11)))
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5])
cond_ = InCondition(hp2, hp1, [0, 1, 2, 3, 4, 5])
self.assertEqual(cond, cond_)
# Test vector value:
self.assertEqual(
cond.vector_values,
[hp1._inverse_transform(i) for i in [0, 1, 2, 3, 4, 5]])
self.assertEqual(cond.vector_values, cond_.vector_values)
cond_reverse = InCondition(hp1, hp2, [0, 1, 2, 3, 4, 5])
self.assertNotEqual(cond, cond_reverse)
self.assertNotEqual(cond, dict())
self.assertEqual("child | parent in {0, 1, 2, 3, 4, 5}", str(cond))
def test_equals_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
cond_ = EqualsCondition(hp2, hp1, 0)
# Test vector value:
self.assertEqual(cond.vector_value, hp1._inverse_transform(0))
self.assertEqual(cond.vector_value, cond_.vector_value)
# Test invalid conditions:
self.assertRaises(TypeError, EqualsCondition, hp2, "parent", 0)
self.assertRaises(TypeError, EqualsCondition, "child", hp1, 0)
self.assertRaises(ValueError, EqualsCondition, hp1, hp1, 0)
self.assertEqual(cond, cond_)
cond_reverse = EqualsCondition(hp1, hp2, 0)
self.assertNotEqual(cond, cond_reverse)
self.assertNotEqual(cond, dict())
self.assertEqual("child | parent == 0", str(cond))
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel",
["linear", "rbf", "poly", "sigmoid"],
default_value="poly")
cs.add_hyperparameter(kernel)
# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"],
default_value="true")
cs.add_hyperparameters([C, shrinking])
# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter(
"degree", 1, 5, default_value=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0,
default_value=0.0) # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter(
"gamma", ["auto", "value"],
default_value="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value",
0.0001,
8,
# * Neither the name of the <organization> nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
int_a = UniformIntegerHyperparameter("int_a", -1, 6)
class TestIraceWriter(unittest.TestCase):
'''
Test IRACE writer
'''
def test_write_illegal_argument(self):
sp = {"a": int_a}
self.assertRaisesRegexp(
TypeError, "irace.write expects an "
"instance of "
"<class "
"'ConfigSpaceNNI.configuration_"
"space.ConfigurationSpace'>, you provided "
"'<(type|class) 'dict'>'", irace.write, sp)
def test_merge_foreign_data(self):
''' test smac.utils.merge_foreign_data '''
scenario = Scenario(self.test_scenario_dict)
scenario_2 = Scenario(self.test_scenario_dict)
scenario_2.feature_dict = {"inst_new": [4]}
# init cs
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='a', lower=0, upper=100))
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='b', lower=0, upper=100))
# build runhistory
rh_merge = RunHistory(aggregate_func=average_cost)
config = Configuration(cs, values={'a': 1, 'b': 2})
rh_merge.add(config=config,
instance_id="inst_new",
cost=10,
time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# "d" is an instance in <scenario>
rh_merge.add(config=config,
instance_id="d",
cost=5,
time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# build empty rh
rh_base = RunHistory(aggregate_func=average_cost)
merge_foreign_data(scenario=scenario,
runhistory=rh_base,
in_scenario_list=[scenario_2],
in_runhistory_list=[rh_merge])
# both runs should be in the runhistory
# but we should not use the data to update the cost of config
self.assertTrue(len(rh_base.data) == 2)
self.assertTrue(np.isnan(rh_base.get_cost(config)))
# we should not get direct access to external run data
runs = rh_base.get_runs_for_config(config)
self.assertTrue(len(runs) == 0)
rh_merge.add(config=config,
instance_id="inst_new_2",
cost=10,
time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
self.assertRaises(
ValueError, merge_foreign_data, **{
"scenario": scenario,
"runhistory": rh_base,
"in_scenario_list": [scenario_2],
"in_runhistory_list": [rh_merge]
})
def test_normalint_to_uniform(self):
f1 = NormalIntegerHyperparameter("param", 0, 10, q=0.1)
f1_expected = UniformIntegerHyperparameter("param", -30, 30)
f1_actual = f1.to_uniform()
self.assertEqual(f1_expected, f1_actual)
def test_greater_and_less_condition(self):
child = Constant('child', 'child')
hp1 = UniformFloatHyperparameter("float", 0, 5)
hp2 = UniformIntegerHyperparameter("int", 0, 5)
hp3 = OrdinalHyperparameter("ord", list(range(6)))
for hp in [hp1, hp2, hp3]:
hyperparameter_idx = {child.name: 0, hp.name: 1}
gt = GreaterThanCondition(child, hp, 1)
gt.set_vector_idx(hyperparameter_idx)
self.assertFalse(gt.evaluate({hp.name: 0}))
self.assertTrue(gt.evaluate({hp.name: 2}))
self.assertFalse(gt.evaluate({hp.name: None}))
# Evaluate vector
test_value = hp._inverse_transform(2)
self.assertFalse(gt.evaluate_vector(np.array([np.NaN, 0])))
self.assertTrue(gt.evaluate_vector(np.array([np.NaN, test_value])))
self.assertFalse(gt.evaluate_vector(np.array([np.NaN, np.NaN])))
lt = LessThanCondition(child, hp, 1)
lt.set_vector_idx(hyperparameter_idx)
self.assertTrue(lt.evaluate({hp.name: 0}))
self.assertFalse(lt.evaluate({hp.name: 2}))
self.assertFalse(lt.evaluate({hp.name: None}))
# Evaluate vector
test_value = hp._inverse_transform(2)
self.assertTrue(lt.evaluate_vector(np.array([np.NaN, 0, 0, 0])))
self.assertFalse(lt.evaluate_vector(np.array([np.NaN,
test_value])))
self.assertFalse(lt.evaluate_vector(np.array([np.NaN, np.NaN])))
hp4 = CategoricalHyperparameter("cat", list(range(6)))
self.assertRaisesRegexp(
ValueError, "Parent hyperparameter in a > or < "
"condition must be a subclass of "
"NumericalHyperparameter or "
"OrdinalHyperparameter, but is "
"<cdef class 'ConfigSpaceNNI.hyperparameters.CategoricalHyperparameter'>",
GreaterThanCondition, child, hp4, 1)
self.assertRaisesRegexp(
ValueError, "Parent hyperparameter in a > or < "
"condition must be a subclass of "
"NumericalHyperparameter or "
"OrdinalHyperparameter, but is "
"<cdef class 'ConfigSpaceNNI.hyperparameters.CategoricalHyperparameter'>",
LessThanCondition, child, hp4, 1)
hp5 = OrdinalHyperparameter("ord",
['cold', 'luke warm', 'warm', 'hot'])
hyperparameter_idx = {child.name: 0, hp5.name: 1}
gt = GreaterThanCondition(child, hp5, 'warm')
gt.set_vector_idx(hyperparameter_idx)
self.assertTrue(gt.evaluate({hp5.name: 'hot'}))
self.assertFalse(gt.evaluate({hp5.name: 'cold'}))
self.assertTrue(gt.evaluate_vector(np.array([np.NaN, 3])))
self.assertFalse(gt.evaluate_vector(np.array([np.NaN, 0])))
lt = LessThanCondition(child, hp5, 'warm')
lt.set_vector_idx(hyperparameter_idx)
self.assertTrue(lt.evaluate({hp5.name: 'luke warm'}))
self.assertFalse(lt.evaluate({hp5.name: 'warm'}))
self.assertTrue(lt.evaluate_vector(np.array([np.NaN, 1])))
self.assertFalse(lt.evaluate_vector(np.array([np.NaN, 2])))
def test_uniforminteger(self):
# TODO: rounding or converting or error message?
f1 = UniformIntegerHyperparameter("param", 0.0, 5.0)
f1_ = UniformIntegerHyperparameter("param", 0, 5)
self.assertEqual(f1, f1_)
self.assertEqual(
"param, Type: UniformInteger, Range: [0, 5], "
"Default: 2", str(f1))
f2 = UniformIntegerHyperparameter("param", 0, 10, q=0.1)
f2_ = UniformIntegerHyperparameter("param", 0, 10, q=0.1)
self.assertEqual(f2, f2_)
self.assertEqual(
"param, Type: UniformInteger, Range: [0, 10], Default: 5", str(f2))
#f2_large_q = UniformIntegerHyperparameter("param", 0, 10, q=2)
#f2_large_q_ = UniformIntegerHyperparameter("param", 0, 10, q=2)
#self.assertEqual(f2_large_q, f2_large_q_)
#self.assertEqual(
# "param, Type: UniformInteger, Range: [0, 10], Default: 5, Q: 2",
# str(f2_large_q))
f3 = UniformIntegerHyperparameter("param", 1, 10, log=True)
f3_ = UniformIntegerHyperparameter("param", 1, 10, log=True)
self.assertEqual(f3, f3_)
self.assertEqual(
"param, Type: UniformInteger, Range: [1, 10], Default: 3, "
"on log-scale", str(f3))
f4 = UniformIntegerHyperparameter("param",
1,
10,
default_value=1,
log=True)
f4_ = UniformIntegerHyperparameter("param",
1,
10,
default_value=1,
log=True)
self.assertEqual(f4, f4_)
self.assertEqual(
"param, Type: UniformInteger, Range: [1, 10], Default: 1, "
"on log-scale", str(f4))
f5 = UniformIntegerHyperparameter("param", 1, 10, default_value=1, q=0.1,\
log=True)
f5_ = UniformIntegerHyperparameter("param", 1, 10, default_value=1, q=0.1,\
log=True)
self.assertEqual(f5, f5_)
self.assertEqual(
"param, Type: UniformInteger, Range: [1, 10], Default: 1, "
"on log-scale", str(f5))
#self.assertNotEqual(f2, f2_large_q)
self.assertNotEqual(f1, "UniformFloat")
# test that meta-data is stored correctly
f_meta = UniformIntegerHyperparameter("param",
1,
10,
q=0.1,
log=True,
default_value=1,
meta=dict(self.meta_data))
self.assertEqual(f_meta.meta, self.meta_data)
from ConfigSpaceNNI.hyperparameters import CategoricalHyperparameter, \
UniformIntegerHyperparameter, UniformFloatHyperparameter, OrdinalHyperparameter
from ConfigSpaceNNI.conditions import EqualsCondition, InCondition, \
AndConjunction, OrConjunction, NotEqualsCondition, \
GreaterThanCondition
from ConfigSpaceNNI.forbidden import ForbiddenInClause, ForbiddenAndConjunction
# More complex search space
classifier = CategoricalHyperparameter("classifier", ["svm", "nn"])
kernel = CategoricalHyperparameter("kernel", ["rbf", "poly", "sigmoid"])
kernel_condition = EqualsCondition(kernel, classifier, "svm")
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True)
C_condition = EqualsCondition(C, classifier, "svm")
gamma = UniformFloatHyperparameter("gamma", 0.000030518, 8, log=True)
gamma_condition = EqualsCondition(gamma, kernel, "rbf")
degree = UniformIntegerHyperparameter("degree", 1, 5)
degree_condition = InCondition(degree, kernel, ["poly", "sigmoid"])
neurons = UniformIntegerHyperparameter("neurons", 16, 1024)
neurons_condition = EqualsCondition(neurons, classifier, "nn")
lr = UniformFloatHyperparameter("lr", 0.0001, 1.0)
lr_condition = EqualsCondition(lr, classifier, "nn")
preprocessing = CategoricalHyperparameter("preprocessing", ["None", "pca"])
conditional_space = ConfigurationSpace()
conditional_space.add_hyperparameter(preprocessing)
conditional_space.add_hyperparameter(classifier)
conditional_space.add_hyperparameter(kernel)
conditional_space.add_hyperparameter(C)
conditional_space.add_hyperparameter(neurons)
conditional_space.add_hyperparameter(lr)
conditional_space.add_hyperparameter(degree)
conditional_space.add_hyperparameter(gamma)
def test_forbidden_equals_clause(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = CategoricalHyperparameter("grandchild", ["hot", "cold"])
self.assertRaisesRegexp(
TypeError,
"Argument 'hyperparameter' has incorrect type \(expected ConfigSpaceNNI.hyperparameters.Hyperparameter, got str\)",
ForbiddenEqualsClause,
"HP1",
1,
)
self.assertRaisesRegexp(
ValueError,
"Forbidden clause must be instantiated with a legal hyperparameter value for "
"'parent, Type: Categorical, Choices: \{0, 1\}, Default: 0', but got '2'",
ForbiddenEqualsClause,
hp1,
2,
)
forb1 = ForbiddenEqualsClause(hp1, 1)
forb1_ = ForbiddenEqualsClause(hp1, 1)
forb1__ = ForbiddenEqualsClause(hp1, 0)
forb2 = ForbiddenEqualsClause(hp2, 10)
forb3 = ForbiddenEqualsClause(hp3, "hot")
forb3_ = ForbiddenEqualsClause(hp3, "hot")
self.assertEqual(forb3, forb3_)
# print("\eq0:", 1, 1)
# self.assertEqual(1, 1)
# print("\neq1:", forb1, forb1_)
self.assertEqual(forb1, forb1_)
# print("\nneq2:", forb1, "forb1")
self.assertNotEqual(forb1, "forb1")
# print("\nneq3:", forb1, forb2)
self.assertNotEqual(forb1, forb2)
# print("\nneq4:", forb1_, forb1)
self.assertNotEqual(forb1__, forb1)
# print("\neq5:", "Forbidden: parent == 1", str(forb1))
self.assertEqual("Forbidden: parent == 1", str(forb1))
# print("\nraisereg6:")
self.assertRaisesRegexp(
ValueError, "Is_forbidden must be called with the "
"instanstatiated hyperparameter in the "
"forbidden clause; you are missing "
"'parent'", forb1.is_forbidden, {1: hp2}, True)
# print("\nneq7:")
self.assertFalse(forb1.is_forbidden({'child': 1}, strict=False))
# print("\nneq8:")
self.assertFalse(forb1.is_forbidden({'parent': 0}, True))
# print("\nneq9:")
self.assertTrue(forb1.is_forbidden({'parent': 1}, True))
# Test forbidden on vector values
hyperparameter_idx = {hp1.name: 0, hp2.name: 1}
forb1.set_vector_idx(hyperparameter_idx)
# print("\nneq10:")
self.assertFalse(
forb1.is_forbidden_vector(np.array([np.NaN, np.NaN]),
strict=False))
# print("\nneq11:")
self.assertFalse(
forb1.is_forbidden_vector(np.array([0., np.NaN]), strict=False))
# print("\nneq12:")
self.assertTrue(
forb1.is_forbidden_vector(np.array([1., np.NaN]), strict=False))
def test_in_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1, 2, 3, 4])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("child2", 0, 10)
hp4 = CategoricalHyperparameter("grandchild", ["hot", "cold", "warm"])
self.assertRaisesRegexp(
TypeError,
"Argument 'hyperparameter' has incorrect type \(expected ConfigSpaceNNI.hyperparameters.Hyperparameter, got str\)",
ForbiddenInClause,
"HP1",
1,
)
self.assertRaisesRegexp(
ValueError,
"Forbidden clause must be instantiated with a "
"legal hyperparameter value for "
"'parent, Type: Categorical, Choices: {0, 1, 2, 3, 4}, "
"Default: 0', but got '5'",
ForbiddenInClause,
hp1,
[5],
)
forb1 = ForbiddenInClause(hp2, [5, 6, 7, 8, 9])
forb1_ = ForbiddenInClause(hp2, [9, 8, 7, 6, 5])
forb2 = ForbiddenInClause(hp2, [5, 6, 7, 8])
forb3 = ForbiddenInClause(hp3, [5, 6, 7, 8, 9])
forb4 = ForbiddenInClause(hp4, ["hot", "cold"])
forb4_ = ForbiddenInClause(hp4, ["hot", "cold"])
forb5 = ForbiddenInClause(hp1, [3, 4])
forb5_ = ForbiddenInClause(hp1, [3, 4])
self.assertEqual(forb5, forb5_)
self.assertEqual(forb4, forb4_)
# print("\nTest1:")
self.assertEqual(forb1, forb1_)
# print("\nTest2:")
self.assertNotEqual(forb1, forb2)
# print("\nTest3:")
self.assertNotEqual(forb1, forb3)
# print("\nTest4:")
self.assertEqual("Forbidden: child in {5, 6, 7, 8, 9}", str(forb1))
# print("\nTest5:")
self.assertRaisesRegexp(
ValueError, "Is_forbidden must be called with the "
"instanstatiated hyperparameter in the "
"forbidden clause; you are missing "
"'child'", forb1.is_forbidden, {'parent': 1}, True)
# print("\nTest6:")
self.assertFalse(forb1.is_forbidden({'parent': 1}, strict=False))
# print("\nTest7:")
for i in range(0, 5):
self.assertFalse(forb1.is_forbidden({'child': i}, True))
# print("\nTest8:")
for i in range(5, 10):
self.assertTrue(forb1.is_forbidden({'child': i}, True))
# Test forbidden on vector values
hyperparameter_idx = {hp1.name: 0, hp2.name: 1}
forb1.set_vector_idx(hyperparameter_idx)
# print("\nTest9:")
self.assertFalse(
forb1.is_forbidden_vector(np.array([np.NaN, np.NaN]),
strict=False))
# print("\nTest10:")
self.assertFalse(
forb1.is_forbidden_vector(np.array([np.NaN, 0]), strict=False))
correct_vector_value = hp2._inverse_transform(6)
# print("\nTest11:")
print(correct_vector_value, np.array([np.NaN, correct_vector_value]))
self.assertTrue(
forb1.is_forbidden_vector(np.array([np.NaN, correct_vector_value]),
strict=False))