def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter(
"classifier", ["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
# check backward compatibility with checking configurations instead of vectors
cs.check_configuration(configuration)
def _check_and_cast_configuration(configuration: Union[Dict, ConfigSpace.Configuration],
configuration_space: ConfigSpace.ConfigurationSpace) \
-> ConfigSpace.Configuration:
""" Helper-function to evaluate the given configuration.
Cast it to a ConfigSpace.Configuration and evaluate if it violates its boundaries.
Note:
We remove inactive hyperparameters from the given configuration. Inactive hyperparameters are
hyperparameters that are not relevant for a configuration, e.g. hyperparameter A is only relevant if
hyperparameter B=1 and if B!=1 then A is inactive and will be removed from the configuration.
Since the authors of the benchmark removed those parameters explicitly, they should also handle the
cases that inactive parameters are not present in the input-configuration.
"""
if isinstance(configuration, dict):
configuration = ConfigSpace.Configuration(configuration_space, configuration,
allow_inactive_with_values=True)
elif isinstance(configuration, ConfigSpace.Configuration):
configuration = configuration
else:
raise TypeError(f'Configuration has to be from type List, np.ndarray, dict, or '
f'ConfigSpace.Configuration but was {type(configuration)}')
all_hps = set(configuration_space.get_hyperparameter_names())
active_hps = configuration_space.get_active_hyperparameters(configuration)
inactive_hps = all_hps - active_hps
if len(inactive_hps) != 0:
logger.debug(f'There are inactive {len(inactive_hps)} hyperparameter: {inactive_hps}'
'Going to remove them from the configuration.')
configuration = deactivate_inactive_hyperparameters(configuration, configuration_space)
configuration_space.check_configuration(configuration)
return configuration
def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter("classifier",
["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
# check backward compatibility with checking configurations instead of vectors
cs.check_configuration(configuration)
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def _check_and_cast_configuration(configuration: Union[Dict, ConfigSpace.Configuration],
configuration_space: ConfigSpace.ConfigurationSpace) \
-> ConfigSpace.Configuration:
""" Helper-function to evaluate the given configuration.
Cast it to a ConfigSpace.Configuration and evaluate if it violates some boundaries.
"""
if isinstance(configuration, dict):
configuration = ConfigSpace.Configuration(configuration_space, configuration)
elif isinstance(configuration, ConfigSpace.Configuration):
configuration = configuration
else:
raise TypeError(f'Configuration has to be from type List, np.ndarray, dict, or '
f'ConfigSpace.Configuration but was {type(configuration)}')
configuration_space.check_configuration(configuration)
return configuration
def _check_and_cast_fidelity(fidelity: Union[dict, ConfigSpace.Configuration, None],
fidelity_space: ConfigSpace.ConfigurationSpace, **kwargs) \
-> ConfigSpace.Configuration:
""" Helper-function to evaluate the given fidelity object.
Similar to the checking and casting from above, we validate the fidelity object. To do so, we cast it to a
ConfigSpace.Configuration object.
If the fidelity is not specified (None), then we use the default fidelity of the benchmark.
If the benchmark is a multi-multi-fidelity benchmark and only a subset of the available fidelities is
specified, we fill the missing ones with their default values.
"""
# Make a check, that no fidelities are in the kwargs.
f_in_kwargs = []
for f in fidelity_space.get_hyperparameters():
if f.name in kwargs:
f_in_kwargs.append(f.name)
if len(f_in_kwargs) != 0:
raise ValueError(f'Fidelity parameters {", ".join(f_in_kwargs)} should not be part of kwargs\n'
f'Fidelity: {fidelity}\n Kwargs: {kwargs}')
default_fidelities = fidelity_space.get_default_configuration()
if fidelity is None:
fidelity = default_fidelities
if isinstance(fidelity, dict):
default_fidelities_cfg = default_fidelities.get_dictionary()
fidelity_copy = fidelity.copy()
fidelity = {k: fidelity_copy.pop(k, v) for k, v in default_fidelities_cfg.items()}
assert len(fidelity_copy) == 0, 'Provided fidelity dict contained unknown fidelity ' \
f'values: {fidelity_copy.keys()}'
fidelity = ConfigSpace.Configuration(fidelity_space, fidelity)
elif isinstance(fidelity, ConfigSpace.Configuration):
fidelity = fidelity
else:
raise TypeError(f'Fidelity has to be an instance of type None, dict, or '
f'ConfigSpace.Configuration but was {type(fidelity)}')
# Ensure that the extracted fidelity values play well with the defined fidelity space
fidelity_space.check_configuration(fidelity)
return fidelity