def set_hyperparameters(
self,
configuration: Configuration,
init_params: Optional[Dict[str, Any]] = None) -> 'OHEChoice':
new_params = {}
params = configuration.get_dictionary()
choice = params['__choice__']
del params['__choice__']
for param, value in params.items():
param = param.replace(choice, '').replace(':', '')
new_params[param] = value
if init_params is not None:
for param, value in init_params.items():
# These next two lines are different than in the base class -
# they allow removing the categorical feature indicator array
# in order to not pass it to the no encoding
if choice not in param:
continue
param = param.replace(choice, '').replace(':', '')
new_params[param] = value
new_params['random_state'] = self.random_state
self.new_params = new_params
self.choice = self.get_components()[choice](**new_params)
return self
def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(NormalFloatHyperparameter('b', 1, 2,
log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(
NormalFloatHyperparameter('b', 1, 2, log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def insert_runhistory(self, config: Configuration, cost: float, time: float,
status: StatusType, instance_id: str = "",
seed: int = 0,
additional_info: dict = frozendict(),
origin: DataOrigin = DataOrigin.INTERNAL):
config_id = get_id_of_config(config)
run_id = self.get_run_id(instance_id, config_id)
if instance_id is None:
instance_id = ""
try:
self.Model(
run_id=run_id,
config_id=config_id,
config=config.get_dictionary(),
config_origin=config.origin,
config_bin=pickle.dumps(config),
cost=cost,
time=time,
instance_id=instance_id,
seed=seed,
status=status.value,
additional_info=dict(additional_info),
origin=origin.value,
).save()
except Exception as e:
pass
self.timestamp = datetime.datetime.now()
def set_child_hyperparameters(self,
children: List[Tuple[str, EstimatorComponent]],
configuration: Dict = None,
init_params=None):
for node_idx, (node_name, node) in enumerate(children):
sub_configuration_space = node.get_hyperparameter_search_space()
sub_config_dict = {}
for param in configuration:
if param.startswith(f'{node_name}:'):
value = configuration[param]
new_name = param.replace(f'{node_name}:', '', 1)
sub_config_dict[new_name] = value
sub_configuration = Configuration(sub_configuration_space, values=sub_config_dict)
if init_params is not None:
sub_init_params_dict = {}
for param in init_params:
if param.startswith(f'{node_name}:'):
value = init_params[param]
new_name = param.replace(f'{node_name}:', '', 1)
sub_init_params_dict[new_name] = value
else:
sub_init_params_dict = None
if isinstance(node, (ComponentChoice, EstimatorComponent)):
node.set_hyperparameters(configuration=sub_configuration.get_dictionary(),
init_params=sub_init_params_dict)
else:
raise NotImplementedError('Not supported yet!')
def process_config_info_pair(self, config: Configuration, info_dict: dict,
budget):
self.budget2obvs[budget]["locks"].append(config.get_array().copy())
info_dict = deepcopy(info_dict)
if config.origin is None:
config.origin = "unknown"
info_dict.update({"origin": config.origin})
return config.get_dictionary(), info_dict
def remove_resource(
self, config_ext: CS.Configuration,
as_dict: bool=False) -> Union[CS.Configuration, dict]:
x_dct = copy.copy(config_ext.get_dictionary())
del x_dct[self.resource_attr_name]
if as_dict:
return x_dct
else:
return CS.Configuration(self.hp_ranges.config_space, values=x_dct)
def get(self, config: CS.Configuration, resource: int) -> CS.Configuration:
"""
Create extended config with resource added.
:param config:
:param resource:
:return: Extended config
"""
values = copy.deepcopy(config.get_dictionary())
values[self.resource_attr_name] = resource
return CS.Configuration(self.hp_ranges_ext.config_space, values=values)
def test_remove_inactive_parameter():
configuration_space = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("hp1", choices=[0, 1])
hp2 = CategoricalHyperparameter("hp2", choices=['a'])
hp3 = UniformIntegerHyperparameter("hp3",
lower=0,
upper=5,
default_value=5)
configuration_space.add_hyperparameters([hp1, hp2, hp3])
# If hp1 = 0, then don't allow hp2
not_condition = NotEqualsCondition(hp2, hp1, 0)
configuration_space.add_condition(not_condition)
allowed_cfg = Configuration(configuration_space, {
'hp1': 1,
'hp2': 'a',
'hp3': 5
})
not_allowed = {'hp1': 0, 'hp2': 'a', 'hp3': 5}
with pytest.raises(ValueError):
Configuration(configuration_space, not_allowed)
# No inactive hp - case: config is CS.configuration
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg, configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# No inactive hp - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg.get_dictionary(), configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# Remove inactive: - case: config is CS.configuration
not_allowed_cs = Configuration(configuration_space, {
'hp1': 0,
'hp2': 'a',
'hp3': 5
},
allow_inactive_with_values=True)
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed_cs, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
# Remove inactive: - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
def split(self, config_ext: CS.Configuration, as_dict: bool=False) -> \
(Union[CS.Configuration, dict], int):
"""
Split extended config into normal config and resource value.
:param config_ext: Extended config
:param as_dict: Return config as dict?
:return: (config, resource_value)
"""
x_res = copy.copy(config_ext.get_dictionary())
resource_value = int(x_res[self.resource_attr_name])
del x_res[self.resource_attr_name]
if not as_dict:
x_res = CS.Configuration(self.hp_ranges.config_space, values=x_res)
return x_res, resource_value
def remove_resource(
self,
config_ext: CS.Configuration,
as_dict: bool = False) -> Union[CS.Configuration, dict]:
"""
Strips away resource attribute and returns normal config
:param config_ext: Extended config
:param as_dict: Return as dict?
:return: config_ext without resource attribute
"""
x_dct = copy.copy(config_ext.get_dictionary())
del x_dct[self.resource_attr_name]
if as_dict:
return x_dct
else:
return CS.Configuration(self.hp_ranges.config_space, values=x_dct)
def transform_dataset(self, ds: Dataset, cid: CandidateId, component: EstimatorComponent,
config: Configuration) -> Tuple[np.ndarray, Optional[float]]:
component.set_hyperparameters(config.get_dictionary())
if is_classifier(component):
score, y_pred, y_prob, models = self._score(ds, component)
# TODO crude fix for holdout score. Fix this
if y_pred.shape != ds.y.shape:
y_pred = models[0].predict(ds.X)
y_prob = models[0].predict_proba(ds.X)
X = np.hstack((ds.X, y_prob, np.reshape(y_pred, (-1, 1))))
else:
models = [component.fit(ds.X, ds.y)]
X = models[0].transform(ds.X)
score = [None, None]
self._store_models(cid, models)
return X, score
def get_random_neighborhood(configuration: Configuration, num: int,
seed: int) -> List[Configuration]:
configuration_space = configuration.configuration_space
conf_dict_data = configuration.get_dictionary()
array_data = configuration.get_array()
neighbor_dict = dict()
for key, value in conf_dict_data.items():
neighbor_dict[key] = [
array_data[configuration_space._hyperparameter_idx[key]]
]
for hp in configuration.configuration_space.get_hyperparameters():
# trans_data = hp._inverse_transform(conf_dict_data[hp.name])
# neighbors = hp.get_neighbors(trans_data, np.random.RandomState(seed), num, False)
# neighbor_dict[hp.name].extend(neighbors)
if hp.name not in conf_dict_data:
continue
neighbors = get_hp_neighbors(hp,
conf_dict_data,
num,
transform=False,
seed=seed)
neighbor_dict[hp.name].extend(neighbors)
neighborhood = []
conf_num = 0
cnt = 0
while conf_num < num and cnt < 5 * num:
cnt += 1
data = array_data.copy()
# TODO: one exchange neighborhood
for key in conf_dict_data.keys():
data[configuration_space._hyperparameter_idx[key]] = random.choice(
neighbor_dict[key])
# data[configuration_space._hyperparameter_idx[key]] = sample_hp(neighbor_dict[key], seed)
try:
config = Configuration(configuration_space, vector=data)
config.is_valid_configuration()
except Exception as e:
pass
if config not in neighborhood:
neighborhood.append(config)
conf_num += 1
assert (len(neighborhood) >= 1)
return neighborhood
def remap_resource(self,
config_ext: CS.Configuration,
resource: int,
as_dict: bool = False) -> Union[CS.Configuration, dict]:
"""
Re-assigns resource value for extended config.
:param config_ext: Extended config
:param resource: New resource value
:param as_dict: Return as dict?
:return:
"""
x_dct = copy.copy(config_ext.get_dictionary())
x_dct[self.resource_attr_name] = resource
if as_dict:
return x_dct
else:
return CS.Configuration(self.hp_ranges_ext.config_space,
values=x_dct)
def cleanup_pending(self, config: CS.Configuration):
"""
Removes all pending candidates whose configuration (i.e., lacking the
resource attribute) is equal to config.
This should be called after an evaluation terminates. For various
reasons (e.g., termination due to convergence), pending candidates
for this evaluation may still be present.
It is also called for a failed evaluation.
:param config: See above
"""
config_dct = config.get_dictionary()
def filter_pred(x: PendingEvaluation) -> bool:
x_dct = self.configspace_ext.remove_resource(
x.candidate, as_dict=True)
return x_dct != config_dct
self.state_transformer.filter_pending_evaluations(filter_pred)
for uhp in unconditional_hyperparameters:
children = configuration_space._children_of[uhp]
if len(children) > 0:
hyperparameters_with_children.append(uhp)
hps.extendleft(hyperparameters_with_children)
inactive = set()
while len(hps) > 0:
hp = hps.pop()
children = configuration_space._children_of[hp]
for child in children:
conditions = configuration_space._parent_conditions_of[child.name]
for condition in conditions:
if not condition.evaluate_vector(configuration.get_array()):
dic = configuration.get_dictionary()
try:
del dic[child.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
inactive.add(child.name)
hps.appendleft(child.name)
for hp in hyperparameters:
if hp.name in inactive:
dic = configuration.get_dictionary()
try:
def get_resource(self, config_ext: CS.Configuration) -> int:
"""
:param config_ext: Extended config
:return: Value of resource attribute
"""
return int(config_ext.get_dictionary()[self.resource_attr_name])
def get_random_neighbor(configuration: Configuration, seed: int) -> Configuration:
"""Draw a random neighbor by changing one parameter of a configuration.
* If the parameter is categorical, it changes it to another value.
* If the parameter is ordinal, it changes it to the next higher or lower
value.
* If parameter is a float, draw a random sample
If changing a parameter activates new parameters or deactivates
previously active parameters, the configuration will be rejected. If more
than 10000 configurations were rejected, this function raises a
ValueError.
Parameters
----------
configuration : Configuration
seed : int
Used to generate a random state.
Returns
-------
Configuration
The new neighbor.
"""
random = np.random.RandomState(seed)
rejected = True
values = copy.deepcopy(configuration.get_dictionary())
while rejected:
# First, choose an active hyperparameter
active = False
iteration = 0
while not active:
iteration += 1
if configuration._num_hyperparameters > 1:
rand_idx = random.randint(0,
configuration._num_hyperparameters - 1)
else:
rand_idx = 0
value = configuration.get_array()[rand_idx]
if np.isfinite(value):
active = True
hp_name = configuration.configuration_space \
.get_hyperparameter_by_idx(rand_idx)
hp = configuration.configuration_space.get_hyperparameter(hp_name)
# Only choose if there is a possibility of finding a neigboor
if not hp.has_neighbors():
active = False
if iteration > 10000:
raise ValueError('Probably caught in an infinite loop.')
# Get a neighboor and adapt the rest of the configuration if necessary
neighbor = hp.get_neighbors(value, random, number=1, transform=True)[0]
previous_value = values[hp.name]
values[hp.name] = neighbor
try:
new_configuration = Configuration(
configuration.configuration_space, values=values)
rejected = False
except ValueError as e:
values[hp.name] = previous_value
return new_configuration
def filter_pred(x: CS.Configuration) -> bool:
x_dct = x.get_dictionary()
return (x_dct[self.name_last_pos] == self.value_for_last_pos)
def _transform_config(self, config: CS.Configuration) -> CS.Configuration:
values = config.get_dictionary() # No copy is done here
values[self.name_last_pos] = self.value_for_last_pos
return CS.Configuration(self.config_space, values=values)
def get_random_neighbor(configuration: Configuration, seed: int) -> Configuration:
"""Draw a random neighbor by changing one parameter of a configuration.
* If the parameter is categorical, it changes it to another value.
* If the parameter is ordinal, it changes it to the next higher or lower
value.
* If parameter is a float, draw a random sample
If changing a parameter activates new parameters or deactivates
previously active parameters, the configuration will be rejected. If more
than 10000 configurations were rejected, this function raises a
ValueError.
Parameters
----------
configuration : Configuration
seed : int
Used to generate a random state.
Returns
-------
Configuration
The new neighbor.
"""
random = np.random.RandomState(seed)
rejected = True
values = copy.deepcopy(configuration.get_dictionary())
while rejected:
# First, choose an active hyperparameter
active = False
iteration = 0
while not active:
iteration += 1
if configuration._num_hyperparameters > 1:
rand_idx = random.randint(0,
configuration._num_hyperparameters - 1)
else:
rand_idx = 0
value = configuration.get_array()[rand_idx]
if np.isfinite(value):
active = True
hp_name = configuration.configuration_space \
.get_hyperparameter_by_idx(rand_idx)
hp = configuration.configuration_space.get_hyperparameter(hp_name)
# Only choose if there is a possibility of finding a neigboor
if not hp.has_neighbors():
active = False
if iteration > 10000:
raise ValueError('Probably caught in an infinite loop.')
# Get a neighboor and adapt the rest of the configuration if necessary
neighbor = hp.get_neighbors(value, random, number=1, transform=True)[0]
previous_value = values[hp.name]
values[hp.name] = neighbor
try:
new_configuration = Configuration(
configuration.configuration_space, values=values)
rejected = False
except ValueError as e:
values[hp.name] = previous_value
return new_configuration
