def test_acts_as_mapping(self):
"""
This tests checks that a Configuration can be used as a a dictionary by
checking indexing[], iteration ..., items, keys
"""
names = ['parent', 'child', 'friend']
values = [1, 2, 3]
values_dict = dict(zip(names, values))
config = Configuration(self.cs, values=values_dict)
# Test indexing
assert (config['parent'] == values_dict['parent']
and config['child'] == values_dict['child'])
# Test dict methods
assert set(config.keys()) == set(names)
assert set(config.values()) == set(values)
assert set(config.items()) == set(values_dict.items())
assert len(config) == 3
# Test __iter__
assert set(iter(config)) == set(names)
# Test unpacking
d = {**config}
assert d == values_dict
def test_order_of_hyperparameters_is_same_as_config_space(self):
"""
Test the keys respect the contract that they follow the same order that
is present in the ConfigurationSpace.
"""
# Deliberatily different values
config = Configuration(self.cs,
values={
'child': 2,
'parent': 1,
'friend': 3
})
assert config.keys() == list(self.cs.keys())
def get_one_exchange_neighbourhood(configuration: Configuration,
seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(
np.isfinite(configuration.get_array()))
configuration_space = configuration.configuration_space
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration_space.get_hyperparameter(hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(
configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index],
random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
new_array = check_neighbouring_config_vector(
configuration, new_array, neighbor_value, hp_name)
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(configuration_space,
vector=new_array)
# Only rigorously check every tenth configuration (
# because moving around in the neighborhood should
# just work!)
if np.random.random() > 0.9:
new_configuration.is_valid_configuration()
else:
configuration_space._check_forbidden(new_array)
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ForbiddenValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
def get_one_exchange_neighbourhood(configuration: Configuration,
seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(
np.isfinite(configuration.get_array()))
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration.configuration_space.get_hyperparameter(
hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(
configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index],
random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
# Activate hyperparameters if their parent node got activated
children = configuration.configuration_space.get_children_of(
hp_name)
if len(children) > 0:
to_visit = deque() #type: deque
to_visit.extendleft(children)
visited = set() #type: Set[str]
activated_values = dict(
) #type: Dict[str, Union[int, float, str]]
while len(to_visit) > 0:
current = to_visit.pop()
if current.name in visited:
continue
visited.add(current.name)
current_idx = configuration.configuration_space. \
get_idx_by_hyperparameter_name(current.name)
current_value = new_array[current_idx]
conditions = configuration.configuration_space.\
_get_parent_conditions_of(current.name)
active = True
for condition in conditions:
parent_names = [
c.parent.name for c in condition.
get_descendant_literal_conditions()
]
parents = {
parent_name: configuration[parent_name]
for parent_name in parent_names
}
# parents come from the original configuration.
# We change at least one parameter. In order set
# other parameters which are conditional on this,
# we have to activate this
if hp_name in parents:
parents[hp_name] = neighbor_value
# Hyperparameters which are in depth 1 of the
# hyperparameter tree might have children which
# have to be activated as well. Once we set hp in
# level 1 to active, it's value changes from the
# value of the original configuration and this
# must be done here
for parent_name in parent_names:
if parent_name in activated_values:
parents[
parent_name] = activated_values[
parent_name]
# if one of the parents is None, the hyperparameter cannot be
# active! Else we have to check this
if any([
parent_value is None
for parent_value in parents.values()
]):
active = False
break
else:
if not condition.evaluate(parents):
active = False
break
if active and (current_value is None
or not np.isfinite(current_value)):
default = current._inverse_transform(
current.default)
new_array[current_idx] = default
children = configuration.configuration_space.get_children_of(
current.name)
if len(children) > 0:
to_visit.extendleft(children)
activated_values[
current.name] = current.default
if not active and (current_value is not None
or np.isfinite(current_value)):
new_array[current_idx] = np.NaN
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(
configuration.configuration_space,
vector=new_array)
new_configuration.is_valid_configuration()
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(np.isfinite(configuration.get_array()))
configuration_space = configuration.configuration_space
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration_space.get_hyperparameter(hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index], random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
# Hyperparameters which are going to be set to inactive
disabled = []
# Activate hyperparameters if their parent node got activated
children = configuration_space._children_of[hp_name]
if len(children) > 0:
to_visit = deque() #type: deque
to_visit.extendleft(children)
visited = set() #type: Set[str]
activated_values = dict() #type: Dict[str, Union[int, float, str]]
while len(to_visit) > 0:
current = to_visit.pop()
if current.name in visited:
continue
visited.add(current.name)
if current.name in disabled:
continue
current_idx = configuration_space.get_idx_by_hyperparameter_name(current.name)
current_value = new_array[current_idx]
conditions = configuration.configuration_space.\
_parent_conditions_of[current.name]
active = True
for condition in conditions:
parent_names = [parent.name for parent in
configuration_space._parents_of[current.name]]
parents = {parent_name: configuration[parent_name] for
parent_name in parent_names}
# parents come from the original configuration.
# We change at least one parameter. In order set
# other parameters which are conditional on this,
# we have to activate this
if hp_name in parents:
parents[hp_name] = neighbor_value
# Hyperparameters which are in depth 1 of the
# hyperparameter tree might have children which
# have to be activated as well. Once we set hp in
# level 1 to active, it's value changes from the
# value of the original configuration and this
# must be done here
for parent_name in parent_names:
if parent_name in activated_values:
parents[parent_name] = activated_values[
parent_name]
# if one of the parents is None, the hyperparameter cannot be
# active! Else we have to check this
if any([parent_value is None for parent_value in
parents.values()]):
active = False
break
else:
if not condition.evaluate(parents):
active = False
break
if active and (current_value is None or
not np.isfinite(current_value)):
default = current._inverse_transform(current.default)
new_array[current_idx] = default
children_ = configuration_space._children_of[current.name]
if len(children_) > 0:
to_visit.extendleft(children_)
activated_values[current.name] = current.default
# If the hyperparameter was made inactive,
# all its children need to be deactivade as well
if not active and (current_value is not None
or np.isfinite(current_value)):
new_array[current_idx] = np.NaN
children = configuration.configuration_space._children_of[current.name]
if len(children) > 0:
to_disable = set()
for ch in children:
to_disable.add(ch.name)
while len(to_disable) > 0:
child = to_disable.pop()
child_idx = configuration.configuration_space. \
get_idx_by_hyperparameter_name(child)
disabled.append(child_idx)
children = configuration.configuration_space._children_of[child]
for ch in children:
to_disable.add(ch.name)
for idx in disabled:
new_array[idx] = np.NaN
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(configuration_space, vector=new_array)
new_configuration.is_valid_configuration()
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ForbiddenValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
