def test_acts_as_mapping(self): """ This tests checks that a Configuration can be used as a a dictionary by checking indexing[], iteration ..., items, keys """ names = ['parent', 'child', 'friend'] values = [1, 2, 3] values_dict = dict(zip(names, values)) config = Configuration(self.cs, values=values_dict) # Test indexing assert (config['parent'] == values_dict['parent'] and config['child'] == values_dict['child']) # Test dict methods assert set(config.keys()) == set(names) assert set(config.values()) == set(values) assert set(config.items()) == set(values_dict.items()) assert len(config) == 3 # Test __iter__ assert set(iter(config)) == set(names) # Test unpacking d = {**config} assert d == values_dict
def test_order_of_hyperparameters_is_same_as_config_space(self): """ Test the keys respect the contract that they follow the same order that is present in the ConfigurationSpace. """ # Deliberatily different values config = Configuration(self.cs, values={ 'child': 2, 'parent': 1, 'friend': 3 }) assert config.keys() == list(self.cs.keys())
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]: """Return all configurations in a one-exchange neighborhood. The method is implemented as defined by: Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown Sequential Model-Based Optimization for General Algorithm Configuration In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5) """ random = np.random.RandomState(seed) hyperparameters_list = list(configuration.keys()) hyperparameters_list_length = len(hyperparameters_list) neighbors_to_return = dict() hyperparameters_used = list() number_of_usable_hyperparameters = sum( np.isfinite(configuration.get_array())) configuration_space = configuration.configuration_space while len(hyperparameters_used) != number_of_usable_hyperparameters: index = random.randint(hyperparameters_list_length) hp_name = hyperparameters_list[index] if hp_name in neighbors_to_return: random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_ else: neighbourhood = [] number_of_sampled_neighbors = 0 array = configuration.get_array() if not np.isfinite(array[index]): continue iteration = 0 while True: hp = configuration_space.get_hyperparameter(hp_name) configuration._populate_values() num_neighbors = hp.get_num_neighbors( configuration.get(hp_name)) # Obtain neigbors differently for different possible numbers of # neighbors if num_neighbors == 0: break # No infinite loops elif iteration > 100: break elif np.isinf(num_neighbors): if number_of_sampled_neighbors >= 4: break num_samples_to_go = 4 - number_of_sampled_neighbors neighbors = hp.get_neighbors(array[index], random, number=num_samples_to_go) else: if iteration > 0: break neighbors = hp.get_neighbors(array[index], random) # Check all newly obtained neigbors for neighbor in neighbors: new_array = array.copy() new_array[index] = neighbor neighbor_value = hp._transform(neighbor) new_array = check_neighbouring_config_vector( configuration, new_array, neighbor_value, hp_name) try: # Populating a configuration from an array does not check # if it is a legal configuration - check this (slow) new_configuration = Configuration(configuration_space, vector=new_array) # Only rigorously check every tenth configuration ( # because moving around in the neighborhood should # just work!) if np.random.random() > 0.9: new_configuration.is_valid_configuration() else: configuration_space._check_forbidden(new_array) neighbourhood.append(new_configuration) number_of_sampled_neighbors += 1 # todo: investigate why tests fail when ForbiddenValueError is caught here except ForbiddenValueError as e: pass # Count iterations to not run into an infinite loop when # sampling floats/ints and there is large amount of forbidden # values; also to find out if we tried to get a neighbor for # a categorical hyperparameter, and the only possible # neighbor is forbidden together with another active # value/default hyperparameter iteration += 1 if len(neighbourhood) == 0: hyperparameters_used.append(hp_name) else: if hp_name not in hyperparameters_used: neighbors_to_return[hp_name] = neighbourhood random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]: """Return all configurations in a one-exchange neighborhood. The method is implemented as defined by: Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown Sequential Model-Based Optimization for General Algorithm Configuration In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5) """ random = np.random.RandomState(seed) hyperparameters_list = list(configuration.keys()) hyperparameters_list_length = len(hyperparameters_list) neighbors_to_return = dict() hyperparameters_used = list() number_of_usable_hyperparameters = sum( np.isfinite(configuration.get_array())) while len(hyperparameters_used) != number_of_usable_hyperparameters: index = random.randint(hyperparameters_list_length) hp_name = hyperparameters_list[index] if hp_name in neighbors_to_return: random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_ else: neighbourhood = [] number_of_sampled_neighbors = 0 array = configuration.get_array() if not np.isfinite(array[index]): continue iteration = 0 while True: hp = configuration.configuration_space.get_hyperparameter( hp_name) configuration._populate_values() num_neighbors = hp.get_num_neighbors( configuration.get(hp_name)) # Obtain neigbors differently for different possible numbers of # neighbors if num_neighbors == 0: break # No infinite loops elif iteration > 100: break elif np.isinf(num_neighbors): if number_of_sampled_neighbors >= 4: break num_samples_to_go = 4 - number_of_sampled_neighbors neighbors = hp.get_neighbors(array[index], random, number=num_samples_to_go) else: if iteration > 0: break neighbors = hp.get_neighbors(array[index], random) # Check all newly obtained neigbors for neighbor in neighbors: new_array = array.copy() new_array[index] = neighbor neighbor_value = hp._transform(neighbor) # Activate hyperparameters if their parent node got activated children = configuration.configuration_space.get_children_of( hp_name) if len(children) > 0: to_visit = deque() #type: deque to_visit.extendleft(children) visited = set() #type: Set[str] activated_values = dict( ) #type: Dict[str, Union[int, float, str]] while len(to_visit) > 0: current = to_visit.pop() if current.name in visited: continue visited.add(current.name) current_idx = configuration.configuration_space. \ get_idx_by_hyperparameter_name(current.name) current_value = new_array[current_idx] conditions = configuration.configuration_space.\ _get_parent_conditions_of(current.name) active = True for condition in conditions: parent_names = [ c.parent.name for c in condition. get_descendant_literal_conditions() ] parents = { parent_name: configuration[parent_name] for parent_name in parent_names } # parents come from the original configuration. # We change at least one parameter. In order set # other parameters which are conditional on this, # we have to activate this if hp_name in parents: parents[hp_name] = neighbor_value # Hyperparameters which are in depth 1 of the # hyperparameter tree might have children which # have to be activated as well. Once we set hp in # level 1 to active, it's value changes from the # value of the original configuration and this # must be done here for parent_name in parent_names: if parent_name in activated_values: parents[ parent_name] = activated_values[ parent_name] # if one of the parents is None, the hyperparameter cannot be # active! Else we have to check this if any([ parent_value is None for parent_value in parents.values() ]): active = False break else: if not condition.evaluate(parents): active = False break if active and (current_value is None or not np.isfinite(current_value)): default = current._inverse_transform( current.default) new_array[current_idx] = default children = configuration.configuration_space.get_children_of( current.name) if len(children) > 0: to_visit.extendleft(children) activated_values[ current.name] = current.default if not active and (current_value is not None or np.isfinite(current_value)): new_array[current_idx] = np.NaN try: # Populating a configuration from an array does not check # if it is a legal configuration - check this (slow) new_configuration = Configuration( configuration.configuration_space, vector=new_array) new_configuration.is_valid_configuration() neighbourhood.append(new_configuration) number_of_sampled_neighbors += 1 # todo: investigate why tests fail when ForbiddenValueError is caught here except ValueError as e: pass # Count iterations to not run into an infinite loop when # sampling floats/ints and there is large amount of forbidden # values; also to find out if we tried to get a neighbor for # a categorical hyperparameter, and the only possible # neighbor is forbidden together with another active # value/default hyperparameter iteration += 1 if len(neighbourhood) == 0: hyperparameters_used.append(hp_name) else: if hp_name not in hyperparameters_used: neighbors_to_return[hp_name] = neighbourhood random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]: """Return all configurations in a one-exchange neighborhood. The method is implemented as defined by: Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown Sequential Model-Based Optimization for General Algorithm Configuration In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5) """ random = np.random.RandomState(seed) hyperparameters_list = list(configuration.keys()) hyperparameters_list_length = len(hyperparameters_list) neighbors_to_return = dict() hyperparameters_used = list() number_of_usable_hyperparameters = sum(np.isfinite(configuration.get_array())) configuration_space = configuration.configuration_space while len(hyperparameters_used) != number_of_usable_hyperparameters: index = random.randint(hyperparameters_list_length) hp_name = hyperparameters_list[index] if hp_name in neighbors_to_return: random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_ else: neighbourhood = [] number_of_sampled_neighbors = 0 array = configuration.get_array() if not np.isfinite(array[index]): continue iteration = 0 while True: hp = configuration_space.get_hyperparameter(hp_name) configuration._populate_values() num_neighbors = hp.get_num_neighbors(configuration.get(hp_name)) # Obtain neigbors differently for different possible numbers of # neighbors if num_neighbors == 0: break # No infinite loops elif iteration > 100: break elif np.isinf(num_neighbors): if number_of_sampled_neighbors >= 4: break num_samples_to_go = 4 - number_of_sampled_neighbors neighbors = hp.get_neighbors(array[index], random, number=num_samples_to_go) else: if iteration > 0: break neighbors = hp.get_neighbors(array[index], random) # Check all newly obtained neigbors for neighbor in neighbors: new_array = array.copy() new_array[index] = neighbor neighbor_value = hp._transform(neighbor) # Hyperparameters which are going to be set to inactive disabled = [] # Activate hyperparameters if their parent node got activated children = configuration_space._children_of[hp_name] if len(children) > 0: to_visit = deque() #type: deque to_visit.extendleft(children) visited = set() #type: Set[str] activated_values = dict() #type: Dict[str, Union[int, float, str]] while len(to_visit) > 0: current = to_visit.pop() if current.name in visited: continue visited.add(current.name) if current.name in disabled: continue current_idx = configuration_space.get_idx_by_hyperparameter_name(current.name) current_value = new_array[current_idx] conditions = configuration.configuration_space.\ _parent_conditions_of[current.name] active = True for condition in conditions: parent_names = [parent.name for parent in configuration_space._parents_of[current.name]] parents = {parent_name: configuration[parent_name] for parent_name in parent_names} # parents come from the original configuration. # We change at least one parameter. In order set # other parameters which are conditional on this, # we have to activate this if hp_name in parents: parents[hp_name] = neighbor_value # Hyperparameters which are in depth 1 of the # hyperparameter tree might have children which # have to be activated as well. Once we set hp in # level 1 to active, it's value changes from the # value of the original configuration and this # must be done here for parent_name in parent_names: if parent_name in activated_values: parents[parent_name] = activated_values[ parent_name] # if one of the parents is None, the hyperparameter cannot be # active! Else we have to check this if any([parent_value is None for parent_value in parents.values()]): active = False break else: if not condition.evaluate(parents): active = False break if active and (current_value is None or not np.isfinite(current_value)): default = current._inverse_transform(current.default) new_array[current_idx] = default children_ = configuration_space._children_of[current.name] if len(children_) > 0: to_visit.extendleft(children_) activated_values[current.name] = current.default # If the hyperparameter was made inactive, # all its children need to be deactivade as well if not active and (current_value is not None or np.isfinite(current_value)): new_array[current_idx] = np.NaN children = configuration.configuration_space._children_of[current.name] if len(children) > 0: to_disable = set() for ch in children: to_disable.add(ch.name) while len(to_disable) > 0: child = to_disable.pop() child_idx = configuration.configuration_space. \ get_idx_by_hyperparameter_name(child) disabled.append(child_idx) children = configuration.configuration_space._children_of[child] for ch in children: to_disable.add(ch.name) for idx in disabled: new_array[idx] = np.NaN try: # Populating a configuration from an array does not check # if it is a legal configuration - check this (slow) new_configuration = Configuration(configuration_space, vector=new_array) new_configuration.is_valid_configuration() neighbourhood.append(new_configuration) number_of_sampled_neighbors += 1 # todo: investigate why tests fail when ForbiddenValueError is caught here except ForbiddenValueError as e: pass # Count iterations to not run into an infinite loop when # sampling floats/ints and there is large amount of forbidden # values; also to find out if we tried to get a neighbor for # a categorical hyperparameter, and the only possible # neighbor is forbidden together with another active # value/default hyperparameter iteration += 1 if len(neighbourhood) == 0: hyperparameters_used.append(hp_name) else: if hp_name not in hyperparameters_used: neighbors_to_return[hp_name] = neighbourhood random.shuffle(neighbors_to_return[hp_name]) n_ = neighbors_to_return[hp_name].pop() if len(neighbors_to_return[hp_name]) == 0: del neighbors_to_return[hp_name] hyperparameters_used.append(hp_name) yield n_