def new_result(self, job: Job, update_model=True):
##############################
### 1. update observations ###
##############################
if job.result is None:
# One could skip crashed results, but we decided to
# assign a +inf loss and count them as bad configurations
loss = np.inf
else:
# same for non numeric losses.
# Note that this means losses of minus infinity will count as bad!
loss = job.result["loss"] if np.isfinite(
job.result["loss"]) else np.inf
budget = job.kwargs["budget"]
config_dict = job.kwargs["config"]
# config_info = job.kwargs["config_info"]
config = Configuration(self.config_space, config_dict)
# add lock (It may be added twice, but it does not affect)
self.budget2obvs[budget]["locks"].append(config.get_array().copy())
self.budget2obvs[budget]["configs"].append(deepcopy(config))
self.budget2obvs[budget]["vectors"].append(config.get_array())
self.budget2obvs[budget]["losses"].append(loss)
losses = np.array(self.budget2obvs[budget]["losses"])
vectors = np.array(self.budget2obvs[budget]["vectors"])
###################################################################
### 2. Judge whether the EPM training conditions are satisfied ###
###################################################################
if not update_model:
return
self.new_result_(budget, vectors, losses)
def test_check_neighbouring_config_diamond_str(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', ['red', 'green'])
left = CategoricalHyperparameter('left', ['red', 'green'])
right = CategoricalHyperparameter('right',
['red', 'green', 'blue', 'yellow'])
bottom = CategoricalHyperparameter('bottom', ['red', 'green'])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 'red'))
diamond.add_condition(EqualsCondition(right, head, 'red'))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 'green'),
EqualsCondition(bottom, right, 'green')))
config = Configuration(diamond, {
'bottom': 'red',
'head': 'red',
'left': 'green',
'right': 'green'
})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = ConfigSpace.c_util.change_hp_value(diamond,
config.get_array(),
hp_name, neighbor_value,
index)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_check_neighbouring_config_diamond(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1, 2, 3])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 1),
EqualsCondition(bottom, right, 1)))
config = Configuration(diamond, {
'bottom': 0,
'head': 0,
'left': 1,
'right': 1
})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = change_hp_value(diamond, config.get_array(), hp_name,
neighbor_value, index)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_check_neighbouring_config_diamond_str(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', ['red', 'green'])
left = CategoricalHyperparameter('left', ['red', 'green'])
right = CategoricalHyperparameter('right', ['red', 'green', 'blue', 'yellow'])
bottom = CategoricalHyperparameter('bottom', ['red', 'green'])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 'red'))
diamond.add_condition(EqualsCondition(right, head, 'red'))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 'green'),
EqualsCondition(bottom, right, 'green')))
config = Configuration(diamond, {'bottom': 'red', 'head': 'red', 'left': 'green', 'right': 'green'})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = ConfigSpace.c_util.change_hp_value(
diamond,
config.get_array(),
hp_name,
neighbor_value,
index
)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def process_config_info_pair(self, config: Configuration, info_dict: dict,
budget):
self.budget2obvs[budget]["locks"].append(config.get_array().copy())
info_dict = deepcopy(info_dict)
if config.origin is None:
config.origin = "unknown"
info_dict.update({"origin": config.origin})
return config.get_dictionary(), info_dict
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def get_id_of_config(config: Configuration):
# todo:, instance="", seed=0
X: np.ndarray = config.get_array()
m = hashlib.md5()
if X.flags['C_CONTIGUOUS']:
m.update(X.data)
m.update(str(X.shape).encode('utf8'))
else:
X_tmp = np.ascontiguousarray(X.T)
m.update(X_tmp.data)
m.update(str(X_tmp.shape).encode('utf8'))
# m.update(instance.encode())
# m.update(str(seed).encode())
hash_value = m.hexdigest()
return hash_value
def get_random_neighborhood(configuration: Configuration, num: int,
seed: int) -> List[Configuration]:
configuration_space = configuration.configuration_space
conf_dict_data = configuration.get_dictionary()
array_data = configuration.get_array()
neighbor_dict = dict()
for key, value in conf_dict_data.items():
neighbor_dict[key] = [
array_data[configuration_space._hyperparameter_idx[key]]
]
for hp in configuration.configuration_space.get_hyperparameters():
# trans_data = hp._inverse_transform(conf_dict_data[hp.name])
# neighbors = hp.get_neighbors(trans_data, np.random.RandomState(seed), num, False)
# neighbor_dict[hp.name].extend(neighbors)
if hp.name not in conf_dict_data:
continue
neighbors = get_hp_neighbors(hp,
conf_dict_data,
num,
transform=False,
seed=seed)
neighbor_dict[hp.name].extend(neighbors)
neighborhood = []
conf_num = 0
cnt = 0
while conf_num < num and cnt < 5 * num:
cnt += 1
data = array_data.copy()
# TODO: one exchange neighborhood
for key in conf_dict_data.keys():
data[configuration_space._hyperparameter_idx[key]] = random.choice(
neighbor_dict[key])
# data[configuration_space._hyperparameter_idx[key]] = sample_hp(neighbor_dict[key], seed)
try:
config = Configuration(configuration_space, vector=data)
config.is_valid_configuration()
except Exception as e:
pass
if config not in neighborhood:
neighborhood.append(config)
conf_num += 1
assert (len(neighborhood) >= 1)
return neighborhood
def is_config_exist(self, budget, config: Configuration):
vectors_list = []
budgets = [budget_ for budget_ in list(self.budget2obvs.keys()) if budget_ >= budget]
for budget_ in budgets:
vectors = np.array(self.budget2obvs[budget_]["locks"])
if vectors.size:
vectors_list.append(vectors)
if len(vectors_list) == 0:
return False
vectors = np.vstack(vectors_list)
if np.any(np.array(vectors.shape) == 0):
return False
vectors[np.isnan(vectors)] = -1
vector = config.get_array().copy()
vector[np.isnan(vector)] = -1
if np.any(np.all(vector == vectors, axis=1)):
return True
return False
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(
list(configuration.configuration_space._hyperparameters.keys())
)
hyperparameters_list_length = len(hyperparameters_list)
hyperparameters_used = [hp.name for hp in configuration.configuration_space.get_hyperparameters()
if hp.get_num_neighbors(configuration.get(hp.name)) == 0 and configuration.get(hp.name) is not None]
number_of_usable_hyperparameters = sum(np.isfinite(configuration.get_array()))
n_neighbors_per_hp = {
hp.name: 4 if np.isinf(hp.get_num_neighbors(configuration.get(hp.name))) else hp.get_num_neighbors(configuration.get(hp.name))
for hp in configuration.configuration_space.get_hyperparameters()
}
finite_neighbors_stack = {}
configuration_space = configuration.configuration_space
while len(hyperparameters_used) < number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if n_neighbors_per_hp[hp_name] == 0:
continue
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
hp = configuration_space.get_hyperparameter(hp_name)
num_neighbors = hp.get_num_neighbors(configuration.get(hp_name))
while True:
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 1:
break
neighbor = hp.get_neighbors(array[index], random,
number=1)[0]
else:
if iteration > 0:
break
if hp_name not in finite_neighbors_stack:
neighbors = hp.get_neighbors(array[index], random)
random.shuffle(neighbors)
finite_neighbors_stack[hp_name] = neighbors
else:
neighbors = finite_neighbors_stack[hp_name]
neighbor = neighbors.pop()
# Check all newly obtained neigbors
new_array = array.copy()
new_array = ConfigSpace.c_util.change_hp_value(
configuration_space=configuration_space,
configuration_array=new_array,
hp_name=hp_name,
hp_value=neighbor,
index=index)
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(configuration_space,
vector=new_array)
# Only rigorously check every tenth configuration (
# because moving around in the neighborhood should
# just work!)
if np.random.random() > 0.95:
new_configuration.is_valid_configuration()
else:
configuration_space._check_forbidden(new_array)
neighbourhood.append(new_configuration)
except ForbiddenValueError as e:
pass
iteration += 1
if len(neighbourhood) > 0:
number_of_sampled_neighbors += 1
# Some infinite loop happened and no valid neighbor was found OR
# no valid neighbor is available for a categorical
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
n_neighbors_per_hp[hp_name] = 0
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
n_ = neighbourhood.pop()
n_neighbors_per_hp[hp_name] -= 1
if n_neighbors_per_hp[hp_name] == 0:
hyperparameters_used.append(hp_name)
yield n_
def get_random_neighbor(configuration: Configuration, seed: int) -> Configuration:
"""Draw a random neighbor by changing one parameter of a configuration.
* If the parameter is categorical, it changes it to another value.
* If the parameter is ordinal, it changes it to the next higher or lower
value.
* If parameter is a float, draw a random sample
If changing a parameter activates new parameters or deactivates
previously active parameters, the configuration will be rejected. If more
than 10000 configurations were rejected, this function raises a
ValueError.
Parameters
----------
configuration : Configuration
seed : int
Used to generate a random state.
Returns
-------
Configuration
The new neighbor.
"""
random = np.random.RandomState(seed)
rejected = True
values = copy.deepcopy(configuration.get_dictionary())
while rejected:
# First, choose an active hyperparameter
active = False
iteration = 0
while not active:
iteration += 1
if configuration._num_hyperparameters > 1:
rand_idx = random.randint(0,
configuration._num_hyperparameters - 1)
else:
rand_idx = 0
value = configuration.get_array()[rand_idx]
if np.isfinite(value):
active = True
hp_name = configuration.configuration_space \
.get_hyperparameter_by_idx(rand_idx)
hp = configuration.configuration_space.get_hyperparameter(hp_name)
# Only choose if there is a possibility of finding a neigboor
if not hp.has_neighbors():
active = False
if iteration > 10000:
raise ValueError('Probably caught in an infinite loop.')
# Get a neighboor and adapt the rest of the configuration if necessary
neighbor = hp.get_neighbors(value, random, number=1, transform=True)[0]
previous_value = values[hp.name]
values[hp.name] = neighbor
try:
new_configuration = Configuration(
configuration.configuration_space, values=values)
rejected = False
except ValueError as e:
values[hp.name] = previous_value
return new_configuration
def get_one_exchange_neighbourhood(configuration: Configuration,
seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(
np.isfinite(configuration.get_array()))
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration.configuration_space.get_hyperparameter(
hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(
configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index],
random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
# Activate hyperparameters if their parent node got activated
children = configuration.configuration_space.get_children_of(
hp_name)
if len(children) > 0:
to_visit = deque() #type: deque
to_visit.extendleft(children)
visited = set() #type: Set[str]
activated_values = dict(
) #type: Dict[str, Union[int, float, str]]
while len(to_visit) > 0:
current = to_visit.pop()
if current.name in visited:
continue
visited.add(current.name)
current_idx = configuration.configuration_space. \
get_idx_by_hyperparameter_name(current.name)
current_value = new_array[current_idx]
conditions = configuration.configuration_space.\
_get_parent_conditions_of(current.name)
active = True
for condition in conditions:
parent_names = [
c.parent.name for c in condition.
get_descendant_literal_conditions()
]
parents = {
parent_name: configuration[parent_name]
for parent_name in parent_names
}
# parents come from the original configuration.
# We change at least one parameter. In order set
# other parameters which are conditional on this,
# we have to activate this
if hp_name in parents:
parents[hp_name] = neighbor_value
# Hyperparameters which are in depth 1 of the
# hyperparameter tree might have children which
# have to be activated as well. Once we set hp in
# level 1 to active, it's value changes from the
# value of the original configuration and this
# must be done here
for parent_name in parent_names:
if parent_name in activated_values:
parents[
parent_name] = activated_values[
parent_name]
# if one of the parents is None, the hyperparameter cannot be
# active! Else we have to check this
if any([
parent_value is None
for parent_value in parents.values()
]):
active = False
break
else:
if not condition.evaluate(parents):
active = False
break
if active and (current_value is None
or not np.isfinite(current_value)):
default = current._inverse_transform(
current.default)
new_array[current_idx] = default
children = configuration.configuration_space.get_children_of(
current.name)
if len(children) > 0:
to_visit.extendleft(children)
activated_values[
current.name] = current.default
if not active and (current_value is not None
or np.isfinite(current_value)):
new_array[current_idx] = np.NaN
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(
configuration.configuration_space,
vector=new_array)
new_configuration.is_valid_configuration()
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
def get_one_exchange_neighbourhood(configuration: Configuration,
seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(
np.isfinite(configuration.get_array()))
configuration_space = configuration.configuration_space
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration_space.get_hyperparameter(hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(
configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index],
random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
new_array = check_neighbouring_config_vector(
configuration, new_array, neighbor_value, hp_name)
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(configuration_space,
vector=new_array)
# Only rigorously check every tenth configuration (
# because moving around in the neighborhood should
# just work!)
if np.random.random() > 0.9:
new_configuration.is_valid_configuration()
else:
configuration_space._check_forbidden(new_array)
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ForbiddenValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
def get_one_exchange_neighbourhood(configuration: Configuration, seed: int) -> List[Configuration]:
"""Return all configurations in a one-exchange neighborhood.
The method is implemented as defined by:
Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown
Sequential Model-Based Optimization for General Algorithm Configuration
In: Proceedings of the conference on Learning and Intelligent OptimizatioN (LION 5)
"""
random = np.random.RandomState(seed)
hyperparameters_list = list(configuration.keys())
hyperparameters_list_length = len(hyperparameters_list)
neighbors_to_return = dict()
hyperparameters_used = list()
number_of_usable_hyperparameters = sum(np.isfinite(configuration.get_array()))
configuration_space = configuration.configuration_space
while len(hyperparameters_used) != number_of_usable_hyperparameters:
index = random.randint(hyperparameters_list_length)
hp_name = hyperparameters_list[index]
if hp_name in neighbors_to_return:
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
else:
neighbourhood = []
number_of_sampled_neighbors = 0
array = configuration.get_array()
if not np.isfinite(array[index]):
continue
iteration = 0
while True:
hp = configuration_space.get_hyperparameter(hp_name)
configuration._populate_values()
num_neighbors = hp.get_num_neighbors(configuration.get(hp_name))
# Obtain neigbors differently for different possible numbers of
# neighbors
if num_neighbors == 0:
break
# No infinite loops
elif iteration > 100:
break
elif np.isinf(num_neighbors):
if number_of_sampled_neighbors >= 4:
break
num_samples_to_go = 4 - number_of_sampled_neighbors
neighbors = hp.get_neighbors(array[index], random,
number=num_samples_to_go)
else:
if iteration > 0:
break
neighbors = hp.get_neighbors(array[index], random)
# Check all newly obtained neigbors
for neighbor in neighbors:
new_array = array.copy()
new_array[index] = neighbor
neighbor_value = hp._transform(neighbor)
# Hyperparameters which are going to be set to inactive
disabled = []
# Activate hyperparameters if their parent node got activated
children = configuration_space._children_of[hp_name]
if len(children) > 0:
to_visit = deque() #type: deque
to_visit.extendleft(children)
visited = set() #type: Set[str]
activated_values = dict() #type: Dict[str, Union[int, float, str]]
while len(to_visit) > 0:
current = to_visit.pop()
if current.name in visited:
continue
visited.add(current.name)
if current.name in disabled:
continue
current_idx = configuration_space.get_idx_by_hyperparameter_name(current.name)
current_value = new_array[current_idx]
conditions = configuration.configuration_space.\
_parent_conditions_of[current.name]
active = True
for condition in conditions:
parent_names = [parent.name for parent in
configuration_space._parents_of[current.name]]
parents = {parent_name: configuration[parent_name] for
parent_name in parent_names}
# parents come from the original configuration.
# We change at least one parameter. In order set
# other parameters which are conditional on this,
# we have to activate this
if hp_name in parents:
parents[hp_name] = neighbor_value
# Hyperparameters which are in depth 1 of the
# hyperparameter tree might have children which
# have to be activated as well. Once we set hp in
# level 1 to active, it's value changes from the
# value of the original configuration and this
# must be done here
for parent_name in parent_names:
if parent_name in activated_values:
parents[parent_name] = activated_values[
parent_name]
# if one of the parents is None, the hyperparameter cannot be
# active! Else we have to check this
if any([parent_value is None for parent_value in
parents.values()]):
active = False
break
else:
if not condition.evaluate(parents):
active = False
break
if active and (current_value is None or
not np.isfinite(current_value)):
default = current._inverse_transform(current.default)
new_array[current_idx] = default
children_ = configuration_space._children_of[current.name]
if len(children_) > 0:
to_visit.extendleft(children_)
activated_values[current.name] = current.default
# If the hyperparameter was made inactive,
# all its children need to be deactivade as well
if not active and (current_value is not None
or np.isfinite(current_value)):
new_array[current_idx] = np.NaN
children = configuration.configuration_space._children_of[current.name]
if len(children) > 0:
to_disable = set()
for ch in children:
to_disable.add(ch.name)
while len(to_disable) > 0:
child = to_disable.pop()
child_idx = configuration.configuration_space. \
get_idx_by_hyperparameter_name(child)
disabled.append(child_idx)
children = configuration.configuration_space._children_of[child]
for ch in children:
to_disable.add(ch.name)
for idx in disabled:
new_array[idx] = np.NaN
try:
# Populating a configuration from an array does not check
# if it is a legal configuration - check this (slow)
new_configuration = Configuration(configuration_space, vector=new_array)
new_configuration.is_valid_configuration()
neighbourhood.append(new_configuration)
number_of_sampled_neighbors += 1
# todo: investigate why tests fail when ForbiddenValueError is caught here
except ForbiddenValueError as e:
pass
# Count iterations to not run into an infinite loop when
# sampling floats/ints and there is large amount of forbidden
# values; also to find out if we tried to get a neighbor for
# a categorical hyperparameter, and the only possible
# neighbor is forbidden together with another active
# value/default hyperparameter
iteration += 1
if len(neighbourhood) == 0:
hyperparameters_used.append(hp_name)
else:
if hp_name not in hyperparameters_used:
neighbors_to_return[hp_name] = neighbourhood
random.shuffle(neighbors_to_return[hp_name])
n_ = neighbors_to_return[hp_name].pop()
if len(neighbors_to_return[hp_name]) == 0:
del neighbors_to_return[hp_name]
hyperparameters_used.append(hp_name)
yield n_
def get_random_neighbor(configuration: Configuration, seed: int) -> Configuration:
"""Draw a random neighbor by changing one parameter of a configuration.
* If the parameter is categorical, it changes it to another value.
* If the parameter is ordinal, it changes it to the next higher or lower
value.
* If parameter is a float, draw a random sample
If changing a parameter activates new parameters or deactivates
previously active parameters, the configuration will be rejected. If more
than 10000 configurations were rejected, this function raises a
ValueError.
Parameters
----------
configuration : Configuration
seed : int
Used to generate a random state.
Returns
-------
Configuration
The new neighbor.
"""
random = np.random.RandomState(seed)
rejected = True
values = copy.deepcopy(configuration.get_dictionary())
while rejected:
# First, choose an active hyperparameter
active = False
iteration = 0
while not active:
iteration += 1
if configuration._num_hyperparameters > 1:
rand_idx = random.randint(0,
configuration._num_hyperparameters - 1)
else:
rand_idx = 0
value = configuration.get_array()[rand_idx]
if np.isfinite(value):
active = True
hp_name = configuration.configuration_space \
.get_hyperparameter_by_idx(rand_idx)
hp = configuration.configuration_space.get_hyperparameter(hp_name)
# Only choose if there is a possibility of finding a neigboor
if not hp.has_neighbors():
active = False
if iteration > 10000:
raise ValueError('Probably caught in an infinite loop.')
# Get a neighboor and adapt the rest of the configuration if necessary
neighbor = hp.get_neighbors(value, random, number=1, transform=True)[0]
previous_value = values[hp.name]
values[hp.name] = neighbor
try:
new_configuration = Configuration(
configuration.configuration_space, values=values)
rejected = False
except ValueError as e:
values[hp.name] = previous_value
return new_configuration
hyperparameters_with_children = list()
for uhp in unconditional_hyperparameters:
children = configuration_space._children_of[uhp]
if len(children) > 0:
hyperparameters_with_children.append(uhp)
hps.extendleft(hyperparameters_with_children)
inactive = set()
while len(hps) > 0:
hp = hps.pop()
children = configuration_space._children_of[hp]
for child in children:
conditions = configuration_space._parent_conditions_of[child.name]
for condition in conditions:
if not condition.evaluate_vector(configuration.get_array()):
dic = configuration.get_dictionary()
try:
del dic[child.name]
except KeyError:
continue
configuration = Configuration(
configuration_space=configuration_space,
values=dic,
allow_inactive_with_values=True)
inactive.add(child.name)
hps.appendleft(child.name)
for hp in hyperparameters:
if hp.name in inactive:
dic = configuration.get_dictionary()
def new_result(self, job: Job, update_model=True):
super().new_result(job)
##############################
### 1. update observations ###
##############################
if job.result is None:
# One could skip crashed results, but we decided to
# assign a +inf loss and count them as bad configurations
loss = np.inf
else:
# same for non numeric losses.
# Note that this means losses of minus infinity will count as bad!
loss = job.result["loss"] if np.isfinite(
job.result["loss"]) else np.inf
budget = job.kwargs["budget"]
config_dict = job.kwargs["config"]
config_info = job.kwargs["config_info"]
config = Configuration(self.config_space, config_dict)
###############################
### 3. update observations ###
###############################
self.budget2obvs[budget]["configs"].append(deepcopy(config))
self.budget2obvs[budget]["vectors"].append(config.get_array())
self.budget2obvs[budget]["losses"].append(loss)
losses = np.array(self.budget2obvs[budget]["losses"])
vectors = np.array(self.budget2obvs[budget]["vectors"])
#####################################
### 2. update beta distributions ###
#####################################
if config_info.get("thompson_sampling", False):
sorted_losses = np.sort(losses)
L = losses.size
if L >= 2:
if loss <= sorted_losses[max(
0, int(round(L * (self.hit_top_n_percent / 100))))]:
state = "hit"
self.budget2theta[budget]["alpha"] += 1
else:
state = "miss"
self.budget2theta[budget]["beta"] += 1
alpha = self.budget2theta[budget]["alpha"]
beta = self.budget2theta[budget]["beta"]
self.logger.info(
f"Updated budget {pprint_budget(budget)} 's beta distributions, state = {state}, "
f"alpha = {alpha}, beta = {beta} .")
###################################################################
### 3. Judge whether the EPM training conditions are satisfied ###
###################################################################
# If the number of observations is too few, the condition of training model is not satisfied
if len(losses) < self.min_points_in_model:
return
if not update_model:
return
##############################################################################
### 4. transform X_obvs, do one-hot-encoding, imputing or other operators ###
##############################################################################
X_obvs = self.config_transformer.transform(vectors)
y_obvs = self.loss_transformer.fit_transform(losses)
################################################
### 5. training empirical performance model ###
################################################
if self.budget2epm[budget] is None:
epm = deepcopy(self.epm)
else:
epm = self.budget2epm[budget]
self.budget2epm[budget] = epm.fit(X_obvs, y_obvs)
###########################################
### 6. update surragete model's weight ###
###########################################
self.update_weight()