def __call__(self,
configurations: Union[List[Configuration], np.ndarray],
convert=True,
**kwargs):
"""Computes the acquisition value for a given X
Parameters
----------
configurations : list
The configurations where the acquisition function
should be evaluated.
convert : bool
Returns
-------
np.ndarray(N, 1)
acquisition values for X
"""
if convert:
X = convert_configurations_to_array(configurations)
else:
X = configurations # to be compatible with multi-objective acq to call single acq
if len(X.shape) == 1:
X = X[np.newaxis, :]
acq = self._compute(X, **kwargs)
if np.any(np.isnan(acq)):
idx = np.where(np.isnan(acq))[0]
acq[idx, :] = -np.finfo(np.float).max
return acq
def get_regressor(self):
# Train transfer learning regressor.
for idx, hpo_evaluation_data in enumerate(self.source_hpo_data):
print('Build the %d-th residual GPs.' % idx)
_X, _y = list(), list()
for _config, _config_perf in list(
hpo_evaluation_data.items())[:self.num_src_hpo_trial]:
_X.append(_config)
_y.append(_config_perf)
X = convert_configurations_to_array(_X)
y = np.array(_y, dtype=np.float64)
self.train_regressor(X, y)
def get_suggestion(self):
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
num_failed_trial = len(self.failed_configurations)
failed_perfs = list() if self.max_y is None else [self.max_y
] * num_failed_trial
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
num_config_evaluated = len(self.perfs + self.failed_configurations)
if num_config_evaluated < self.init_num:
return self.initial_configurations[num_config_evaluated]
if self.optimization_strategy == 'random':
return self.sample_random_configs(1)[0]
elif self.optimization_strategy == 'bo':
self.surrogate_model.train(X, Y)
incumbent_value = self.history_container.get_incumbents()[0][1]
self.acquisition_function.update(model=self.surrogate_model,
eta=incumbent_value,
num_data=num_config_evaluated)
challengers = self.optimizer.maximize(
runhistory=self.history_container, num_points=5000)
is_repeated_config = True
repeated_time = 0
cur_config = None
while is_repeated_config:
cur_config = challengers.challengers[repeated_time]
if cur_config in (self.configurations +
self.failed_configurations):
repeated_time += 1
else:
is_repeated_config = False
return cur_config
else:
raise ValueError('Unknown optimization strategy: %s.' %
self.optimization_strategy)
def build_source_surrogates(self, normalize):
if self.source_hpo_data is None:
self.logger.warning(
'No history BO data provided, resort to naive BO optimizer without TL.'
)
return
self.logger.info('Start to train base surrogates.')
start_time = time.time()
self.source_surrogates = list()
for hpo_evaluation_data in self.source_hpo_data:
print('.', end='')
model = build_surrogate(self.surrogate_type, self.config_space,
np.random.RandomState(self.random_seed))
_X, _y = list(), list()
for _config, _config_perf in hpo_evaluation_data.items():
_X.append(_config)
_y.append(_config_perf)
X = convert_configurations_to_array(_X)
y = np.array(_y, dtype=np.float64)
if self.num_src_hpo_trial != -1:
X = X[:self.num_src_hpo_trial]
y = y[:self.num_src_hpo_trial]
if normalize == 'standardize':
if (y == y[0]).all():
y[0] += 1e-4
y, _, _ = zero_mean_unit_var_normalization(y)
elif normalize == 'scale':
if (y == y[0]).all():
y[0] += 1e-4
y, _, _ = zero_one_normalization(y)
y = 2 * y - 1.
else:
raise ValueError('Invalid parameter in norm.')
self.eta_list.append(np.min(y))
model.train(X, y)
self.source_surrogates.append(model)
self.logger.info('Building base surrogates took %.3fs.' %
(time.time() - start_time))
def get_suggestions(self):
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
num_failed_trial = len(self.failed_configurations)
failed_perfs = list() if self.max_y is None else [self.max_y
] * num_failed_trial
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
all_considered_configs = self.configurations + self.failed_configurations
num_config_evaluated = len(all_considered_configs)
batch_configs_list = list()
if num_config_evaluated < self.init_num:
if self.initial_configurations is not None: # self.init_num equals to len(self.initial_configurations)
return self.initial_configurations
else:
return self.sample_random_configs(self.init_num)
if self.optimization_strategy == 'random':
return self.sample_random_configs(self.batch_size)
if self.batch_strategy == 'median_imputation':
estimated_y = np.median(Y)
batch_history_container = copy.deepcopy(self.history_container)
for i in range(self.batch_size):
self.surrogate_model.train(X, Y)
incumbent_value = batch_history_container.get_incumbents(
)[0][1]
self.acquisition_function.update(
model=self.surrogate_model,
eta=incumbent_value,
num_data=len(batch_history_container.data))
challengers = self.optimizer.maximize(
runhistory=batch_history_container, num_points=5000)
is_repeated_config = True
repeated_time = 0
curr_batch_config = None
while is_repeated_config:
curr_batch_config = challengers.challengers[repeated_time]
if curr_batch_config in all_considered_configs:
is_repeated_config = True
repeated_time += 1
else:
is_repeated_config = False
batch_history_container.add(curr_batch_config, estimated_y)
batch_configs_list.append(curr_batch_config)
all_considered_configs.append(curr_batch_config)
X = np.append(X,
curr_batch_config.get_array().reshape(1, -1),
axis=0)
Y = np.append(Y, estimated_y)
elif self.batch_strategy == 'local_penalization':
self.surrogate_model.train(X, Y)
incumbent_value = self.history_container.get_incumbents()[0][1]
# L = self.estimate_L(X)
for i in range(self.batch_size):
self.acquisition_function.update(
model=self.surrogate_model,
eta=incumbent_value,
num_data=len(self.history_container.data),
batch_configs=batch_configs_list)
challengers = self.optimizer.maximize(
runhistory=self.history_container, num_points=5000)
batch_configs_list.append(challengers.challengers[0])
else:
raise ValueError('Invalid sampling strategy - %s.' %
self.batch_strategy)
return batch_configs_list
def iterate(self):
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
failed_perfs = list() if self.max_y is None else [self.max_y] * len(
self.failed_configurations)
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
config = self.choose_next(X, Y)
trial_state = SUCCESS
trial_info = None
if config not in (self.configurations + self.failed_configurations):
# Evaluate this configuration.
try:
args, kwargs = (config, ), dict()
timeout_status, _result = time_limit(self.objective_function,
self.time_limit_per_trial,
args=args,
kwargs=kwargs)
if timeout_status:
raise TimeoutException(
'Timeout: time limit for this evaluation is %.1fs' %
self.time_limit_per_trial)
else:
perf = MAXINT if _result is None else _result
except Exception as e:
if isinstance(e, TimeoutException):
trial_state = TIMEOUT
else:
traceback.print_exc(file=sys.stdout)
trial_state = FAILED
perf = MAXINT
trial_info = str(e)
self.logger.error(trial_info)
if trial_state == SUCCESS and perf < MAXINT:
if len(self.configurations) == 0:
self.default_obj_value = perf
self.configurations.append(config)
self.perfs.append(perf)
self.history_container.add(config, perf)
self.perc = np.percentile(self.perfs, self.scale_perc)
self.min_y = np.min(self.perfs)
self.max_y = np.max(self.perfs)
else:
self.failed_configurations.append(config)
else:
self.logger.debug(
'This configuration has been evaluated! Skip it.')
if config in self.configurations:
config_idx = self.configurations.index(config)
trial_state, perf = SUCCESS, self.perfs[config_idx]
else:
trial_state, perf = FAILED, MAXINT
self.iteration_id += 1
self.logger.info(
'Iteration-%d, objective improvement: %.4f' %
(self.iteration_id, max(0, self.default_obj_value - perf)))
return config, trial_state, perf, trial_info
def get_suggestion(self):
# Check if turbo needs to be restarted
if self.use_trust_region and self.turbo_state.restart_triggered:
self.configurations = list()
self.failed_configurations = list()
self.perfs = list()
self.history_container.restart()
print('-' * 30)
print('Restart!')
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
num_failed_trial = len(self.failed_configurations)
failed_perfs = list() if self.max_y is None else [self.max_y
] * num_failed_trial
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
num_config_evaluated = len(self.perfs + self.failed_configurations)
if num_config_evaluated < self.init_num:
return self.initial_configurations[num_config_evaluated]
if self.optimization_strategy == 'random':
return self.sample_random_configs(1)[0]
elif self.optimization_strategy == 'bo':
# train surrogate model
if self.num_objs == 1:
self.surrogate_model.train(X, Y)
else: # multi-objectives
for i in range(self.num_objs):
self.surrogate_model[i].train(X, Y[:, i])
# train constraint model
cX = None
if self.num_constraints > 0:
cX = []
for c in self.constraint_perfs:
failed_c = list(
) if num_failed_trial == 0 else [max(c)] * num_failed_trial
cX.append(np.array(c + failed_c, dtype=np.float64))
for i, model in enumerate(self.constraint_models):
model.train(X, cX[i])
# update acquisition function
if self.num_objs == 1: # MC-EI
incumbent_value = self.history_container.get_incumbents()[0][1]
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
eta=incumbent_value)
else: # MC-ParEGO or MC-EHVI
if self.acq_type.startswith('mcparego'):
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models)
elif self.acq_type.startswith('mcehvi'):
partitioning = NondominatedPartitioning(self.num_objs, Y)
cell_bounds = partitioning.get_hypercell_bounds(
ref_point=self.ref_point)
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
cell_lower_bounds=cell_bounds[0],
cell_upper_bounds=cell_bounds[1])
# optimize acquisition function
challengers = self.optimizer.maximize(
runhistory=self.history_container,
num_points=5000,
turbo_state=self.turbo_state)
is_repeated_config = True
repeated_time = 0
cur_config = None
while is_repeated_config:
cur_config = challengers.challengers[repeated_time]
if cur_config in (self.configurations +
self.failed_configurations):
repeated_time += 1
else:
is_repeated_config = False
return cur_config
else:
raise ValueError('Unknown optimization strategy: %s.' %
self.optimization_strategy)
def get_suggestion(self):
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
num_failed_trial = len(self.failed_configurations)
failed_perfs = list() if self.max_y is None else [self.max_y
] * num_failed_trial
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
num_config_evaluated = len(self.perfs + self.failed_configurations)
if num_config_evaluated < self.init_num:
return self.initial_configurations[num_config_evaluated]
if self.optimization_strategy == 'random':
return self.sample_random_configs(1)[0]
elif self.optimization_strategy == 'bo':
# train surrogate model
if self.num_objs == 1:
self.surrogate_model.train(X, Y)
elif self.acq_type == 'parego':
weights = np.random.random_sample(self.num_objs)
weights = weights / np.sum(weights)
scalarized_obj = get_chebyshev_scalarization(weights, Y)
self.surrogate_model.train(X, scalarized_obj(Y))
else: # multi-objectives
for i in range(self.num_objs):
self.surrogate_model[i].train(X, Y[:, i])
# train constraint model
cX = None
if self.num_constraints > 0:
cX = []
for c in self.constraint_perfs:
failed_c = list(
) if num_failed_trial == 0 else [max(c)] * num_failed_trial
cX.append(np.array(c + failed_c, dtype=np.float64))
for i, model in enumerate(self.constraint_models):
model.train(X, cX[i])
# update acquisition function
if self.num_objs == 1:
incumbent_value = self.history_container.get_incumbents()[0][1]
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
eta=incumbent_value,
num_data=num_config_evaluated)
else: # multi-objectives
mo_incumbent_value = self.history_container.get_mo_incumbent_value(
)
if self.acq_type.startswith('parego'):
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
eta=scalarized_obj(np.atleast_2d(mo_incumbent_value)),
num_data=num_config_evaluated)
elif self.acq_type.startswith('ehvi'):
partitioning = NondominatedPartitioning(self.num_objs, Y)
cell_bounds = partitioning.get_hypercell_bounds(
ref_point=self.ref_point)
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
cell_lower_bounds=cell_bounds[0],
cell_upper_bounds=cell_bounds[1])
else:
self.acquisition_function.update(
model=self.surrogate_model,
constraint_models=self.constraint_models,
constraint_perfs=cX, # for MESMOC
eta=mo_incumbent_value,
num_data=num_config_evaluated,
X=X,
Y=Y)
# optimize acquisition function
challengers = self.optimizer.maximize(
runhistory=self.history_container, num_points=5000)
is_repeated_config = True
repeated_time = 0
cur_config = None
while is_repeated_config:
cur_config = challengers.challengers[repeated_time]
if cur_config in (self.configurations +
self.failed_configurations):
repeated_time += 1
else:
is_repeated_config = False
return cur_config
else:
raise ValueError('Unknown optimization strategy: %s.' %
self.optimization_strategy)
def get_suggestion(self):
self.logger.info('#Call get_suggestion: %d.' %
len(self.running_configs))
if len(self.configurations) == 0:
X = np.array([])
else:
failed_configs = list(
) if self.max_y is None else self.failed_configurations.copy()
X = convert_configurations_to_array(self.configurations +
failed_configs)
# Failed trial.
num_failed_trial = len(self.failed_configurations)
failed_perfs = list() if self.max_y is None else [self.max_y
] * num_failed_trial
Y = np.array(self.perfs + failed_perfs, dtype=np.float64)
all_considered_configs = self.configurations + self.failed_configurations + self.running_configs
num_config_evaluated = len(all_considered_configs)
if (num_config_evaluated < self.init_num) or \
len(self.history_container.data) <= self.bo_start_n or \
self.optimization_strategy == 'random':
if num_config_evaluated >= len(self.initial_configurations):
_config = self.sample_random_configs(1)[0]
else:
_config = self.initial_configurations[num_config_evaluated]
self.running_configs.append(_config)
return _config
if self.batch_strategy == 'median_imputation':
self.logger.info('Config is sampled from [BO].')
X = convert_configurations_to_array(all_considered_configs)
y_median = np.median(self.perfs)
running_perfs = [y_median] * len(self.running_configs)
Y = np.array(self.perfs + failed_perfs + running_perfs,
dtype=np.float64)
self.surrogate_model.train(X, Y)
incumbent_value = self.history_container.get_incumbents()[0][1]
self.acquisition_function.update(model=self.surrogate_model,
eta=incumbent_value,
num_data=len(
self.history_container.data))
challengers = self.optimizer.maximize(
runhistory=self.history_container, num_points=5000)
is_repeated_config = True
repeated_time = 0
curr_batch_config = None
while is_repeated_config:
curr_batch_config = challengers.challengers[repeated_time]
if curr_batch_config in all_considered_configs:
is_repeated_config = True
repeated_time += 1
else:
is_repeated_config = False
_config = curr_batch_config
elif self.batch_strategy == 'local_penalization':
self.surrogate_model.train(X, Y)
incumbent_value = self.history_container.get_incumbents()[0][1]
self.acquisition_function.update(
model=self.surrogate_model,
eta=incumbent_value,
num_data=len(self.history_container.data),
batch_configs=self.running_configs)
challengers = self.optimizer.maximize(
runhistory=self.history_container, num_points=5000)
_config = challengers.challengers[0]
else:
raise ValueError('Invalid sampling strategy - %s.' %
self.batch_strategy)
self.running_configs.append(_config)
return _config