def maximize(self, batch_size=1):
"""
Maximizes the given acquisition function.
Parameters
----------
batch_size: number of maximizer returned.
Returns
-------
np.ndarray(N,D)
Point with highest acquisition value.
"""
incs_configs = list(
get_one_exchange_neighbourhood(self.objective_func.eta['config'],
seed=self.rng.randint(int(1e6))))
configs_list = list(incs_configs)
rand_incs = convert_configurations_to_array(configs_list)
# Sample random points uniformly over the whole space
rand_configs = sample_configurations(
self.config_space, self.n_samples - rand_incs.shape[0])
rand = convert_configurations_to_array(rand_configs)
configs_list.extend(rand_configs)
X = np.concatenate((rand_incs, rand), axis=0)
y = self.objective_func(X).flatten()
candidate_idxs = list(np.argsort(-y)[:batch_size])
# print(candidate_idxs)
# print(type(candidate_idxs))
# print(configs_list[:5])
return [configs_list[idx] for idx in candidate_idxs]
def fetch_candidate_configurations(self, num_config):
if len(self.target_y[self.iterate_r[-1]]) == 0:
return sample_configurations(self.config_space, num_config)
incumbent = dict()
max_r = self.iterate_r[-1]
# The lower, the better.
best_index = np.argmin(self.target_y[max_r])
incumbent['config'] = self.target_x[max_r][best_index]
approximate_obj = self.weighted_surrogate.predict(convert_configurations_to_array([incumbent['config']]))[0]
incumbent['obj'] = approximate_obj
self.weighted_acquisition_func.update(model=self.weighted_surrogate, eta=incumbent)
config_candidates = self.weighted_acq_optimizer.maximize(batch_size=num_config)
p_threshold = 0.3
n_acq = self.eta * self.eta
if num_config <= n_acq:
return config_candidates
candidates = config_candidates[: n_acq]
idx_acq = n_acq
for _id in range(num_config - n_acq):
if rd.random() < p_threshold:
config = sample_configurations(self.config_space, 1)[0]
else:
config = config_candidates[idx_acq]
idx_acq += 1
candidates.append(config)
return candidates
def _iterate(self, s, budget=MAX_INT, skip_last=0):
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
# initial number of iterations per config
r = int(self.R * self.eta**(-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = self.get_candidate_configurations(n)
time_elapsed = time.time() - start_time
self.logger.info(
"Choosing next batch of configurations took %.2f sec." %
time_elapsed)
with ParallelProcessEvaluator(self.eval_func,
n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() >= budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta**(-i)
n_resource = r * self.eta**i
self.logger.info(
"BOHB: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
val_losses = executor.parallel_execute(
T,
resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
self.time_ticks.extend(
[time.time() - self.global_start_time] * len(T))
# Only update results using maximal resources
if self.config_generator != 'smac':
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.config_gen.new_result(T[_id], _val_loss)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
# Refit the surrogate model.
resource_val = self.iterate_r[-1]
if len(self.target_y[resource_val]) > 1:
if self.config_generator == 'smac':
normalized_y = std_normalization(self.target_y[resource_val])
self.surrogate.train(
convert_configurations_to_array(
self.target_x[resource_val]),
np.array(normalized_y, dtype=np.float64))
def run(self, dl_evaluator):
start_time = time.time()
inc_config, inc_perf = None, np.inf
architecture_candidates = self.architectures.copy()
for _arch in architecture_candidates:
self.eval_hist_configs[_arch] = dict()
self.eval_hist_perfs[_arch] = dict()
self.evaluation_stats['timestamps'] = list()
self.evaluation_stats['val_scores'] = list()
with ParallelProcessEvaluator(dl_evaluator,
n_worker=self.n_jobs) as executor:
terminate_proc = False
while not terminate_proc:
r = 1
C = self.sample_configs_for_archs(
architecture_candidates,
self.N,
sampling_strategy=self.sampling_strategy)
while r < self.R or (r == self.R
and len(architecture_candidates) == 1):
for _arch in architecture_candidates:
if r not in self.eval_hist_configs[_arch]:
self.eval_hist_configs[_arch][r] = list()
self.eval_hist_perfs[_arch][r] = list()
self.logger.info('Evalutions [r=%d]' % r)
self.logger.info(
'Start to evaluate %d configurations with %d resource'
% (len(C), r))
self.logger.info('=' * 20)
_start_time = time.time()
if _start_time >= start_time + self.time_limit:
terminate_proc = True
break
if self.n_jobs > 1:
val_losses = executor.parallel_execute(
C,
resource_ratio=float(r / self.R),
eta=self.eta,
first_iter=(r == 1))
for _id, val_loss in enumerate(val_losses):
if np.isfinite(val_loss):
_arch = C[_id]['estimator']
self.eval_hist_configs[_arch][r].append(C[_id])
self.eval_hist_perfs[_arch][r].append(val_loss)
self.evaluation_stats['timestamps'].append(
time.time() - start_time)
self.evaluation_stats['val_scores'].append(
val_loss)
else:
val_losses = list()
for config in C:
val_loss = dl_evaluator(config,
resource_ratio=float(
r / self.R),
eta=self.eta,
first_iter=(r == 1))
val_losses.append(val_loss)
if np.isfinite(val_loss):
_arch = config['estimator']
self.eval_hist_configs[_arch][r].append(config)
self.eval_hist_perfs[_arch][r].append(val_loss)
self.evaluation_stats['timestamps'].append(
time.time() - start_time)
self.evaluation_stats['val_scores'].append(
val_loss)
self.logger.info('Evaluations [R=%d] took %.2f seconds' %
(r, time.time() - _start_time))
# Train surrogate
if self.sampling_strategy == 'bohb':
if r == self.R:
for i, _config in enumerate(C):
if np.isfinite(val_losses[i]):
_arch = _config['estimator']
self.tpe_config_gen[_arch].new_result(
_config, val_losses[i], r)
elif self.sampling_strategy == 'mfse':
for _arch in architecture_candidates: # Only update surrogate in candidates
normalized_y = std_normalization(
self.eval_hist_perfs[_arch][r])
if len(
self.eval_hist_configs[_arch]
[r]) == 0: # No configs for this architecture
continue
self.mfse_config_gen[_arch]['surrogate'].train(
convert_configurations_to_array(
self.eval_hist_configs[_arch][r]),
np.array(normalized_y, dtype=np.float64),
r=r)
if self.elimination_strategy == 'bandit':
indices = np.argsort(val_losses)
if len(C) >= self.eta:
C = [C[i] for i in indices]
reduced_num = int(len(C) / self.eta)
C = C[0:reduced_num]
else:
C = [C[indices[0]]]
else:
if r > 1:
val_losses_previous_iter = self.query_performance(
C, r // self.eta)
previous_inc_loss = np.min(
val_losses_previous_iter)
indices = np.argsort(val_losses)
C = [
C[idx] for idx in indices
if val_losses[idx] < previous_inc_loss
]
if inc_perf > val_losses[indices[0]]:
inc_perf = val_losses[indices[0]]
inc_config = C[0]
r *= self.eta
# Remove tmp model
if dl_evaluator.continue_training:
for filename in os.listdir(dl_evaluator.model_dir):
# Temporary model
if 'tmp_%s' % dl_evaluator.timestamp in filename:
try:
filepath = os.path.join(
dl_evaluator.model_dir, filename)
os.remove(filepath)
except Exception:
pass
archs, reduced_archs = [config['estimator']
for config in C], list()
# Preserve the partial-relationship order.
for _arch in archs:
if _arch not in reduced_archs:
reduced_archs.append(_arch)
architecture_candidates = reduced_archs
print('=' * 20)
print('Reduced architectures:', architecture_candidates)
print('=' * 20)
return inc_config, inc_perf
def sample_configs_for_archs(self,
include_architectures,
N,
sampling_strategy='uniform'):
configs = list()
for _arch in include_architectures:
_cs = self.get_model_config_space(_arch)
if sampling_strategy == 'uniform':
configs.append(_cs.get_default_configuration())
configs.extend(sample_configurations(_cs, N - 1))
elif sampling_strategy == 'bohb':
if _arch not in self.tpe_config_gen:
self.tpe_config_gen[_arch] = TPE(_cs)
config_candidates = list()
config_left = N
while config_left:
config = self.tpe_config_gen[_arch].get_config()[0]
if config in config_candidates:
continue
config_candidates.append(config)
config_left -= 1
p_threshold = 0.3
idx_acq = 0
for _id in range(N):
if rd.random() < p_threshold:
config = sample_configurations(_cs, 1)[0]
else:
config = config_candidates[idx_acq]
idx_acq += 1
configs.append(config)
else: # mfse
if _arch not in self.mfse_config_gen:
types, bounds = get_types(_cs)
init_weight = [1. / self.s_max] * self.s_max + [0.]
self.mfse_config_gen[_arch] = dict()
self.mfse_config_gen[_arch][
'surrogate'] = WeightedRandomForestCluster(
types, bounds, self.s_max, self.eta, init_weight,
'gpoe')
acq_func = EI(
model=self.mfse_config_gen[_arch]['surrogate'])
self.mfse_config_gen[_arch][
'acq_optimizer'] = RandomSampling(
acq_func,
_cs,
n_samples=2000,
rng=np.random.RandomState(1))
if self.R not in self.eval_hist_perfs[_arch] or len(
self.eval_hist_perfs[_arch][self.R]) == 0:
configs.extend(sample_configurations(_cs, N))
continue
incumbent = dict()
max_r = self.R
# The lower, the better.
best_index = np.argmin(self.eval_hist_perfs[_arch][max_r])
incumbent['config'] = self.eval_hist_configs[_arch][max_r][
best_index]
approximate_obj = self.mfse_config_gen[_arch][
'surrogate'].predict(
convert_configurations_to_array([incumbent['config']
]))[0]
incumbent['obj'] = approximate_obj
self.mfse_config_gen[_arch]['acq_optimizer'].update(
model=self.mfse_config_gen[_arch]['surrogate'],
eta=incumbent)
config_candidates = self.mfse_config_gen[_arch][
'acq_optimizer'].maximize(batch_size=N)
p_threshold = 0.3
n_acq = self.eta * self.eta
if N <= n_acq:
return config_candidates
candidates = config_candidates[:n_acq]
idx_acq = n_acq
for _id in range(N - n_acq):
if rd.random() < p_threshold:
config = sample_configurations(_cs, 1)[0]
else:
config = config_candidates[idx_acq]
idx_acq += 1
candidates.append(config)
return candidates
return configs
def _iterate(self, s, budget=MAX_INT, skip_last=0):
if self.weight_update_id > self.s_max:
self.update_weight()
self.weight_update_id += 1
# Set initial number of configurations
n = int(ceil(self.B / self.R / (s + 1) * self.eta ** s))
# initial number of iterations per config
r = int(self.R * self.eta ** (-s))
# Choose a batch of configurations in different mechanisms.
start_time = time.time()
T = self.fetch_candidate_configurations(n)
time_elapsed = time.time() - start_time
self.logger.info("Choosing next batch of configurations took %.2f sec." % time_elapsed)
with ParallelProcessEvaluator(self.eval_func, n_worker=self.n_workers) as executor:
for i in range((s + 1) - int(skip_last)): # changed from s + 1
if time.time() >= budget + start_time:
break
# Run each of the n configs for <iterations>
# and keep best (n_configs / eta) configurations
n_configs = n * self.eta ** (-i)
n_resource = r * self.eta ** i
self.logger.info("MFSE: %d configurations x size %d / %d each" %
(int(n_configs), n_resource, self.R))
if self.n_workers > 1:
val_losses = executor.parallel_execute(T, resource_ratio=float(n_resource / self.R),
eta=self.eta,
first_iter=(i == 0))
for _id, _val_loss in enumerate(val_losses):
if np.isfinite(_val_loss):
self.target_x[int(n_resource)].append(T[_id])
self.target_y[int(n_resource)].append(_val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(_val_loss)
else:
val_losses = list()
for config in T:
try:
val_loss = self.eval_func(config, resource_ratio=float(n_resource / self.R),
eta=self.eta, first_iter=(i == 0))
except Exception as e:
val_loss = np.inf
val_losses.append(val_loss)
if np.isfinite(val_loss):
self.target_x[int(n_resource)].append(config)
self.target_y[int(n_resource)].append(val_loss)
self.evaluation_stats['timestamps'].append(time.time() - self.global_start_time)
self.evaluation_stats['val_scores'].append(val_loss)
self.exp_output[time.time()] = (int(n_resource), T, val_losses)
if int(n_resource) == self.R:
self.incumbent_configs.extend(T)
self.incumbent_perfs.extend(val_losses)
# Select a number of best configurations for the next loop.
# Filter out early stops, if any.
indices = np.argsort(val_losses)
if len(T) >= self.eta:
T = [T[i] for i in indices]
reduced_num = int(n_configs / self.eta)
T = T[0:reduced_num]
else:
T = [T[indices[0]]]
for item in self.iterate_r[self.iterate_r.index(r):]:
if len(self.target_y[item]) == 0:
continue
normalized_y = std_normalization(self.target_y[item])
self.weighted_surrogate.train(convert_configurations_to_array(self.target_x[item]),
np.array(normalized_y, dtype=np.float64), r=item)
def update_weight(self):
max_r = self.iterate_r[-1]
incumbent_configs = self.target_x[max_r]
if len(incumbent_configs) <= 3:
return
test_x = convert_configurations_to_array(incumbent_configs)
test_y = np.array(self.target_y[max_r], dtype=np.float64)
r_list = self.weighted_surrogate.surrogate_r
K = len(r_list)
if len(test_y) >= 3:
# # p-norm
# # Get previous weights
# preserving_order_p = list()
# preserving_order_nums = list()
# for i, r in enumerate(r_list):
# fold_num = 5
# if i != K - 1:
# mean, var = self.weighted_surrogate.surrogate_container[r].predict(test_x)
# tmp_y = np.reshape(mean, -1)
# preorder_num, pair_num = self.calculate_preserving_order_num(tmp_y, test_y)
# preserving_order_p.append(preorder_num / pair_num)
# preserving_order_nums.append(preorder_num)
# else:
# if len(test_y) < 2 * fold_num:
# preserving_order_p.append(0)
# else:
# # 5-fold cross validation.
# kfold = KFold(n_splits=fold_num)
# cv_pred = np.array([0] * len(test_y))
# for train_idx, valid_idx in kfold.split(test_x):
# train_configs, train_y = test_x[train_idx], test_y[train_idx]
# valid_configs, valid_y = test_x[valid_idx], test_y[valid_idx]
# types, bounds = get_types(self.config_space)
# _surrogate = RandomForestWithInstances(types=types, bounds=bounds)
# _surrogate.train(train_configs, train_y)
# pred, _ = _surrogate.predict(valid_configs)
# cv_pred[valid_idx] = pred.reshape(-1)
# preorder_num, pair_num = self.calculate_preserving_order_num(cv_pred, test_y)
# preserving_order_p.append(preorder_num / pair_num)
# preserving_order_nums.append(preorder_num)
# p = 3
# trans_order_weight = np.array(preserving_order_p)
# power_sum = np.sum(np.power(trans_order_weight, p))
# new_weights = np.power(trans_order_weight, p) / power_sum
# sample
n_sampling = 100
argmin_cnt = [0] * K
predictive_mu, predictive_std = list(), list()
n_fold = 5
n_instance = len(test_y)
ranking_loss_hist = list()
for i, r in enumerate(r_list):
if i != K - 1:
_mean, _var = self.weighted_surrogate.surrogate_container[r].predict(test_x)
predictive_mu.append(_mean)
predictive_std.append(np.sqrt(_var))
else:
fold_num = n_instance // n_fold
target_mu, target_std = list(), list()
for i in range(n_fold):
instance_indexs = list(range(n_instance))
bound = (n_instance - i * fold_num) if i == (n_fold - 1) else fold_num
start_id = i * fold_num
del instance_indexs[start_id: start_id + bound]
types, bounds = get_types(self.config_space)
_surrogate = RandomForestWithInstances(types=types, bounds=bounds)
_surrogate.train(test_x[instance_indexs, :], test_y[instance_indexs])
_mu, _var = _surrogate.predict(test_x[start_id: start_id + bound])
target_mu.extend(_mu.flatten())
target_std.extend(np.sqrt(_var).flatten())
predictive_mu.append(target_mu)
predictive_std.append(target_std)
for _ in range(n_sampling):
ranking_loss_list = list()
for i, r in enumerate(r_list):
sampled_y = np.random.normal(predictive_mu[i], predictive_std[i])
rank_loss = 0
for i in range(len(test_y)):
for j in range(len(test_y)):
if (test_y[i] < test_y[j]) ^ (sampled_y[i] < sampled_y[j]):
rank_loss += 1
ranking_loss_list.append(rank_loss)
ranking_loss_hist.append(ranking_loss_list)
argmin_id = np.argmin(ranking_loss_list)
argmin_cnt[argmin_id] += 1
new_weights = np.array(argmin_cnt) / n_sampling
else:
old_weights = list()
for i, r in enumerate(r_list):
_weight = self.weighted_surrogate.surrogate_weight[r]
old_weights.append(_weight)
new_weights = old_weights.copy()
self.logger.info('Model weights[%d]: %s' % (self.weight_changed_cnt, str(new_weights)))
# Assign the weight to each basic surrogate.
for i, r in enumerate(r_list):
self.weighted_surrogate.surrogate_weight[r] = new_weights[i]
self.weight_changed_cnt += 1
# Save the weight data.
self.hist_weights.append(new_weights)