def __call__(self, configurations: Union[List[Configuration], np.ndarray], convert=True, **kwargs): """Computes the acquisition value for a given X Parameters ---------- configurations : list The configurations where the acquisition function should be evaluated. convert : bool Returns ------- np.ndarray(N, 1) acquisition values for X """ if convert: X = convert_configurations_to_array(configurations) else: X = configurations # to be compatible with multi-objective acq to call single acq if len(X.shape) == 1: X = X[np.newaxis, :] acq = self._compute(X, **kwargs) if np.any(np.isnan(acq)): idx = np.where(np.isnan(acq))[0] acq[idx, :] = -np.finfo(np.float).max return acq
def update_observation(self, config, perf, n_iteration): rung_id = self.get_rung_id(self.bracket, n_iteration) updated = False for job in self.bracket[rung_id]['jobs']: _job_status, _config, _perf, _extra_conf = job if _config == config: assert _job_status == RUNNING job[0] = COMPLETED job[2] = perf updated = True break assert updated # print('=== bracket after update_observation:', self.get_bracket_status(self.bracket)) configs_running = list() for _config in self.bracket[rung_id]['configs']: if _config not in self.target_x[n_iteration]: configs_running.append(_config) value_imputed = np.median(self.target_y[n_iteration]) n_iteration = int(n_iteration) self.target_x[n_iteration].append(config) self.target_y[n_iteration].append(perf) if n_iteration == self.R: self.incumbent_configs.append(config) self.incumbent_perfs.append(perf) # Update history container. self.history_container.add(config, perf) # Refit the ensemble surrogate model. configs_train = self.target_x[n_iteration] + configs_running results_train = self.target_y[n_iteration] + [value_imputed ] * len(configs_running) results_train = np.array(std_normalization(results_train), dtype=np.float64) if not self.use_bohb_strategy: self.surrogate.train( convert_configurations_to_array(configs_train), results_train, r=n_iteration) else: if n_iteration == self.R: self.surrogate.train( convert_configurations_to_array(configs_train), results_train)
def get_suggestion(self, history_container=None): self.logger.info('#Call get_suggestion. len of running configs = %d.' % len(self.running_configs)) if history_container is None: history_container = self.history_container num_config_all = len(history_container.configurations) + len(self.running_configs) num_config_successful = len(history_container.successful_perfs) if (num_config_all < self.init_num) or \ num_config_successful < self.bo_start_n or \ self.optimization_strategy == 'random': if num_config_all >= len(self.initial_configurations): _config = self.sample_random_configs(1, history_container)[0] else: _config = self.initial_configurations[num_config_all] self.running_configs.append(_config) return _config X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs() # cY = history_container.get_transformed_constraint_perfs() if self.batch_strategy == 'median_imputation': # set bilog_transform=False to get real cY for estimating median cY = history_container.get_transformed_constraint_perfs(bilog_transform=False) estimated_y = np.median(Y, axis=0).reshape(-1).tolist() estimated_c = np.median(cY, axis=0).tolist() if self.num_constraints > 0 else None batch_history_container = copy.deepcopy(history_container) # imputation for config in self.running_configs: observation = Observation(config, SUCCESS, estimated_c, estimated_y, None) batch_history_container.update_observation(observation) # use super class get_suggestion _config = super().get_suggestion(batch_history_container) elif self.batch_strategy == 'local_penalization': # local_penalization only supports single objective with no constraint self.surrogate_model.train(X, Y) incumbent_value = history_container.get_incumbents()[0][1] self.acquisition_function.update(model=self.surrogate_model, eta=incumbent_value, num_data=len(history_container.data), batch_configs=self.running_configs) challengers = self.optimizer.maximize( runhistory=history_container, num_points=5000 ) _config = challengers.challengers[0] else: raise ValueError('Invalid sampling strategy - %s.' % self.batch_strategy) self.running_configs.append(_config) return _config
def fit_kde_models(self, history_container): num_config_successful = len(history_container.successful_perfs) if num_config_successful <= self.min_points_in_model - 1: self.logger.debug( "Only %i run(s) available, need more than %s -> can't build model!" % (num_config_successful, self.min_points_in_model + 1)) return train_configs = convert_configurations_to_array( history_container.configurations) train_losses = history_container.get_transformed_perfs() n_good = max(self.min_points_in_model, (self.top_n_percent * train_configs.shape[0]) // 100) # n_bad = min(max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100), 10) n_bad = max(self.min_points_in_model, ((100 - self.top_n_percent) * train_configs.shape[0]) // 100) # Refit KDE for the current budget idx = np.argsort(train_losses) train_data_good = self.impute_conditional_data( train_configs[idx[:n_good]]) train_data_bad = self.impute_conditional_data( train_configs[idx[n_good:n_good + n_bad]]) if train_data_good.shape[0] <= train_data_good.shape[1]: return if train_data_bad.shape[0] <= train_data_bad.shape[1]: return # more expensive crossvalidation method # bw_estimation = 'cv_ls' # quick rule of thumb bw_estimation = 'normal_reference' bad_kde = sm.nonparametric.KDEMultivariate(data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) good_kde = sm.nonparametric.KDEMultivariate(data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) self.kde_models = {'good': good_kde, 'bad': bad_kde} # update probs for the categorical parameters for later sampling self.logger.debug( 'done building a new model based on %i/%i split\nBest loss for this budget:%f\n\n\n\n\n' % (n_good, n_bad, np.min(train_losses)))
def get_regressor(self): # Train transfer learning regressor. for idx, hpo_evaluation_data in enumerate(self.source_hpo_data): print('Build the %d-th residual GPs.' % idx) _X, _y = list(), list() for _config, _config_perf in list( hpo_evaluation_data.items())[:self.num_src_hpo_trial]: _X.append(_config) _y.append(_config_perf) X = convert_configurations_to_array(_X) y = np.array(_y, dtype=np.float64) self.train_regressor(X, y)
def get_suggestion(self): if len(self.configurations) == 0: X = np.array([]) else: failed_configs = list( ) if self.max_y is None else self.failed_configurations.copy() X = convert_configurations_to_array(self.configurations + failed_configs) num_failed_trial = len(self.failed_configurations) failed_perfs = list() if self.max_y is None else [self.max_y ] * num_failed_trial Y = np.array(self.perfs + failed_perfs, dtype=np.float64) num_config_evaluated = len(self.perfs + self.failed_configurations) if num_config_evaluated < self.init_num: return self.initial_configurations[num_config_evaluated] if self.optimization_strategy == 'random': return self.sample_random_configs(1)[0] elif self.optimization_strategy == 'bo': self.surrogate_model.train(X, Y) incumbent_value = self.history_container.get_incumbents()[0][1] self.acquisition_function.update(model=self.surrogate_model, eta=incumbent_value, num_data=num_config_evaluated) challengers = self.optimizer.maximize( runhistory=self.history_container, num_points=5000) is_repeated_config = True repeated_time = 0 cur_config = None while is_repeated_config: cur_config = challengers.challengers[repeated_time] if cur_config in (self.configurations + self.failed_configurations): repeated_time += 1 else: is_repeated_config = False return cur_config else: raise ValueError('Unknown optimization strategy: %s.' % self.optimization_strategy)
def build_source_surrogates(self, normalize): if self.source_hpo_data is None: self.logger.warning( 'No history BO data provided, resort to naive BO optimizer without TL.' ) return self.logger.info('Start to train base surrogates.') start_time = time.time() self.source_surrogates = list() for hpo_evaluation_data in self.source_hpo_data: print('.', end='') model = build_surrogate(self.surrogate_type, self.config_space, np.random.RandomState(self.random_seed)) _X, _y = list(), list() for _config, _config_perf in hpo_evaluation_data.items(): _X.append(_config) _y.append(_config_perf) X = convert_configurations_to_array(_X) y = np.array(_y, dtype=np.float64) if self.num_src_hpo_trial != -1: X = X[:self.num_src_hpo_trial] y = y[:self.num_src_hpo_trial] if normalize == 'standardize': if (y == y[0]).all(): y[0] += 1e-4 y, _, _ = zero_mean_unit_var_normalization(y) elif normalize == 'scale': if (y == y[0]).all(): y[0] += 1e-4 y, _, _ = zero_one_normalization(y) y = 2 * y - 1. else: raise ValueError('Invalid parameter in norm.') self.eta_list.append(np.min(y)) model.train(X, y) self.source_surrogates.append(model) self.logger.info('Building base surrogates took %.3fs.' % (time.time() - start_time))
def _get_suggestion(self, history_container=None): if history_container is None: history_container = self.history_container num_config_all = len(history_container.configurations) + len( self.running_configs) num_config_successful = len(history_container.successful_perfs) if (num_config_all < self.init_num) or \ num_config_successful < self.bo_start_n or \ self.optimization_strategy == 'random': if num_config_all >= len(self.initial_configurations): _config = self.sample_random_configs(1, history_container)[0] else: _config = self.initial_configurations[num_config_all] return _config # sample random configuration proportionally if self.rng.random() < self.rand_prob: self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob) return self.sample_random_configs( 1, history_container, excluded_configs=self.running_configs)[0] X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs(transform=None) # cY = history_container.get_transformed_constraint_perfs(transform='bilog') if self.batch_strategy == 'median_imputation': # set bilog_transform=False to get real cY for estimating median cY = history_container.get_transformed_constraint_perfs( transform=None) estimated_y = np.median(Y, axis=0).reshape(-1).tolist() estimated_c = np.median( cY, axis=0).tolist() if self.num_constraints > 0 else None batch_history_container = copy.deepcopy(history_container) # imputation for config in self.running_configs: observation = Observation(config=config, objs=estimated_y, constraints=estimated_c, trial_state=SUCCESS, elapsed_time=None) batch_history_container.update_observation(observation) # use super class get_suggestion return super().get_suggestion(batch_history_container) elif self.batch_strategy == 'local_penalization': # local_penalization only supports single objective with no constraint self.surrogate_model.train(X, Y) incumbent_value = history_container.get_incumbents()[0][1] self.acquisition_function.update( model=self.surrogate_model, eta=incumbent_value, num_data=len(history_container.data), batch_configs=self.running_configs) challengers = self.optimizer.maximize(runhistory=history_container, num_points=5000) return challengers.challengers[0] elif self.batch_strategy == 'default': # select first N candidates candidates = super().get_suggestion(history_container, return_list=True) for config in candidates: if config not in self.running_configs and config not in history_container.configurations: return config self.logger.warning( 'Cannot get non duplicate configuration from BO candidates (len=%d). ' 'Sample random config.' % (len(candidates), )) return self.sample_random_configs( 1, history_container, excluded_configs=self.running_configs)[0] else: raise ValueError('Invalid sampling strategy - %s.' % self.batch_strategy)
def get_suggestion(self, history_container=None): """ Generate a configuration (suggestion) for this query. Returns ------- A configuration. """ if history_container is None: history_container = self.history_container num_config_evaluated = len(history_container.configurations) num_config_successful = len(history_container.successful_perfs) if num_config_evaluated < self.init_num: return self.initial_configurations[num_config_evaluated] if self.optimization_strategy == 'random': return self.sample_random_configs(1, history_container)[0] X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs() cY = history_container.get_transformed_constraint_perfs() if self.optimization_strategy == 'bo': if num_config_successful < max(self.init_num, 1): self.logger.warning('No enough successful initial trials! Sample random configuration.') return self.sample_random_configs(1, history_container)[0] # train surrogate model if self.num_objs == 1: self.surrogate_model.train(X, Y) elif self.acq_type == 'parego': weights = self.rng.random_sample(self.num_objs) weights = weights / np.sum(weights) scalarized_obj = get_chebyshev_scalarization(weights, Y) self.surrogate_model.train(X, scalarized_obj(Y)) else: # multi-objectives for i in range(self.num_objs): self.surrogate_model[i].train(X, Y[:, i]) # train constraint model for i in range(self.num_constraints): self.constraint_models[i].train(X, cY[:, i]) # update acquisition function if self.num_objs == 1: incumbent_value = history_container.get_incumbents()[0][1] self.acquisition_function.update(model=self.surrogate_model, constraint_models=self.constraint_models, eta=incumbent_value, num_data=num_config_evaluated) else: # multi-objectives mo_incumbent_value = history_container.get_mo_incumbent_value() if self.acq_type == 'parego': self.acquisition_function.update(model=self.surrogate_model, constraint_models=self.constraint_models, eta=scalarized_obj(np.atleast_2d(mo_incumbent_value)), num_data=num_config_evaluated) elif self.acq_type.startswith('ehvi'): partitioning = NondominatedPartitioning(self.num_objs, Y) cell_bounds = partitioning.get_hypercell_bounds(ref_point=self.ref_point) self.acquisition_function.update(model=self.surrogate_model, constraint_models=self.constraint_models, cell_lower_bounds=cell_bounds[0], cell_upper_bounds=cell_bounds[1]) else: self.acquisition_function.update(model=self.surrogate_model, constraint_models=self.constraint_models, constraint_perfs=cY, # for MESMOC eta=mo_incumbent_value, num_data=num_config_evaluated, X=X, Y=Y) # optimize acquisition function challengers = self.optimizer.maximize(runhistory=history_container, num_points=5000) is_repeated_config = True repeated_time = 0 cur_config = None while is_repeated_config: cur_config = challengers.challengers[repeated_time] # todo: test small space if cur_config in history_container.configurations: repeated_time += 1 else: is_repeated_config = False return cur_config else: raise ValueError('Unknown optimization strategy: %s.' % self.optimization_strategy)
def get_suggestion(self, history_container=None): if history_container is None: history_container = self.history_container # Check if turbo needs to be restarted if self.use_trust_region and self.turbo_state.restart_triggered: history_container.restart() print('-' * 30) print('Restart!') num_config_evaluated = len(history_container.configurations) num_config_successful = len(history_container.successful_perfs) if num_config_evaluated < self.init_num: return self.initial_configurations[num_config_evaluated] if self.optimization_strategy == 'random': return self.sample_random_configs(1)[0] X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs(transform=None) cY = history_container.get_transformed_constraint_perfs( transform='bilog') if self.optimization_strategy == 'bo': if num_config_successful < max(self.init_num, 1): self.logger.warning( 'No enough successful initial trials! Sample random configuration.' ) return self.sample_random_configs(1)[0] # train surrogate model if self.num_objs == 1: self.surrogate_model.train(X, Y) else: # multi-objectives for i in range(self.num_objs): self.surrogate_model[i].train(X, Y[:, i]) # train constraint model for i in range(self.num_constraints): self.constraint_models[i].train(X, cY[:, i]) # update acquisition function if self.num_objs == 1: # MC-EI incumbent_value = history_container.get_incumbents()[0][1] self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, eta=incumbent_value) else: # MC-ParEGO or MC-EHVI if self.acq_type.startswith('mcparego'): self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models) elif self.acq_type.startswith('mcehvi'): partitioning = NondominatedPartitioning(self.num_objs, Y) cell_bounds = partitioning.get_hypercell_bounds( ref_point=self.ref_point) self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, cell_lower_bounds=cell_bounds[0], cell_upper_bounds=cell_bounds[1]) # optimize acquisition function challengers = self.optimizer.maximize(runhistory=history_container, num_points=5000, turbo_state=self.turbo_state) is_repeated_config = True repeated_time = 0 cur_config = None while is_repeated_config: cur_config = challengers.challengers[repeated_time] if cur_config in history_container.configurations: repeated_time += 1 else: is_repeated_config = False return cur_config else: raise ValueError('Unknown optimization strategy: %s.' % self.optimization_strategy)
def get_suggestion(self, history_container=None, return_list=False): """ Generate a configuration (suggestion) for this query. Returns ------- A configuration. """ if history_container is None: history_container = self.history_container self.alter_model(history_container) num_config_evaluated = len(history_container.configurations) num_config_successful = len(history_container.successful_perfs) if num_config_evaluated < self.init_num: return self.initial_configurations[num_config_evaluated] if self.optimization_strategy == 'random': return self.sample_random_configs(1, history_container)[0] if (not return_list) and self.rng.random() < self.rand_prob: self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob) return self.sample_random_configs(1, history_container)[0] X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs(transform=None) cY = history_container.get_transformed_constraint_perfs( transform='bilog') if self.optimization_strategy == 'bo': if num_config_successful < max(self.init_num, 1): self.logger.warning( 'No enough successful initial trials! Sample random configuration.' ) return self.sample_random_configs(1, history_container)[0] # train surrogate model if self.num_objs == 1: self.surrogate_model.train(X, Y) elif self.acq_type == 'parego': weights = self.rng.random_sample(self.num_objs) weights = weights / np.sum(weights) scalarized_obj = get_chebyshev_scalarization(weights, Y) self.surrogate_model.train(X, scalarized_obj(Y)) else: # multi-objectives for i in range(self.num_objs): self.surrogate_model[i].train(X, Y[:, i]) # train constraint model for i in range(self.num_constraints): self.constraint_models[i].train(X, cY[:, i]) # update acquisition function if self.num_objs == 1: incumbent_value = history_container.get_incumbents()[0][1] self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, eta=incumbent_value, num_data=num_config_evaluated) else: # multi-objectives mo_incumbent_value = history_container.get_mo_incumbent_value() if self.acq_type == 'parego': self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, eta=scalarized_obj(np.atleast_2d(mo_incumbent_value)), num_data=num_config_evaluated) elif self.acq_type.startswith('ehvi'): partitioning = NondominatedPartitioning(self.num_objs, Y) cell_bounds = partitioning.get_hypercell_bounds( ref_point=self.ref_point) self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, cell_lower_bounds=cell_bounds[0], cell_upper_bounds=cell_bounds[1]) else: self.acquisition_function.update( model=self.surrogate_model, constraint_models=self.constraint_models, constraint_perfs=cY, # for MESMOC eta=mo_incumbent_value, num_data=num_config_evaluated, X=X, Y=Y) # optimize acquisition function challengers = self.optimizer.maximize(runhistory=history_container, num_points=5000) if return_list: # Caution: return_list doesn't contain random configs sampled according to rand_prob return challengers.challengers for config in challengers.challengers: if config not in history_container.configurations: return config self.logger.warning( 'Cannot get non duplicate configuration from BO candidates (len=%d). ' 'Sample random config.' % (len(challengers.challengers), )) return self.sample_random_configs(1, history_container)[0] else: raise ValueError('Unknown optimization strategy: %s.' % self.optimization_strategy)
def iterate(self, skip_last=0): for s in reversed(range(self.s_max + 1)): if self.update_enable and self.weight_update_id > self.s_max: self.update_weight() self.weight_update_id += 1 # Set initial number of configurations n = int(ceil(self.B / self.R / (s + 1) * self.eta ** s)) # initial number of iterations per config r = int(self.R * self.eta ** (-s)) # Choose a batch of configurations in different mechanisms. start_time = time.time() T = self.choose_next(n) time_elapsed = time.time() - start_time self.logger.info("[%s] Choosing next configurations took %.2f sec." % (self.method_name, time_elapsed)) extra_info = None last_run_num = None for i in range((s + 1) - int(skip_last)): # changed from s + 1 # Run each of the n configs for <iterations> # and keep best (n_configs / eta) configurations n_configs = n * self.eta ** (-i) n_iteration = r * self.eta ** (i) n_iter = n_iteration if last_run_num is not None and not self.restart_needed: n_iter -= last_run_num last_run_num = n_iteration self.logger.info("%s: %d configurations x %d iterations each" % (self.method_name, int(n_configs), int(n_iteration))) ret_val, early_stops = self.run_in_parallel(T, n_iter, extra_info) val_losses = [item['loss'] for item in ret_val] ref_list = [item['ref_id'] for item in ret_val] self.target_x[int(n_iteration)].extend(T) self.target_y[int(n_iteration)].extend(val_losses) if int(n_iteration) == self.R: self.incumbent_configs.extend(T) self.incumbent_perfs.extend(val_losses) # Update history container. for _config, _perf in zip(T, val_losses): self.history_container.add(_config, _perf) # Select a number of best configurations for the next loop. # Filter out early stops, if any. indices = np.argsort(val_losses) if len(T) == sum(early_stops): break if len(T) >= self.eta: indices = [i for i in indices if not early_stops[i]] T = [T[i] for i in indices] extra_info = [ref_list[i] for i in indices] reduced_num = int(n_configs / self.eta) T = T[0:reduced_num] extra_info = extra_info[0:reduced_num] else: T = [T[indices[0]]] # todo: confirm no filter early stops? extra_info = [ref_list[indices[0]]] val_losses = [val_losses[i] for i in indices][0:len(T)] # update: sorted incumbent_loss = val_losses[0] self.add_stage_history(self.stage_id, min(self.global_incumbent, incumbent_loss)) self.stage_id += 1 # self.remove_immediate_model() for item in self.iterate_r[self.iterate_r.index(r):]: # NORMALIZE Objective value: normalization normalized_y = std_normalization(self.target_y[item]) self.weighted_surrogate.train(convert_configurations_to_array(self.target_x[item]), np.array(normalized_y, dtype=np.float64), r=item)
def get_suggestions(self, batch_size=None, history_container=None): if batch_size is None: batch_size = self.batch_size assert batch_size >= 1 if history_container is None: history_container = self.history_container num_config_evaluated = len(history_container.configurations) num_config_successful = len(history_container.successful_perfs) if num_config_evaluated < self.init_num: if self.initial_configurations is not None: # self.init_num equals to len(self.initial_configurations) next_configs = self.initial_configurations[ num_config_evaluated:num_config_evaluated + batch_size] if len(next_configs) < batch_size: next_configs.extend( self.sample_random_configs( batch_size - len(next_configs), history_container)) return next_configs else: return self.sample_random_configs(batch_size, history_container) if self.optimization_strategy == 'random': return self.sample_random_configs(batch_size, history_container) if num_config_successful < max(self.init_num, 1): self.logger.warning( 'No enough successful initial trials! Sample random configurations.' ) return self.sample_random_configs(batch_size, history_container) X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs(transform=None) # cY = history_container.get_transformed_constraint_perfs(transform='bilog') batch_configs_list = list() if self.batch_strategy == 'median_imputation': # set bilog_transform=False to get real cY for estimating median cY = history_container.get_transformed_constraint_perfs( transform=None) estimated_y = np.median(Y, axis=0).reshape(-1).tolist() estimated_c = np.median( cY, axis=0).tolist() if self.num_constraints > 0 else None batch_history_container = copy.deepcopy(history_container) for batch_i in range(batch_size): # use super class get_suggestion curr_batch_config = super().get_suggestion( batch_history_container) # imputation observation = Observation(config=curr_batch_config, objs=estimated_y, constraints=estimated_c, trial_state=SUCCESS, elapsed_time=None) batch_history_container.update_observation(observation) batch_configs_list.append(curr_batch_config) elif self.batch_strategy == 'local_penalization': # local_penalization only supports single objective with no constraint self.surrogate_model.train(X, Y) incumbent_value = history_container.get_incumbents()[0][1] # L = self.estimate_L(X) for i in range(batch_size): if self.rng.random() < self.rand_prob: # sample random configuration proportionally self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob) cur_config = self.sample_random_configs( 1, history_container, excluded_configs=batch_configs_list)[0] else: self.acquisition_function.update( model=self.surrogate_model, eta=incumbent_value, num_data=len(history_container.data), batch_configs=batch_configs_list) challengers = self.optimizer.maximize( runhistory=history_container, num_points=5000, ) cur_config = challengers.challengers[0] batch_configs_list.append(cur_config) elif self.batch_strategy == 'reoptimization': surrogate_trained = False for i in range(batch_size): if self.rng.random() < self.rand_prob: # sample random configuration proportionally self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob) cur_config = self.sample_random_configs( 1, history_container, excluded_configs=batch_configs_list)[0] else: if not surrogate_trained: # set return_list=True to ensure surrogate trained candidates = super().get_suggestion(history_container, return_list=True) surrogate_trained = True else: # re-optimize acquisition function challengers = self.optimizer.maximize( runhistory=history_container, num_points=5000) candidates = challengers.challengers cur_config = None for config in candidates: if config not in batch_configs_list and config not in history_container.configurations: cur_config = config break if cur_config is None: self.logger.warning( 'Cannot get non duplicate configuration from BO candidates (len=%d). ' 'Sample random config.' % (len(candidates), )) cur_config = self.sample_random_configs( 1, history_container, excluded_configs=batch_configs_list)[0] batch_configs_list.append(cur_config) elif self.batch_strategy == 'default': # select first N candidates candidates = super().get_suggestion(history_container, return_list=True) idx = 0 while len(batch_configs_list) < batch_size: if idx >= len(candidates): self.logger.warning( 'Cannot get non duplicate configuration from BO candidates (len=%d). ' 'Sample random config.' % (len(candidates), )) cur_config = self.sample_random_configs( 1, history_container, excluded_configs=batch_configs_list)[0] elif self.rng.random() < self.rand_prob: # sample random configuration proportionally self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob) cur_config = self.sample_random_configs( 1, history_container, excluded_configs=batch_configs_list)[0] else: cur_config = None while idx < len(candidates): conf = candidates[idx] idx += 1 if conf not in batch_configs_list and conf not in history_container.configurations: cur_config = conf break if cur_config is not None: batch_configs_list.append(cur_config) else: raise ValueError('Invalid sampling strategy - %s.' % self.batch_strategy) return batch_configs_list
def get_suggestions(self, batch_size=None, history_container=None): if batch_size is None: batch_size = self.batch_size if history_container is None: history_container = self.history_container num_config_evaluated = len(history_container.configurations) num_config_successful = len(history_container.successful_perfs) if num_config_evaluated < self.init_num: if self.initial_configurations is not None: # self.init_num equals to len(self.initial_configurations) return self.initial_configurations else: return self.sample_random_configs(self.init_num, history_container) if self.optimization_strategy == 'random': return self.sample_random_configs(batch_size, history_container) if num_config_successful < max(self.init_num, 1): self.logger.warning( 'No enough successful initial trials! Sample random configurations.' ) return self.sample_random_configs(batch_size, history_container) X = convert_configurations_to_array(history_container.configurations) Y = history_container.get_transformed_perfs() # cY = history_container.get_transformed_constraint_perfs() batch_configs_list = list() if self.batch_strategy == 'median_imputation': # set bilog_transform=False to get real cY for estimating median cY = history_container.get_transformed_constraint_perfs( bilog_transform=False) estimated_y = np.median(Y, axis=0).reshape(-1).tolist() estimated_c = np.median( cY, axis=0).tolist() if self.num_constraints > 0 else None batch_history_container = copy.deepcopy(history_container) for batch_i in range(batch_size): # use super class get_suggestion curr_batch_config = super().get_suggestion( batch_history_container) # imputation observation = Observation(curr_batch_config, SUCCESS, estimated_c, estimated_y, None) batch_history_container.update_observation(observation) batch_configs_list.append(curr_batch_config) elif self.batch_strategy == 'local_penalization': # local_penalization only supports single objective with no constraint self.surrogate_model.train(X, Y) incumbent_value = history_container.get_incumbents()[0][1] # L = self.estimate_L(X) for i in range(batch_size): self.acquisition_function.update( model=self.surrogate_model, eta=incumbent_value, num_data=len(history_container.data), batch_configs=batch_configs_list) challengers = self.optimizer.maximize( runhistory=history_container, num_points=5000, ) batch_configs_list.append(challengers.challengers[0]) else: raise ValueError('Invalid sampling strategy - %s.' % self.batch_strategy) return batch_configs_list
def get_suggestions(self, n_suggestions): if len(self.configurations) == 0: X = np.array([]) else: failed_configs = list( ) if self.max_y is None else self.failed_configurations.copy() X = convert_configurations_to_array(self.configurations + failed_configs) num_failed_trial = len(self.failed_configurations) failed_perfs = list() if self.max_y is None else [self.max_y ] * num_failed_trial Y = np.array(self.perfs + failed_perfs, dtype=np.float64) num_config_evaluated = len(self.perfs) batch_configs_list = list() if num_config_evaluated < self.init_num or self.optimization_strategy == 'random': return self.sample_random_configs(n_suggestions) if self.batch_strategy == 'median_imputation': estimated_y = np.median(Y) batch_history_container = copy.deepcopy(self.history_container) for i in range(n_suggestions): self.surrogate_model.train(X, Y, snapshot=(i == 0)) incumbent_value = batch_history_container.get_incumbents( )[0][1] self.acquisition_function.update( model=self.surrogate_model, eta=incumbent_value, num_data=len(batch_history_container.data)) challengers = self.optimizer.maximize( runhistory=batch_history_container, num_points=5000) is_repeated_config = True repeated_time = 0 curr_batch_config = None while is_repeated_config: try: curr_batch_config = challengers.challengers[ repeated_time] batch_history_container.add(curr_batch_config, estimated_y) except ValueError: is_repeated_config = True repeated_time += 1 else: is_repeated_config = False batch_configs_list.append(curr_batch_config) X = np.append(X, convert_configurations_to_array( [curr_batch_config]), axis=0) Y = np.append(Y, estimated_y) elif self.batch_strategy == 'local_penalization': self.surrogate_model.train(X, Y) incumbent_value = self.history_container.get_incumbents()[0][1] # L = self.estimate_L(X) for i in range(n_suggestions): self.acquisition_function.update( model=self.surrogate_model, eta=incumbent_value, num_data=len(self.history_container.data), batch_configs=batch_configs_list) challengers = self.optimizer.maximize( runhistory=self.history_container, num_points=5000) batch_configs_list.append(challengers.challengers[0]) else: raise ValueError('Invalid sampling strategy - %s.' % self.batch_strategy) return batch_configs_list
def iterate(self): if len(self.configurations) == 0: X = np.array([]) else: failed_configs = list() if self.max_y is None else self.failed_configurations.copy() X = convert_configurations_to_array(self.configurations + failed_configs) failed_perfs = list() if self.max_y is None else [self.max_y] * len(self.failed_configurations) Y = np.array(self.perfs + failed_perfs, dtype=np.float64) config_list = self.choose_next(X, Y) trial_state_list = list() trial_info_list = list() perf_list = list() for i, config in enumerate(config_list): trial_state_list.append(SUCCESS) trial_info_list.append(None) perf_list.append(None) if config not in (self.configurations + self.failed_configurations): # Evaluate this configuration. try: args, kwargs = (config,), dict() timeout_status, _result = time_limit(self.objective_function, self.time_limit_per_trial, args=args, kwargs=kwargs) if timeout_status: raise TimeoutException( 'Timeout: time limit for this evaluation is %.1fs' % self.time_limit_per_trial) else: perf_list[i] = _result except Exception as e: if isinstance(e, TimeoutException): trial_state_list[i] = TIMEOUT else: traceback.print_exc(file=sys.stdout) trial_state_list[i] = FAILED perf_list[i] = MAXINT trial_info_list[i] = str(e) self.logger.error(trial_info_list[i]) if trial_state_list[i] == SUCCESS and perf_list[i] < MAXINT: if len(self.configurations) == 0: self.default_obj_value = perf_list[i] self.configurations.append(config) self.perfs.append(perf_list[i]) self.history_container.add(config, perf_list[i]) self.perc = np.percentile(self.perfs, self.scale_perc) self.min_y = np.min(self.perfs) self.max_y = np.max(self.perfs) else: self.failed_configurations.append(config) else: self.logger.debug('This configuration has been evaluated! Skip it.') if config in self.configurations: config_idx = self.configurations.index(config) trial_state_list[i], perf_list[i] = SUCCESS, self.perfs[config_idx] else: trial_state_list[i], perf_list[i] = FAILED, MAXINT self.iteration_id += 1 self.logger.info( 'Iteration-%d, objective improvement: %.4f' % ( self.iteration_id, max(0, self.default_obj_value - min(perf_list)))) return config_list, trial_state_list, perf_list, trial_info_list
def update_weight(self): start_time = time.time() max_r = self.iterate_r[-1] incumbent_configs = self.target_x[max_r] test_x = convert_configurations_to_array(incumbent_configs) test_y = np.array(self.target_y[max_r], dtype=np.float64) r_list = self.surrogate.surrogate_r K = len(r_list) if len(test_y) >= 3: # Get previous weights if self.weight_method == 'rank_loss_p_norm': preserving_order_p = list() preserving_order_nums = list() for i, r in enumerate(r_list): fold_num = 5 if i != K - 1: mean, var = self.surrogate.surrogate_container[ r].predict(test_x) tmp_y = np.reshape(mean, -1) preorder_num, pair_num = self.calculate_preserving_order_num( tmp_y, test_y) preserving_order_p.append(preorder_num / pair_num) preserving_order_nums.append(preorder_num) else: if len(test_y) < 2 * fold_num: preserving_order_p.append(0) else: # 5-fold cross validation. kfold = KFold(n_splits=fold_num) cv_pred = np.array([0] * len(test_y)) for train_idx, valid_idx in kfold.split(test_x): train_configs, train_y = test_x[ train_idx], test_y[train_idx] valid_configs, valid_y = test_x[ valid_idx], test_y[valid_idx] types, bounds = get_types(self.config_space) _surrogate = RandomForestWithInstances( types=types, bounds=bounds) _surrogate.train(train_configs, train_y) pred, _ = _surrogate.predict(valid_configs) cv_pred[valid_idx] = pred.reshape(-1) preorder_num, pair_num = self.calculate_preserving_order_num( cv_pred, test_y) preserving_order_p.append(preorder_num / pair_num) preserving_order_nums.append(preorder_num) trans_order_weight = np.array(preserving_order_p) power_sum = np.sum(np.power(trans_order_weight, self.power_num)) new_weights = np.power(trans_order_weight, self.power_num) / power_sum elif self.weight_method == 'rank_loss_prob': # For basic surrogate i=1:K-1. mean_list, var_list = list(), list() for i, r in enumerate(r_list[:-1]): mean, var = self.surrogate.surrogate_container[r].predict( test_x) mean_list.append(np.reshape(mean, -1)) var_list.append(np.reshape(var, -1)) sample_num = 100 min_probability_array = [0] * K for _ in range(sample_num): order_preseving_nums = list() # For basic surrogate i=1:K-1. for idx in range(K - 1): sampled_y = self.rng.normal(mean_list[idx], var_list[idx]) _num, _ = self.calculate_preserving_order_num( sampled_y, test_y) order_preseving_nums.append(_num) fold_num = 5 # For basic surrogate i=K. cv if len(test_y) < 2 * fold_num: order_preseving_nums.append(0) else: # 5-fold cross validation. kfold = KFold(n_splits=fold_num) cv_pred = np.array([0] * len(test_y)) for train_idx, valid_idx in kfold.split(test_x): train_configs, train_y = test_x[train_idx], test_y[ train_idx] valid_configs, valid_y = test_x[valid_idx], test_y[ valid_idx] types, bounds = get_types(self.config_space) _surrogate = RandomForestWithInstances( types=types, bounds=bounds) _surrogate.train(train_configs, train_y) _pred, _var = _surrogate.predict(valid_configs) sampled_pred = self.rng.normal( _pred.reshape(-1), _var.reshape(-1)) cv_pred[valid_idx] = sampled_pred _num, _ = self.calculate_preserving_order_num( cv_pred, test_y) order_preseving_nums.append(_num) max_id = np.argmax(order_preseving_nums) min_probability_array[max_id] += 1 new_weights = np.array(min_probability_array) / sample_num else: raise ValueError('Invalid weight method: %s!' % self.weight_method) else: old_weights = list() for i, r in enumerate(r_list): _weight = self.surrogate.surrogate_weight[r] old_weights.append(_weight) new_weights = old_weights.copy() self.logger.info( '[%s] %d-th Updating weights: %s' % (self.weight_method, self.weight_changed_cnt, str(new_weights))) # Assign the weight to each basic surrogate. for i, r in enumerate(r_list): self.surrogate.surrogate_weight[r] = new_weights[i] self.weight_changed_cnt += 1 # Save the weight data. self.hist_weights.append(new_weights) dir_path = os.path.join(self.data_directory, 'saved_weights') file_name = 'mfes_weights_%s.npy' % (self.method_name, ) if not os.path.exists(dir_path): os.makedirs(dir_path) np.save(os.path.join(dir_path, file_name), np.asarray(self.hist_weights)) self.logger.info( 'update_weight() cost %.2fs. new weights are saved to %s' % (time.time() - start_time, os.path.join(dir_path, file_name)))