def fetch_candidate_configurations(self, num_config): if len(self.target_y[self.iterate_r[-1]]) == 0: return sample_configurations(self.config_space, num_config) incumbent = dict() max_r = self.iterate_r[-1] # The lower, the better. best_index = np.argmin(self.target_y[max_r]) incumbent['config'] = self.target_x[max_r][best_index] approximate_obj = self.weighted_surrogate.predict(convert_configurations_to_array([incumbent['config']]))[0] incumbent['obj'] = approximate_obj self.weighted_acquisition_func.update(model=self.weighted_surrogate, eta=incumbent) config_candidates = self.weighted_acq_optimizer.maximize(batch_size=num_config) p_threshold = 0.3 n_acq = self.eta * self.eta if num_config <= n_acq: return config_candidates candidates = config_candidates[: n_acq] idx_acq = n_acq for _id in range(num_config - n_acq): if rd.random() < p_threshold: config = sample_configurations(self.config_space, 1)[0] else: config = config_candidates[idx_acq] idx_acq += 1 candidates.append(config) return candidates
def maximize(self, batch_size=1): """ Maximizes the given acquisition function. Parameters ---------- batch_size: number of maximizer returned. Returns ------- np.ndarray(N,D) Point with highest acquisition value. """ incs_configs = list( get_one_exchange_neighbourhood(self.objective_func.eta['config'], seed=self.rng.randint(int(1e6)))) configs_list = list(incs_configs) rand_incs = convert_configurations_to_array(configs_list) # Sample random points uniformly over the whole space rand_configs = sample_configurations( self.config_space, self.n_samples - rand_incs.shape[0]) rand = convert_configurations_to_array(rand_configs) configs_list.extend(rand_configs) X = np.concatenate((rand_incs, rand), axis=0) y = self.objective_func(X).flatten() candidate_idxs = list(np.argsort(-y)[:batch_size]) # print(candidate_idxs) # print(type(candidate_idxs)) # print(configs_list[:5]) return [configs_list[idx] for idx in candidate_idxs]
def _iterate(self, s, budget=MAX_INT, skip_last=0): # Set initial number of configurations n = int(ceil(self.B / self.R / (s + 1) * self.eta**s)) # initial number of iterations per config r = int(self.R * self.eta**(-s)) # Choose a batch of configurations in different mechanisms. start_time = time.time() T = sample_configurations(self.config_space, n) time_elapsed = time.time() - start_time self.logger.info( "Choosing next batch of configurations took %.2f sec." % time_elapsed) with ParallelProcessEvaluator(self.eval_func, n_worker=self.n_workers) as executor: for i in range((s + 1) - int(skip_last)): # changed from s + 1 if time.time() >= budget + start_time: break # Run each of the n configs for <iterations> # and keep best (n_configs / eta) configurations n_configs = n * self.eta**(-i) n_resource = r * self.eta**i self.logger.info( "MFSE: %d configurations x size %d / %d each" % (int(n_configs), n_resource, self.R)) val_losses = executor.parallel_execute( T, resource_ratio=float(n_resource / self.R), eta=self.eta, first_iter=(i == 0)) for _id, _val_loss in enumerate(val_losses): if np.isfinite(_val_loss): self.target_x[int(n_resource)].append(T[_id]) self.target_y[int(n_resource)].append(_val_loss) self.exp_output[time.time()] = (int(n_resource), T, val_losses) if int(n_resource) == self.R: self.incumbent_configs.extend(T) self.incumbent_perfs.extend(val_losses) # Select a number of best configurations for the next loop. # Filter out early stops, if any. indices = np.argsort(val_losses) if len(T) >= self.eta: T = [T[i] for i in indices] reduced_num = int(n_configs / self.eta) T = T[0:reduced_num] else: T = [T[indices[0]]]
def choose_next(self, X: np.ndarray, Y: np.ndarray): _config_num = X.shape[0] if _config_num < self.init_num: if self.initial_configurations is None: if _config_num == 0: return self.config_space.get_default_configuration() else: return sample_configurations(self.config_space, 1)[0] else: return self.initial_configurations[_config_num] else: if self.surrogate_model == 'tpe': config, _ = self.model.get_config() elif self.surrogate_model == 'random_search': config = sample_configurations(self.config_space, 1)[0] else: raise ValueError('Invalid surrogate model - %s.' % self.surrogate_model) return config
def smac_get_candidate_configurations(self, num_config): if len(self.target_y[self.iterate_r[-1]]) <= 3: return sample_configurations(self.config_space, num_config) incumbent = dict() max_r = self.iterate_r[-1] best_index = np.argmin(self.target_y[max_r]) incumbent['config'] = self.target_x[max_r][best_index] incumbent['obj'] = self.target_y[max_r][best_index] self.acquisition_func.update(model=self.surrogate, eta=incumbent) config_candidates = self.acq_optimizer.maximize(batch_size=num_config) p_threshold = 0.3 candidates = list() idx_acq = 0 for _id in range(num_config): if rd.random() < p_threshold or _id >= len(config_candidates): config = sample_configurations(self.config_space, 1)[0] else: config = config_candidates[idx_acq] idx_acq += 1 candidates.append(config) return candidates
def baseline_get_candidate_configurations(self, num_config): config_candidates = list() config_left = num_config while config_left: config = self.config_gen.get_config()[0] if config in config_candidates: continue config_candidates.append(config) config_left -= 1 p_threshold = 0.3 candidates = list() idx_acq = 0 for _id in range(num_config): if rd.random() < p_threshold: config = sample_configurations(self.config_space, 1)[0] else: config = config_candidates[idx_acq] idx_acq += 1 candidates.append(config) return candidates
def sample_configs_for_archs(self, include_architectures, N, sampling_strategy='uniform'): configs = list() for _arch in include_architectures: _cs = self.get_model_config_space(_arch) if sampling_strategy == 'uniform': configs.append(_cs.get_default_configuration()) configs.extend(sample_configurations(_cs, N - 1)) elif sampling_strategy == 'bohb': if _arch not in self.tpe_config_gen: self.tpe_config_gen[_arch] = TPE(_cs) config_candidates = list() config_left = N while config_left: config = self.tpe_config_gen[_arch].get_config()[0] if config in config_candidates: continue config_candidates.append(config) config_left -= 1 p_threshold = 0.3 idx_acq = 0 for _id in range(N): if rd.random() < p_threshold: config = sample_configurations(_cs, 1)[0] else: config = config_candidates[idx_acq] idx_acq += 1 configs.append(config) else: # mfse if _arch not in self.mfse_config_gen: types, bounds = get_types(_cs) init_weight = [1. / self.s_max] * self.s_max + [0.] self.mfse_config_gen[_arch] = dict() self.mfse_config_gen[_arch][ 'surrogate'] = WeightedRandomForestCluster( types, bounds, self.s_max, self.eta, init_weight, 'gpoe') acq_func = EI( model=self.mfse_config_gen[_arch]['surrogate']) self.mfse_config_gen[_arch][ 'acq_optimizer'] = RandomSampling( acq_func, _cs, n_samples=2000, rng=np.random.RandomState(1)) if self.R not in self.eval_hist_perfs[_arch] or len( self.eval_hist_perfs[_arch][self.R]) == 0: configs.extend(sample_configurations(_cs, N)) continue incumbent = dict() max_r = self.R # The lower, the better. best_index = np.argmin(self.eval_hist_perfs[_arch][max_r]) incumbent['config'] = self.eval_hist_configs[_arch][max_r][ best_index] approximate_obj = self.mfse_config_gen[_arch][ 'surrogate'].predict( convert_configurations_to_array([incumbent['config'] ]))[0] incumbent['obj'] = approximate_obj self.mfse_config_gen[_arch]['acq_optimizer'].update( model=self.mfse_config_gen[_arch]['surrogate'], eta=incumbent) config_candidates = self.mfse_config_gen[_arch][ 'acq_optimizer'].maximize(batch_size=N) p_threshold = 0.3 n_acq = self.eta * self.eta if N <= n_acq: return config_candidates candidates = config_candidates[:n_acq] idx_acq = n_acq for _id in range(N - n_acq): if rd.random() < p_threshold: config = sample_configurations(_cs, 1)[0] else: config = config_candidates[idx_acq] idx_acq += 1 candidates.append(config) return candidates return configs
def sample_configs_for_archs(self, include_architectures, N): configs = list() for _arch in include_architectures: _cs = self.get_model_config_space(_arch) configs.extend(sample_configurations(_cs, N)) return configs