def sample_configurations(configuration_space: ConfigurationSpace, sample_size: int, historical_configs: List[Configuration], seed=1): configuration_space.seed(seed) result = [] sample_cnt = 0 if len(historical_configs) == 0: result.append(configuration_space.get_default_configuration()) while len(result) < sample_size: config = configuration_space.sample_configuration(1) if config not in result and config not in historical_configs: result.append(config) sample_cnt += 1 if sample_cnt > 50 * sample_size: break # if len(result) == 0: # hist_num = len(historical_configs) # if hist_num > sample_size: # idxs = random.sample(range(len(historical_configs)), sample_size) # result = [historical_configs[idx] for idx in idxs] # else: # result = historical_configs.copy() return result
def _get_configuration_space(self) -> ConfigurationSpace: """Get the configuration space for the random forest. Returns ------- ConfigurationSpace """ cfg = ConfigurationSpace() cfg.seed(int(self.rs.randint(0, 1000))) num_trees = Constant("num_trees", value=N_TREES) bootstrap = CategoricalHyperparameter( "do_bootstrapping", choices=(self.bootstrap, ), default_value=self.bootstrap, ) max_feats = CategoricalHyperparameter("max_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1) min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2) min_leavs = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1) cfg.add_hyperparameters( [num_trees, bootstrap, max_feats, min_split, min_leavs]) return cfg
def get_random_initial_configs(shps: ConfigurationSpace, n_configs, random_state=42) -> List[Configuration]: None_name = "None:NoneType" shps = deepcopy(shps) shps.seed(random_state) for config in shps.get_hyperparameters(): name: str = config.name if name.startswith(PHASE1) and name.endswith("__choice__") and ( None_name in config.choices): # fixme 重构之后 None_name是不是改变了? config.default_value = None_name model_choice = shps.get_hyperparameter(f"{PHASE2}:__choice__") result = [] for choice in model_choice.choices: cur_phps = deepcopy(shps) cur_phps.get_hyperparameter( f"{PHASE2}:__choice__").default_value = choice default = cur_phps.get_default_configuration() result.append(default) if len(result) < n_configs: result.extend(shps.sample_configuration(n_configs - len(result))) elif len(result) > n_configs: result = random.sample(result, n_configs) return result
def get_random_initial_configs(shps: ConfigurationSpace, n_configs, random_state=42) -> List[Configuration]: shps = deepcopy(shps) shps.seed(random_state) results = shps.sample_configuration(n_configs) if not isinstance(results, list): results = [results] return results
def main(): for synthetic_function_cls in synthetic_functions: meta_info = synthetic_function_cls.get_meta_information() if "num_function_evals" in meta_info: max_iter = meta_info["num_function_evals"] else: max_iter = 200 # 构造超参空间 config_space = ConfigurationSpace() config_space.generate_all_continuous_from_bounds( synthetic_function_cls.get_meta_information()['bounds']) synthetic_function = synthetic_function_cls() # 定义目标函数 def evaluation(config: dict): config = Configuration(config_space, values=config) return synthetic_function.objective_function(config)["function_value"] - \ synthetic_function.get_meta_information()["f_opt"] # 对experiment_param的删除等操作放在存储后面 res = pd.DataFrame(columns=[f"trial-{i}" for i in range(10)], index=range(max_iter)) for trial in range(10): random_state = 50 + trial * 10 # 设置超参空间的随机种子(会影响后面的采样) config_space.seed(random_state) print("==========================") print(f"= Trial -{trial:01d}- =") print("==========================") print('iter | loss | config origin') print('----------------------------') cg = SamplingSortOptimizer(config_space, [1], min_points_in_model=25, n_samples=2500) loss = np.inf for ix in range(max_iter): config, config_info = cg.get_config(1) cur_loss = evaluation(config) loss = min(loss, cur_loss) print(f" {ix:03d} {loss:.4f} {config_info.get('origin')}") job = Job("") job.result = {"loss": cur_loss} job.kwargs = { "budget": 1, "config": config, "config_info": config_info } cg.new_result(job) res.loc[ix, f"trial-{trial}"] = cur_loss res = raw2min(res) m = res.mean(1) s = res.std(1) name = synthetic_function.get_meta_information()["name"] final_result[name] = {"mean": m.tolist(), "std": s.tolist()} Path("EETPE.json").write_text(json.dumps(final_result))
def _test_random_neigbor(self, hp): cs = ConfigurationSpace() if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) cs.seed(1) config = cs.sample_configuration() for i in range(100): new_config = get_random_neighbor(config, i) self.assertNotEqual(config, new_config)
def _test_random_neigbor(self, hp): cs = ConfigurationSpace() if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) cs.seed(1) config = cs.sample_configuration() for i in range(100): new_config = get_random_neighbor(config, i) self.assertNotEqual(config, new_config)
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def main(): for synthetic_function_cls in synthetic_functions: meta_info = synthetic_function_cls.get_meta_information() if "num_function_evals" in meta_info: max_iter = meta_info["num_function_evals"] else: max_iter = base_max_iter # 构造超参空间 config_space = ConfigurationSpace() config_space.generate_all_continuous_from_bounds( synthetic_function_cls.get_meta_information()['bounds']) synthetic_function = synthetic_function_cls() space = CS2HyperoptSpace(config_space) # 定义目标函数 def evaluation(config: dict): config = Configuration(config_space, values=config) return synthetic_function.objective_function(config)["function_value"] - \ synthetic_function.get_meta_information()["f_opt"] # 对experiment_param的删除等操作放在存储后面 res = pd.DataFrame(columns=[f"trial-{i}" for i in range(repetitions)], index=range(max_iter)) for trial in range(repetitions): random_state = base_random_state + trial * 10 # 设置超参空间的随机种子(会影响后面的采样) config_space.seed(random_state) trials = Trials() best = fmin( evaluation, space, algo=partial(tpe.suggest, n_startup_jobs=20), max_evals=max_iter, rstate=np.random.RandomState(random_state), trials=trials, ) losses = trials.losses() res[f"trial-{trial}"] = losses res = raw2min(res) m = res.mean(1) s = res.std(1) name = synthetic_function.get_meta_information()["name"] final_result[name] = {"mean": m.tolist(), "std": s.tolist()} Path("hyperopt.json").write_text(json.dumps(final_result))
def _test_get_one_exchange_neighbourhood(self, hp): cs = ConfigurationSpace() num_neighbors = 0 if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) if np.isinf(hp_.get_num_neighbors()): num_neighbors += 4 else: num_neighbors += hp_.get_num_neighbors() cs.seed(1) config = cs.get_default_configuration() all_neighbors = [] for i in range(100): neighborhood = get_one_exchange_neighbourhood(config, i) for new_config in neighborhood: self.assertNotEqual(config, new_config) all_neighbors.append(new_config) return all_neighbors
def _test_get_one_exchange_neighbourhood(self, hp): cs = ConfigurationSpace() num_neighbors = 0 if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) if np.isinf(hp_.get_num_neighbors()): num_neighbors += 4 else: num_neighbors += hp_.get_num_neighbors() cs.seed(1) config = cs.get_default_configuration() all_neighbors = [] for i in range(100): neighborhood = get_one_exchange_neighbourhood(config, i) for new_config in neighborhood: self.assertNotEqual(config, new_config) all_neighbors.append(new_config) return all_neighbors