def sample_configurations(configuration_space: ConfigurationSpace,
sample_size: int,
historical_configs: List[Configuration],
seed=1):
configuration_space.seed(seed)
result = []
sample_cnt = 0
if len(historical_configs) == 0:
result.append(configuration_space.get_default_configuration())
while len(result) < sample_size:
config = configuration_space.sample_configuration(1)
if config not in result and config not in historical_configs:
result.append(config)
sample_cnt += 1
if sample_cnt > 50 * sample_size:
break
# if len(result) == 0:
# hist_num = len(historical_configs)
# if hist_num > sample_size:
# idxs = random.sample(range(len(historical_configs)), sample_size)
# result = [historical_configs[idx] for idx in idxs]
# else:
# result = historical_configs.copy()
return result
def _get_configuration_space(self) -> ConfigurationSpace:
"""Get the configuration space for the random forest.
Returns
-------
ConfigurationSpace
"""
cfg = ConfigurationSpace()
cfg.seed(int(self.rs.randint(0, 1000)))
num_trees = Constant("num_trees", value=N_TREES)
bootstrap = CategoricalHyperparameter(
"do_bootstrapping",
choices=(self.bootstrap, ),
default_value=self.bootstrap,
)
max_feats = CategoricalHyperparameter("max_features",
choices=(3 / 6, 4 / 6, 5 / 6, 1),
default_value=1)
min_split = UniformIntegerHyperparameter("min_samples_to_split",
lower=1,
upper=10,
default_value=2)
min_leavs = UniformIntegerHyperparameter("min_samples_in_leaf",
lower=1,
upper=10,
default_value=1)
cfg.add_hyperparameters(
[num_trees, bootstrap, max_feats, min_split, min_leavs])
return cfg
def get_random_initial_configs(shps: ConfigurationSpace,
n_configs,
random_state=42) -> List[Configuration]:
None_name = "None:NoneType"
shps = deepcopy(shps)
shps.seed(random_state)
for config in shps.get_hyperparameters():
name: str = config.name
if name.startswith(PHASE1) and name.endswith("__choice__") and (
None_name in config.choices): # fixme 重构之后 None_name是不是改变了？
config.default_value = None_name
model_choice = shps.get_hyperparameter(f"{PHASE2}:__choice__")
result = []
for choice in model_choice.choices:
cur_phps = deepcopy(shps)
cur_phps.get_hyperparameter(
f"{PHASE2}:__choice__").default_value = choice
default = cur_phps.get_default_configuration()
result.append(default)
if len(result) < n_configs:
result.extend(shps.sample_configuration(n_configs - len(result)))
elif len(result) > n_configs:
result = random.sample(result, n_configs)
return result
def get_random_initial_configs(shps: ConfigurationSpace,
n_configs,
random_state=42) -> List[Configuration]:
shps = deepcopy(shps)
shps.seed(random_state)
results = shps.sample_configuration(n_configs)
if not isinstance(results, list):
results = [results]
return results
def main():
for synthetic_function_cls in synthetic_functions:
meta_info = synthetic_function_cls.get_meta_information()
if "num_function_evals" in meta_info:
max_iter = meta_info["num_function_evals"]
else:
max_iter = 200
# 构造超参空间
config_space = ConfigurationSpace()
config_space.generate_all_continuous_from_bounds(
synthetic_function_cls.get_meta_information()['bounds'])
synthetic_function = synthetic_function_cls()
# 定义目标函数
def evaluation(config: dict):
config = Configuration(config_space, values=config)
return synthetic_function.objective_function(config)["function_value"] - \
synthetic_function.get_meta_information()["f_opt"]
# 对experiment_param的删除等操作放在存储后面
res = pd.DataFrame(columns=[f"trial-{i}" for i in range(10)],
index=range(max_iter))
for trial in range(10):
random_state = 50 + trial * 10
# 设置超参空间的随机种子（会影响后面的采样）
config_space.seed(random_state)
print("==========================")
print(f"= Trial -{trial:01d}- =")
print("==========================")
print('iter | loss | config origin')
print('----------------------------')
cg = SamplingSortOptimizer(config_space, [1],
min_points_in_model=25,
n_samples=2500)
loss = np.inf
for ix in range(max_iter):
config, config_info = cg.get_config(1)
cur_loss = evaluation(config)
loss = min(loss, cur_loss)
print(f" {ix:03d} {loss:.4f} {config_info.get('origin')}")
job = Job("")
job.result = {"loss": cur_loss}
job.kwargs = {
"budget": 1,
"config": config,
"config_info": config_info
}
cg.new_result(job)
res.loc[ix, f"trial-{trial}"] = cur_loss
res = raw2min(res)
m = res.mean(1)
s = res.std(1)
name = synthetic_function.get_meta_information()["name"]
final_result[name] = {"mean": m.tolist(), "std": s.tolist()}
Path("EETPE.json").write_text(json.dumps(final_result))
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def main():
for synthetic_function_cls in synthetic_functions:
meta_info = synthetic_function_cls.get_meta_information()
if "num_function_evals" in meta_info:
max_iter = meta_info["num_function_evals"]
else:
max_iter = base_max_iter
# 构造超参空间
config_space = ConfigurationSpace()
config_space.generate_all_continuous_from_bounds(
synthetic_function_cls.get_meta_information()['bounds'])
synthetic_function = synthetic_function_cls()
space = CS2HyperoptSpace(config_space)
# 定义目标函数
def evaluation(config: dict):
config = Configuration(config_space, values=config)
return synthetic_function.objective_function(config)["function_value"] - \
synthetic_function.get_meta_information()["f_opt"]
# 对experiment_param的删除等操作放在存储后面
res = pd.DataFrame(columns=[f"trial-{i}" for i in range(repetitions)],
index=range(max_iter))
for trial in range(repetitions):
random_state = base_random_state + trial * 10
# 设置超参空间的随机种子（会影响后面的采样）
config_space.seed(random_state)
trials = Trials()
best = fmin(
evaluation,
space,
algo=partial(tpe.suggest, n_startup_jobs=20),
max_evals=max_iter,
rstate=np.random.RandomState(random_state),
trials=trials,
)
losses = trials.losses()
res[f"trial-{trial}"] = losses
res = raw2min(res)
m = res.mean(1)
s = res.std(1)
name = synthetic_function.get_meta_information()["name"]
final_result[name] = {"mean": m.tolist(), "std": s.tolist()}
Path("hyperopt.json").write_text(json.dumps(final_result))
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors