def test_serial_warm_start_multi_fidelity(self):
optimizer = "ETPE"
n_iterations = 15
n_jobs = 3
parallel_strategy = "AsyncComm"
multi_fidelity_iter_generator = HyperBandIterGenerator(25, 100, 2)
for warm_start_strategy in valid_warm_start_strategies:
p_res = fmin(
evaluation,
config_space,
optimizer=optimizer,
n_jobs=n_jobs,
n_iterations=n_iterations,
parallel_strategy=parallel_strategy,
multi_fidelity_iter_generator=multi_fidelity_iter_generator)
print(p_res)
for i in range(3):
res = fmin(
evaluation,
config_space,
warm_start_strategy=warm_start_strategy,
n_jobs=n_jobs,
n_iterations=n_iterations,
previous_result=p_res,
parallel_strategy=parallel_strategy,
multi_fidelity_iter_generator=multi_fidelity_iter_generator
)
p_res = res
print(p_res)
assert len(res["budget2obvs"][25]["losses"]) == 20 * (i + 2)
def test_hyperband_iter_generator(self):
hp_iter_gen = HyperBandIterGenerator(1 / 16, 1, 4, SH_only=True)
res = hp_iter_gen.num_all_configs(3)
assert res == 21 * 3
res = hp_iter_gen.num_all_configs(5)
assert res == 21 * 5
hp_iter_gen = HyperBandIterGenerator(1 / 16, 1, 4)
res = hp_iter_gen.num_all_configs(3)
assert res == 29
res = hp_iter_gen.num_all_configs(5)
assert res == 55
def test_all_iter_generator(self):
iter_gens = [
HyperBandIterGenerator(1 / 16, 1, 4),
SuccessiveHalvingIterGenerator(1 / 16, 1, 4),
CustomIterGenerator([4, 2, 1], [1 / 4, 1 / 2, 1])
]
for iter_gen in iter_gens:
print(iter_gen)
print()
print("get_budgets", iter_gen.get_budgets())
print("num_all_configs", iter_gen.num_all_configs(3))
print("get_next_iteration", iter_gen.get_next_iteration(4))
def test_all_methods(self):
for optimizer in valid_optimizers:
for parallel_strategie in valid_parallel_strategies:
if parallel_strategie == "AsyncComm":
multi_fidelity_iter_generators = [
HyperBandIterGenerator(50, 100, 2),
SuccessiveHalvingIterGenerator(50, 100, 2)
]
else:
multi_fidelity_iter_generators = [None]
for multi_fidelity_iter_generator in multi_fidelity_iter_generators:
print(optimizer, parallel_strategie,
multi_fidelity_iter_generator)
ret = fmin(evaluate,
config_space,
optimizer=optimizer,
n_iterations=2,
n_jobs=2,
parallel_strategy=parallel_strategie,
multi_fidelity_iter_generator=
multi_fidelity_iter_generator)
print(ret)
# -------------------------------------------------------------
# 采用在对【 5折交叉验证中的训练集 】进行采样的方法，采样率为 budget
sample_ratio = budget
scores = []
for i, (train_ix, valid_ix) in enumerate(self.cv.split(X, y)):
rng = np.random.RandomState(i)
size = int(train_ix.size * sample_ratio)
train_ix = rng.choice(train_ix, size, replace=False)
X_train = X[train_ix, :]
y_train = y[train_ix]
X_valid = X[valid_ix, :]
y_valid = y[valid_ix]
ML_model.fit(X_train, y_train)
y_pred = ML_model.predict(X_valid)
score = eval(f"sklearn.metrics.{self.metric}_score")(y_valid,
y_pred)
scores.append(score)
# -------------------------------------------------------------
score = np.mean(scores)
return 1 - score
evaluator = Evaluator(X, y)
iter_generator = HyperBandIterGenerator(min_budget=1 / 4, max_budget=1, eta=2)
result = fmin(evaluator,
HDL,
optimizer="ETPE",
n_iterations=30,
multi_fidelity_iter_generator=iter_generator,
n_jobs=3)
print(result)
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Author : qichun tang # @Date : 2020-12-27 # @Contact : [email protected] from ultraopt import fmin from ultraopt.multi_fidelity import HyperBandIterGenerator from ultraopt.tests.mock import evaluate, config_space import pylab as plt res = fmin(evaluate, config_space, n_jobs=3, multi_fidelity_iter_generator=HyperBandIterGenerator(25, 100, 2)) res.plot_convergence_over_time(yscale="log") plt.show() res.plot_concurrent_over_time(num_points=100) plt.show() res.plot_finished_over_time() plt.show() res.plot_correlation_across_budgets() plt.show() print()
raise NotImplementedError
output_path = os.path.join(args.output_path, f"{args.benchmark}-ultraopt_{args.optimizer}")
os.makedirs(os.path.join(output_path), exist_ok=True)
def objective_function(config: dict, budget: int = 100):
loss, cost = b.objective_function(config, int(budget))
return float(loss)
cs = b.get_configuration_space()
HB = False
if args.optimizer == "BOHB":
optimizer = "ETPE"
iter_generator = HyperBandIterGenerator(min_budget=3, max_budget=100, eta=3)
HB = True
elif args.optimizer == "HyperBand":
optimizer = "Random"
iter_generator = HyperBandIterGenerator(min_budget=3, max_budget=100, eta=3)
HB = True
else:
optimizer = args.optimizer
iter_generator = None
fmin_result = fmin(objective_function, cs, optimizer, n_iterations=args.n_iters, random_state=args.run_id,
multi_fidelity_iter_generator=iter_generator)
print(fmin_result)
# dump(fmin_result, os.path.join(output_path, 'run_%d.pkl' % args.run_id))
res = b.get_results()
fh = open(os.path.join(output_path, 'run_%d.json' % args.run_id), 'w')