def test_bo(cs, id):
bo = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=n_work)
bo.runtime_limit = runtime_limit
method_name = "BO-fcnet-%d" % id
bo.set_method_name(method_name)
bo.run()
bo.plot_statistics(method="BO-fcnet-%d" % id)
result = bo.get_incumbent(5)
print(result)
return result
def test_bo(cs, id):
method_id = "BO-resnet-%d" % id
bo = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=n_work)
bo.restart_needed = True
bo.runtime_limit = args.b
bo.run()
bo.plot_statistics(method=method_id)
result = bo.get_incumbent(5)
save_result(method_id, result)
print(result)
return result
def test_vanilla_bo(cs, n_id):
bo = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=1)
method_name = "Vanilla-BO-fcnet-%d" % n_id
bo.set_method_name(method_name)
bo.runtime_limit = runtime_limit
bo.run()
result = bo.get_incumbent(5)
print(result)
return result
def test_vanilla_bo(cs, id):
model = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=1)
model.method_name = "Vanilla-BO-xgb-%d" % id
model.runtime_limit = runtime_limit
model.restart_needed = True
model.run()
result = model.get_incumbent(5)
print(result)
return result
def test_bo(cs, id):
bo = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=n_work)
bo.run()
bo.plot_statistics(method="BO-rnn-%d" % id)
result = bo.get_incumbent(5)
print(result)
return result
def evaluate_baseline(baseline_id, cs, id):
_seed = seeds[id]
method_name = "%s-%s-%d-%d-%d" % (baseline_id, benchmark_id, id, runtime_limit, n_worker)
if baseline_id == 'hb':
optimizer = Hyperband(cs, train, maximal_iter, num_iter=iter_num,
n_workers=n_worker, random_state=_seed, method_id=method_name)
elif baseline_id == 'bohb':
optimizer = BOHB(cs, train, maximal_iter, num_iter=iter_num,
p=0.3, n_workers=n_worker, random_state=_seed, method_id=method_name)
elif baseline_id == 'mbhb':
optimizer = MBHB(cs, train, maximal_iter, num_iter=iter_num,
n_workers=n_worker, random_state=_seed, method_id=method_name)
elif baseline_id == 'mfse':
optimizer = MFSE(cs, train, maximal_iter, num_iter=iter_num, weight_method='rank_loss_p_norm',
n_workers=n_worker, random_state=_seed, method_id=method_name, power_num=3)
elif baseline_id == 'smac':
optimizer = SMAC(cs, train, maximal_iter, num_iter=iter_num,
n_workers=1, random_state=_seed, method_id=method_name)
elif baseline_id == 'batch_bo':
optimizer = SMAC(cs, train, maximal_iter, num_iter=iter_num,
n_workers=n_worker, random_state=_seed, method_id=method_name)
elif baseline_id == 'boes':
optimizer = SMAC_ES(cs, train, maximal_iter, num_iter=iter_num,
n_workers=n_worker, random_state=_seed, method_id=method_name)
elif baseline_id == 'random_search':
optimizer = RandomSearch(cs, train, maximal_iter, num_iter=iter_num, n_workers=n_worker,
random_state=_seed, method_id=method_name)
elif baseline_id == 'fabolas': # only for xgb R=27
optimizer = FABOLAS(method_id=method_name)
elif baseline_id == 'tse':
optimizer = TSE(n_workers=n_worker, method_id=method_name)
else:
raise ValueError('Invalid baseline name: %s' % baseline_id)
if 'covtype' in benchmark_id or 'mnist' in benchmark_id or 'sys' in benchmark_id:
optimizer.restart_needed = True
optimizer.runtime_limit = runtime_limit
optimizer.run()
print(optimizer.get_incumbent(5))
return optimizer.get_incumbent(5)
def test_methods_mean_variance(cs):
iterations = args.iter_c
for i in range(1, 1 + iterations):
weight = [1.0, 0.5, 0.25, 0.125, 0.125]
bo = SMAC(cs, train, maximal_iter, num_iter=iter_num, n_workers=n_work)
bo.run()
bo.plot_statistics(method="BO-lenet-%d" % i)
hyperband = Hyperband(cs,
train,
maximal_iter,
num_iter=iter_num,
n_workers=n_work)
hyperband.set_restart()
hyperband.run()
hyperband.plot_statistics(method="HB-lenet-%d" % i)
bohb = BOHB(cs,
train,
maximal_iter,
num_iter=iter_num,
p=0.2,
n_workers=n_work)
bohb.set_restart()
bohb.run()
bohb.plot_statistics(method="BOHB-lenet-%d" % i)
xfhb = XFHB(cs,
train,
maximal_iter,
num_iter=iter_num,
n_workers=n_work,
info_type='Weighted',
init_weight=weight,
random_mode=False)
xfhb.set_restart()
xfhb.run()
xfhb.plot_statistics(method="XFHB-lenet-disable_w-%d" % i)
xfhb = XFHB(cs,
train,
maximal_iter,
num_iter=iter_num,
n_workers=n_work,
info_type='Weighted',
update_enable=True,
update_delta=1,
init_weight=weight,
random_mode=False)
xfhb.set_restart()
xfhb.run()
xfhb.plot_statistics(method="XFHB-lenet-update_w-%d" % i)