"deterministic": "true",
})
runhistory = RunHistory_modified(None) # aggregate_func handled by smac_facade.SMAC
smac = SMAC(scenario=scenario, runhistory=runhistory,
tae_runner=objective_function, run_id=seed, # set run_id for smac output_dir
rng=np.random.RandomState(seed))
smac.optimize()
# keys = [k.config_id for k in smac.runhistory.data.keys()]
# perfs = [v.cost for v in smac.runhistory.data.values()]
config_list = smac.runhistory.config_list
perf_list = smac.runhistory.perf_list
time_list = smac.runhistory.time_list
return config_list, perf_list, time_list
with timeit('%s all' % (mth,)):
for run_i in range(start_id, start_id + rep):
seed = seeds[run_i]
with timeit('%s %d %d' % (mth, run_i, seed)):
# Evaluate
config_list, perf_list, time_list = evaluate(mth, run_i, seed)
# Save result
print('=' * 20)
print(seed, mth, config_list, perf_list, time_list)
print(seed, mth, 'best perf', np.min(perf_list))
timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time()))
dir_path = 'logs/so_benchmark_%s_%d/%s/' % (problem_str, max_runs, mth)
file = 'benchmark_%s_%04d_%s.pkl' % (mth, seed, timestamp)
if not os.path.exists(dir_path):
global_start_time = time.time()
for i in range(max_runs):
config, trial_state, objs, trial_info = bo.iterate()
global_time = time.time() - global_start_time
print(seed, i, objs, config, trial_state, trial_info, 'time=',
global_time)
config_list.append(config)
perf_list.append(objs[0])
time_list.append(global_time)
if global_time >= runtime_limit:
break
return config_list, perf_list, time_list
with timeit('%s all' % (mth, )):
for dataset in test_datasets:
problem_str = 'lgb_%s' % (dataset, )
problem = get_problem(problem_str, n_jobs=n_jobs, data_dir=data_dir)
cs = problem.get_configspace(optimizer='smac')
task_id = '%s-%d_%s' % (mth, batch_size, problem_str)
for run_i in range(start_id, start_id + rep):
seed = seeds[run_i]
with timeit('%s-%d-%s-%d-%d' %
(mth, batch_size, problem_str, run_i, seed)):
# Evaluate
if batch_size > 1:
config_list, perf_list, time_list = evaluate_parallel(
problem, mth, batch_size, seed, ip, port)
else: