"deterministic": "true", }) runhistory = RunHistory_modified(None) # aggregate_func handled by smac_facade.SMAC smac = SMAC(scenario=scenario, runhistory=runhistory, tae_runner=objective_function, run_id=seed, # set run_id for smac output_dir rng=np.random.RandomState(seed)) smac.optimize() # keys = [k.config_id for k in smac.runhistory.data.keys()] # perfs = [v.cost for v in smac.runhistory.data.values()] config_list = smac.runhistory.config_list perf_list = smac.runhistory.perf_list time_list = smac.runhistory.time_list return config_list, perf_list, time_list with timeit('%s all' % (mth,)): for run_i in range(start_id, start_id + rep): seed = seeds[run_i] with timeit('%s %d %d' % (mth, run_i, seed)): # Evaluate config_list, perf_list, time_list = evaluate(mth, run_i, seed) # Save result print('=' * 20) print(seed, mth, config_list, perf_list, time_list) print(seed, mth, 'best perf', np.min(perf_list)) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time.time())) dir_path = 'logs/so_benchmark_%s_%d/%s/' % (problem_str, max_runs, mth) file = 'benchmark_%s_%04d_%s.pkl' % (mth, seed, timestamp) if not os.path.exists(dir_path):
global_start_time = time.time() for i in range(max_runs): config, trial_state, objs, trial_info = bo.iterate() global_time = time.time() - global_start_time print(seed, i, objs, config, trial_state, trial_info, 'time=', global_time) config_list.append(config) perf_list.append(objs[0]) time_list.append(global_time) if global_time >= runtime_limit: break return config_list, perf_list, time_list with timeit('%s all' % (mth, )): for dataset in test_datasets: problem_str = 'lgb_%s' % (dataset, ) problem = get_problem(problem_str, n_jobs=n_jobs, data_dir=data_dir) cs = problem.get_configspace(optimizer='smac') task_id = '%s-%d_%s' % (mth, batch_size, problem_str) for run_i in range(start_id, start_id + rep): seed = seeds[run_i] with timeit('%s-%d-%s-%d-%d' % (mth, batch_size, problem_str, run_i, seed)): # Evaluate if batch_size > 1: config_list, perf_list, time_list = evaluate_parallel( problem, mth, batch_size, seed, ip, port) else: