def load_model_data(path, exp_dir=experiment_directory()):
if not os.path.exists(path):
path = os.path.join(exp_dir, path)
logfile_dir = os.path.join(path, 'log')
model_filename = os.path.join(path, 'model.pt')
cfg_data_filename = os.path.join(path, 'data_config.pkl')
model = torch.load(model_filename)
cfg_data_file = open(cfg_data_filename, 'r')
cfg_data = pickle.load(cfg_data_file)
for key, value in pickle.load(cfg_data_file).iteritems():
if key != 'logfile_dir':
exec(key + '=value')
cfg_data_file.close()
return model, cfg_data, logfile_dir
def run_bo(exp_dirname, task, store_data, parallel):
bo_data_filename = os.path.join(experiment_directory(), exp_dirname,
'bo_data.pt')
bo_data = torch.load(bo_data_filename)
surrogate_model = bo_data['surrogate_model']
eval_inputs = bo_data['eval_inputs']
eval_outputs = bo_data['eval_outputs']
n_vertices = bo_data['n_vertices']
adj_mat_list = bo_data['adj_mat_list']
log_beta = bo_data['log_beta']
sorted_partition = bo_data['sorted_partition']
time_list = bo_data['time_list']
elapse_list = bo_data['elapse_list']
pred_mean_list = bo_data['pred_mean_list']
pred_std_list = bo_data['pred_std_list']
pred_var_list = bo_data['pred_var_list']
acquisition_func = bo_data['acquisition_func']
objective = bo_data['objective']
updated = False
if eval_inputs.size(0) == eval_outputs.size(0) and task in [
'suggest', 'both'
]:
suggestion, log_beta, sorted_partition, processing_time = run_suggest(
surrogate_model=surrogate_model,
eval_inputs=eval_inputs,
eval_outputs=eval_outputs,
n_vertices=n_vertices,
adj_mat_list=adj_mat_list,
log_beta=log_beta,
sorted_partition=sorted_partition,
acquisition_func=acquisition_func,
parallel=parallel)
next_input, pred_mean, pred_std, pred_var = suggestion
eval_inputs = torch.cat([eval_inputs, next_input.view(1, -1)], 0)
elapse_list.append(processing_time)
pred_mean_list.append(pred_mean.item())
pred_std_list.append(pred_std.item())
pred_var_list.append(pred_var.item())
updated = True
if eval_inputs.size(0) - 1 == eval_outputs.size(0) and task in [
'evaluate', 'both'
]:
next_output = objective.evaluate(eval_inputs[-1]).view(1, 1)
eval_outputs = torch.cat([eval_outputs, next_output])
assert not torch.isnan(eval_outputs).any()
time_list.append(time.time())
updated = True
if updated:
bo_data = {
'surrogate_model': surrogate_model,
'eval_inputs': eval_inputs,
'eval_outputs': eval_outputs,
'n_vertices': n_vertices,
'adj_mat_list': adj_mat_list,
'log_beta': log_beta,
'sorted_partition': sorted_partition,
'objective': objective,
'acquisition_func': acquisition_func,
'time_list': time_list,
'elapse_list': elapse_list,
'pred_mean_list': pred_mean_list,
'pred_std_list': pred_std_list,
'pred_var_list': pred_var_list
}
torch.save(bo_data, bo_data_filename)
displaying_and_logging(
os.path.join(experiment_directory(), exp_dirname,
'log'), eval_inputs, eval_outputs, pred_mean_list,
pred_std_list, pred_var_list, time_list, elapse_list, store_data)
print('Optimizing %s with regularization %.2E, random seed : %s' %
(objective.__class__.__name__, objective.lamda if hasattr(
objective, 'lamda') else 0, objective.random_seed_info
if hasattr(objective, 'random_seed_info') else 'none'))
return eval_outputs.size(0)
def COMBO(objective=None,
n_eval=200,
dir_name=None,
parallel=False,
store_data=False,
task='both',
**kwargs):
"""
:param objective:
:param n_eval:
:param dir_name:
:param parallel:
:param store_data:
:param task:
:param kwargs:
:return:
"""
assert task in ['suggest', 'evaluate', 'both']
# GOLD continues from info given in 'path' or starts minimization of 'objective'
assert (dir_name is None) != (objective is None)
acquisition_func = expected_improvement
if objective is not None:
exp_dir = experiment_directory()
objective_id_list = [objective.__class__.__name__]
if hasattr(objective, 'random_seed_info'):
objective_id_list.append(objective.random_seed_info)
if hasattr(objective, 'lamda'):
objective_id_list.append('%.1E' % objective.lamda)
if hasattr(objective, 'data_type'):
objective_id_list.append(objective.data_type)
objective_id_list.append('COMBO')
objective_name = '_'.join(objective_id_list)
exp_dirname = bo_exp_dirname(exp_dir=exp_dir,
objective_name=objective_name)
n_vertices = objective.n_vertices
adj_mat_list = objective.adjacency_mat
grouped_log_beta = torch.ones(len(objective.fourier_freq))
fourier_freq_list = objective.fourier_freq
fourier_basis_list = objective.fourier_basis
suggested_init = objective.suggested_init # suggested_init should be 2d tensor
n_init = suggested_init.size(0)
kernel = DiffusionKernel(grouped_log_beta=grouped_log_beta,
fourier_freq_list=fourier_freq_list,
fourier_basis_list=fourier_basis_list)
surrogate_model = GPRegression(kernel=kernel)
eval_inputs = suggested_init
eval_outputs = torch.zeros(eval_inputs.size(0),
1,
device=eval_inputs.device)
for i in range(eval_inputs.size(0)):
eval_outputs[i] = objective.evaluate(eval_inputs[i])
assert not torch.isnan(eval_outputs).any()
log_beta = eval_outputs.new_zeros(eval_inputs.size(1))
sorted_partition = [[m] for m in range(eval_inputs.size(1))]
time_list = [time.time()] * n_init
elapse_list = [0] * n_init
pred_mean_list = [0] * n_init
pred_std_list = [0] * n_init
pred_var_list = [0] * n_init
surrogate_model.init_param(eval_outputs)
print('(%s) Burn-in' % time.strftime('%H:%M:%S', time.gmtime()))
sample_posterior = posterior_sampling(surrogate_model,
eval_inputs,
eval_outputs,
n_vertices,
adj_mat_list,
log_beta,
sorted_partition,
n_sample=1,
n_burn=99,
n_thin=1)
log_beta = sample_posterior[1][0]
sorted_partition = sample_posterior[2][0]
print('')
bo_data = {
'surrogate_model': surrogate_model,
'eval_inputs': eval_inputs,
'eval_outputs': eval_outputs,
'n_vertices': n_vertices,
'adj_mat_list': adj_mat_list,
'log_beta': log_beta,
'sorted_partition': sorted_partition,
'time_list': time_list,
'elapse_list': elapse_list,
'pred_mean_list': pred_mean_list,
'pred_std_list': pred_std_list,
'pred_var_list': pred_var_list,
'acquisition_func': acquisition_func,
'objective': objective
}
torch.save(bo_data, os.path.join(exp_dirname, 'bo_data.pt'))
eval_cnt = 0
while eval_cnt < n_eval:
eval_cnt = run_bo(
exp_dirname=dir_name if objective is None else exp_dirname,
store_data=store_data,
task=task,
parallel=parallel)
from COMBO.experiments.random_seed_config import generate_random_seed_pestcontrol, \
generate_random_seed_pair_centroid, generate_random_seed_maxsat, generate_random_seed_pair_ising, \
generate_random_seed_pair_contamination
from COMBO.experiments.test_functions.discretized_continuous import Branin
from COMBO.experiments.test_functions.binary_categorical import ISING_N_EDGES, CONTAMINATION_N_STAGES
from COMBO.experiments.test_functions.binary_categorical import Ising, Contamination
from COMBO.experiments.test_functions.multiple_categorical import PESTCONTROL_N_STAGES, CENTROID_GRID, \
CENTROID_N_EDGES, CENTROID_N_CHOICE, PESTCONTROL_N_CHOICE
from COMBO.experiments.test_functions.multiple_categorical import PestControl, Centroid, \
edge_choice, partition, ising_dense
from COMBO.experiments.MaxSAT.maximum_satisfiability import MaxSAT28, MaxSAT43, MaxSAT60
from COMBO.experiments.exp_utils import sample_init_points
from COMBO.config import experiment_directory, SMAC_exp_dir
EXP_DIR = experiment_directory()
RESULT_DIR = SMAC_exp_dir()
def ising(n_eval, lamda, random_seed_pair):
evaluator = Ising(lamda, random_seed_pair)
name_tag = '_'.join([
'ising', ('%.2E' % lamda),
datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")
])
cs = ConfigurationSpace()
for i in range(ISING_N_EDGES):
car_var = CategoricalHyperparameter('x' + str(i + 1).zfill(2),
[str(elm) for elm in range(2)],
default_value='0')
cs.add_hyperparameter(car_var)
def COMBO(objective=None,
n_eval=200,
path=None,
parallel=False,
store_data=False,
**kwargs):
"""
:param objective:
:param n_eval:
:param path:
:param parallel:
:param kwargs:
:return:
"""
# GOLD continues from info given in 'path' or starts minimization of 'objective'
assert (path is None) != (objective is None)
acquisition_func = expected_improvement
n_vertices = adj_mat_list = None
eval_inputs = eval_outputs = log_beta = sorted_partition = None
time_list = elapse_list = pred_mean_list = pred_std_list = pred_var_list = None
if objective is not None:
exp_dir = experiment_directory()
objective_id_list = [objective.__class__.__name__]
if hasattr(objective, 'random_seed_info'):
objective_id_list.append(objective.random_seed_info)
if hasattr(objective, 'lamda'):
objective_id_list.append('%.1E' % objective.lamda)
if hasattr(objective, 'data_type'):
objective_id_list.append(objective.data_type)
objective_id_list.append('COMBO')
objective_name = '_'.join(objective_id_list)
model_filename, data_cfg_filaname, logfile_dir = model_data_filenames(
exp_dir=exp_dir, objective_name=objective_name)
n_vertices = objective.n_vertices
adj_mat_list = objective.adjacency_mat
grouped_log_beta = torch.ones(len(objective.fourier_freq))
fourier_freq_list = objective.fourier_freq
fourier_basis_list = objective.fourier_basis
suggested_init = objective.suggested_init # suggested_init should be 2d tensor
n_init = suggested_init.size(0)
kernel = DiffusionKernel(grouped_log_beta=grouped_log_beta,
fourier_freq_list=fourier_freq_list,
fourier_basis_list=fourier_basis_list)
surrogate_model = GPRegression(kernel=kernel)
eval_inputs = suggested_init
eval_outputs = torch.zeros(eval_inputs.size(0),
1,
device=eval_inputs.device)
for i in range(eval_inputs.size(0)):
eval_outputs[i] = objective.evaluate(eval_inputs[i])
assert not torch.isnan(eval_outputs).any()
log_beta = eval_outputs.new_zeros(eval_inputs.size(1))
sorted_partition = [[m] for m in range(eval_inputs.size(1))]
time_list = [time.time()] * n_init
elapse_list = [0] * n_init
pred_mean_list = [0] * n_init
pred_std_list = [0] * n_init
pred_var_list = [0] * n_init
surrogate_model.init_param(eval_outputs)
print('(%s) Burn-in' % time.strftime('%H:%M:%S', time.gmtime()))
sample_posterior = posterior_sampling(surrogate_model,
eval_inputs,
eval_outputs,
n_vertices,
adj_mat_list,
log_beta,
sorted_partition,
n_sample=1,
n_burn=99,
n_thin=1)
log_beta = sample_posterior[1][0]
sorted_partition = sample_posterior[2][0]
print('')
else:
surrogate_model, cfg_data, logfile_dir = load_model_data(
path, exp_dir=experiment_directory())
for _ in range(n_eval):
start_time = time.time()
reference = torch.min(eval_outputs, dim=0)[0].item()
print('(%s) Sampling' % time.strftime('%H:%M:%S', time.gmtime()))
sample_posterior = posterior_sampling(surrogate_model,
eval_inputs,
eval_outputs,
n_vertices,
adj_mat_list,
log_beta,
sorted_partition,
n_sample=10,
n_burn=0,
n_thin=1)
hyper_samples, log_beta_samples, partition_samples, freq_samples, basis_samples, edge_mat_samples = sample_posterior
log_beta = log_beta_samples[-1]
sorted_partition = partition_samples[-1]
print('')
x_opt = eval_inputs[torch.argmin(eval_outputs)]
inference_samples = inference_sampling(eval_inputs, eval_outputs,
n_vertices, hyper_samples,
log_beta_samples,
partition_samples, freq_samples,
basis_samples)
suggestion = next_evaluation(x_opt, eval_inputs, inference_samples,
partition_samples, edge_mat_samples,
n_vertices, acquisition_func, reference,
parallel)
next_eval, pred_mean, pred_std, pred_var = suggestion
processing_time = time.time() - start_time
eval_inputs = torch.cat([eval_inputs, next_eval.view(1, -1)], 0)
eval_outputs = torch.cat(
[eval_outputs,
objective.evaluate(eval_inputs[-1]).view(1, 1)])
assert not torch.isnan(eval_outputs).any()
time_list.append(time.time())
elapse_list.append(processing_time)
pred_mean_list.append(pred_mean.item())
pred_std_list.append(pred_std.item())
pred_var_list.append(pred_var.item())
displaying_and_logging(logfile_dir, eval_inputs, eval_outputs,
pred_mean_list, pred_std_list, pred_var_list,
time_list, elapse_list, store_data)
print(
'Optimizing %s with regularization %.2E up to %4d visualization random seed : %s'
% (objective.__class__.__name__, objective.lamda if hasattr(
objective, 'lamda') else 0, n_eval, objective.random_seed_info
if hasattr(objective, 'random_seed_info') else 'none'))