def next_suggestion(self, input_configuration, output_evaluation): """ According to the information you need, you can change return value :param input_configuration: n by p tensor :param output_evaluation: n by 1 tensor :return: """ inference = self.surrogate_inference( (self.original2cube(input_configuration), output_evaluation)) reference, ref_ind = torch.min(output_evaluation, 0) reference = reference.data.squeeze()[0] gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) inferences = deepcopy_inference(inference, gp_hyper_params) x0_cand = optimization_candidates(input_configuration, output_evaluation, -1, 1) x0, sample_info = optimization_init_points(x0_cand, reference=reference, inferences=inferences) next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest( x0=x0, reference=reference, inferences=inferences, acquisition_function=self.acquisition_function, bounds=sphere_bound(self.radius), pool=self.pool) return self.cube2original(next_x_point)
def BO(n_eval=200, path=None, func=None, ndim=None): assert (path is None) != (func is None) if path is not None: if not os.path.exists(path): path = os.path.join(EXPERIMENT_DIR, path) model_filename = os.path.join(path, 'model.pt') data_config_filename = os.path.join(path, 'data_config.pkl') model = torch.load(model_filename) data_config_file = open(data_config_filename, 'r') for key, value in pickle.load(data_config_file).iteritems(): exec(key + '=value') data_config_file.close() inference = ShadowInference((x_input, output), model) else: assert (func.dim == 0) != (ndim is None) if ndim is None: ndim = func.dim dir_list = [ elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm)) ] folder_name = func.__name__ + '_D' + str( ndim) + '_' + exp_str + '_' + datetime.now().strftime( '%Y%m%d-%H:%M:%S:%f') os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') search_sphere_radius = ndim**0.5 x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) output = Variable(torch.zeros(x_input.size(0), 1)) for i in range(x_input.size(0)): output[i] = func(x_input[i]) model = GPRegression(kernel=RadializationKernel( max_power=3, search_radius=search_sphere_radius)) time_list = [time.time()] * 2 elapse_list = [0, 0] pred_mean_list = [0, 0] pred_std_list = [0, 0] pred_var_list = [0, 0] pred_stdmax_list = [1, 1] pred_varmax_list = [1, 1] reference_list = [output.data.squeeze()[0]] * 2 refind_list = [1, 1] dist_to_ref_list = [0, 0] sample_info_list = [(10, 0, 10)] * 2 inference = ShadowInference((x_input, output), model) inference.init_parameters() inference.sampling(n_sample=1, n_burnin=99, n_thin=1) stored_variable_names = locals().keys() ignored_variable_names = [ 'n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 'next_ind', 'model_filename', 'data_config_filename', 'i', 'kernel_input_map', 'model', 'inference' ] stored_variable_names = set(stored_variable_names).difference( set(ignored_variable_names)) bnd = radial_bound(search_sphere_radius) for _ in range(3): print('Experiment based on data in ' + os.path.split(model_filename)[0]) for _ in range(n_eval): inference = ShadowInference((x_input, output), model) reference, ref_ind = torch.min(output, 0) reference = reference.data.squeeze()[0] gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) inferences = deepcopy_inference(inference, gp_hyper_params) x0_cand = optimization_candidates(x_input, output, -1, 1) x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest( inferences, x0=x0, bounds=bnd, reference=reference) time_list.append(time.time()) elapse_list.append(time_list[-1] - time_list[-2]) pred_mean_list.append(pred_mean.squeeze()[0]) pred_std_list.append(pred_std.squeeze()[0]) pred_var_list.append(pred_var.squeeze()[0]) pred_stdmax_list.append(pred_stdmax.squeeze()[0]) pred_varmax_list.append(pred_varmax.squeeze()[0]) reference_list.append(reference) refind_list.append(ref_ind.data.squeeze()[0] + 1) dist_to_ref_list.append( torch.sum((next_x_point - x_input[ref_ind].data)**2)**0.5) sample_info_list.append(sample_info) x_input = torch.cat([x_input, Variable(next_x_point)], 0) output = torch.cat([output, func(x_input[-1])]) min_ind = torch.min(output, 0)[1] min_loc = x_input[min_ind] min_val = output[min_ind] dist_to_suggest = torch.sum((x_input - x_input[-1]).data**2, 1)**0.5 dist_to_min = torch.sum((x_input - min_loc).data**2, 1)**0.5 out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) print('') for i in range(x_input.size(0)): time_str = time.strftime('%H:%M:%S', time.gmtime( time_list[i])) + '(' + time.strftime( '%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' data_str = ( '%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i]**2) **0.5, out_of_box[i], refind_list[i], reference_list[i], sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) min_str = ' <========= MIN' if i == min_ind.data.squeeze( )[0] else '' print(time_str + data_str + min_str) print(model.kernel.__class__.__name__) torch.save(model, model_filename) stored_variable = dict() for key in stored_variable_names: stored_variable[key] = locals()[key] f = open(data_config_filename, 'w') pickle.dump(stored_variable, f) f.close() for _ in range(3): print('Experiment based on data in ' + os.path.split(model_filename)[0]) return os.path.split(model_filename)[0]
def BO(geometry=None, n_eval=200, path=None, func=None, ndim=None, boundary=False, ard=False, origin=False, warping=False, parallel=False): assert (path is None) != (func is None) if path is None: assert (func.dim == 0) != (ndim is None) assert geometry is not None if ndim is None: ndim = func.dim exp_conf_str = geometry if geometry == 'sphere': assert not ard exp_conf_str += 'warping' if warping else '' radius_input_map = Kumaraswamy(ndim=1, max_input=ndim** 0.5) if warping else None model = GPRegression( kernel=RadializationKernel(max_power=3, search_radius=ndim**0.5, radius_input_map=radius_input_map)) inference_method = None if origin and boundary: inference_method = both_ShadowInference exp_conf_str += 'both' elif origin: inference_method = origin_ShadowInference exp_conf_str += 'origin' elif boundary: inference_method = satellite_ShadowInference exp_conf_str += 'boundary' else: inference_method = Inference exp_conf_str += 'none' bnd = sphere_bound(ndim**0.5) elif geometry == 'cube': assert not origin exp_conf_str += ('ard' if ard else '') + ('boundary' if boundary else '') model = GPRegression(kernel=Matern52(ndim=ndim, ard=ard)) inference_method = satellite_ShadowInference if boundary else Inference bnd = (-1, 1) if not os.path.isdir(EXPERIMENT_DIR): raise ValueError( 'In file : ' + os.path.realpath(__file__) + '\nEXPERIMENT_DIR variable is not properly assigned. Please check it.' ) dir_list = [ elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm)) ] folder_name = func.__name__ + '_D' + str( ndim) + '_' + exp_conf_str + '_' + datetime.now().strftime( '%Y%m%d-%H:%M:%S:%f') os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) logfile_dir = os.path.join(EXPERIMENT_DIR, folder_name, 'log') os.makedirs(logfile_dir) model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') x_input = Variable( torch.stack( [torch.zeros(ndim), torch.FloatTensor(ndim).uniform_(-1, 1)])) if func.func_name == 'stochastic_depth_resnet_cifar100': special_init_point = torch.FloatTensor([ -0.88672996375809265, -0.83845553984377363, -0.80082455589209434, -0.76868080609344613, -0.74002860499719103, -0.71384507914214379, -0.6895229479156415, -0.66666666534211871, -0.64500158781765049, -0.62432778870160499, -0.60449429448743319, -0.58538383736427368, -0.56690311453886821, -0.54897644926147593, -0.53154137077618735, -0.51454570980003023, -0.49794520561122835, -0.4817019618876005, -0.46578329447738975, -0.45016063464220946, -0.43480887900991927, -0.41970588594137237, -0.40483184457290511, -0.39016909932337462, -0.37570168000845294, -0.36141512736958714, -0.34729635533386094, -0.33333334161175654, -0.31951507564952675, -0.30583136944490208, -0.29227292909996905, -0.27883100126437665, -0.26549747264739709, -0.25226475894331168, -0.23912574658399377, -0.22607369983030123, -0.2131023835975443, -0.20020577167418563, -0.18737817967669568, -0.1746141913340078, -0.16190858934371632, -0.14925649319813961, -0.13665309066289877, -0.12409378040195429, -0.11157411163518405, -0.099089726169870107, -0.086636502479268351, -0.074210299199806373, -0.061807101474520065, -0.049422967019945307, -0.037054013082912562, -0.024696364163967699, -0.012346298973719083, 0 ]) x_input = torch.cat([x_input, Variable(special_init_point)]) n_init_eval = x_input.size(0) output = Variable(torch.zeros(n_init_eval, 1)) for i in range(n_init_eval): output[i] = func(x_input[i]) time_list = [time.time()] * n_init_eval elapse_list = [0] * n_init_eval pred_mean_list = [0] * n_init_eval pred_std_list = [0] * n_init_eval pred_var_list = [0] * n_init_eval pred_stdmax_list = [1] * n_init_eval pred_varmax_list = [1] * n_init_eval reference_list = [output.data.squeeze()[0]] * n_init_eval refind_list = [1] * n_init_eval dist_to_ref_list = [0] * n_init_eval sample_info_list = [(10, 0, 10)] * n_init_eval inference = inference_method((x_input, output), model) inference.init_parameters() inference.sampling(n_sample=1, n_burnin=99, n_thin=1) else: if not os.path.exists(path): path = os.path.join(EXPERIMENT_DIR, path) logfile_dir = os.path.join(path, 'log') model_filename = os.path.join(path, 'model.pt') data_config_filename = os.path.join(path, 'data_config.pkl') model = torch.load(model_filename) data_config_file = open(data_config_filename, 'r') for key, value in pickle.load(data_config_file).iteritems(): if key != 'logfile_dir': exec(key + '=value') data_config_file.close() ignored_variable_names = [ 'n_eval', 'path', 'i', 'key', 'value', 'logfile_dir', 'n_init_eval', 'data_config_file', 'dir_list', 'folder_name', 'model_filename', 'data_config_filename', 'kernel', 'model', 'inference', 'parallel', 'pool' ] stored_variable_names = set(locals().keys()).difference( set(ignored_variable_names)) if path is None: torch.save(model, model_filename) stored_variable = dict() for key in stored_variable_names: stored_variable[key] = locals()[key] f = open(data_config_filename, 'w') pickle.dump(stored_variable, f) f.close() print('Experiment based on data in %s' % os.path.split(model_filename)[0]) # multiprocessing conflicts with pytorch linear algebra operation pool = multiprocessing.Pool(N_INIT) if parallel else None for _ in range(n_eval): start_time = time.time() logfile = open( os.path.join(logfile_dir, str(x_input.size(0) + 1).zfill(4) + '.out'), 'w') inference = inference_method((x_input, output), model) reference, ref_ind = torch.min(output, 0) reference = reference.data.squeeze()[0] gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) inferences = deepcopy_inference(inference, gp_hyper_params) x0_cand = optimization_candidates(x_input, output, -1, 1) x0, sample_info = optimization_init_points(x0_cand, reference=reference, inferences=inferences) next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest( x0=x0, reference=reference, inferences=inferences, bounds=bnd, pool=pool) time_list.append(time.time()) elapse_list.append(time_list[-1] - time_list[-2]) pred_mean_list.append(pred_mean.squeeze()[0]) pred_std_list.append(pred_std.squeeze()[0]) pred_var_list.append(pred_var.squeeze()[0]) pred_stdmax_list.append(pred_stdmax.squeeze()[0]) pred_varmax_list.append(pred_varmax.squeeze()[0]) reference_list.append(reference) refind_list.append(ref_ind.data.squeeze()[0] + 1) dist_to_ref_list.append( torch.sum((next_x_point - x_input[ref_ind]).data**2)**0.5) sample_info_list.append(sample_info) x_input = torch.cat([x_input, next_x_point], 0) output = torch.cat([output, func(x_input[-1]).resize(1, 1)]) min_ind = torch.min(output, 0)[1] min_loc = x_input[min_ind] min_val = output[min_ind] dist_to_suggest = torch.sum((x_input - x_input[-1]).data**2, 1)**0.5 dist_to_min = torch.sum((x_input - min_loc).data**2, 1)**0.5 out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) print('') for i in range(x_input.size(0)): time_str = time.strftime('%H:%M:%S', time.gmtime( time_list[i])) + '(' + time.strftime( '%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' data_str = ( '%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i]**2) **0.5, out_of_box[i], refind_list[i], reference_list[i], sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) min_str = ' <========= MIN' if i == min_ind.data.squeeze( )[0] else '' print(time_str + data_str + min_str) logfile.writelines(time_str + data_str + min_str + '\n') logfile.close() torch.save(model, model_filename) stored_variable = dict() for key in stored_variable_names: stored_variable[key] = locals()[key] f = open(data_config_filename, 'w') pickle.dump(stored_variable, f) f.close() if parallel: pool.close() print('Experiment based on data in %s' % os.path.split(model_filename)[0]) return os.path.split(model_filename)[0]