def run_nn_experiment(xhmc_delta_list,input_data,v_fun,test_set,type_problem): out_list = [None]*(len(xhmc_delta_list)+1) for i in range(len(out_list)): model_dict = {"num_units": 50} v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=input_data, model_dict=model_dict) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=False) if i<len(out_list)-1: input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"], "max_tree_depth": [8],"xhmc_delta":[xhmc_delta_list[i]], "metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"]} else: input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"], "max_tree_depth": [8], "metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]} ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10, "kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"} # adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator( precision_type="torch.DoubleTensor").get_model_dim())] dual_args_list = [ep_dual_metadata_argument] other_arguments = other_default_arguments() # tune_settings_dict = tuning_settings([],[],[],[]) tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict) diagnostics_np = sampler1.np_diagnostics() samples_mixed = sampler1.get_samples(permuted=True) te = test_error(target_dataset=test_set,v_obj=v_generator("torch.DoubleTensor"),mcmc_samples=samples_mixed,type=type_problem) out = {"test_error":te,"diagnostics":diagnostics_np} out_list.append(out) te_store = numpy.zeros(len(out_list)) diagnostics_store = numpy.zeros(shape=[len(out_list)]+list(diagnostics_np.shape)) for i in range(len(out_list)): te_store[i] = out_list[i]["test_error"] diagnostics_store[i,...] = out_list[i]["diagnostics"] output_store = {"test_error":te_store,"diagnostics":diagnostics_store} save_name = "xhmc_v_gnuts_8x8mnist.npz" numpy.savez(save_name,**output_store) return(save_name)
from experiments.correctdist_experiments.prototype import check_mean_var_stan from abstract.util import wrap_V_class_with_input_data mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=500, warmup_per_chain=600, is_float=False, isstore_to_disk=False, allow_restart=False, seed=25) input_data = get_data_dict("pima_indian") V_pima_indian_logit = wrap_V_class_with_input_data( class_constructor=V_logistic_regression, input_data=input_data) address = os.environ[ "PYTHONPATH"] + "/experiments/correctdist_experiments/result_from_long_chain.pkl" correct = pickle.load(open(address, 'rb')) correct_mean = correct["correct_mean"] correct_cov = correct["correct_cov"] correct_diag_cov = correct_cov.diagonal() input_dict = { "v_fun": [V_pima_indian_logit], "epsilon": ["dual"], "second_order": [False], "evolve_t": [1.], "metric_name": ["unit_e"], "dynamic": [False],
from input_data.convert_data_to_dict import get_data_dict from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=0, warmup_per_chain=100, is_float=False, isstore_to_disk=False, allow_restart=False) pima_indian_data = get_data_dict("pima_indian") V_generator = wrap_V_class_with_input_data( class_constructor=V_logistic_regression, input_data=pima_indian_data) input_dict = { "v_fun": [V_generator], "epsilon": [0.01], "second_order": [False], "metric_name": ["unit_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"] } other_arguments = other_default_arguments() tune_settings_dict = tuning_settings([], [], [], other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict,
from input_data.convert_data_to_dict import get_data_dict from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=0, warmup_per_chain=100, is_float=False, isstore_to_disk=False, allow_restart=False) input_data = get_data_dict("1-PL", standardize_predictor=False) V_generator = wrap_V_class_with_input_data(class_constructor=V_response_model, input_data=input_data) input_dict = { "v_fun": [V_generator], "epsilon": [0.1], "second_order": [False], "metric_name": ["unit_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"] } other_arguments = other_default_arguments() tune_settings_dict = tuning_settings([], [], [], other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict,
from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics num_p = 100 non_zero_p = 20 seedid = 33034 numpy.random.seed(seedid) torch.manual_seed(seedid) true_p = numpy.zeros(num_p) true_p[:non_zero_p] = numpy.random.randn(non_zero_p)*5 y = true_p + numpy.random.randn(num_p) input_data = {"target":y} v_generator =wrap_V_class_with_input_data(class_constructor=V_student_toy,input_data=input_data) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=4,thin=1,tune_l_per_chain=1000, warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8], "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]} # input_dict = {"v_fun":[v_generator],"epsilon":[0.1],"second_order":[False],"evolve_L":[10], # "metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} ep_dual_metadata_argument = {"name":"epsilon","target":0.9,"gamma":0.05,"t_0":10, "kappa":0.75,"obj_fun":"accept_rate","par_type":"fast"} # adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow",dim=v_generator(precision_type="torch.DoubleTensor").get_model_dim())]
from abstract.util import wrap_V_class_with_input_data from distributions.neural_nets.fc_V_model_1 import V_fc_model_1 from unit_tests.debug_optimization.debug_optimization import gradient_descent import torch input_data = get_data_dict("mnist") # 0.866 when num_units = 10 input_data = { "input": input_data["input"][:5000, :], "target": input_data["target"][:5000] } #input_data = get_data_dict("mnist") prior_dict = {"name": "normal"} model_dict = {"num_units": 300} v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=input_data, prior_dict=prior_dict, model_dict=model_dict) out, explode_grad = gradient_descent( number_of_iter=5000, lr=0.01, v_obj=v_generator(precision_type="torch.DoubleTensor")) #print(out.flattened_tensor) mcmc_samples = torch.zeros(1, len(out.flattened_tensor)) mcmc_samples[0, :] = out.flattened_tensor mcmc_samples = mcmc_samples.numpy() te, predicted = test_error( target_dataset=input_data, v_obj=v_generator(precision_type="torch.DoubleTensor"),
"input": input_data["input"][250:500, ], "target": input_data["target"][250:500] } test_set = { "input": input_data["input"][:-500, ], "target": input_data["target"][:-500] } train_set = { "input": input_data["input"][:250, ], "target": input_data["target"][:250] } model_dict = {"num_units": 50} v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_debug, input_data=train_set, model_dict=model_dict) out, explode_grad = gradient_descent( number_of_iter=2000, lr=0.01, v_obj=v_generator(precision_type="torch.DoubleTensor"), validation_set=validate_set) #print(out.flattened_tensor) mcmc_samples = torch.zeros(1, len(out.flattened_tensor)) mcmc_samples[0, :] = out.flattened_tensor print(max(torch.abs(out.flattened_tensor))) print(min(torch.abs(out.flattened_tensor))) mcmc_samples = mcmc_samples.numpy()
from experiments.correctdist_experiments.prototype import check_mean_var_stan from post_processing.ESS_nuts import ess_stan from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics seed = 1 numpy.random.seed(seed) non_zero_num_p = 20 full_p = 400 num_samples = 100 X_np = numpy.random.randn(num_samples, full_p) * 5 true_beta = numpy.zeros(full_p) true_beta[:non_zero_num_p] = numpy.random.randn(non_zero_num_p) * 5 y_np = X_np.dot(true_beta) + numpy.random.randn(num_samples) input_data = {"input": X_np, "target": y_np} v_generator = wrap_V_class_with_input_data(class_constructor=V_rhs_lr, input_data=input_data) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=2, num_cpu=1, thin=1, tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=False) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
import numpy from experiments.experiment_util import wishart_for_cov from distributions.mvn import V_mvn from abstract.util import wrap_V_class_with_input_data from experiments.float_vs_double.convergence.util import convert_convergence_output_to_numpy from experiments.float_vs_double.convergence.float_vs_double_convergence import convergence_diagnostics from input_data.convert_data_to_dict import get_data_dict from distributions.logistic_regressions.logistic_regression import V_logistic_regression from abstract.util import wrap_V_class_with_input_data ##################################################################################################################################### numpy.random.seed(1) input_data = {"input": wishart_for_cov(dim=10)} V_mvn1 = wrap_V_class_with_input_data(class_constructor=V_mvn, input_data=input_data) V_mvn2 = wrap_V_class_with_input_data(class_constructor=V_mvn, input_data=input_data) #################################################################################################################################### # logisitc regressions input_data_pima_indian = get_data_dict("pima_indian") V_pima_indian = wrap_V_class_with_input_data( class_constructor=V_logistic_regression, input_data=input_data_pima_indian) input_data_australian = get_data_dict("australian") V_australian = wrap_V_class_with_input_data( class_constructor=V_logistic_regression, input_data=input_data_australian) input_data_heart = get_data_dict("heart") V_heart = wrap_V_class_with_input_data(class_constructor=V_logistic_regression, input_data=input_data_heart)
seed = 1 numpy.random.seed(seed) n = 30 dim = 100 X = numpy.random.randn(n, dim) y = [None] * n for i in range(n): y[i] = numpy.asscalar(numpy.random.choice(2, 1)) if y[i] > 0: X[i, 0:2] = numpy.random.randn(2) * 0.5 + 1 else: X[i, 0:2] = numpy.random.randn(2) * 0.5 - 1 input_data = {"target": y, "input": X} v_generator = wrap_V_class_with_input_data( class_constructor=V_logistic_regression_hs, input_data=input_data) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=2, num_cpu=1, thin=1, tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=False) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
#from unit_tests.debug_optimization.debug_nn_raw.model import V_fc_model_1 #from unit_tests.debug_optimization.debug_nn_raw.model import V_fc_model_1 from distributions.neural_nets.fc_V_model_4 import V_fc_model_4 from distributions.neural_nets.fc_V_model_1 import V_fc_model_1 from distributions.neural_nets.fc_V_model_debug import V_fc_model_debug from distributions.neural_nets.fc_V_model_layers import V_fc_model_layers from unit_tests.debug_optimization.debug_optimization import gradient_descent import torch input_data = get_data_dict("8x8mnist") # 0.866 when num_units = 10 test_set = {"input":input_data["input"][600:1000,],"target":input_data["target"][600:1000]} train_set = {"input":input_data["input"][:500,],"target":input_data["target"][:500]} prior_dict = {"name":"normal"} model_dict = {"num_layers":4} v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_layers,prior_dict=prior_dict,input_data=train_set,model_dict=model_dict) out,explode_grad = gradient_descent(number_of_iter=2000,lr=0.0051,v_obj=v_generator(precision_type="torch.DoubleTensor")) print(out.flattened_tensor.shape) mcmc_samples = torch.zeros(1,len(out.flattened_tensor)) mcmc_samples[0,:] = out.flattened_tensor print(max(torch.abs(out.flattened_tensor))) print(min(torch.abs(out.flattened_tensor))) mcmc_samples = mcmc_samples.numpy() v_obj = v_generator(precision_type="torch.DoubleTensor") v_obj.flattened_tensor.copy_(out.flattened_tensor) v_obj.load_flattened_tensor_to_param() print(v_obj.forward())
import numpy, torch from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict from adapt_util.tune_param_classes.tune_param_setting_util import * from experiments.experiment_obj import tuneinput_class from post_processing.get_diagnostics import get_short_diagnostics from post_processing.ESS_nuts import diagnostics_stan from distributions.logistic_regressions.logistic_regression import V_logistic_regression from abstract.util import wrap_V_class_with_input_data from input_data.convert_data_to_dict import get_data_dict from experiments.float_vs_double.convergence.float_vs_double_convergence import convergence_diagnostics input_data = get_data_dict("pima_indian") #input_data = {"input":wishart_for_cov(dim=10)} v_fun = wrap_V_class_with_input_data(class_constructor=V_logistic_regression, input_data=input_data) mcmc_meta_double = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1500, num_chains=2, num_cpu=2, thin=1, tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=True) input_dict = { "v_fun": [v_fun], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
from post_processing.ESS_nuts import ess_stan,diagnostics_stan from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics,get_params_mcmc_tensor,get_short_diagnostics from input_data.convert_data_to_dict import get_data_dict from post_processing.test_error import test_error seed = 10 numpy.random.seed(seed) torch.manual_seed(seed) input_data = get_data_dict("boston") test_set = {"input":input_data["input"][-100:,],"target":input_data["target"][-100:]} train_set = {"input": input_data["input"][:100, ], "target": input_data["target"][:100]} model_dict = {"num_units":50} v_generator =wrap_V_class_with_input_data(class_constructor=V_fc_unscaled_model_regression,input_data=train_set,model_dict=model_dict) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=4,thin=1,tune_l_per_chain=1000, warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False) # input_dict = {"v_fun":[v_generator],"epsilon":[0.001],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} # input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8], # "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["xhmc"],"xhmc_delta":[0.1]} input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8], "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]} # input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"max_tree_depth":[8], # "metric_name":["unit_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
def stability_experiment(priors_list, input_data, num_of_pts, save_name, seed=1): torch.manual_seed(seed) numpy.random.seed(seed) store_outcome = numpy.zeros(shape=[len(priors_list), 2, num_of_pts]) stored = True for i in range(len(priors_list)): model_dict = {"num_units": 35} prior_dict = {"name": priors_list[i]} v_fun = wrap_V_class_with_input_data(class_constructor=V_fc_model_4, prior_dict=prior_dict, model_dict=model_dict, input_data=input_data) out = generate_q_list(v_fun=v_fun, num_of_pts=num_of_pts) list_q_double = out["list_q_double"] list_q_float = out["list_q_float"] list_p_double = [None] * len(list_q_double) list_p_float = [None] * len(list_q_float) for j in range(len(list_q_float)): print(list_q_double[j].flattened_tensor) p_double = list_q_double[j].point_clone() momentum = torch.randn(len( p_double.flattened_tensor)).type("torch.DoubleTensor") p_double.flattened_tensor.copy_(momentum) p_double.load_flatten() p_float = list_q_float[j].point_clone() p_float.flattened_tensor.copy_(momentum.type("torch.FloatTensor")) p_float.load_flatten() list_p_double[j] = p_double list_p_float[j] = p_float out = generate_Hams(v_fun=v_fun) Ham_float = out["float"] Ham_double = out["double"] out_double = leapfrog_stability_test( Ham=Ham_double, epsilon=0.01, L=500, list_q=list_q_double, list_p=list_p_double, precision_type="torch.DoubleTensor") out_float = leapfrog_stability_test(Ham=Ham_float, epsilon=0.01, L=500, list_q=list_q_float, list_p=list_p_float, precision_type="torch.FloatTensor") store_outcome[i, 0, :] = out_double store_outcome[i, 1, :] = out_float to_store = { "output": store_outcome, "priors_list": priors_list, "seed": seed } numpy.savez(save_name, **to_store) return ()
test_set = { "input": input_data["input"][-100:, ], "target": input_data["target"][-100:] } train_set = { "input": input_data["input"][:100, ], "target": input_data["target"][:100] } prior_dict = {"name": "normal"} model_dict = {"num_units": 25} v_generator = wrap_V_class_with_input_data( class_constructor=V_fc_model_1_regression, input_data=train_set, prior_dict=prior_dict, model_dict=model_dict) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=False) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
tune_l_per_chain=1000, warmup_per_chain=1100, is_float=False, isstore_to_disk=False, allow_restart=False) numpy.random.seed(0) Sigma_inv = wishart_for_cov(dim=50) Sigma = numpy.linalg.inv(Sigma_inv) # sd_vec = numpy.sqrt(numpy.diagonal(Sigma)) # print(max(sd_vec)) # print(min(sd_vec)) # exit() input_data = {"input": Sigma_inv} V_generator = wrap_V_class_with_input_data(class_constructor=V_mvn, input_data=input_data) input_dict = { "v_fun": [V_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"], "metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"] } ep_dual_metadata_argument = { "name": "epsilon", "target": 0.65,
def setup_gibbs_v_joint_experiment(num_units_list, train_set, test_set, num_samples, save_name, seed=1): output_names = [ "train_error", "test_error", "train_error_sd", "test_error_sd", "sigma_2_ess", "mean_sigma2", "median_sigma2", "min_ess", "median_ess" ] output_store = numpy.zeros((len(num_units_list), 3, len(output_names))) diagnostics_store = numpy.zeros(shape=[len(num_units_list), 3] + [4, 13]) time_store = numpy.zeros(shape=[len(num_units_list), 3]) for i in range(len(num_units_list)): for j in range(3): start_time = time.time() v_fun = V_fc_model_4 model_dict = {"num_units": num_units_list[i]} mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1000 + num_samples, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=900, warmup_per_chain=1000, is_float=False, isstore_to_disk=False, allow_restart=True, seed=seed + i + 1) if j == 2: v_generator = wrap_V_class_with_input_data( class_constructor=V_fc_gibbs_model_1, input_data=train_set, model_dict=model_dict) v_obj = v_generator(precision_type="torch.DoubleTensor", gibbs=True) metric_obj = metric(name="unit_e", V_instance=v_obj) Ham = Hamiltonian(v_obj, metric_obj) init_q_point = point(V=v_obj) init_hyperparam = torch.abs(torch.randn(1)) + 3 log_obj = log_class() dim = len(init_q_point.flattened_tensor) mcmc_samples_weight = torch.zeros(1, num_samples + 1000, dim) mcmc_samples_hyper = torch.zeros(1, num_samples + 1000, 1) for iter in range(num_samples + 1000): print("iter {}".format(iter)) outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step( init_q_point, init_hyperparam, Ham, 0.01, log_obj) init_q_point.flattened_tensor.copy_(outq.flattened_tensor) init_q_point.load_flatten() init_hyperparam = out_hyperparam mcmc_samples_weight[ 0, iter, :] = outq.flattened_tensor.clone() mcmc_samples_hyper[0, iter, 0] = out_hyperparam mcmc_samples_weight = mcmc_samples_weight[:, 1000:, :].numpy() mcmc_samples_hyper = mcmc_samples_hyper[:, 1000:, :].numpy() te, predicted, te_sd = test_error( test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_weight[0, :, :], type="classification", memory_efficient=False) train_error, _, train_error_sd = test_error( train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_weight[0, :, :], type="classification", memory_efficient=False) sigma2_diagnostics = diagnostics_stan(mcmc_samples_hyper) sigma2_ess = sigma2_diagnostics["ess"] posterior_mean_hidden_in_sigma2 = numpy.mean( mcmc_samples_hyper) posterior_median_hidden_in_sigma2 = numpy.median( mcmc_samples_hyper) weight_ess = diagnostics_stan(mcmc_samples_weight)["ess"] min_ess = min(sigma2_ess, min(weight_ess)) median_ess = numpy.median([sigma2_ess] + list(weight_ess)) output_store[i, j, 0] = train_error output_store[i, j, 1] = te output_store[i, j, 2] = train_error output_store[i, j, 3] = te_sd output_store[i, j, 4] = sigma2_ess output_store[i, j, 5] = posterior_mean_hidden_in_sigma2 output_store[i, j, 6] = posterior_median_hidden_in_sigma2 output_store[i, j, 7] = min_ess output_store[i, j, 8] = median_ess elif j == 0: prior_dict = {"name": "gaussian_inv_gamma_1"} v_generator = wrap_V_class_with_input_data( class_constructor=v_fun, input_data=train_set, prior_dict=prior_dict, model_dict=model_dict) elif j == 1: prior_dict = {"name": "gaussian_inv_gamma_2"} v_generator = wrap_V_class_with_input_data( class_constructor=v_fun, input_data=train_set, prior_dict=prior_dict, model_dict=model_dict) if j == 0 or j == 1: input_dict = { "v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "max_tree_depth": [8], "metric_name": ["unit_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"], "xhmc_delta": [0.1] } ep_dual_metadata_argument = { "name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10, "kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast" } dual_args_list = [ep_dual_metadata_argument] other_arguments = other_default_arguments() tune_settings_dict = tuning_settings(dual_args_list, [], [], other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict) sampler1.start_sampling() np_diagnostics, feature_names = sampler1.np_diagnostics() mcmc_samples_hidden_in = sampler1.get_samples_alt( prior_obj_name="hidden_in", permuted=False) samples = mcmc_samples_hidden_in["samples"] hidden_in_sigma2_indices = mcmc_samples_hidden_in[ "indices_dict"]["sigma2"] sigma2_diagnostics = diagnostics_stan( samples[:, :, hidden_in_sigma2_indices]) sigma2_ess = sigma2_diagnostics["ess"] posterior_mean_hidden_in_sigma2 = numpy.mean( samples[:, :, hidden_in_sigma2_indices].reshape( -1, len(hidden_in_sigma2_indices)), axis=0) posterior_median_hidden_in_sigma2 = numpy.median( samples[:, :, hidden_in_sigma2_indices].reshape( -1, len(hidden_in_sigma2_indices)), axis=0) mcmc_samples_mixed = sampler1.get_samples(permuted=True) te, predicted, te_sd = test_error( test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False) train_error, _, train_error_sd = test_error( train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False) output_store[i, j, 0] = train_error output_store[i, j, 1] = te output_store[i, j, 2] = train_error output_store[i, j, 3] = te_sd output_store[i, j, 4] = sigma2_ess output_store[i, j, 5] = posterior_mean_hidden_in_sigma2 output_store[i, j, 6] = posterior_median_hidden_in_sigma2 diagnostics_store[i, j, :, :] = np_diagnostics output_store[i, j, 7] = np_diagnostics[0, 10] output_store[i, j, 8] = np_diagnostics[0, 11] total_time = time.time() - start_time() time_store[i, j] = total_time to_store = { "diagnostics": diagnostics_store, "output": output_store, "diagnostics_names": feature_names, "output_names": output_names, "seed": seed, "num_units_list": num_units_list, "time_store": time_store } numpy.savez(save_name, **to_store) return ()
def setup_ensemble_experiment(num_unit_list, list_num_ensemble_pts, train_set, validate_set, test_set, save_name, seed=1): output_names = [ "ensemble_train_error", "ensemble_te", "ensemble_train_error_sd", "ensemble_te_sd" ] output_store = numpy.zeros( (len(num_unit_list), len(list_num_ensemble_pts), len(output_names))) diagnostics_store = numpy.zeros(shape=[len(num_unit_list)] + [4, 13]) numpy.random.seed(seed) torch.manual_seed(seed) diver_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts))) time_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts))) for i in range(len(num_unit_list)): model_dict = {"num_units": num_unit_list[i]} for k in range(len(list_num_ensemble_pts)): start_time = time.time() prior_dict = {"name": "normal"} num_ensemble_pts = list_num_ensemble_pts[k] num_diver = 0 ensemble_list = [] v_generator = wrap_V_class_with_input_data( class_constructor=V_fc_model_4, prior_dict=prior_dict, input_data=train_set, model_dict=model_dict) for j in range(num_ensemble_pts): out, explode_grad = gradient_descent( number_of_iter=2000, lr=0.001, validation_set=validate_set, v_obj=v_generator(precision_type="torch.DoubleTensor")) if explode_grad: num_diver += 1 else: ensemble_list.append(out.point_clone()) ensemble_pts = numpy.zeros( (len(ensemble_list), len(ensemble_list[0].flattened_tensor))) for z in range(len(ensemble_list)): ensemble_pts[z, :] = ensemble_list[z].flattened_tensor.numpy() ensemble_te, predicted, ensemble_te_sd = test_error( test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=ensemble_pts, type="classification", memory_efficient=False) ensemble_train_error, _, ensemble_train_error_sd = test_error( train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=ensemble_pts, type="classification", memory_efficient=False) total_time = time.time() - start_time output_store[i, k, 0] = ensemble_train_error output_store[i, k, 1] = ensemble_te output_store[i, k, 2] = ensemble_train_error_sd output_store[i, k, 3] = ensemble_te_sd diver_store[i, k] = num_diver time_store[i, k] = total_time #print(output_store) to_store = { "output": output_store, "output_names": output_names, "num_unit_list": num_unit_list, "seed": seed, "list_num_ensemble_pts": list_num_ensemble_pts, "num_div": diver_store, "time_store": time_store } numpy.savez(save_name, **to_store) return ()
X = numpy.random.randn(n,dim) y = numpy.zeros(n) for i in range(n): y[i] = numpy.asscalar(numpy.random.choice(2,1)) if y[i] > 0: X[i,0:2] = numpy.random.randn(2)*0.5 + 1 else: X[i,0:2] = numpy.random.randn(2)*0.5 -1 input_data = {"target":y,"input":X} prior_dict = {"name":"rhorseshoe_2"} v_generator =wrap_V_class_with_input_data(class_constructor=V_logit,input_data=input_data,prior_dict=prior_dict) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=1,thin=1,tune_l_per_chain=1000, warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8], "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]} # input_dict = {"v_fun":[v_generator],"epsilon":[0.1],"second_order":[False],"evolve_L":[10], # "metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} ep_dual_metadata_argument = {"name":"epsilon","target":0.9,"gamma":0.05,"t_0":10, "kappa":0.75,"obj_fun":"accept_rate","par_type":"fast"} # adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow",dim=v_generator(precision_type="torch.DoubleTensor").get_model_dim())]
def setup_sghmc_experiment(ep_list,L_list,eta_list,train_set,test_set,save_name,seed=1): output_names = ["train_error", "test_error","train_error_sd","test_error_sd"] output_store = numpy.zeros((len(ep_list),len(L_list),len(eta_list), len(output_names))) diagnostics_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)]+[4,13]) model_dict = {"num_units":35} prior_dict = {"name":"normal"} time_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)]) for i in range(len(ep_list)): for j in range(len(L_list)): for k in range(len(eta_list)): start_time = time.time() v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set, prior_dict=prior_dict, model_dict=model_dict) v_obj = v_generator(precision_type="torch.DoubleTensor") metric_obj = metric(name="unit_e", V_instance=v_obj) Ham = Hamiltonian(V=v_obj, metric=metric_obj) full_data = train_set init_q_point = point(V=v_obj) store,explode_grad = sghmc_sampler(init_q_point=init_q_point, epsilon=ep_list[i], L=L_list[j], Ham=Ham, alpha=0.01, eta=eta_list[k], betahat=0, full_data=full_data, num_samples=2000, thin=0, burn_in=1000, batch_size=25) total_time = time.time() - start_time if not explode_grad: v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set, prior_dict=prior_dict, model_dict=model_dict) test_mcmc_samples = store.numpy() te1, predicted1,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=test_mcmc_samples, type="classification", memory_efficient=False) train_error, predicted1, train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=test_mcmc_samples, type="classification", memory_efficient=False) else: train_error = 2 te1 = 2 train_error_sd = 2 te_sd = 2 output_store[i,j,k,0] = train_error output_store[i,j,k,1] = te1 output_store[i,j,k,2] = train_error_sd output_store[i,j,k,3] = te_sd time_store[i,j,k] = total_time to_store = {"diagnostics":diagnostics_store,"output":output_store,"output_names":output_names,"seed":seed, "ep_list":ep_list,"L_list":L_list,"eta_list":eta_list,"num_units":model_dict["num_units"], "prior":prior_dict["name"],"total_store":time_store} numpy.savez(save_name,**to_store) return()
# hmc need to add jitter (0.9 ep, 1.1 ep) num_repeats = 2 num_grid_divides = 3 ep_bounds = [1e-3, 0.2] evolve_t_bounds = [0.15, 50.] evolve_L_bounds = [5, 1000] # add constraints such that L = round(evolove_t/ep) < 1024 ep_list = list(numpy.linspace(ep_bounds[0], ep_bounds[1], num_grid_divides)) evolve_t_list = list( numpy.linspace(evolve_t_bounds[0], evolve_t_bounds[1], num_grid_divides)) evolve_L_list = list( numpy.linspace(evolve_L_bounds[0], evolve_L_bounds[1], num_grid_divides)) input_data = {"input": wishart_for_cov(dim=10)} V_mvn1 = wrap_V_class_with_input_data(class_constructor=V_mvn, input_data=input_data) V_mvn2 = wrap_V_class_with_input_data(class_constructor=V_mvn, input_data=input_data) target_fun = fun_extract_median_ess v_fun_list = [V_mvn1, V_mvn2] # grid computations # experiment_setting = experiment_setting_dict(chain_length=300,num_chains_per_sampler=4,warm_up=150, # tune_l=0,allow_restart=True,max_num_restarts=5,num_cpu_per_sampler=4) # input_dict = { "v_fun": v_fun_list[0:1], "epsilon": ep_list, "second_order": [False], "stepsize_jitter": [True], "evolve_L": evolve_L_list,
import os, numpy,torch import dill as pickle from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict from adapt_util.tune_param_classes.tune_param_setting_util import * from experiments.experiment_obj import tuneinput_class from experiments.correctdist_experiments.prototype import check_mean_var_stan from post_processing.ESS_nuts import ess_stan,diagnostics_stan from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics,get_params_mcmc_tensor,get_short_diagnostics from input_data.convert_data_to_dict import get_data_dict from experiments.neural_net_experiments.gibbs_vs_joint_sampling.V_hierarchical_fc1 import V_fc_gibbs_model_1 from post_processing.test_error import test_error input_data = get_data_dict("8x8mnist") input_data = {"input":input_data["input"][:500,],"target":input_data["target"][:500]} model_dict = {"num_units":15} V_fun = wrap_V_class_with_input_data(class_constructor=V_fc_gibbs_model_1,input_data=input_data,model_dict=model_dict) prior_dict = {"name":"gaussian_inv_gamma_1"} mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=1,num_cpu=1,thin=1,tune_l_per_chain=1000, warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False,seed=133) # input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False], # "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]} # input_dict = {"v_fun":[V_fun],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],"xhmc_delta":[0.1], # "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["xhmc"]} input_dict = {"v_fun":[V_fun],"epsilon":["dual"],"second_order":[False], "metric_name":["unit_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
def setup_xhmc_gnuts_experiment(xhmc_delta_list,train_set,test_set,save_name,seed=1): xhmc_delta_list.append(0) output_names = ["train_error", "test_error","train_error_sd","test_error_sd","min_ess","median_ess"] output_store = numpy.zeros((len(xhmc_delta_list), len(output_names))) diagnostics_store = numpy.zeros(shape=[len(xhmc_delta_list)]+[4,13]) prior_dict = {"name": "normal"} model_dict = {"num_units": 35} time_list = [] for i in range(len(xhmc_delta_list)): start_time = time.time() v_fun = V_fc_model_4 v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=train_set,prior_dict=prior_dict, model_dict=model_dict) mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1, tune_l_per_chain=900, warmup_per_chain=1000, is_float=False, isstore_to_disk=False, allow_restart=True,seed=seed+i+1) if i == len(xhmc_delta_list)-1: input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"], "max_tree_depth": [8], "metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]} else: input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"], "max_tree_depth": [8], "metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"],"xhmc_delta":[xhmc_delta_list[i]]} ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10, "kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"} # # adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator( precision_type="torch.DoubleTensor").get_model_dim())] dual_args_list = [ep_dual_metadata_argument] other_arguments = other_default_arguments() tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict) sampler1.start_sampling() total_time = time.time() - start_time np_diagnostics,feature_names = sampler1.np_diagnostics() mcmc_samples_mixed = sampler1.get_samples(permuted=True) te, predicted,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False) train_error,_,train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"), mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False) output_store[i,0] = train_error output_store[i,1] = te output_store[i,2] = train_error_sd output_store[i,3] = te_sd diagnostics_store[i,:,:] = np_diagnostics output_store[i,4] = np_diagnostics[0,10] output_store[i,5] = np_diagnostics[0,11] time_list.append(total_time) to_store = {"diagnostics":diagnostics_store,"output":output_store,"diagnostics_names":feature_names, "output_names":output_names,"seed":seed,"xhmc_delta_list":xhmc_delta_list,"prior":prior_dict["name"], "num_units":model_dict["num_units"],"time_list":time_list} numpy.savez(save_name,**to_store) return()
from input_data.convert_data_to_dict import get_data_dict from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=1000, num_chains=1, num_cpu=1, thin=1, tune_l_per_chain=0, warmup_per_chain=100, is_float=False, isstore_to_disk=False, allow_restart=False) input_data = get_data_dict("sp500") V_generator = wrap_V_class_with_input_data( class_constructor=V_stochastic_volatility, input_data=input_data) input_dict = { "v_fun": [V_generator], "epsilon": [0.001], "second_order": [False], "metric_name": ["unit_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"] } other_arguments = other_default_arguments() tune_settings_dict = tuning_settings([], [], [], other_arguments) tune_dict = tuneinput_class(input_dict).singleton_tune_dict() sampler1 = mcmc_sampler(tune_dict=tune_dict,