def run_nn_experiment(xhmc_delta_list,input_data,v_fun,test_set,type_problem):
out_list = [None]*(len(xhmc_delta_list)+1)
for i in range(len(out_list)):
model_dict = {"num_units": 50}
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=input_data,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100, is_float=False, isstore_to_disk=False,
allow_restart=False)
if i<len(out_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],"xhmc_delta":[xhmc_delta_list[i]],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
# tune_settings_dict = tuning_settings([],[],[],[])
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
diagnostics_np = sampler1.np_diagnostics()
samples_mixed = sampler1.get_samples(permuted=True)
te = test_error(target_dataset=test_set,v_obj=v_generator("torch.DoubleTensor"),mcmc_samples=samples_mixed,type=type_problem)
out = {"test_error":te,"diagnostics":diagnostics_np}
out_list.append(out)
te_store = numpy.zeros(len(out_list))
diagnostics_store = numpy.zeros(shape=[len(out_list)]+list(diagnostics_np.shape))
for i in range(len(out_list)):
te_store[i] = out_list[i]["test_error"]
diagnostics_store[i,...] = out_list[i]["diagnostics"]
output_store = {"test_error":te_store,"diagnostics":diagnostics_store}
save_name = "xhmc_v_gnuts_8x8mnist.npz"
numpy.savez(save_name,**output_store)
return(save_name)
from experiments.correctdist_experiments.prototype import check_mean_var_stan
from abstract.util import wrap_V_class_with_input_data
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=500,
warmup_per_chain=600,
is_float=False,
isstore_to_disk=False,
allow_restart=False,
seed=25)
input_data = get_data_dict("pima_indian")
V_pima_indian_logit = wrap_V_class_with_input_data(
class_constructor=V_logistic_regression, input_data=input_data)
address = os.environ[
"PYTHONPATH"] + "/experiments/correctdist_experiments/result_from_long_chain.pkl"
correct = pickle.load(open(address, 'rb'))
correct_mean = correct["correct_mean"]
correct_cov = correct["correct_cov"]
correct_diag_cov = correct_cov.diagonal()
input_dict = {
"v_fun": [V_pima_indian_logit],
"epsilon": ["dual"],
"second_order": [False],
"evolve_t": [1.],
"metric_name": ["unit_e"],
"dynamic": [False],
from input_data.convert_data_to_dict import get_data_dict
from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
pima_indian_data = get_data_dict("pima_indian")
V_generator = wrap_V_class_with_input_data(
class_constructor=V_logistic_regression, input_data=pima_indian_data)
input_dict = {
"v_fun": [V_generator],
"epsilon": [0.01],
"second_order": [False],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["gnuts"]
}
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings([], [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
from input_data.convert_data_to_dict import get_data_dict
from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
input_data = get_data_dict("1-PL", standardize_predictor=False)
V_generator = wrap_V_class_with_input_data(class_constructor=V_response_model,
input_data=input_data)
input_dict = {
"v_fun": [V_generator],
"epsilon": [0.1],
"second_order": [False],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["gnuts"]
}
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings([], [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics
num_p = 100
non_zero_p = 20
seedid = 33034
numpy.random.seed(seedid)
torch.manual_seed(seedid)
true_p = numpy.zeros(num_p)
true_p[:non_zero_p] = numpy.random.randn(non_zero_p)*5
y = true_p + numpy.random.randn(num_p)
input_data = {"target":y}
v_generator =wrap_V_class_with_input_data(class_constructor=V_student_toy,input_data=input_data)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=4,thin=1,tune_l_per_chain=1000,
warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],
"metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
# input_dict = {"v_fun":[v_generator],"epsilon":[0.1],"second_order":[False],"evolve_L":[10],
# "metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
ep_dual_metadata_argument = {"name":"epsilon","target":0.9,"gamma":0.05,"t_0":10,
"kappa":0.75,"obj_fun":"accept_rate","par_type":"fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow",dim=v_generator(precision_type="torch.DoubleTensor").get_model_dim())]
from abstract.util import wrap_V_class_with_input_data
from distributions.neural_nets.fc_V_model_1 import V_fc_model_1
from unit_tests.debug_optimization.debug_optimization import gradient_descent
import torch
input_data = get_data_dict("mnist") # 0.866 when num_units = 10
input_data = {
"input": input_data["input"][:5000, :],
"target": input_data["target"][:5000]
}
#input_data = get_data_dict("mnist")
prior_dict = {"name": "normal"}
model_dict = {"num_units": 300}
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1,
input_data=input_data,
prior_dict=prior_dict,
model_dict=model_dict)
out, explode_grad = gradient_descent(
number_of_iter=5000,
lr=0.01,
v_obj=v_generator(precision_type="torch.DoubleTensor"))
#print(out.flattened_tensor)
mcmc_samples = torch.zeros(1, len(out.flattened_tensor))
mcmc_samples[0, :] = out.flattened_tensor
mcmc_samples = mcmc_samples.numpy()
te, predicted = test_error(
target_dataset=input_data,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
"input": input_data["input"][250:500, ],
"target": input_data["target"][250:500]
}
test_set = {
"input": input_data["input"][:-500, ],
"target": input_data["target"][:-500]
}
train_set = {
"input": input_data["input"][:250, ],
"target": input_data["target"][:250]
}
model_dict = {"num_units": 50}
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_debug,
input_data=train_set,
model_dict=model_dict)
out, explode_grad = gradient_descent(
number_of_iter=2000,
lr=0.01,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
validation_set=validate_set)
#print(out.flattened_tensor)
mcmc_samples = torch.zeros(1, len(out.flattened_tensor))
mcmc_samples[0, :] = out.flattened_tensor
print(max(torch.abs(out.flattened_tensor)))
print(min(torch.abs(out.flattened_tensor)))
mcmc_samples = mcmc_samples.numpy()
from experiments.correctdist_experiments.prototype import check_mean_var_stan
from post_processing.ESS_nuts import ess_stan
from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics
seed = 1
numpy.random.seed(seed)
non_zero_num_p = 20
full_p = 400
num_samples = 100
X_np = numpy.random.randn(num_samples, full_p) * 5
true_beta = numpy.zeros(full_p)
true_beta[:non_zero_num_p] = numpy.random.randn(non_zero_num_p) * 5
y_np = X_np.dot(true_beta) + numpy.random.randn(num_samples)
input_data = {"input": X_np, "target": y_np}
v_generator = wrap_V_class_with_input_data(class_constructor=V_rhs_lr,
input_data=input_data)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=2000,
num_chains=2,
num_cpu=1,
thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
import numpy
from experiments.experiment_util import wishart_for_cov
from distributions.mvn import V_mvn
from abstract.util import wrap_V_class_with_input_data
from experiments.float_vs_double.convergence.util import convert_convergence_output_to_numpy
from experiments.float_vs_double.convergence.float_vs_double_convergence import convergence_diagnostics
from input_data.convert_data_to_dict import get_data_dict
from distributions.logistic_regressions.logistic_regression import V_logistic_regression
from abstract.util import wrap_V_class_with_input_data
#####################################################################################################################################
numpy.random.seed(1)
input_data = {"input": wishart_for_cov(dim=10)}
V_mvn1 = wrap_V_class_with_input_data(class_constructor=V_mvn,
input_data=input_data)
V_mvn2 = wrap_V_class_with_input_data(class_constructor=V_mvn,
input_data=input_data)
####################################################################################################################################
# logisitc regressions
input_data_pima_indian = get_data_dict("pima_indian")
V_pima_indian = wrap_V_class_with_input_data(
class_constructor=V_logistic_regression, input_data=input_data_pima_indian)
input_data_australian = get_data_dict("australian")
V_australian = wrap_V_class_with_input_data(
class_constructor=V_logistic_regression, input_data=input_data_australian)
input_data_heart = get_data_dict("heart")
V_heart = wrap_V_class_with_input_data(class_constructor=V_logistic_regression,
input_data=input_data_heart)
seed = 1
numpy.random.seed(seed)
n = 30
dim = 100
X = numpy.random.randn(n, dim)
y = [None] * n
for i in range(n):
y[i] = numpy.asscalar(numpy.random.choice(2, 1))
if y[i] > 0:
X[i, 0:2] = numpy.random.randn(2) * 0.5 + 1
else:
X[i, 0:2] = numpy.random.randn(2) * 0.5 - 1
input_data = {"target": y, "input": X}
v_generator = wrap_V_class_with_input_data(
class_constructor=V_logistic_regression_hs, input_data=input_data)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=2000,
num_chains=2,
num_cpu=1,
thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
#from unit_tests.debug_optimization.debug_nn_raw.model import V_fc_model_1
#from unit_tests.debug_optimization.debug_nn_raw.model import V_fc_model_1
from distributions.neural_nets.fc_V_model_4 import V_fc_model_4
from distributions.neural_nets.fc_V_model_1 import V_fc_model_1
from distributions.neural_nets.fc_V_model_debug import V_fc_model_debug
from distributions.neural_nets.fc_V_model_layers import V_fc_model_layers
from unit_tests.debug_optimization.debug_optimization import gradient_descent
import torch
input_data = get_data_dict("8x8mnist") # 0.866 when num_units = 10
test_set = {"input":input_data["input"][600:1000,],"target":input_data["target"][600:1000]}
train_set = {"input":input_data["input"][:500,],"target":input_data["target"][:500]}
prior_dict = {"name":"normal"}
model_dict = {"num_layers":4}
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_layers,prior_dict=prior_dict,input_data=train_set,model_dict=model_dict)
out,explode_grad = gradient_descent(number_of_iter=2000,lr=0.0051,v_obj=v_generator(precision_type="torch.DoubleTensor"))
print(out.flattened_tensor.shape)
mcmc_samples = torch.zeros(1,len(out.flattened_tensor))
mcmc_samples[0,:] = out.flattened_tensor
print(max(torch.abs(out.flattened_tensor)))
print(min(torch.abs(out.flattened_tensor)))
mcmc_samples = mcmc_samples.numpy()
v_obj = v_generator(precision_type="torch.DoubleTensor")
v_obj.flattened_tensor.copy_(out.flattened_tensor)
v_obj.load_flattened_tensor_to_param()
print(v_obj.forward())
import numpy, torch
from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict
from adapt_util.tune_param_classes.tune_param_setting_util import *
from experiments.experiment_obj import tuneinput_class
from post_processing.get_diagnostics import get_short_diagnostics
from post_processing.ESS_nuts import diagnostics_stan
from distributions.logistic_regressions.logistic_regression import V_logistic_regression
from abstract.util import wrap_V_class_with_input_data
from input_data.convert_data_to_dict import get_data_dict
from experiments.float_vs_double.convergence.float_vs_double_convergence import convergence_diagnostics
input_data = get_data_dict("pima_indian")
#input_data = {"input":wishart_for_cov(dim=10)}
v_fun = wrap_V_class_with_input_data(class_constructor=V_logistic_regression,
input_data=input_data)
mcmc_meta_double = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1500,
num_chains=2,
num_cpu=2,
thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100,
is_float=False,
isstore_to_disk=False,
allow_restart=True)
input_dict = {
"v_fun": [v_fun],
"epsilon": ["dual"],
"second_order": [False],
"cov": ["adapt"],
from post_processing.ESS_nuts import ess_stan,diagnostics_stan
from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics,get_params_mcmc_tensor,get_short_diagnostics
from input_data.convert_data_to_dict import get_data_dict
from post_processing.test_error import test_error
seed = 10
numpy.random.seed(seed)
torch.manual_seed(seed)
input_data = get_data_dict("boston")
test_set = {"input":input_data["input"][-100:,],"target":input_data["target"][-100:]}
train_set = {"input": input_data["input"][:100, ], "target": input_data["target"][:100]}
model_dict = {"num_units":50}
v_generator =wrap_V_class_with_input_data(class_constructor=V_fc_unscaled_model_regression,input_data=train_set,model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=4,thin=1,tune_l_per_chain=1000,
warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False)
# input_dict = {"v_fun":[v_generator],"epsilon":[0.001],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
# input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],
# "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["xhmc"],"xhmc_delta":[0.1]}
input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],
"metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
# input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"max_tree_depth":[8],
# "metric_name":["unit_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
def stability_experiment(priors_list,
input_data,
num_of_pts,
save_name,
seed=1):
torch.manual_seed(seed)
numpy.random.seed(seed)
store_outcome = numpy.zeros(shape=[len(priors_list), 2, num_of_pts])
stored = True
for i in range(len(priors_list)):
model_dict = {"num_units": 35}
prior_dict = {"name": priors_list[i]}
v_fun = wrap_V_class_with_input_data(class_constructor=V_fc_model_4,
prior_dict=prior_dict,
model_dict=model_dict,
input_data=input_data)
out = generate_q_list(v_fun=v_fun, num_of_pts=num_of_pts)
list_q_double = out["list_q_double"]
list_q_float = out["list_q_float"]
list_p_double = [None] * len(list_q_double)
list_p_float = [None] * len(list_q_float)
for j in range(len(list_q_float)):
print(list_q_double[j].flattened_tensor)
p_double = list_q_double[j].point_clone()
momentum = torch.randn(len(
p_double.flattened_tensor)).type("torch.DoubleTensor")
p_double.flattened_tensor.copy_(momentum)
p_double.load_flatten()
p_float = list_q_float[j].point_clone()
p_float.flattened_tensor.copy_(momentum.type("torch.FloatTensor"))
p_float.load_flatten()
list_p_double[j] = p_double
list_p_float[j] = p_float
out = generate_Hams(v_fun=v_fun)
Ham_float = out["float"]
Ham_double = out["double"]
out_double = leapfrog_stability_test(
Ham=Ham_double,
epsilon=0.01,
L=500,
list_q=list_q_double,
list_p=list_p_double,
precision_type="torch.DoubleTensor")
out_float = leapfrog_stability_test(Ham=Ham_float,
epsilon=0.01,
L=500,
list_q=list_q_float,
list_p=list_p_float,
precision_type="torch.FloatTensor")
store_outcome[i, 0, :] = out_double
store_outcome[i, 1, :] = out_float
to_store = {
"output": store_outcome,
"priors_list": priors_list,
"seed": seed
}
numpy.savez(save_name, **to_store)
return ()
test_set = {
"input": input_data["input"][-100:, ],
"target": input_data["target"][-100:]
}
train_set = {
"input": input_data["input"][:100, ],
"target": input_data["target"][:100]
}
prior_dict = {"name": "normal"}
model_dict = {"num_units": 25}
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_model_1_regression,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=2000,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
tune_l_per_chain=1000,
warmup_per_chain=1100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
numpy.random.seed(0)
Sigma_inv = wishart_for_cov(dim=50)
Sigma = numpy.linalg.inv(Sigma_inv)
# sd_vec = numpy.sqrt(numpy.diagonal(Sigma))
# print(max(sd_vec))
# print(min(sd_vec))
# exit()
input_data = {"input": Sigma_inv}
V_generator = wrap_V_class_with_input_data(class_constructor=V_mvn,
input_data=input_data)
input_dict = {
"v_fun": [V_generator],
"epsilon": ["dual"],
"second_order": [False],
"cov": ["adapt"],
"metric_name": ["diag_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["gnuts"]
}
ep_dual_metadata_argument = {
"name": "epsilon",
"target": 0.65,
def setup_gibbs_v_joint_experiment(num_units_list,
train_set,
test_set,
num_samples,
save_name,
seed=1):
output_names = [
"train_error", "test_error", "train_error_sd", "test_error_sd",
"sigma_2_ess", "mean_sigma2", "median_sigma2", "min_ess", "median_ess"
]
output_store = numpy.zeros((len(num_units_list), 3, len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_units_list), 3] + [4, 13])
time_store = numpy.zeros(shape=[len(num_units_list), 3])
for i in range(len(num_units_list)):
for j in range(3):
start_time = time.time()
v_fun = V_fc_model_4
model_dict = {"num_units": num_units_list[i]}
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000 +
num_samples,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
seed=seed + i + 1)
if j == 2:
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_gibbs_model_1,
input_data=train_set,
model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor",
gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = torch.abs(torch.randn(1)) + 3
log_obj = log_class()
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(1, num_samples + 1000, dim)
mcmc_samples_hyper = torch.zeros(1, num_samples + 1000, 1)
for iter in range(num_samples + 1000):
print("iter {}".format(iter))
outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step(
init_q_point, init_hyperparam, Ham, 0.01, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
init_q_point.load_flatten()
init_hyperparam = out_hyperparam
mcmc_samples_weight[
0, iter, :] = outq.flattened_tensor.clone()
mcmc_samples_hyper[0, iter, 0] = out_hyperparam
mcmc_samples_weight = mcmc_samples_weight[:, 1000:, :].numpy()
mcmc_samples_hyper = mcmc_samples_hyper[:, 1000:, :].numpy()
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
sigma2_diagnostics = diagnostics_stan(mcmc_samples_hyper)
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
mcmc_samples_hyper)
posterior_median_hidden_in_sigma2 = numpy.median(
mcmc_samples_hyper)
weight_ess = diagnostics_stan(mcmc_samples_weight)["ess"]
min_ess = min(sigma2_ess, min(weight_ess))
median_ess = numpy.median([sigma2_ess] + list(weight_ess))
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
output_store[i, j, 7] = min_ess
output_store[i, j, 8] = median_ess
elif j == 0:
prior_dict = {"name": "gaussian_inv_gamma_1"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
elif j == 1:
prior_dict = {"name": "gaussian_inv_gamma_2"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
if j == 0 or j == 1:
input_dict = {
"v_fun": [v_generator],
"epsilon": ["dual"],
"second_order": [False],
"max_tree_depth": [8],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["xhmc"],
"xhmc_delta": [0.1]
}
ep_dual_metadata_argument = {
"name": "epsilon",
"target": 0.9,
"gamma": 0.05,
"t_0": 10,
"kappa": 0.75,
"obj_fun": "accept_rate",
"par_type": "fast"
}
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [],
other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
np_diagnostics, feature_names = sampler1.np_diagnostics()
mcmc_samples_hidden_in = sampler1.get_samples_alt(
prior_obj_name="hidden_in", permuted=False)
samples = mcmc_samples_hidden_in["samples"]
hidden_in_sigma2_indices = mcmc_samples_hidden_in[
"indices_dict"]["sigma2"]
sigma2_diagnostics = diagnostics_stan(
samples[:, :, hidden_in_sigma2_indices])
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
posterior_median_hidden_in_sigma2 = numpy.median(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
diagnostics_store[i, j, :, :] = np_diagnostics
output_store[i, j, 7] = np_diagnostics[0, 10]
output_store[i, j, 8] = np_diagnostics[0, 11]
total_time = time.time() - start_time()
time_store[i, j] = total_time
to_store = {
"diagnostics": diagnostics_store,
"output": output_store,
"diagnostics_names": feature_names,
"output_names": output_names,
"seed": seed,
"num_units_list": num_units_list,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
def setup_ensemble_experiment(num_unit_list,
list_num_ensemble_pts,
train_set,
validate_set,
test_set,
save_name,
seed=1):
output_names = [
"ensemble_train_error", "ensemble_te", "ensemble_train_error_sd",
"ensemble_te_sd"
]
output_store = numpy.zeros(
(len(num_unit_list), len(list_num_ensemble_pts), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_unit_list)] + [4, 13])
numpy.random.seed(seed)
torch.manual_seed(seed)
diver_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts)))
time_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts)))
for i in range(len(num_unit_list)):
model_dict = {"num_units": num_unit_list[i]}
for k in range(len(list_num_ensemble_pts)):
start_time = time.time()
prior_dict = {"name": "normal"}
num_ensemble_pts = list_num_ensemble_pts[k]
num_diver = 0
ensemble_list = []
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_model_4,
prior_dict=prior_dict,
input_data=train_set,
model_dict=model_dict)
for j in range(num_ensemble_pts):
out, explode_grad = gradient_descent(
number_of_iter=2000,
lr=0.001,
validation_set=validate_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"))
if explode_grad:
num_diver += 1
else:
ensemble_list.append(out.point_clone())
ensemble_pts = numpy.zeros(
(len(ensemble_list), len(ensemble_list[0].flattened_tensor)))
for z in range(len(ensemble_list)):
ensemble_pts[z, :] = ensemble_list[z].flattened_tensor.numpy()
ensemble_te, predicted, ensemble_te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=ensemble_pts,
type="classification",
memory_efficient=False)
ensemble_train_error, _, ensemble_train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=ensemble_pts,
type="classification",
memory_efficient=False)
total_time = time.time() - start_time
output_store[i, k, 0] = ensemble_train_error
output_store[i, k, 1] = ensemble_te
output_store[i, k, 2] = ensemble_train_error_sd
output_store[i, k, 3] = ensemble_te_sd
diver_store[i, k] = num_diver
time_store[i, k] = total_time
#print(output_store)
to_store = {
"output": output_store,
"output_names": output_names,
"num_unit_list": num_unit_list,
"seed": seed,
"list_num_ensemble_pts": list_num_ensemble_pts,
"num_div": diver_store,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
X = numpy.random.randn(n,dim)
y = numpy.zeros(n)
for i in range(n):
y[i] = numpy.asscalar(numpy.random.choice(2,1))
if y[i] > 0:
X[i,0:2] = numpy.random.randn(2)*0.5 + 1
else:
X[i,0:2] = numpy.random.randn(2)*0.5 -1
input_data = {"target":y,"input":X}
prior_dict = {"name":"rhorseshoe_2"}
v_generator =wrap_V_class_with_input_data(class_constructor=V_logit,input_data=input_data,prior_dict=prior_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=4,num_cpu=1,thin=1,tune_l_per_chain=1000,
warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],
"metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
# input_dict = {"v_fun":[v_generator],"epsilon":[0.1],"second_order":[False],"evolve_L":[10],
# "metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
ep_dual_metadata_argument = {"name":"epsilon","target":0.9,"gamma":0.05,"t_0":10,
"kappa":0.75,"obj_fun":"accept_rate","par_type":"fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow",dim=v_generator(precision_type="torch.DoubleTensor").get_model_dim())]
def setup_sghmc_experiment(ep_list,L_list,eta_list,train_set,test_set,save_name,seed=1):
output_names = ["train_error", "test_error","train_error_sd","test_error_sd"]
output_store = numpy.zeros((len(ep_list),len(L_list),len(eta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)]+[4,13])
model_dict = {"num_units":35}
prior_dict = {"name":"normal"}
time_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)])
for i in range(len(ep_list)):
for j in range(len(L_list)):
for k in range(len(eta_list)):
start_time = time.time()
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor")
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(V=v_obj, metric=metric_obj)
full_data = train_set
init_q_point = point(V=v_obj)
store,explode_grad = sghmc_sampler(init_q_point=init_q_point, epsilon=ep_list[i], L=L_list[j], Ham=Ham, alpha=0.01, eta=eta_list[k],
betahat=0, full_data=full_data, num_samples=2000, thin=0, burn_in=1000,
batch_size=25)
total_time = time.time() - start_time
if not explode_grad:
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
test_mcmc_samples = store.numpy()
te1, predicted1,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
train_error, predicted1, train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
else:
train_error = 2
te1 = 2
train_error_sd = 2
te_sd = 2
output_store[i,j,k,0] = train_error
output_store[i,j,k,1] = te1
output_store[i,j,k,2] = train_error_sd
output_store[i,j,k,3] = te_sd
time_store[i,j,k] = total_time
to_store = {"diagnostics":diagnostics_store,"output":output_store,"output_names":output_names,"seed":seed,
"ep_list":ep_list,"L_list":L_list,"eta_list":eta_list,"num_units":model_dict["num_units"],
"prior":prior_dict["name"],"total_store":time_store}
numpy.savez(save_name,**to_store)
return()
# hmc need to add jitter (0.9 ep, 1.1 ep)
num_repeats = 2
num_grid_divides = 3
ep_bounds = [1e-3, 0.2]
evolve_t_bounds = [0.15, 50.]
evolve_L_bounds = [5, 1000]
# add constraints such that L = round(evolove_t/ep) < 1024
ep_list = list(numpy.linspace(ep_bounds[0], ep_bounds[1], num_grid_divides))
evolve_t_list = list(
numpy.linspace(evolve_t_bounds[0], evolve_t_bounds[1], num_grid_divides))
evolve_L_list = list(
numpy.linspace(evolve_L_bounds[0], evolve_L_bounds[1], num_grid_divides))
input_data = {"input": wishart_for_cov(dim=10)}
V_mvn1 = wrap_V_class_with_input_data(class_constructor=V_mvn,
input_data=input_data)
V_mvn2 = wrap_V_class_with_input_data(class_constructor=V_mvn,
input_data=input_data)
target_fun = fun_extract_median_ess
v_fun_list = [V_mvn1, V_mvn2]
# grid computations
# experiment_setting = experiment_setting_dict(chain_length=300,num_chains_per_sampler=4,warm_up=150,
# tune_l=0,allow_restart=True,max_num_restarts=5,num_cpu_per_sampler=4)
#
input_dict = {
"v_fun": v_fun_list[0:1],
"epsilon": ep_list,
"second_order": [False],
"stepsize_jitter": [True],
"evolve_L": evolve_L_list,
import os, numpy,torch
import dill as pickle
from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict
from adapt_util.tune_param_classes.tune_param_setting_util import *
from experiments.experiment_obj import tuneinput_class
from experiments.correctdist_experiments.prototype import check_mean_var_stan
from post_processing.ESS_nuts import ess_stan,diagnostics_stan
from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics,get_params_mcmc_tensor,get_short_diagnostics
from input_data.convert_data_to_dict import get_data_dict
from experiments.neural_net_experiments.gibbs_vs_joint_sampling.V_hierarchical_fc1 import V_fc_gibbs_model_1
from post_processing.test_error import test_error
input_data = get_data_dict("8x8mnist")
input_data = {"input":input_data["input"][:500,],"target":input_data["target"][:500]}
model_dict = {"num_units":15}
V_fun = wrap_V_class_with_input_data(class_constructor=V_fc_gibbs_model_1,input_data=input_data,model_dict=model_dict)
prior_dict = {"name":"gaussian_inv_gamma_1"}
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=2000,num_chains=1,num_cpu=1,thin=1,tune_l_per_chain=1000,
warmup_per_chain=1100,is_float=False,isstore_to_disk=False,allow_restart=False,seed=133)
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
# input_dict = {"v_fun":[V_fun],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],"xhmc_delta":[0.1],
# "metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["xhmc"]}
input_dict = {"v_fun":[V_fun],"epsilon":["dual"],"second_order":[False],
"metric_name":["unit_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
def setup_xhmc_gnuts_experiment(xhmc_delta_list,train_set,test_set,save_name,seed=1):
xhmc_delta_list.append(0)
output_names = ["train_error", "test_error","train_error_sd","test_error_sd","min_ess","median_ess"]
output_store = numpy.zeros((len(xhmc_delta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(xhmc_delta_list)]+[4,13])
prior_dict = {"name": "normal"}
model_dict = {"num_units": 35}
time_list = []
for i in range(len(xhmc_delta_list)):
start_time = time.time()
v_fun = V_fc_model_4
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=train_set,prior_dict=prior_dict,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000, is_float=False, isstore_to_disk=False,
allow_restart=True,seed=seed+i+1)
if i == len(xhmc_delta_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"],"xhmc_delta":[xhmc_delta_list[i]]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
# #
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
total_time = time.time() - start_time
np_diagnostics,feature_names = sampler1.np_diagnostics()
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
train_error,_,train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
output_store[i,0] = train_error
output_store[i,1] = te
output_store[i,2] = train_error_sd
output_store[i,3] = te_sd
diagnostics_store[i,:,:] = np_diagnostics
output_store[i,4] = np_diagnostics[0,10]
output_store[i,5] = np_diagnostics[0,11]
time_list.append(total_time)
to_store = {"diagnostics":diagnostics_store,"output":output_store,"diagnostics_names":feature_names,
"output_names":output_names,"seed":seed,"xhmc_delta_list":xhmc_delta_list,"prior":prior_dict["name"],
"num_units":model_dict["num_units"],"time_list":time_list}
numpy.savez(save_name,**to_store)
return()
from input_data.convert_data_to_dict import get_data_dict
from post_processing.get_diagnostics import energy_diagnostics, process_diagnostics, get_params_mcmc_tensor, get_short_diagnostics
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000,
num_chains=1,
num_cpu=1,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=100,
is_float=False,
isstore_to_disk=False,
allow_restart=False)
input_data = get_data_dict("sp500")
V_generator = wrap_V_class_with_input_data(
class_constructor=V_stochastic_volatility, input_data=input_data)
input_dict = {
"v_fun": [V_generator],
"epsilon": [0.001],
"second_order": [False],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["gnuts"]
}
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings([], [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
