def min_ess_gnuts(v_fun, ep):
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=10000,
num_chains=4,
num_cpu=1,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
max_num_restarts=5)
tune_settings_dict = tuning_settings([], [], [], [])
input_dict = {
"v_fun": [v_fun],
"epsilon": [ep],
"second_order": [False],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [None],
"criterion": ["gnuts"]
}
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler.start_sampling()
out = get_min_ess_and_esjds(ran_sampler=sampler)
return (out)
def match_convergence_test(v_fun,seed,correct_mean,correct_cov):
mcmc_meta_double = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100, is_float=False, isstore_to_disk=False,
allow_restart=True, seed=seed)
mcmc_meta_float = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100, is_float=True, isstore_to_disk=False,
allow_restart=True, seed=seed)
input_dict = {"v_fun": [v_fun], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False],
"criterion": ["gnuts"]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.8, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
dim = len(v_fun(precision_type="torch.DoubleTensor").flattened_tensor)
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=dim)]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler_double = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta_double,
tune_settings_dict=tune_settings_dict)
sampler_float = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta_float,
tune_settings_dict=tune_settings_dict)
sampler_double.start_sampling()
sampler_float.start_sampling()
sampler_double.remove_failed_chains()
sampler_float.remove_failed_chains()
float_samples = sampler_float.get_samples(permuted=False)
double_samples = sampler_double.get_samples(permuted=False)
short_diagnostics_float = check_mean_var_stan(mcmc_samples=float_samples,
correct_mean=correct_mean.astype(numpy.float32),
correct_cov=correct_cov.astype(numpy.float32))
short_diagnostics_double = check_mean_var_stan(mcmc_samples=double_samples,correct_mean=correct_mean,correct_cov=correct_cov)
samples_double_cast_to_float = double_samples.astype(numpy.float32)
# samples_float = output_float["samples"]
# combined_samples = torch.cat([samples_double_cast_to_float,float_samples],dim=0)
combined_samples = numpy.concatenate([samples_double_cast_to_float, float_samples], axis=0)
short_diagnostics_combined = check_mean_var_stan(mcmc_samples=combined_samples,correct_mean=correct_mean.astype(numpy.float32),
correct_cov=correct_cov.astype(numpy.float32))
out = {"diag_combined_mean": short_diagnostics_combined["pc_of_mean"], "diag_float_mean": short_diagnostics_float["pc_of_mean"],
"diag_double_mean": short_diagnostics_double["pc_of_mean"],"diag_combined_cov":short_diagnostics_combined["pc_of_cov"],
"diag_float_cov":short_diagnostics_float["pc_of_cov"],"diag_double_cov":short_diagnostics_double["pc_of_cov"]}
return(out)
def run_nn_experiment(xhmc_delta_list,input_data,v_fun,test_set,type_problem):
out_list = [None]*(len(xhmc_delta_list)+1)
for i in range(len(out_list)):
model_dict = {"num_units": 50}
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=input_data,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100, is_float=False, isstore_to_disk=False,
allow_restart=False)
if i<len(out_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],"xhmc_delta":[xhmc_delta_list[i]],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
# tune_settings_dict = tuning_settings([],[],[],[])
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
diagnostics_np = sampler1.np_diagnostics()
samples_mixed = sampler1.get_samples(permuted=True)
te = test_error(target_dataset=test_set,v_obj=v_generator("torch.DoubleTensor"),mcmc_samples=samples_mixed,type=type_problem)
out = {"test_error":te,"diagnostics":diagnostics_np}
out_list.append(out)
te_store = numpy.zeros(len(out_list))
diagnostics_store = numpy.zeros(shape=[len(out_list)]+list(diagnostics_np.shape))
for i in range(len(out_list)):
te_store[i] = out_list[i]["test_error"]
diagnostics_store[i,...] = out_list[i]["diagnostics"]
output_store = {"test_error":te_store,"diagnostics":diagnostics_store}
save_name = "xhmc_v_gnuts_8x8mnist.npz"
numpy.savez(save_name,**output_store)
return(save_name)
def opt_experiment_ep_t(v_fun_list, ep_list, evolve_L_list, num_of_opt_steps,
objective, input_dict, max_L):
# given list of v_fun, epsilon, evolve_t's, number of bayesian optimization steps, an objective function
# and an input dict , find optimal (ep,t) by repeatedly trying different combinations , sample long chain and
# compare performance using different objective functions
assert objective in ("median_ess_normalized", "max_ess_normalized",
"min_ess_normalized", "median_ess", "max_ess",
"min_ess", "esjd", "esjd_normalized")
num_grid_divides = len(ep_list)
ep_bounds = [ep_list[0], ep_list[-1]]
evolve_L_bounds = [evolve_L_list[0], evolve_L_list[-1]]
chosen_init = [
ep_list[numpy.asscalar(numpy.random.choice(num_grid_divides, 1))],
evolve_L_list[numpy.asscalar(numpy.random.choice(num_grid_divides, 1))]
]
this_opt_state = opt_state(bounds=[ep_bounds, evolve_L_bounds],
init=chosen_init)
cur_ep = chosen_init[0]
cur_evolve_L = chosen_init[1]
input_dict.update({"epsilon": [cur_ep], "evolve_L": [cur_evolve_L]})
for j in range(num_of_opt_steps):
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=300,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=150,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
max_num_restarts=5)
tune_settings_dict = tuning_settings([], [], [], [])
input_dict.update({"epsilon": [cur_ep], "evolve_L": [cur_evolve_L]})
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler.start_sampling()
out = get_ess_and_esjds(ran_sampler=sampler)
#L = max(1,round(cur_evolve_t/cur_ep))
# need to use actual number of transitions
this_opt_state.update(new_y=-out[objective])
cur_ep = this_opt_state.X_step[-1][0]
cur_evolve_L = round(this_opt_state.X_step[-1][1])
return (this_opt_state)
def choose_optimal_L(v_fun, fixed_ep, L_list, windowed):
store_median_ess = [None] * len(L_list)
for i in range(len(L_list)):
L = L_list[i]
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=2000,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=0,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
max_num_restarts=5)
tune_settings_dict = tuning_settings([], [], [], [])
input_dict = {
"v_fun": [v_fun],
"epsilon": [fixed_ep],
"second_order": [False],
"evolve_L": [L],
"metric_name": ["unit_e"],
"dynamic": [False],
"windowed": [windowed],
"criterion": [None]
}
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler.start_sampling()
out = get_ess_and_esjds(ran_sampler=sampler)
#samples = sampler.get_samples(permuted=False)
store_median_ess[i] = out["median_ess"]
best_median_ess = max(store_median_ess)
best_L = L_list[numpy.argmax(store_median_ess)]
out = {"best_median_ess": best_median_ess, "best_L": best_L}
return (out)
def setup_xhmc_gnuts_experiment(xhmc_delta_list,train_set,test_set,save_name,seed=1):
xhmc_delta_list.append(0)
output_names = ["train_error", "test_error","train_error_sd","test_error_sd","min_ess","median_ess"]
output_store = numpy.zeros((len(xhmc_delta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(xhmc_delta_list)]+[4,13])
prior_dict = {"name": "normal"}
model_dict = {"num_units": 35}
time_list = []
for i in range(len(xhmc_delta_list)):
start_time = time.time()
v_fun = V_fc_model_4
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=train_set,prior_dict=prior_dict,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000, is_float=False, isstore_to_disk=False,
allow_restart=True,seed=seed+i+1)
if i == len(xhmc_delta_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"],"xhmc_delta":[xhmc_delta_list[i]]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
# #
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
total_time = time.time() - start_time
np_diagnostics,feature_names = sampler1.np_diagnostics()
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
train_error,_,train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
output_store[i,0] = train_error
output_store[i,1] = te
output_store[i,2] = train_error_sd
output_store[i,3] = te_sd
diagnostics_store[i,:,:] = np_diagnostics
output_store[i,4] = np_diagnostics[0,10]
output_store[i,5] = np_diagnostics[0,11]
time_list.append(total_time)
to_store = {"diagnostics":diagnostics_store,"output":output_store,"diagnostics_names":feature_names,
"output_names":output_names,"seed":seed,"xhmc_delta_list":xhmc_delta_list,"prior":prior_dict["name"],
"num_units":model_dict["num_units"],"time_list":time_list}
numpy.savez(save_name,**to_store)
return()
# input_dict = {"v_fun":[V_pima_inidan_logit],"epsilon":[0.1],"second_order":[False],
# "evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_dict = {"v_fun":[v_generator],"epsilon":["dual"],"second_order":[False],"cov":["adapt"],"max_tree_depth":[8],
"metric_name":["diag_e"],"dynamic":[True],"windowed":[False],"criterion":["gnuts"]}
# input_dict = {"v_fun":[v_generator],"epsilon":[0.1],"second_order":[False],"evolve_L":[10],
# "metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
ep_dual_metadata_argument = {"name":"epsilon","target":0.8,"gamma":0.05,"t_0":10,
"kappa":0.75,"obj_fun":"accept_rate","par_type":"fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow",dim=v_generator(precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
#tune_settings_dict = tuning_settings([],[],[],[])
tune_settings_dict = tuning_settings(dual_args_list,[],adapt_cov_arguments,other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,mcmc_settings_dict=mcmc_meta,tune_settings_dict=tune_settings_dict)
store_name = 'student2_fc1_cp_sampler.pkl'
sampled = False
if sampled:
sampler1 = pickle.load(open(store_name, 'rb'))
else:
sampler1.start_sampling()
with open(store_name, 'wb') as f:
pickle.dump(sampler1, f)
#out = sampler1.start_sampling()
def setup_gibbs_v_joint_experiment(num_units_list,
train_set,
test_set,
num_samples,
save_name,
seed=1):
output_names = [
"train_error", "test_error", "train_error_sd", "test_error_sd",
"sigma_2_ess", "mean_sigma2", "median_sigma2", "min_ess", "median_ess"
]
output_store = numpy.zeros((len(num_units_list), 3, len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_units_list), 3] + [4, 13])
time_store = numpy.zeros(shape=[len(num_units_list), 3])
for i in range(len(num_units_list)):
for j in range(3):
start_time = time.time()
v_fun = V_fc_model_4
model_dict = {"num_units": num_units_list[i]}
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000 +
num_samples,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
seed=seed + i + 1)
if j == 2:
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_gibbs_model_1,
input_data=train_set,
model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor",
gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = torch.abs(torch.randn(1)) + 3
log_obj = log_class()
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(1, num_samples + 1000, dim)
mcmc_samples_hyper = torch.zeros(1, num_samples + 1000, 1)
for iter in range(num_samples + 1000):
print("iter {}".format(iter))
outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step(
init_q_point, init_hyperparam, Ham, 0.01, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
init_q_point.load_flatten()
init_hyperparam = out_hyperparam
mcmc_samples_weight[
0, iter, :] = outq.flattened_tensor.clone()
mcmc_samples_hyper[0, iter, 0] = out_hyperparam
mcmc_samples_weight = mcmc_samples_weight[:, 1000:, :].numpy()
mcmc_samples_hyper = mcmc_samples_hyper[:, 1000:, :].numpy()
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
sigma2_diagnostics = diagnostics_stan(mcmc_samples_hyper)
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
mcmc_samples_hyper)
posterior_median_hidden_in_sigma2 = numpy.median(
mcmc_samples_hyper)
weight_ess = diagnostics_stan(mcmc_samples_weight)["ess"]
min_ess = min(sigma2_ess, min(weight_ess))
median_ess = numpy.median([sigma2_ess] + list(weight_ess))
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
output_store[i, j, 7] = min_ess
output_store[i, j, 8] = median_ess
elif j == 0:
prior_dict = {"name": "gaussian_inv_gamma_1"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
elif j == 1:
prior_dict = {"name": "gaussian_inv_gamma_2"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
if j == 0 or j == 1:
input_dict = {
"v_fun": [v_generator],
"epsilon": ["dual"],
"second_order": [False],
"max_tree_depth": [8],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["xhmc"],
"xhmc_delta": [0.1]
}
ep_dual_metadata_argument = {
"name": "epsilon",
"target": 0.9,
"gamma": 0.05,
"t_0": 10,
"kappa": 0.75,
"obj_fun": "accept_rate",
"par_type": "fast"
}
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [],
other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
np_diagnostics, feature_names = sampler1.np_diagnostics()
mcmc_samples_hidden_in = sampler1.get_samples_alt(
prior_obj_name="hidden_in", permuted=False)
samples = mcmc_samples_hidden_in["samples"]
hidden_in_sigma2_indices = mcmc_samples_hidden_in[
"indices_dict"]["sigma2"]
sigma2_diagnostics = diagnostics_stan(
samples[:, :, hidden_in_sigma2_indices])
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
posterior_median_hidden_in_sigma2 = numpy.median(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
diagnostics_store[i, j, :, :] = np_diagnostics
output_store[i, j, 7] = np_diagnostics[0, 10]
output_store[i, j, 8] = np_diagnostics[0, 11]
total_time = time.time() - start_time()
time_store[i, j] = total_time
to_store = {
"diagnostics": diagnostics_store,
"output": output_store,
"diagnostics_names": feature_names,
"output_names": output_names,
"seed": seed,
"num_units_list": num_units_list,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
correct_mean = correct["correct_mean"]
correct_cov = correct["correct_cov"]
correct_diag_cov = correct_cov.diagonal()
ideal_diag_cov = 1/(torch.from_numpy(correct_diag_cov).type("torch.FloatTensor"))
input_dict = {"v_fun":[V_pima_indian_logit],"epsilon":[0.1],"second_order":[False],
"evolve_L":[10],"metric_name":["diag_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_dict2 = {"v_fun":[V_pima_indian_logit],"epsilon":[0.1],"second_order":[False],
"evolve_L":[10],"metric_name":["diag_e"],"dynamic":[False],"windowed":[True],"criterion":[None]}
input_dict3 = {"v_fun":[V_pima_indian_logit],"epsilon":[0.1],"second_order":[False],"cov":[ideal_diag_cov],
"evolve_L":[10],"metric_name":["diag_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
tune_settings_dict = tuning_settings([],[],[],[])
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
tune_dict2 = tuneinput_class(input_dict2).singleton_tune_dict()
tune_dict3 = tuneinput_class(input_dict3).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,mcmc_settings_dict=mcmc_meta,tune_settings_dict=tune_settings_dict)
sampler2 = mcmc_sampler(tune_dict=tune_dict2,mcmc_settings_dict=mcmc_meta,tune_settings_dict=tune_settings_dict)
sampler3 = mcmc_sampler(tune_dict=tune_dict3,mcmc_settings_dict=mcmc_meta,tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
sampler2.start_sampling()
sampler3.start_sampling()
mcmc_samples1 = sampler1.get_samples(permuted=False)
out = check_mean_var_stan(mcmc_samples=mcmc_samples1,correct_mean=correct_mean,correct_cov=correct_cov,diag_only=False)
max_num_restarts=5,
num_cpu_per_sampler=4)
input_dict_gnuts = {
"v_fun": v_fun_list,
"epsilon": ["dual"],
"second_order": [False],
"cov": ["adapt"],
"metric_name": ["diag_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["gnuts"],
"max_tree_depth": [8]
}
input_object_gnuts = tuneinput_class(input_dict_gnuts)
experiment_instance_gnuts = experiment(
input_object=input_object_gnuts,
experiment_setting=experiment_setting_gnuts,
fun_per_sampler=target_fun)
experiment_instance_gnuts.run()
np_store, col_names, output_names = experiment_instance_gnuts.np_output()
np_store_diagnostics, diagnostics_names = experiment_instance_gnuts.np_diagnostics(
)
np_diagnostics_gnuts[i] = np_store_diagnostics
np_store_gnuts[i] = np_store
np_store_gnuts = numpy.stack(np_store_gnuts, axis=0)
np_diagnostics_gnuts = numpy.stack(np_diagnostics_gnuts, axis=0)
converted_t_bounds = (min(L_bounds)*min(ep_bounds),max(L_bounds)*max(ep_bounds))
ep_list = list(numpy.linspace(ep_bounds[0],ep_bounds[1],num_grid_divides))
evolve_L_list = list(numpy.linspace(L_bounds[0],L_bounds[1],num_grid_divides))
evolve_t_list = list(numpy.linspace(converted_t_bounds[0],converted_t_bounds[1],num_grid_divides))
#print(converted_t_bounds)
#####################################################################################################################################
experiment_setting_ep_L = experiment_setting_dict(chain_length=10000,num_chains_per_sampler=4,warm_up=1000,
tune_l=0,allow_restart=True,max_num_restarts=5)
input_dict_ep_L = {"v_fun":v_fun_list,"epsilon":ep_list,"second_order":[False],
"evolve_t":evolve_t_list,"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_object_ep_L = tuneinput_class(input_dict_ep_L)
experiment_instance_ep_L = experiment(input_object=input_object_ep_L,experiment_setting=experiment_setting_ep_L,fun_per_sampler=function)
experiment_instance_ep_L.run()
result_grid_ep_L= experiment_instance_ep_L.experiment_result_grid_obj
##########################################################################################################################################
experiment_setting_ep_t = experiment_setting_dict(chain_length=10000,num_chains_per_sampler=4,warm_up=1000,
tune_l=0,allow_restart=True,max_num_restarts=5)
input_dict_ep_t = {"v_fun":v_fun_list,"epsilon":ep_list,"second_order":[False],
"evolve_t":evolve_t_list,"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_object_ep_t = tuneinput_class(input_dict_ep_t)
experiment_instance_ep_t = experiment(input_object=input_object_ep_t,experiment_setting=experiment_setting_ep_t,fun_per_sampler=function)
num_grid_divides = 2
ep_list = list(numpy.linspace(1e-2, 0.1, num_grid_divides))
evolve_t_list = list(numpy.linspace(0.15, 5.0, num_grid_divides))
v_fun_list = []
input_dict = {
"v_fun": v_fun_list,
"epsilon": ep_list,
"second_order": [False],
"evolve_t": evolve_t_list,
"metric_name": ["unit_e"],
"dynamic": [False],
"windowed": [False],
"criterion": [None]
}
experiment_setting = experiment_setting_dict(chain_length=10000,
num_repeat=20,
num_chains_per_sampler=4,
warm_up=1000,
tune_l=0,
save_name="temp_experiment.pkl")
input_object = tuneinput_class(input_dict)
experiment_instance = experiment(input_object=input_object,
experiment_setting=experiment_setting,
fun_per_sampler=function)
experiment.run()
def generate_q_list(v_fun,num_of_pts):
# extract number of (q,p) points given v_fun
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=200, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=100,
warmup_per_chain=110, is_float=False, isstore_to_disk=False,allow_restart=True)
input_dict = {"v_fun": [v_fun], "epsilon": ["dual"], "second_order": [False],
"metric_name": ["unit_e"], "dynamic": [True], "windowed": [False],"max_tree_depth":[6],
"criterion": ["xhmc"],"xhmc_delta":[0.1]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.65, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
dim = len(v_fun(precision_type="torch.DoubleTensor").flattened_tensor)
#adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=dim)]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict)
out = sampler1.start_sampling()
sampler1.remove_failed_chains()
print("num chains removed {}".format(sampler1.metadata.num_chains_removed))
print("num restarts {}".format(sampler1.metadata.num_restarts))
samples = sampler1.get_samples(permuted=True)
num_mcmc_samples = samples.shape[0]
indices = numpy.random.choice(a=num_mcmc_samples,size=num_of_pts,replace=False)
chosen_samples = samples[indices,:]
list_q_double = [None]*num_of_pts
#list_p_double = [None]*num_of_pts
for i in range(num_of_pts):
q_point = point(V=v_fun(precision_type="torch.DoubleTensor"))
flattened_tensor = torch.from_numpy(chosen_samples[i,:]).type("torch.DoubleTensor")
q_point.flattened_tensor.copy_(flattened_tensor)
q_point.load_flatten()
#p_point = point(list_tensor=q_point.list_tensor,pointtype="p",need_flatten=q_point.need_flatten)
# p_point.flattened_tensor.normal_()
# p_point.load_flatten()
list_q_double[i] = q_point
#list_p_double[i] = p_point
list_q_float = [None] * num_of_pts
#list_p_float = [None] * num_of_pts
for i in range(num_of_pts):
q_point = point(V=v_fun(precision_type="torch.FloatTensor"))
flattened_tensor = torch.from_numpy(chosen_samples[i, :]).type("torch.FloatTensor")
q_point.flattened_tensor.copy_(flattened_tensor)
q_point.load_flatten()
# p_point = point(list_tensor=q_point.list_tensor,pointtype="p",need_flatten=q_point.need_flatten)
# p_point.flattened_tensor.normal_()
# p_point.load_flatten()
list_q_float[i] = q_point
out = {"list_q_double":list_q_double,"list_q_float":list_q_float}
return(out)
from experiments.experiment_obj import tuneinput_class, experiment
#input_dict = {"v_fun":[V_logistic_regression],"alpha":[0],"epsilon":[0.1,0.2,0.3],"second_order":[False],"Cov":[torch.zeros(2)]}
input_dict = {
"v_fun": [V_logistic_regression],
"epsilon": [0.1, 0.05],
"second_order": [False],
"evolve_L": [10],
"metric_name": ["unit_e"],
"dynamic": [False],
"windowed": [False],
"criterion": [None]
}
input_obj = tuneinput_class(input_dict)
#print(input_obj.__dict__["grid_shape"])
#exit()
exper_obj = experiment(input_object=input_obj)
exper_obj.run()
#print(len(exper_obj.id_to_multi_index))
print(exper_obj.id_to_multi_index[0])
print(exper_obj.id_to_multi_index[1])
print(exper_obj.store_grid_obj[exper_obj.id_to_multi_index[0]])
print(exper_obj.store_grid_obj[exper_obj.id_to_multi_index[1]])
#exit()
#print(exper_obj.store_grid_obj[0,0,0,0,0])
from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict
from adapt_util.tune_param_classes.tune_param_setting_util import *
from experiments.experiment_obj import tuneinput_class
from experiments.experiment_obj import experiment,experiment_setting_dict
from experiments.correctdist_experiments.prototype import check_mean_var
num_per_model = 20
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,samples_per_chain=500,num_chains=1,num_cpu=1,thin=1,tune_l_per_chain=0,
warmup_per_chain=100,is_float=False,isstore_to_disk=False)
input_dict = {"v_fun":[V_funnel_cp],"epsilon":[0.1],"alpha":[1e6,1e2],"second_order":[True],
"evolve_L":[10],"metric_name":["softabs"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_dict2 = {"v_fun":[V_funnel_ncp],"epsilon":[0.1],"second_order":[False],
"evolve_L":[10],"metric_name":["unit_e"],"dynamic":[False],"windowed":[False],"criterion":[None]}
input_obj = tuneinput_class(input_dict)
input_obj2 = tuneinput_class(input_dict2)
experiment_setting_dict = experiment_setting_dict(chain_length=10000,num_repeat=num_per_model)
experiment_obj = experiment(input_object=input_obj,experiment_setting=experiment_setting_dict)
experiment_obj.run()
experiment_obj2 = experiment(input_object=input_obj2,experiment_setting=experiment_setting_dict)
experiment_obj2.run()
"second_order": [True],
"xhmc_delta": [0.1],
"metric_name": [
"softabs", "softabs_diag", "softabs_outer_product",
"softabs_diag_outer_product"
],
"dynamic": [False],
"windowed": [False],
"criterion": ["xhmc"]
}
tune_settings_dict = tuning_settings([], [], [], [])
#tune_dict_fo = tuneinput_class(input_dict_fo).singleton_tune_dict()
#tune_dict_so = tuneinput_class(input_dict_so).singleton_tune_dict()
tune_dict_fo = tuneinput_class(input_dict_fo)
tune_dict_so = tuneinput_class(input_dict_so)
#sampler_fo = mcmc_sampler(tune_dict=tune_dict_fo,mcmc_settings_dict=mcmc_meta,tune_settings_dict=tune_settings_dict)
#out = sampler1.start_sampling()
#####################################################################################################################
mcmc_samples = sampler1.get_samples(permuted=True)
correct = pickle.load(open("result_from_long_chain.pkl", 'rb'))
correct_mean = correct["correct_mean"]
correct_cov = correct["correct_cov"]
correct_diag_cov = correct_cov.diagonal()
out = check_mean_var(mcmc_samples=mcmc_samples,
correct_mean=correct_mean,
correct_cov=correct_cov,
