def run_nn_experiment(xhmc_delta_list,input_data,v_fun,test_set,type_problem):
out_list = [None]*(len(xhmc_delta_list)+1)
for i in range(len(out_list)):
model_dict = {"num_units": 50}
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=input_data,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=1000,
warmup_per_chain=1100, is_float=False, isstore_to_disk=False,
allow_restart=False)
if i<len(out_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],"xhmc_delta":[xhmc_delta_list[i]],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
#
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
# tune_settings_dict = tuning_settings([],[],[],[])
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
diagnostics_np = sampler1.np_diagnostics()
samples_mixed = sampler1.get_samples(permuted=True)
te = test_error(target_dataset=test_set,v_obj=v_generator("torch.DoubleTensor"),mcmc_samples=samples_mixed,type=type_problem)
out = {"test_error":te,"diagnostics":diagnostics_np}
out_list.append(out)
te_store = numpy.zeros(len(out_list))
diagnostics_store = numpy.zeros(shape=[len(out_list)]+list(diagnostics_np.shape))
for i in range(len(out_list)):
te_store[i] = out_list[i]["test_error"]
diagnostics_store[i,...] = out_list[i]["diagnostics"]
output_store = {"test_error":te_store,"diagnostics":diagnostics_store}
save_name = "xhmc_v_gnuts_8x8mnist.npz"
numpy.savez(save_name,**output_store)
return(save_name)
def gradient_descent(number_of_iter,
lr,
v_obj,
validation_set,
validate_interval=10):
# random initialization
init_point = point(V=v_obj)
init_point.flattened_tensor.normal_()
init_point.load_flatten()
theta = init_point.point_clone()
store_v = []
best_validate_error = 10
explode_grad = False
till_validate = validate_interval
validate_continue = True
for cur in range(number_of_iter):
print("iter {}".format(cur))
if not validate_continue:
break
else:
#print(cur)
cur_v = v_obj.evaluate_scalar(theta)
print("v val {}".format(cur_v))
#print(theta.flattened_tensor)
grad, explode_grad = v_obj.dq(theta.flattened_tensor)
if not explode_grad:
theta.flattened_tensor -= lr * grad
theta.load_flatten()
store_v.append(v_obj.evaluate_scalar(theta))
if till_validate == 0:
temp_mcmc_samples = numpy.zeros(
(1, len(theta.flattened_tensor)))
temp_mcmc_samples[0, :] = theta.flattened_tensor.numpy()
validate_error, _, _ = test_error(
target_dataset=validation_set,
v_obj=v_obj,
mcmc_samples=temp_mcmc_samples,
type="classification")
print("validate error {}".format(validate_error))
if validate_error > best_validate_error:
validate_continue = False
else:
till_validate = validate_interval
best_validate_error = validate_error
else:
till_validate -= 1
diff = abs(store_v[-1] - cur_v)
if diff < 1e-6:
break
else:
break
return (theta, explode_grad)
def setup_xhmc_gnuts_experiment(xhmc_delta_list,train_set,test_set,save_name,seed=1):
xhmc_delta_list.append(0)
output_names = ["train_error", "test_error","train_error_sd","test_error_sd","min_ess","median_ess"]
output_store = numpy.zeros((len(xhmc_delta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(xhmc_delta_list)]+[4,13])
prior_dict = {"name": "normal"}
model_dict = {"num_units": 35}
time_list = []
for i in range(len(xhmc_delta_list)):
start_time = time.time()
v_fun = V_fc_model_4
v_generator = wrap_V_class_with_input_data(class_constructor=v_fun, input_data=train_set,prior_dict=prior_dict,
model_dict=model_dict)
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=2000, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000, is_float=False, isstore_to_disk=False,
allow_restart=True,seed=seed+i+1)
if i == len(xhmc_delta_list)-1:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["gnuts"]}
else:
input_dict = {"v_fun": [v_generator], "epsilon": ["dual"], "second_order": [False], "cov": ["adapt"],
"max_tree_depth": [8],
"metric_name": ["diag_e"], "dynamic": [True], "windowed": [False], "criterion": ["xhmc"],"xhmc_delta":[xhmc_delta_list[i]]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.9, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
# #
adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=v_generator(
precision_type="torch.DoubleTensor").get_model_dim())]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], adapt_cov_arguments, other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
total_time = time.time() - start_time
np_diagnostics,feature_names = sampler1.np_diagnostics()
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
train_error,_,train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed, type="classification", memory_efficient=False)
output_store[i,0] = train_error
output_store[i,1] = te
output_store[i,2] = train_error_sd
output_store[i,3] = te_sd
diagnostics_store[i,:,:] = np_diagnostics
output_store[i,4] = np_diagnostics[0,10]
output_store[i,5] = np_diagnostics[0,11]
time_list.append(total_time)
to_store = {"diagnostics":diagnostics_store,"output":output_store,"diagnostics_names":feature_names,
"output_names":output_names,"seed":seed,"xhmc_delta_list":xhmc_delta_list,"prior":prior_dict["name"],
"num_units":model_dict["num_units"],"time_list":time_list}
numpy.savez(save_name,**to_store)
return()
input_data = {
"input": input_data["input"][:5000, :],
"target": input_data["target"][:5000]
}
#input_data = get_data_dict("mnist")
prior_dict = {"name": "normal"}
model_dict = {"num_units": 300}
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1,
input_data=input_data,
prior_dict=prior_dict,
model_dict=model_dict)
out, explode_grad = gradient_descent(
number_of_iter=5000,
lr=0.01,
v_obj=v_generator(precision_type="torch.DoubleTensor"))
#print(out.flattened_tensor)
mcmc_samples = torch.zeros(1, len(out.flattened_tensor))
mcmc_samples[0, :] = out.flattened_tensor
mcmc_samples = mcmc_samples.numpy()
te, predicted = test_error(
target_dataset=input_data,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples,
type="classification")
print(te)
out = sampler1.get_diagnostics(permuted=False)
print("divergent")
processed_diag = process_diagnostics(out,name_list=["divergent"])
print(processed_diag.sum(axis=1))
#print(processed_diag.shape)
#processed_energy = process_diagnostics(out,name_list=["prop_H"])
print(energy_diagnostics(diagnostics_obj=out))
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
#target_dataset = get_data_dict("8x8mnist")
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1,input_data=input_data,prior_dict=prior_dict,model_dict=model_dict)
precision_type = "torch.DoubleTensor"
te2,predicted2 = test_error(input_data,v_obj=v_generator(precision_type=precision_type),mcmc_samples=mcmc_samples_mixed,type="classification",memory_efficient=False)
print(te2)
mixed_mcmc_tensor = sampler1.get_samples(permuted=True)
print(mixed_mcmc_tensor)
mcmc_cov = numpy.cov(mixed_mcmc_tensor,rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
print("mcmc problem difficulty")
print(max(mcmc_sd_vec)/min(mcmc_sd_vec)) # val = 2.25
def setup_gibbs_v_joint_experiment(num_units_list,
train_set,
test_set,
num_samples,
save_name,
seed=1):
output_names = [
"train_error", "test_error", "train_error_sd", "test_error_sd",
"sigma_2_ess", "mean_sigma2", "median_sigma2", "min_ess", "median_ess"
]
output_store = numpy.zeros((len(num_units_list), 3, len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_units_list), 3] + [4, 13])
time_store = numpy.zeros(shape=[len(num_units_list), 3])
for i in range(len(num_units_list)):
for j in range(3):
start_time = time.time()
v_fun = V_fc_model_4
model_dict = {"num_units": num_units_list[i]}
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000 +
num_samples,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
seed=seed + i + 1)
if j == 2:
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_gibbs_model_1,
input_data=train_set,
model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor",
gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = torch.abs(torch.randn(1)) + 3
log_obj = log_class()
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(1, num_samples + 1000, dim)
mcmc_samples_hyper = torch.zeros(1, num_samples + 1000, 1)
for iter in range(num_samples + 1000):
print("iter {}".format(iter))
outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step(
init_q_point, init_hyperparam, Ham, 0.01, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
init_q_point.load_flatten()
init_hyperparam = out_hyperparam
mcmc_samples_weight[
0, iter, :] = outq.flattened_tensor.clone()
mcmc_samples_hyper[0, iter, 0] = out_hyperparam
mcmc_samples_weight = mcmc_samples_weight[:, 1000:, :].numpy()
mcmc_samples_hyper = mcmc_samples_hyper[:, 1000:, :].numpy()
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
sigma2_diagnostics = diagnostics_stan(mcmc_samples_hyper)
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
mcmc_samples_hyper)
posterior_median_hidden_in_sigma2 = numpy.median(
mcmc_samples_hyper)
weight_ess = diagnostics_stan(mcmc_samples_weight)["ess"]
min_ess = min(sigma2_ess, min(weight_ess))
median_ess = numpy.median([sigma2_ess] + list(weight_ess))
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
output_store[i, j, 7] = min_ess
output_store[i, j, 8] = median_ess
elif j == 0:
prior_dict = {"name": "gaussian_inv_gamma_1"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
elif j == 1:
prior_dict = {"name": "gaussian_inv_gamma_2"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
if j == 0 or j == 1:
input_dict = {
"v_fun": [v_generator],
"epsilon": ["dual"],
"second_order": [False],
"max_tree_depth": [8],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["xhmc"],
"xhmc_delta": [0.1]
}
ep_dual_metadata_argument = {
"name": "epsilon",
"target": 0.9,
"gamma": 0.05,
"t_0": 10,
"kappa": 0.75,
"obj_fun": "accept_rate",
"par_type": "fast"
}
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [],
other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
np_diagnostics, feature_names = sampler1.np_diagnostics()
mcmc_samples_hidden_in = sampler1.get_samples_alt(
prior_obj_name="hidden_in", permuted=False)
samples = mcmc_samples_hidden_in["samples"]
hidden_in_sigma2_indices = mcmc_samples_hidden_in[
"indices_dict"]["sigma2"]
sigma2_diagnostics = diagnostics_stan(
samples[:, :, hidden_in_sigma2_indices])
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
posterior_median_hidden_in_sigma2 = numpy.median(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
diagnostics_store[i, j, :, :] = np_diagnostics
output_store[i, j, 7] = np_diagnostics[0, 10]
output_store[i, j, 8] = np_diagnostics[0, 11]
total_time = time.time() - start_time()
time_store[i, j] = total_time
to_store = {
"diagnostics": diagnostics_store,
"output": output_store,
"diagnostics_names": feature_names,
"output_names": output_names,
"seed": seed,
"num_units_list": num_units_list,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
print("energy diagnostics")
print(energy_diagnostics(diagnostics_obj=out))
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
target_dataset = get_data_dict("pima_indian")
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1,
input_data=input_data,
prior_dict=prior_dict,
model_dict=model_dict)
precision_type = "torch.DoubleTensor"
te1, predicted1 = test_error(target_dataset,
v_obj=v_generator(precision_type=precision_type),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
print(te1)
mixed_mcmc_tensor = sampler1.get_samples(permuted=True)
print(mixed_mcmc_tensor)
mcmc_cov = numpy.cov(mixed_mcmc_tensor, rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
print("mcmc problem difficulty")
print(max(mcmc_sd_vec) / min(mcmc_sd_vec)) # val = 3.66
#
# v_obj = v_generator(precision_type="torch.DoubleTensor")
# v_obj.flattened_tensor.copy_(torch.from_numpy(mcmc_samples_mixed[605,:]))
# v_obj.load_flattened_tensor_to_param()
# print(v_obj.forward())
#
# #print((v_obj.flattened_tensor*v_obj.flattened_tensor).sum())
# out = v_obj.predict(inputX=input_data["input"])
# prediction = torch.max(out,1)[1]
# #print(out.shape)
# #print(out)
# print(prediction[60:80].numpy())
# print(input_data["target"][60:80])
te2, predicted2 = test_error(test_set,
v_obj=v_generator(precision_type=precision_type),
mcmc_samples=mcmc_samples_mixed,
type="regression",
memory_efficient=False)
print(te2)
mixed_mcmc_tensor = sampler1.get_samples(permuted=True)
print(mixed_mcmc_tensor)
mcmc_cov = numpy.cov(mixed_mcmc_tensor, rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
print("mcmc problem difficulty")
print(max(mcmc_sd_vec) / min(mcmc_sd_vec)) # val = 1.1
print("num hit max tree depth after warmup")
processed_diag = process_diagnostics(out, name_list=["hit_max_tree_depth"])
print(processed_diag.sum(axis=1))
print("average number of leapfrog steps after warmup")
processed_diag = process_diagnostics(out, name_list=["num_transitions"])
print(processed_diag.mean(axis=1))
#processed_energy = process_diagnostics(out,name_list=["prop_H"])
print("energy diagnostics")
print(energy_diagnostics(diagnostics_obj=out))
mixed_mcmc_tensor = sampler1.get_samples(permuted=True)
print(mixed_mcmc_tensor)
mcmc_cov = numpy.cov(mixed_mcmc_tensor, rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
print("mcmc problem difficulty")
print(max(mcmc_sd_vec) / min(mcmc_sd_vec)) # val = 1.82
test_mcmc_samples = sampler1.get_samples(permuted=True)
te2, predicted2 = test_error(input_data,
v_obj=V_fun(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples,
type="classification",
memory_efficient=False)
print(te2)
def setup_sghmc_experiment(ep_list,L_list,eta_list,train_set,test_set,save_name,seed=1):
output_names = ["train_error", "test_error","train_error_sd","test_error_sd"]
output_store = numpy.zeros((len(ep_list),len(L_list),len(eta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)]+[4,13])
model_dict = {"num_units":35}
prior_dict = {"name":"normal"}
time_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)])
for i in range(len(ep_list)):
for j in range(len(L_list)):
for k in range(len(eta_list)):
start_time = time.time()
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor")
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(V=v_obj, metric=metric_obj)
full_data = train_set
init_q_point = point(V=v_obj)
store,explode_grad = sghmc_sampler(init_q_point=init_q_point, epsilon=ep_list[i], L=L_list[j], Ham=Ham, alpha=0.01, eta=eta_list[k],
betahat=0, full_data=full_data, num_samples=2000, thin=0, burn_in=1000,
batch_size=25)
total_time = time.time() - start_time
if not explode_grad:
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
test_mcmc_samples = store.numpy()
te1, predicted1,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
train_error, predicted1, train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
else:
train_error = 2
te1 = 2
train_error_sd = 2
te_sd = 2
output_store[i,j,k,0] = train_error
output_store[i,j,k,1] = te1
output_store[i,j,k,2] = train_error_sd
output_store[i,j,k,3] = te_sd
time_store[i,j,k] = total_time
to_store = {"diagnostics":diagnostics_store,"output":output_store,"output_names":output_names,"seed":seed,
"ep_list":ep_list,"L_list":L_list,"eta_list":eta_list,"num_units":model_dict["num_units"],
"prior":prior_dict["name"],"total_store":time_store}
numpy.savez(save_name,**to_store)
return()
print("debug")
#print((v_obj.flattened_tensor*v_obj.flattened_tensor).sum())
v_obj = v_generator(precision_type="torch.DoubleTensor")
v_obj.flattened_tensor.copy_(torch.from_numpy(mcmc_samples_mixed[605,:]))
v_obj.load_flattened_tensor_to_param()
print(v_obj.forward())
#print((v_obj.flattened_tensor*v_obj.flattened_tensor).sum())
out = v_obj.predict(inputX=input_data["input"])
prediction = torch.max(out,1)[1]
#print(out.shape)
#print(out)
print(prediction[60:80].numpy())
print(input_data["target"][60:80])
te2,predicted2 = test_error(test_set,v_obj=v_generator(precision_type=precision_type),mcmc_samples=test_mcmc_samples,type="classification",memory_efficient=False)
print(te2)
mixed_mcmc_tensor = sampler1.get_samples(permuted=True)
print(mixed_mcmc_tensor)
mcmc_cov = numpy.cov(mixed_mcmc_tensor,rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
print("mcmc problem difficulty")
print(max(mcmc_sd_vec)/min(mcmc_sd_vec)) # val = 1.1
def setup_ensemble_experiment(num_unit_list,
list_num_ensemble_pts,
train_set,
validate_set,
test_set,
save_name,
seed=1):
output_names = [
"ensemble_train_error", "ensemble_te", "ensemble_train_error_sd",
"ensemble_te_sd"
]
output_store = numpy.zeros(
(len(num_unit_list), len(list_num_ensemble_pts), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_unit_list)] + [4, 13])
numpy.random.seed(seed)
torch.manual_seed(seed)
diver_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts)))
time_store = numpy.zeros((len(num_unit_list), len(list_num_ensemble_pts)))
for i in range(len(num_unit_list)):
model_dict = {"num_units": num_unit_list[i]}
for k in range(len(list_num_ensemble_pts)):
start_time = time.time()
prior_dict = {"name": "normal"}
num_ensemble_pts = list_num_ensemble_pts[k]
num_diver = 0
ensemble_list = []
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_model_4,
prior_dict=prior_dict,
input_data=train_set,
model_dict=model_dict)
for j in range(num_ensemble_pts):
out, explode_grad = gradient_descent(
number_of_iter=2000,
lr=0.001,
validation_set=validate_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"))
if explode_grad:
num_diver += 1
else:
ensemble_list.append(out.point_clone())
ensemble_pts = numpy.zeros(
(len(ensemble_list), len(ensemble_list[0].flattened_tensor)))
for z in range(len(ensemble_list)):
ensemble_pts[z, :] = ensemble_list[z].flattened_tensor.numpy()
ensemble_te, predicted, ensemble_te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=ensemble_pts,
type="classification",
memory_efficient=False)
ensemble_train_error, _, ensemble_train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=ensemble_pts,
type="classification",
memory_efficient=False)
total_time = time.time() - start_time
output_store[i, k, 0] = ensemble_train_error
output_store[i, k, 1] = ensemble_te
output_store[i, k, 2] = ensemble_train_error_sd
output_store[i, k, 3] = ensemble_te_sd
diver_store[i, k] = num_diver
time_store[i, k] = total_time
#print(output_store)
to_store = {
"output": output_store,
"output_names": output_names,
"num_unit_list": num_unit_list,
"seed": seed,
"list_num_ensemble_pts": list_num_ensemble_pts,
"num_div": diver_store,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
#print(store_samples)
sigma2_tensor = numpy.zeros(
(1, store_samples.shape[0], store_samples.shape[1]))
sigma2_tensor[0, :, :] = numpy.exp(store_samples)
print(diagnostics_stan(mcmc_samples_tensor=sigma2_tensor))
print("sigma2 diagnostics gibbs")
print(numpy.mean(mcmc_samples_hyper))
print(numpy.var(mcmc_samples_hyper))
sigma2_tensor = numpy.zeros((1, len(mcmc_samples_hyper), 1))
sigma2_tensor[0, :, 0] = mcmc_samples_hyper
print(diagnostics_stan(mcmc_samples_tensor=sigma2_tensor))
print("weight diagnostics gibbs")
print(numpy.mean(mcmc_samples_weight, axis=0))
weight_tensor = numpy.zeros(
(1, mcmc_samples_weight.shape[0], mcmc_samples_weight.shape[1]))
weight_tensor[0, :, :] = mcmc_samples_weight
print(diagnostics_stan(mcmc_samples_tensor=weight_tensor))
exit()
te1, predicted1 = test_error(data_dict,
v_obj=v_obj,
mcmc_samples=mcmc_samples,
type="classification",
memory_efficient=False)
print(te1)
#print(out.flattened_tensor)
from input_data.convert_data_to_dict import get_data_dict
from post_processing.test_error import map_prediction, test_error
from distributions.linear_regressions.linear_regression import V_linear_regression
import pickle
with open("debug_test_error_mcmc_regression.pkl", 'rb') as f:
mcmc_samples = pickle.load(f)
#print(mcmc_samples.shape)
target_dataset = get_data_dict("boston")
te1, predicted1 = test_error(target_dataset,
v_obj=V_linear_regression(),
mcmc_samples=mcmc_samples,
type="regression",
memory_efficient=False)
te2, predicted2 = test_error(target_dataset,
v_obj=V_linear_regression(),
mcmc_samples=mcmc_samples,
type="regression",
memory_efficient=True)
print(te1)
print(te2)
#print(sum(predicted1!=predicted2))