def default_init_q_point_list(v_fun,num_chains,same_init=False,precision_type="torch.DoubleTensor",seed=None):
v_obj = v_fun(precision_type=precision_type)
init_q_point_list = [None]*num_chains
assert not seed is None
torch.manual_seed(seed)
if same_init:
#print("yes")
temp_point = point(V=v_obj)
temp_point.flattened_tensor.copy_(torch.randn(len(temp_point.flattened_tensor)))
temp_point.load_flatten()
#print(temp_point.flattened_tensor)
#print(temp_point)
#exit()
for i in range(num_chains):
init_q_point_list[i] = temp_point.point_clone()
#print(init_q_point_list[i].flattened_tensor)
else:
for i in range(num_chains):
#temp_point = v_obj.q_point.point_clone()
temp_point = point(V=v_obj)
temp_point.flattened_tensor.copy_(torch.randn(len(temp_point.flattened_tensor)))
temp_point.load_flatten()
init_q_point_list[i] = temp_point
return(init_q_point_list)
def __init__(self, linkedV):
#self.metric = metric
self.linkedV = linkedV
self.dim = self.linkedV.dim
self.store_lens = self.linkedV.store_lens
self.store_shapes = self.linkedV.store_shapes
self.num_var = self.linkedV.num_var
self.list_tensor = numpy.empty(self.linkedV.num_var,
dtype=type(self.linkedV.list_tensor[0]))
for i in range(len(self.list_tensor)):
self.list_tensor[i] = torch.zeros(
self.linkedV.list_tensor[i].shape)
self.list_var = numpy.empty(self.linkedV.num_var,
dtype=type(self.linkedV.list_var[0]))
for i in range(len(self.list_var)):
self.list_var[i] = Variable(self.list_tensor[i],
requires_grad=False)
# need gradient for every metric except unit_e
self.gradient = numpy.empty(len(self.store_shapes),
dtype=torch.FloatTensor)
# definitely need faltten for softabs,
# for other metrics depend on
self.need_flatten = self.linkedV.need_flatten
if self.need_flatten:
self.flattened_gradient = torch.zeros(self.dim)
self.flattened_tensor = torch.zeros(self.dim)
else:
self.flattened_tensor = self.list_var[0].data
if self.metric.name == "unit_e":
self.flattened_gradient = self.flattened_tensor
self.p_point = point(T=self)
def generate_momentum(self, q):
dV = self.linkedV.getdV_tensor(q)
msoftabsalpha = self.metric.msoftabsalpha
gg = torch.dot(dV, dV)
agg = msoftabsalpha * gg
#print(gg)
#print(agg)
exit()
dV = dV * math.sqrt((numpy.cosh(agg) - 1) / gg)
mH = torch.zeros(len(dV), len(dV))
for i in range(len(dV)):
v = dV[i]
L = 1.
r = math.sqrt(L * L + v * v)
c = L / r
s = v / r
mH[i, i] = r
for j in range(len(dV)):
vprime = dV[j]
Lprime = mH[i, j]
dV[j] = c * vprime - s * Lprime
mH[i, j] = s * vprime + c * Lprime
mH = mH * math.sqrt(gg / numpy.sinh(agg))
#print(mH)
exit()
mHL = torch.potrf(mH, upper=False)
out = point(None, self)
out.flattened_tensor.copy_(torch.mv(mHL, torch.randn(len(dV))))
out.load_flatten()
return (out)
def generate_momentum(self, q):
out = point(list_tensor=self.list_tensor,
pointtype="p",
need_flatten=self.need_flatten)
out.flattened_tensor.copy_(torch.randn(self.dim))
out.load_flatten()
return (out)
def gradient_descent(number_of_iter,lr,v_obj):
# random initialization
init_point = point(V=v_obj)
init_point.flattened_tensor.normal_()
init_point.load_flatten()
theta = init_point.point_clone()
store_v = []
explode_grad = False
for cur in range(number_of_iter):
print(cur)
cur_v = v_obj.evaluate_scalar(theta)
print(cur_v)
#print(theta.flattened_tensor)
grad,explode_grad = v_obj.dq(theta.flattened_tensor)
if not explode_grad:
theta.flattened_tensor -= lr * grad
theta.load_flatten()
store_v.append(v_obj.evaluate_scalar(theta))
diff = abs(store_v[-1]-cur_v)
if diff < 1e-6:
break
else:
break
return(theta,explode_grad)
def generate_momentum(self, q):
_, mdiagH = self.linkedV.getdiagH_tensor(q)
mlambda, _ = self.fcomputeMetric(mdiagH)
out = point(None, self)
out.flattened_tensor.copy_(
torch.randn(len(mlambda)) / torch.sqrt(mlambda))
out.load_flatten()
return (out)
def run(self):
if hasattr(self, "log_obj"):
#print("yes")
out = self.one_step_function(
input_point_obj=point(V=self.Ham.V),
Ham_obj=self.Ham,
tune_param_objs_dict=self.tune_param_objs_dict,
log_obj=self.log_obj)
#out = self.one_step_function(self.Ham.V.q_point,self.Ham,self.tuneable_param_dict,self.log_obj)
else:
out = self.one_step_function(
input_point_obj=point(V=self.Ham.V),
Ham_obj=self.Ham,
tune_param_objs_dict=self.tune_param_objs_dict)
#out = self.one_step_function(self.Ham.V.q_point, self.Ham, self.tuneable_param_dict)
self.Ham.V.load_point(out[0])
return (out[0].point_clone())
def convert_mcmc_tensor_to_list_points(mcmc_tensor, v_obj):
num_samples = mcmc_tensor.shape[0]
out = [None] * num_samples
for i in range(num_samples):
this_point = point(V=v_obj)
this_point.flattened_tensor.copy_(mcmc_tensor[i, :])
this_point.load_flatten()
out[i] = this_point
return (out)
def generate_momentum(self, mlambda=None):
if mlambda == None:
mlambda, _ = self.fcomputemetric()
# computed by fcomputemetric
#out = torch.randn(len(self.dim)) / torch.sqrt(mlambda)
out = point(None, self)
out.flattened_tensor.copy_(
torch.randn(len(self.dim)) / torch.sqrt(mlambda))
out.load_flatten()
return (out)
def gradient_descent(number_of_iter,
lr,
v_obj,
validation_set,
validate_interval=10):
# random initialization
init_point = point(V=v_obj)
init_point.flattened_tensor.normal_()
init_point.load_flatten()
theta = init_point.point_clone()
store_v = []
best_validate_error = 10
explode_grad = False
till_validate = validate_interval
validate_continue = True
for cur in range(number_of_iter):
print("iter {}".format(cur))
if not validate_continue:
break
else:
#print(cur)
cur_v = v_obj.evaluate_scalar(theta)
print("v val {}".format(cur_v))
#print(theta.flattened_tensor)
grad, explode_grad = v_obj.dq(theta.flattened_tensor)
if not explode_grad:
theta.flattened_tensor -= lr * grad
theta.load_flatten()
store_v.append(v_obj.evaluate_scalar(theta))
if till_validate == 0:
temp_mcmc_samples = numpy.zeros(
(1, len(theta.flattened_tensor)))
temp_mcmc_samples[0, :] = theta.flattened_tensor.numpy()
validate_error, _, _ = test_error(
target_dataset=validation_set,
v_obj=v_obj,
mcmc_samples=temp_mcmc_samples,
type="classification")
print("validate error {}".format(validate_error))
if validate_error > best_validate_error:
validate_continue = False
else:
till_validate = validate_interval
best_validate_error = validate_error
else:
till_validate -= 1
diff = abs(store_v[-1] - cur_v)
if diff < 1e-6:
break
else:
break
return (theta, explode_grad)
def __init__(self):
super(V, self).__init__()
self.V_setup()
if self.explicit_gradient is None:
raise ValueError(
"self.explicit_gradient need to be defined in V_setup")
if self.need_higherorderderiv is None:
raise ValueError(
"self.need_higherorderderiv need to be defined in V_setup")
#################################################################################
self.decides_if_flattened()
self.V_higherorder_setup()
self.q_point = point(V=self)
self.diagnostics = None
def generate_momentum(self, q):
#if lam == None or Q == None:
# H_ = self.linkedV.getH_tensor()
# lam, Q = eigen(H_)
_, H_ = self.linkedV.getH_tensor(q)
lam, Q = eigen(H_)
temp = torch.mm(
Q,
torch.diag(torch.sqrt(softabs_map(lam,
self.metric.msoftabsalpha))))
out = point(None, self)
out.flattened_tensor.copy_(torch.mv(temp, torch.randn(len(lam))))
out.load_flatten()
return (out)
def generate_momentum(self,q):
#if lam == None or Q == None:
# H_ = self.linkedV.getH_tensor()
# lam, Q = eigen(H_)
_, H_ = self.linkedV.getH_tensor(q)
lam, Q = eigen(H_)
#print(lam)
#print(Q)
#print(lam.shape)
#print(type(lam))
#print(type(lam[0]))
#exit()
temp = torch.mm(Q, torch.diag(torch.sqrt(softabs_map(lam, self.metric.msoftabsalpha))))
#print(temp)
#exit()
out = point(list_tensor=self.list_tensor,pointtype="p",need_flatten=self.need_flatten)
out.flattened_tensor.copy_(torch.mv(temp, torch.randn(len(lam))))
out.load_flatten()
return(out)
def find_reasonable_ep(Ham):
# integrator can be leapfrog or gleapfrog using any possible metric
counter = 0
#q,p = one_chain_obj.cur_q,one_chain_obj.cur_p
q = point(V=Ham.V)
#print(q.flattened_tensor)
#exit()
p = Ham.T.generate_momentum(q)
integrator = Ham.integrator
epsilon = 1
H_cur = Ham.evaluate(q, p)["H"]
qprime, pprime, gleapfrog_stat = integrator(q=q.point_clone(),
p=p.point_clone(),
epsilon=epsilon,
Ham=Ham)
if gleapfrog_stat["explode_grad"]:
prop_H = float("Inf")
else:
prop_H = Ham.evaluate(qprime, pprime)["H"]
a = 2 * (-prop_H + H_cur > math.log(0.5)) - 1
while a * (-prop_H + H_cur) > (-a * math.log(2)):
epsilon = math.exp(a) * epsilon
qprime, pprime, gleapfrog_stat = integrator(q=q.point_clone(),
p=p.point_clone(),
epsilon=epsilon,
Ham=Ham)
if gleapfrog_stat["explode_grad"]:
prop_H = float("Inf")
else:
prop_H = Ham.evaluate(qprime, pprime)["H"]
counter += 1
if counter > 100:
raise ValueError("find_reasonable_ep takes too long. check")
return (epsilon)
def setup_sghmc_experiment(ep_list,L_list,eta_list,train_set,test_set,save_name,seed=1):
output_names = ["train_error", "test_error","train_error_sd","test_error_sd"]
output_store = numpy.zeros((len(ep_list),len(L_list),len(eta_list), len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)]+[4,13])
model_dict = {"num_units":35}
prior_dict = {"name":"normal"}
time_store = numpy.zeros(shape=[len(ep_list),len(L_list),len(eta_list)])
for i in range(len(ep_list)):
for j in range(len(L_list)):
for k in range(len(eta_list)):
start_time = time.time()
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor")
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(V=v_obj, metric=metric_obj)
full_data = train_set
init_q_point = point(V=v_obj)
store,explode_grad = sghmc_sampler(init_q_point=init_q_point, epsilon=ep_list[i], L=L_list[j], Ham=Ham, alpha=0.01, eta=eta_list[k],
betahat=0, full_data=full_data, num_samples=2000, thin=0, burn_in=1000,
batch_size=25)
total_time = time.time() - start_time
if not explode_grad:
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1, input_data=train_set,
prior_dict=prior_dict, model_dict=model_dict)
test_mcmc_samples = store.numpy()
te1, predicted1,te_sd = test_error(test_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
train_error, predicted1, train_error_sd = test_error(train_set, v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=test_mcmc_samples, type="classification",
memory_efficient=False)
else:
train_error = 2
te1 = 2
train_error_sd = 2
te_sd = 2
output_store[i,j,k,0] = train_error
output_store[i,j,k,1] = te1
output_store[i,j,k,2] = train_error_sd
output_store[i,j,k,3] = te_sd
time_store[i,j,k] = total_time
to_store = {"diagnostics":diagnostics_store,"output":output_store,"output_names":output_names,"seed":seed,
"ep_list":ep_list,"L_list":L_list,"eta_list":eta_list,"num_units":model_dict["num_units"],
"prior":prior_dict["name"],"total_store":time_store}
numpy.savez(save_name,**to_store)
return()
samples = sampler1.get_samples(permuted=True)
num_samples = samples.shape[0]
num_chosen = 3
seed = 12
numpy.random.seed(seed)
indices = numpy.random.uniform(0, num_samples, num_chosen)
indices = indices.astype(int)
indices = [numpy.asscalar(y) for y in indices]
q_point_list = [None] * list
for i in range(len(indices)):
flattened_tensor = samples[:, indices[i]]
flattened_tensor = torch.from_numpy(flattened_tensor)
q_point = point(V=V_mvn1(precision_type="torch.DoubleTensor"))
q_point.flattened_tensor.copy_(flattened_tensor)
q_point.load_flatten()
q_point_list[i] = q_point
#############################################################################################################################
# q_point = point(V=V_mvn1(precision_type="torch.DoubleTensor"))
#
# q_point.flattened_tensor.normal_()
# q_point.load_flatten()
# save_name = "pic.png"
# out = generate_H_V_T(epsilon=0.001,L=11000,vo=V_mvn1(precision_type="torch.DoubleTensor"),q_point=q_point)
#
# V_vec = numpy.array(out["V_list"])
# T_vec = numpy.array(out["T_list"])
# # print(len(V_vec))
# # print(len(T_vec))
# compare ess for hyperparameter
input_data = get_data_dict("8x8mnist")
input_data = {
"input": input_data["input"][:500, ],
"target": input_data["target"][:500]
}
model_dict = {"num_units": 25}
V_fun = wrap_V_class_with_input_data(class_constructor=V_fc_gibbs_model_1,
input_data=input_data,
model_dict=model_dict)
v_obj = V_fun(precision_type="torch.DoubleTensor", gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = torch.abs(torch.randn(1))
log_obj = log_class()
#print(init_q_point.flattened_tensor)
num_samples = 1000
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(1, num_samples, dim)
mcmc_samples_hyper = torch.zeros(1, num_samples, 1)
for i in range(num_samples):
print("loop {}".format(i))
#outq,out_hyperparam = update_param_and_hyperparam_one_step(init_q_point,init_hyperparam,Ham,0.1,60,log_obj)
outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step(
init_q_point, init_hyperparam, Ham, 0.0001, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
def load_explicit_diagH(self):
return ()
def load_explicit_graddiagH(self):
return ()
v_obj = V_hierarchical_logistic_gibbs(precision_type="torch.DoubleTensor",
gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = [torch.abs(torch.randn(1)) + 3]
log_obj = log_class()
#print(init_q_point.flattened_tensor)
num_samples = 4000
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(num_samples, dim)
mcmc_samples_hyper = torch.zeros(num_samples)
for i in range(num_samples):
outq, out_hyperparam = update_param_and_hyperparam_one_step(
init_q_point, init_hyperparam, Ham, 0.1, 10, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
init_q_point.load_flatten()
init_hyperparam = out_hyperparam
import torch
import os
from abstract.mcmc_sampler import mcmc_sampler, mcmc_sampler_settings_dict
from adapt_util.tune_param_classes.tune_param_setting_util import *
from distributions.neural_nets.fc_V_model_debug import V_fc_model_debug
from experiments.experiment_obj import tuneinput_class
from distributions.two_d_normal import V_2dnormal
from experiments.correctdist_experiments.prototype import check_mean_var_stan
from post_processing.ESS_nuts import ess_stan
from abstract.abstract_class_point import point
from input_data.convert_data_to_dict import get_data_dict
seedid = 30
numpy.random.seed(seedid)
torch.manual_seed(seedid)
model_dict = {"num_units":20}
input_data = get_data_dict("8x8mnist")
v_obj = V_fc_model_debug(precision_type="torch.DoubleTensor",model_dict=model_dict,input_data=input_data)
q = point(V=v_obj)
print(q.flattened_tensor)
q_clone = q.point_clone()
print(q_clone.flattened_tensor)
def generate_q_list(v_fun,num_of_pts):
# extract number of (q,p) points given v_fun
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0, samples_per_chain=200, num_chains=4, num_cpu=4, thin=1,
tune_l_per_chain=100,
warmup_per_chain=110, is_float=False, isstore_to_disk=False,allow_restart=True)
input_dict = {"v_fun": [v_fun], "epsilon": ["dual"], "second_order": [False],
"metric_name": ["unit_e"], "dynamic": [True], "windowed": [False],"max_tree_depth":[6],
"criterion": ["xhmc"],"xhmc_delta":[0.1]}
ep_dual_metadata_argument = {"name": "epsilon", "target": 0.65, "gamma": 0.05, "t_0": 10,
"kappa": 0.75, "obj_fun": "accept_rate", "par_type": "fast"}
dim = len(v_fun(precision_type="torch.DoubleTensor").flattened_tensor)
#adapt_cov_arguments = [adapt_cov_default_arguments(par_type="slow", dim=dim)]
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict, mcmc_settings_dict=mcmc_meta, tune_settings_dict=tune_settings_dict)
out = sampler1.start_sampling()
sampler1.remove_failed_chains()
print("num chains removed {}".format(sampler1.metadata.num_chains_removed))
print("num restarts {}".format(sampler1.metadata.num_restarts))
samples = sampler1.get_samples(permuted=True)
num_mcmc_samples = samples.shape[0]
indices = numpy.random.choice(a=num_mcmc_samples,size=num_of_pts,replace=False)
chosen_samples = samples[indices,:]
list_q_double = [None]*num_of_pts
#list_p_double = [None]*num_of_pts
for i in range(num_of_pts):
q_point = point(V=v_fun(precision_type="torch.DoubleTensor"))
flattened_tensor = torch.from_numpy(chosen_samples[i,:]).type("torch.DoubleTensor")
q_point.flattened_tensor.copy_(flattened_tensor)
q_point.load_flatten()
#p_point = point(list_tensor=q_point.list_tensor,pointtype="p",need_flatten=q_point.need_flatten)
# p_point.flattened_tensor.normal_()
# p_point.load_flatten()
list_q_double[i] = q_point
#list_p_double[i] = p_point
list_q_float = [None] * num_of_pts
#list_p_float = [None] * num_of_pts
for i in range(num_of_pts):
q_point = point(V=v_fun(precision_type="torch.FloatTensor"))
flattened_tensor = torch.from_numpy(chosen_samples[i, :]).type("torch.FloatTensor")
q_point.flattened_tensor.copy_(flattened_tensor)
q_point.load_flatten()
# p_point = point(list_tensor=q_point.list_tensor,pointtype="p",need_flatten=q_point.need_flatten)
# p_point.flattened_tensor.normal_()
# p_point.load_flatten()
list_q_float[i] = q_point
out = {"list_q_double":list_q_double,"list_q_float":list_q_float}
return(out)
from abstract.abstract_class_point import point
from explicit.leapfrog_ult_util import leapfrog_ult
#y_np= numpy.random.binomial(n=1,p=0.5,size=num_ob)
#X_np = numpy.random.randn(num_ob,dim)
seedid = 33
numpy.random.seed(seedid)
torch.manual_seed(seedid)
import os
inputq = torch.randn(7)
inputp = torch.randn(7)
v_obj = V_pima_inidan_logit()
metric_obj = metric("unit_e", v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
q_point = point(V=Ham.V)
p_point = point(T=Ham.T)
q_point.flattened_tensor.copy_(inputq)
p_point.flattened_tensor.copy_(inputp)
q_point.load_flatten()
p_point.load_flatten()
print("abstract H {}".format(Ham.evaluate(q_point, p_point)))
print("input q {}".format(q_point.flattened_tensor))
print("input p {}".format(p_point.flattened_tensor))
outq_a, outp_a, stat = abstract_leapfrog_ult(q_point, p_point, 0.1, Ham)
print("output q {} ".format(outq_a.flattened_tensor))
print("output p {}".format(outp_a.flattened_tensor))
def generate_momentum(self, q):
out = point(None, self)
out.flattened_tensor.copy_(self.metric._sd_vec * torch.randn(self.dim))
out.load_flatten()
return (out)
def setup_gibbs_v_joint_experiment(num_units_list,
train_set,
test_set,
num_samples,
save_name,
seed=1):
output_names = [
"train_error", "test_error", "train_error_sd", "test_error_sd",
"sigma_2_ess", "mean_sigma2", "median_sigma2", "min_ess", "median_ess"
]
output_store = numpy.zeros((len(num_units_list), 3, len(output_names)))
diagnostics_store = numpy.zeros(shape=[len(num_units_list), 3] + [4, 13])
time_store = numpy.zeros(shape=[len(num_units_list), 3])
for i in range(len(num_units_list)):
for j in range(3):
start_time = time.time()
v_fun = V_fc_model_4
model_dict = {"num_units": num_units_list[i]}
mcmc_meta = mcmc_sampler_settings_dict(mcmc_id=0,
samples_per_chain=1000 +
num_samples,
num_chains=4,
num_cpu=4,
thin=1,
tune_l_per_chain=900,
warmup_per_chain=1000,
is_float=False,
isstore_to_disk=False,
allow_restart=True,
seed=seed + i + 1)
if j == 2:
v_generator = wrap_V_class_with_input_data(
class_constructor=V_fc_gibbs_model_1,
input_data=train_set,
model_dict=model_dict)
v_obj = v_generator(precision_type="torch.DoubleTensor",
gibbs=True)
metric_obj = metric(name="unit_e", V_instance=v_obj)
Ham = Hamiltonian(v_obj, metric_obj)
init_q_point = point(V=v_obj)
init_hyperparam = torch.abs(torch.randn(1)) + 3
log_obj = log_class()
dim = len(init_q_point.flattened_tensor)
mcmc_samples_weight = torch.zeros(1, num_samples + 1000, dim)
mcmc_samples_hyper = torch.zeros(1, num_samples + 1000, 1)
for iter in range(num_samples + 1000):
print("iter {}".format(iter))
outq, out_hyperparam = update_param_and_hyperparam_dynamic_one_step(
init_q_point, init_hyperparam, Ham, 0.01, log_obj)
init_q_point.flattened_tensor.copy_(outq.flattened_tensor)
init_q_point.load_flatten()
init_hyperparam = out_hyperparam
mcmc_samples_weight[
0, iter, :] = outq.flattened_tensor.clone()
mcmc_samples_hyper[0, iter, 0] = out_hyperparam
mcmc_samples_weight = mcmc_samples_weight[:, 1000:, :].numpy()
mcmc_samples_hyper = mcmc_samples_hyper[:, 1000:, :].numpy()
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_weight[0, :, :],
type="classification",
memory_efficient=False)
sigma2_diagnostics = diagnostics_stan(mcmc_samples_hyper)
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
mcmc_samples_hyper)
posterior_median_hidden_in_sigma2 = numpy.median(
mcmc_samples_hyper)
weight_ess = diagnostics_stan(mcmc_samples_weight)["ess"]
min_ess = min(sigma2_ess, min(weight_ess))
median_ess = numpy.median([sigma2_ess] + list(weight_ess))
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
output_store[i, j, 7] = min_ess
output_store[i, j, 8] = median_ess
elif j == 0:
prior_dict = {"name": "gaussian_inv_gamma_1"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
elif j == 1:
prior_dict = {"name": "gaussian_inv_gamma_2"}
v_generator = wrap_V_class_with_input_data(
class_constructor=v_fun,
input_data=train_set,
prior_dict=prior_dict,
model_dict=model_dict)
if j == 0 or j == 1:
input_dict = {
"v_fun": [v_generator],
"epsilon": ["dual"],
"second_order": [False],
"max_tree_depth": [8],
"metric_name": ["unit_e"],
"dynamic": [True],
"windowed": [False],
"criterion": ["xhmc"],
"xhmc_delta": [0.1]
}
ep_dual_metadata_argument = {
"name": "epsilon",
"target": 0.9,
"gamma": 0.05,
"t_0": 10,
"kappa": 0.75,
"obj_fun": "accept_rate",
"par_type": "fast"
}
dual_args_list = [ep_dual_metadata_argument]
other_arguments = other_default_arguments()
tune_settings_dict = tuning_settings(dual_args_list, [], [],
other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
sampler1.start_sampling()
np_diagnostics, feature_names = sampler1.np_diagnostics()
mcmc_samples_hidden_in = sampler1.get_samples_alt(
prior_obj_name="hidden_in", permuted=False)
samples = mcmc_samples_hidden_in["samples"]
hidden_in_sigma2_indices = mcmc_samples_hidden_in[
"indices_dict"]["sigma2"]
sigma2_diagnostics = diagnostics_stan(
samples[:, :, hidden_in_sigma2_indices])
sigma2_ess = sigma2_diagnostics["ess"]
posterior_mean_hidden_in_sigma2 = numpy.mean(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
posterior_median_hidden_in_sigma2 = numpy.median(
samples[:, :, hidden_in_sigma2_indices].reshape(
-1, len(hidden_in_sigma2_indices)),
axis=0)
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
te, predicted, te_sd = test_error(
test_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
train_error, _, train_error_sd = test_error(
train_set,
v_obj=v_generator(precision_type="torch.DoubleTensor"),
mcmc_samples=mcmc_samples_mixed,
type="classification",
memory_efficient=False)
output_store[i, j, 0] = train_error
output_store[i, j, 1] = te
output_store[i, j, 2] = train_error
output_store[i, j, 3] = te_sd
output_store[i, j, 4] = sigma2_ess
output_store[i, j, 5] = posterior_mean_hidden_in_sigma2
output_store[i, j, 6] = posterior_median_hidden_in_sigma2
diagnostics_store[i, j, :, :] = np_diagnostics
output_store[i, j, 7] = np_diagnostics[0, 10]
output_store[i, j, 8] = np_diagnostics[0, 11]
total_time = time.time() - start_time()
time_store[i, j] = total_time
to_store = {
"diagnostics": diagnostics_store,
"output": output_store,
"diagnostics_names": feature_names,
"output_names": output_names,
"seed": seed,
"num_units_list": num_units_list,
"time_store": time_store
}
numpy.savez(save_name, **to_store)
return ()
seedid = 3
numpy.random.seed(seedid)
torch.manual_seed(seedid)
alpha = 1e-4
#debug_dict = {"abstract":None,"explicit":None}
#debug_dict.update({"explicit":y.data.clone()})
# first verify they have the same Hamiltonian function
inputq = torch.randn(7)
v_obj = V_pima_inidan_logit()
metric_obj = metric("softabs_diag", v_obj, alpha)
Ham = Hamiltonian(v_obj, metric_obj)
q_point = point(V=Ham.V)
q_point.flattened_tensor.copy_(inputq)
q_point.load_flatten()
p_point = Ham.T.generate_momentum(q_point)
print("abstract H {}".format(Ham.evaluate(q_point, p_point)))
print("abstract V {}".format(Ham.V.evaluate_scalar(q_point)))
print("abstract T {}".format(Ham.T.evaluate_scalar(q_point, p_point)))
L = 5000
mcmc_samples = torch.zeros(L, 7)
for i in range(L):
out = abstract_static_one_step(epsilon=0.1,
init_q=q_point,
Ham=Ham,
