def np_diagnostics(self):
feature_names = ["num_restarts", "num_divergent","hit_max_tree_depth","ave_num_transitions","total_num_transitions","bfmi","lp_ess",
"lp_rhat", "difficulty","num_chains_removed","min_ess","median_ess","percent_rhat"]
self.remove_failed_chains()
out = self.get_diagnostics(permuted=False)
num_restarts = self.metadata.num_restarts
num_chains_removed = self.metadata.num_chains_removed
if self.tune_dict["dynamic"]:
processed_diag = process_diagnostics(out, name_list=["hit_max_tree_depth"])
hit_max_tree_depth = numpy.squeeze(processed_diag.sum(axis=1))
else:
hit_max_tree_depth = 0
processed_diag = process_diagnostics(out, name_list=["divergent"])
num_divergent = numpy.squeeze(processed_diag.sum(axis=1))
processed_diag = process_diagnostics(out, name_list=["num_transitions"])
total_num_transitions = numpy.sum(processed_diag)
ave_num_transitions = numpy.squeeze(processed_diag.mean(axis=1))
energy_summary = energy_diagnostics(diagnostics_obj=out)
mixed_mcmc_tensor = self.get_samples(permuted=True)
mcmc_cov = numpy.cov(mixed_mcmc_tensor, rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
difficulty = max(mcmc_sd_vec) / min(mcmc_sd_vec)
mcmc_samples = self.get_samples(permuted=False)
out_dict = diagnostics_stan(mcmc_samples)
min_ess = min(out_dict["ess"])
median_ess = numpy.median(out_dict["ess"])
percent_rhat = sum(out_dict["rhat"]<1.05)/len(out_dict["rhat"])
num_id = self.num_chains
output = numpy.zeros((num_id, len(feature_names)))
output[:, 0] = num_restarts
output[:, 1] = num_divergent
output[:, 2] = hit_max_tree_depth
output[:, 3] = ave_num_transitions
output[:, 4] = total_num_transitions
output[:, 5] = energy_summary["bfmi_list"]
output[:, 6] = energy_summary["ess"]
output[:, 7] = energy_summary["rhat"]
output[:, 8] = difficulty
output[:, 9] = num_chains_removed
output[:,10] = min_ess
output[:,11] = median_ess
output[:,12] = percent_rhat
return(output,feature_names)
def get_diagnostics(sampler):
#
# want to return numpy array of dimensions [num_chains,9]
# as well as list of column names
feature_names = [
"num_restarts", "num_divergent", "num_hit_max_tree_depth",
"ave_num_transitions", "bfmi", "lp_ess", "lp_rhat", "difficulty"
]
feature_names = ["num_chains_removed"]
sampler.remove_failed_chains()
out = sampler.get_diagnostics(permuted=False)
num_restarts = sampler.metadata.num_restarts
num_chains_removed = sampler.metadata.num_chains_removed
processed_diag = process_diagnostics(out, name_list=["divergent"])
num_divergent = processed_diag.sum(axis=1)
processed_diag = process_diagnostics(out, name_list=["hit_max_tree_depth"])
hix_max_tree_depth = processed_diag.sum(axis=1)
processed_diag = process_diagnostics(out, name_list=["num_transitions"])
ave_num_transitions = processed_diag.mean(axis=1)
energy_summary = energy_diagnostics(diagnostics_obj=out)
mixed_mcmc_tensor = sampler.get_samples(permuted=True)
mcmc_cov = numpy.cov(mixed_mcmc_tensor, rowvar=False)
mcmc_sd_vec = numpy.sqrt(numpy.diagonal(mcmc_cov))
difficulty = max(mcmc_sd_vec) / min(mcmc_sd_vec)
num_id = sampler.num_chains
output = numpy.zeros(num_id, len(feature_names))
output[:, 0] = num_restarts
output[:, 1] = num_divergent
output[:, 2] = hix_max_tree_depth
output[:, 3] = ave_num_transitions
output[:, 4] = energy_summary["bfmi_list"]
output[:, 5] = energy_summary["ess"]
output[:, 6] = energy_summary["rhat"]
output[:, 7] = difficulty
output[:, 8] = num_chains_removed
return (output, feature_names)
def get_ess_and_esjds(ran_sampler):
# get max,min,median ess and normalized esjd for sampler on unconstrained space
ran_sampler.remove_failed_chains()
sampler_diag_check = {
"num_chains_removed": ran_sampler.metadata.num_chains_removed,
"num_restarts": ran_sampler.metadata.num_restarts
}
if ran_sampler.metadata.num_chains_removed == 0:
diag = ran_sampler.get_diagnostics()
num_transitions = process_diagnostics(diag,
name_list=["num_transitions"])
total_num_transitions = sum(num_transitions)
samples_combined = ran_sampler.get_samples(permuted=True)
esjd = ESJD(samples_combined)
esjd_normalized = esjd / math.sqrt(total_num_transitions)
ess = ess_stan(ran_sampler.get_samples(permuted=False))
min_ess = min(ess)
max_ess = max(ess)
median_ess = numpy.median(ess)
median_ess_normalized = median_ess / math.sqrt(total_num_transitions)
min_ess_normalized = min_ess / math.sqrt(total_num_transitions)
max_ess_normalized = max_ess / math.sqrt(total_num_transitions)
out = {
"median_ess": median_ess,
"max_ess": max_ess,
"min_ess": min_ess,
"esjd": 0,
"esjd_normalized": 0,
"median_ess_normalized": median_ess_normalized,
"min_ess_normalized": min_ess_normalized,
"max_ess_normalized": max_ess_normalized
}
else:
out = {
"median_ess": 0,
"max_ess": 0,
"min_ess": 0,
"esjd": 0,
"esjd_normalized": 0,
"median_ess_normalized": 0,
"min_ess_normalized": 0,
"max_ess_normalized": 0
}
out.update({"sampler_diag_check": sampler_diag_check})
return (out)
print(diagnostics_stan(samples[:,:,hidden_in_sigma2_indices]))
print("posterior mean sigma2 {}".format(posterior_mean_hidden_in_sigma2))
print("posterior median sigma2 {}".format(posterior_median_hidden_in_sigma2))
#print(mcmc_samples_beta["indices_dict"])
full_mcmc_tensor = get_params_mcmc_tensor(sampler=sampler1)
print(get_short_diagnostics(full_mcmc_tensor))
out = sampler1.get_diagnostics(permuted=False)
print("divergent")
processed_diag = process_diagnostics(out,name_list=["divergent"])
print(processed_diag.sum(axis=1))
#print(processed_diag.shape)
#processed_energy = process_diagnostics(out,name_list=["prop_H"])
print(energy_diagnostics(diagnostics_obj=out))
mcmc_samples_mixed = sampler1.get_samples(permuted=True)
#target_dataset = get_data_dict("8x8mnist")
v_generator = wrap_V_class_with_input_data(class_constructor=V_fc_model_1,input_data=input_data,prior_dict=prior_dict,model_dict=model_dict)
precision_type = "torch.DoubleTensor"
te2,predicted2 = test_error(input_data,v_obj=v_generator(precision_type=precision_type),mcmc_samples=mcmc_samples_mixed,type="classification",memory_efficient=False)
full_mcmc_tensor = get_params_mcmc_tensor(sampler=sampler1)
print(get_short_diagnostics(full_mcmc_tensor))
#print(mcmc_samples_beta["indices_dict"])
print("overall diagnostics")
full_mcmc_tensor = get_params_mcmc_tensor(sampler=sampler1)
print(get_short_diagnostics(full_mcmc_tensor))
#print(mcmc_samples_beta["indices_dict"])
out = sampler1.get_diagnostics(permuted=False)
print("num divergences after warmup")
processed_diag = process_diagnostics(out,name_list=["divergent"])
print(processed_diag.sum(axis=1))
print("num hit max tree depth after warmup")
processed_diag = process_diagnostics(out,name_list=["hit_max_tree_depth"])
print(processed_diag.sum(axis=1))
print("average number of leapfrog steps after warmup")
processed_diag = process_diagnostics(out,name_list=["num_transitions"])
print(processed_diag.mean(axis=1))
#processed_energy = process_diagnostics(out,name_list=["prop_H"])
print("energy diagnostics")
print(energy_diagnostics(diagnostics_obj=out))
tune_settings_dict = tuning_settings([], [], [], other_arguments)
tune_dict = tuneinput_class(input_dict).singleton_tune_dict()
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
store_name = 'pima_indian_logit_sampler.pkl'
sampled = False
if sampled:
sampler1 = pickle.load(open(store_name, 'rb'))
else:
sampler1.start_sampling()
with open(store_name, 'wb') as f:
pickle.dump(sampler1, f)
print("overall diagnostics")
full_mcmc_tensor = sampler1.get_samples(permuted=False)
print(get_short_diagnostics(full_mcmc_tensor))
out = sampler1.get_diagnostics(permuted=False)
print("average acceptance rate after warmup")
processed_diag = process_diagnostics(out, name_list=["accept_rate"])
average_accept_rate = numpy.mean(processed_diag, axis=1)
print(average_accept_rate)
print("energy diagnostics")
print(energy_diagnostics(diagnostics_obj=out))
sampler1 = mcmc_sampler(tune_dict=tune_dict,
mcmc_settings_dict=mcmc_meta,
tune_settings_dict=tune_settings_dict)
store_name = 'one_pl_sampler.pkl'
sampled = False
if sampled:
sampler1 = pickle.load(open(store_name, 'rb'))
else:
sampler1.start_sampling()
with open(store_name, 'wb') as f:
pickle.dump(sampler1, f)
print("overall diagnostics")
full_mcmc_tensor = sampler1.get_samples(permuted=False)
print(get_short_diagnostics(full_mcmc_tensor))
out = sampler1.get_diagnostics(permuted=False)
print("num hit max tree depth")
processed_diag = process_diagnostics(out, name_list=["hit_max_tree_depth"])
print(processed_diag.sum(axis=1))
print("average acceptance rate after warmup")
processed_diag = process_diagnostics(out, name_list=["accept_rate"])
average_accept_rate = numpy.mean(processed_diag, axis=1)
print(average_accept_rate)
print("energy diagnostics")
print(energy_diagnostics(diagnostics_obj=out))
import dill as pickle
import numpy
from post_processing.get_diagnostics import energy_diagnostics,process_diagnostics
sampler1 = pickle.load(open('temp_save_sampler1.pkl', 'rb'))
mcmc_samples_beta = sampler1.get_samples_alt(prior_obj_name="beta",permuted=False)
samples = mcmc_samples_beta["samples"]
posterior_mean = numpy.mean(samples.reshape(-1,101),axis=0)
out = sampler1.get_diagnostics(permuted=False,include_warmup=True)
processed_diag = process_diagnostics(out,name_list=["num_transitions"])
print(processed_diag[0,0:1000,0])