def psispeffk(log_lik, name):
loo, loos, kw = psisloo(log_lik)
print("\n")
print("psis-loo:", loo)
err_05 = 0
err_07 = 0
err_1 = 0
err_inf = 0
for i in range(0, len(kw)):
if kw[i] <= 0.5:
err_05 += 1
elif kw[i] <= 0.7:
err_07 += 1
elif kw[i] <= 1.0:
err_1 += 1
else:
err_inf += 1
_sum = 0
for i in range(0, 30):
_sum += np.log(np.mean(np.exp(log_lik[:, i])))
PSIS_COMPARISONS[name] = "psis-loo: " + str(
loo
) + "\nK-VALUES:\n" + "\n" + " " + "(-inf;0.5] &" + " " + str(
err_05
) + " " + "&" + " " + str(
100 * err_05 / len(kw)
) + "\n" + " " + "(0.5;0.7] &" + " " + str(err_07) + " " + "&" + " " + str(
100 * err_07 / len(kw)
) + "\n" + " " + "(0.7;1.0] &" + " " + str(err_1) + " " + "&" + " " + str(
100 * err_1 / len(kw)) + "\n" + " " + "(1.0;inf) &" + " " + str(
err_inf) + " " + "&" + " " + str(
100 * err_inf / len(kw)
) + "\n" + " " + "\n" + "\n" + " " + "p_eff:" + str(_sum - loo)
return 0
def psis(self, test_data, NSAMPLES=1):
log_lik = np.zeros((NSAMPLES, test_data.shape[0]))
for n in range(NSAMPLES):
t = np.random.gamma(
self.gamma_t,
1. / self.rho_t) # gamma_t is shape and rho_t is rate
b = np.random.gamma(
self.gamma_b,
1.0 / self.rho_b) # gamma_b is shape and rho_b is rate
lambdas = np.inner(t, b).T
log_lik[n, :] = stats.poisson(
lambdas[test_data[:, 0],
test_data[:, 1]]).logpmf(test_data[:, 2]).reshape(-1)
loo, loos, ks = psis.psisloo(log_lik)
return loo
def print_loo_and_ks(samples):
"""
Print PSIS loo and summary of k values.
:param samples: with key 'log_lik'
:return:
"""
from psis import psisloo
loglik = samples['log_lik']
loo, loos, ks = psisloo(loglik)
print("Loo: %.2f" % loo)
ks_sum = [[(ks <= 0.5).sum(),
sum([1 for k in ks if 0.5 < k <= 0.7]), (ks > 0.7).sum()]]
ks_df = pd.DataFrame(ks_sum, columns=["k<=0.5", "0.5<k<=0.7", "0.7<k"])
print(ks_df)
def compute_loodiff(log_liks, use_mean=False):
loos = []
measure = []
for model in log_liks:
loosum_model, loos_model, ks = psis.psisloo(model)
loos.append(loos_model)
if use_mean:
measure.append(loosum_model / loos_model.size)
else:
measure.append(loosum_model)
if use_mean:
return (measure[0] - measure[1],
(loos[0] - loos[1]).std() / np.sqrt(loos[0].size))
else:
return (measure[0] - measure[1],
np.sqrt( (loos[0] - loos[1]).var() * loos[0].size ))
def get_external_validity(self):
'''
Model performance metrics
psisloo reweights samples of pointwise posterior likelihoods to approximate data likelihood under leave one out cross validation
ref: https://arxiv.org/pdf/1507.04544.pdf
lmbd measures the extent to which parameters are pooled across individuals
ref: http://www.stat.columbia.edu/~gelman/research/published/rsquared.pdf
'''
assert self.data is not None, 'train the model first'
eps_smps = []
llk_smps = []
for pars in self.get_posterior():
llk_smps.append(
self.get_distr(self.data, pars).log_prob(self.data['Y']))
eps_smps.append(pars['ind'])
loo, loos, ks = psisloo(torch.stack(llk_smps).t().numpy())
eps_smps = torch.stack(eps_smps)
lmbd = 1 - (eps_smps.mean(0).var(0) / eps_smps.var(1).mean(0))
return lmbd.squeeze(), loo, (ks < .7).mean()
def loo(data, groups):
"""
Parameters
----------
dtaa: OrderedDict of samples(extract) from different models
groups: Groups to compare
"""
print("PSIS leave-one-out cross validation.")
print("Estimates are unreliable if k > 0.7")
for model, samples in data.items():
print('\n{:8s}'.format(model))
print('{:2s} {:9s}'.format('-', ' ELPD'))
for i, group in enumerate(groups):
elpd = psisloo(samples[i]['log_lik'])[0]
print('{:2s} {: 5.3f}'.format(group, elpd), end=" ")
unreliable = np.where(
psisloo(samples[i]['log_lik'])[2] > 0.7)[0].size
if unreliable > 0:
print(f'Note: there are {unreliable} unreliable points.')
print(
"\n--------------------------\nModel Comparisons: Estimate of ELPD difference"
)
pairs = itertools.combinations(data.items(), 2)
for pair in pairs:
model0, samples0 = pair[0]
model1, samples1 = pair[1]
print('\n{:7s} - {:7s}'.format(model0, model1))
print('{:2s} {:8s} {:5s}'.format('', 'estimate', 'SE'))
for i, group in enumerate(groups):
elpd_diff = psisloo(samples0[i]['log_lik'])[0] - psisloo(
samples1[i]['log_lik'])[0]
elpd_diff_pointwise = psisloo(samples0[i]['log_lik'])[1] - psisloo(
samples1[i]['log_lik'])[1]
elpd_diff_se = np.sqrt(
len(elpd_diff_pointwise) * np.var(elpd_diff_pointwise))
print('{:2s} {: 5.3f} {:5.3f}'.format(group, elpd_diff,
elpd_diff_se))
def loo_values(log_lik):
LOO_PSIS = psisloo(log_lik)
loo, loos, ks = LOO_PSIS[0], LOO_PSIS[1], LOO_PSIS[2]
ks_thr = 1.0
ks_large_percent = ((ks > ks_thr).sum() / ks.size) * 100
return loo, ks, ks_large_percent
