def get_waic_and_loo(fit):
"""Compute WAIC and LOO from a fit instance"""
idata = az.from_pystan(fit, log_likelihood="llx")
result = {}
result.update(dict(az.loo(idata, scale='deviance')))
result.update(dict(az.waic(idata, scale='deviance')))
return result
def get_pareto_k(fit, k_thresh=.7):
x = az.loo(fit, pointwise=True)
k_val = x['pareto_k']
k_mask = k_val > k_thresh
inds = np.where(k_mask)[0]
trls = fit.observed_data.y[k_mask]
return k_val, inds, trls
def plot_ppc_and_score(trace, data, ax=None, title='PPC', paras=None):
# Sample PPC
ppc_trace = pm.sample_posterior_predictive(trace=trace, var_names=['y'])
# Calculate LOO score
loo = az.loo(trace).loo
loo_text = "LOO = %.2f"%loo
# Aggregate binary responses
new_trace = []
for soa in sorted(set((data.SOA_IN_FRAMES))):
new_trace.append(ppc_trace['y'][:,(data.SOA_IN_FRAMES==soa) &
(data.PROBE_SALIENT==0)].mean(axis=1))
new_trace.append(ppc_trace['y'][:,(data.SOA_IN_FRAMES==soa) &
(data.PROBE_SALIENT==1)].mean(axis=1))
ppc_trace = {'y': np.array(new_trace).T}
# Prepare axes if none provided
if ax is None: f,ax= plt.subplots()
# Get SOAs and condition mask from data
SOAs = sorted(set(data['SOA_IN_MS']))
cond = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT'])['PROBE_SALIENT'].min().values
# Plot
az.plot_hdi(y=ppc_trace['y'][:,cond==0],x=SOAs, color='k', ax=ax,
hdi_prob=0.95, fill_kwargs={'alpha' : 0.23})
az.plot_hdi(y=ppc_trace['y'][:,cond==1],x=SOAs, color='g', ax=ax,
hdi_prob=0.95, fill_kwargs={'alpha' : 0.23})
ax.plot(SOAs, np.mean(ppc_trace['y'][:,cond==0],axis=0), color='k')
ax.plot(SOAs, np.mean(ppc_trace['y'][:,cond==1],axis=0), color='g')
pf_mean = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).mean().PROBE_FIRST_RESPONSE
pf_count = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).sum().PROBE_FIRST_RESPONSE
pf_obs = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).count().PROBE_FIRST_RESPONSE
pf_ci = abs(np.array(prop_ci(pf_count.values, pf_obs.values)) - pf_mean.values)
ax.plot(SOAs, pf_mean.values[::2], 'k.')
ax.errorbar(np.array(SOAs)-0.5, pf_mean.values[::2],
pf_ci[:,::2], fmt='none', color='k', alpha=0.5)
ax.plot(SOAs, pf_mean.values[1::2], 'g.')
ax.errorbar(np.array(SOAs)+0.5, pf_mean.values[1::2],
pf_ci[:,1::2], fmt='none', color='g', alpha=0.5)
ax.axvline(0, linestyle='dashed')
ax.axhline(0.5, linestyle='dashed')
ax.text(-20,0, loo_text)
if paras is not None:
for i, varname in enumerate(paras):
stats = az.summary(trace, var_names=[varname], hdi_prob=.95)
for j, s in enumerate(stats['mean']):
text = r'$' + varname + r'$: %.2f [%.2f, %.2f]'
text = text%(s, stats['hdi_2.5%'][j], stats['hdi_97.5%'][j])
posx, posy = .1 + .5 - (1 - j) * .5, 0.95 - (.05*i) - ((1-j)*.5)
ax.text(posx, posy, text, transform = ax.transAxes, color=['k','g'][j])
ax.set_title(title)
def generate_samples(
study_name: str,
measurements: pd.DataFrame,
model_configurations: List[ModelConfiguration],
) -> None:
"""Run cmdstanpy.CmdStanModel.sample, do diagnostics and save results.
:param study_name: a string
"""
infds = {}
for model_config in model_configurations:
fit_name = f"{study_name}-{model_config.name}"
print(f"Fitting model {fit_name}...")
loo_file = os.path.join(LOO_DIR, f"loo_{fit_name}.pkl")
infd_file = os.path.join(INFD_DIR, f"infd_{fit_name}.ncdf")
json_file = os.path.join(JSON_DIR, f"input_data_{fit_name}.json")
stan_input = model_config.stan_input_function(measurements)
print(f"Writing input data to {json_file}")
jsondump(json_file, stan_input)
model = CmdStanModel(
model_name=fit_name, stan_file=model_config.stan_file
)
print(f"Writing csv files to {SAMPLES_DIR}...")
mcmc = model.sample(
data=stan_input,
output_dir=SAMPLES_DIR,
**model_config.sample_kwargs,
)
print(mcmc.diagnose().replace("\n\n", "\n"))
infd = az.from_cmdstanpy(
mcmc, **model_config.infd_kwargs_function(measurements)
)
print(az.summary(infd))
infds[fit_name] = infd
print(f"Writing inference data to {infd_file}")
infd.to_netcdf(infd_file)
print(f"Writing psis-loo results to {loo_file}\n")
az.loo(infd, pointwise=True).to_pickle(loo_file)
if len(infds) > 1:
comparison = az.compare(infds)
print(f"Loo comparison:\n{comparison}")
comparison.to_csv(os.path.join(LOO_DIR, "loo_comparison.csv"))
def compare_models(dir_, p1, p2, include_sus=True, exclude=('divergence', )):
md1 = read_models(dir_, p1)
md2 = read_models(dir_, p2)
common_mods = set(md1.keys()).intersection(set(md2.keys()))
print(md2.keys())
print(common_mods)
comparisons = {}
for k in common_mods:
m1, _, d1 = md1[k]
m2, _, d2 = md2[k]
list(d1.pop(e) for e in exclude)
list(d2.pop(e) for e in exclude)
if include_sus or (np.all(list(d1.values()))
and np.all(list(d2.values()))):
l1 = av.loo(m1.arviz)
l2 = av.loo(m2.arviz)
d = {'p1': m1.arviz, 'p2': m2.arviz}
c = av.compare(d)
comparisons[k] = (l1, l2, c)
return comparisons
def run_validate_arviz(inferred):
az.plot_ppc(inferred, data_pairs={'y': 'y_hat'})
loo = az.loo(inferred, pointwise=True)
az.plot_khat(loo)
az.plot_loo_pit(inferred, y='y', y_hat='y_hat')
loo_pit = az.loo_pit(inferred, y='y', y_hat='y_hat')
bfmi = az.bfmi(inferred)
if any(bfmi < 0.5):
print("BFMI warning:", bfmi < 0.5)
print("LOO analysis:\n", loo)
return loo, loo_pit, bfmi
def plot_k_trials(fit_az,
k_thresh=.7,
dim_red=None,
ax=None,
k_color=(1, 0, 0),
reg_color=(.5, .5, .5),
plot_nk=False):
if ax is None:
f, ax = plt.subplots(1, 1)
x = az.loo(fit_az, pointwise=True)
k_vals = x['pareto_k']
k_mask = k_vals > k_thresh
k_trls = fit_az.observed_data.y[k_mask]
nk_trls = fit_az.observed_data.y[np.logical_not(k_mask)]
if dim_red is not None:
k_trls = dim_red(k_trls.T)
nk_trls = dim_red(nk_trls.T)
ax.plot(*k_trls, 'o', color=k_color)
if plot_nk:
ax.plot(*nk_trls, 'o', color=reg_color)
def get_fit_quality(fit) -> dict:
"""Compute Widely-Available Information Criterion (WAIC) and
Leave One Out (LOO) from a fit instance using Arviz.
Args:
fit: A PyStan4model instance (i.e. a PyStan fit).
Returns:
dict: WAIC and LOO statistics (and se's) for this fit.
"""
result = {}
try:
idata = az.from_pystan(fit, log_likelihood="llx")
except KeyError as e:
warn("'%s' not found; waic and loo will not be computed" % str(e),
stacklevel=2)
result.update({'waic': 0, 'loo': 0})
else:
result.update(dict(az.loo(idata, scale='deviance')))
result.update(dict(az.waic(idata, scale='deviance')))
result.update({'lp__rhat': get_rhat(fit)})
return result
def extract_ic(model_data: TaskModel,
ic: str = 'both',
ncore: int = 2) \
-> Dict:
"""Extract model comparison estimates.
Parameters
----------
model_data
hBayesDM output objects from running model functions.
ic
Information criterion. 'looic', 'waic', or 'both'. Defaults to 'both'.
ncore
Number of cores to use when computing LOOIC. Defaults to 2.
Returns
-------
Dict
Leave-One-Out and/or Watanabe-Akaike information criterion estimates.
"""
ic_options = ('looic', 'waic', 'both')
if ic not in ic_options:
raise RuntimeError('Information Criterion (ic) must be one of ' +
repr(ic_options))
dat = az.from_pystan(model_data.fit, log_likelihood='log_lik')
ret = {}
if ic in ['looic', 'both']:
ret['looic'] = az.loo(dat)['loo']
if ic in ['waic', 'both']:
ret['waic'] = az.waic(dat)['waic']
return ret
import arviz as az
idata = az.load_arviz_data("radon")
log_lik = idata.sample_stats[["log_likelihood"]]
loo_obs = az.loo(idata, pointwise=True)
print(loo_obs)
idata.sample_stats = log_lik.groupby("observed_county").sum()
loo_county = az.loo(idata, pointwise=True)
print(loo_county)
"""
Pareto Shape Plot
=================
_thumb: .7, .5
_example_title: k̂ plot
"""
import matplotlib.pyplot as plt
import arviz as az
az.style.use("arviz-darkgrid")
idata = az.load_arviz_data("radon")
loo = az.loo(idata, pointwise=True)
az.plot_khat(loo, show_bins=True)
plt.show()
sigma = pm.Exponential("sigma", 1.0)
#Likelihood
y = pm.Normal("y",
mu,
sigma=sigma,
observed=log_electricity,
dims="obs_id")
#Fitting
with partial_pooling_2:
approx = pm.fit(n=50000,
method='fullrank_advi',
callbacks=[CheckParametersConvergence(tolerance=0.01)])
partial_pooled_loo = az.loo(partial_pooling_trace, partial_pooling_2)
partial_pooled_waic = az.waic(partial_pooling_trace, partial_pooling_2)
with pm.Model(coords=coords_2) as no_pooling_2:
profile_cluster_idx = pm.Data("profile_cluster_idx",
clusters,
dims="obs_id")
heat_temp_cluster_idx = pm.Data("heat_temp_cluster_idx",
heat_clusters,
dims="obs_id")
cool_temp_cluster_idx = pm.Data("cool_temp_cluster_idx",
cool_clusters,
dims="obs_id")
daypart = pm.Data("daypart", dayparts, dims="obs_id")
fs_sin_1 = pm.Data("fs_sin_1", daypart_fs_sin_1, dims="obs_id")
def run(n=1000):
if n == "short":
n = 50
with schools:
tr = sample(n)
loo(tr)
model = TwTi_RoRi(parameters, hold_order=1)
reg = Regressor(model)
# Dynamic Hamiltonian Monte Carlo with multinomial sampling
fit = reg.fit(df=df, inputs=inputs, outputs='T_int')
# Compute the posterior predictive distribution
ym = reg.posterior_predictive(trace=fit.posterior, df=df, inputs=inputs)[0]
sns.set_style('darkgrid')
sns.set_context('talk')
percentile_plot(
df.index,
ym,
n=10,
percentile_min=0,
percentile_max=100,
plot_median=True,
plot_mean=False,
color='darkblue',
line_color='navy',
)
plt.plot(df.index, df['T_int'], color='darkred')
plt.tight_layout()
sns.despine()
# Compute Leave-one out cross validation with pareto smoothing importance sampling
loglik = reg.pointwise_log_likelihood(trace=fit.posterior, df=df, inputs=inputs, outputs='T_int')
az_data = az.from_dict(posterior=fit.posterior, sample_stats=loglik)
psis = az.loo(az_data, pointwise=True)
model = pkl.load(open(CODE_DIR+'assignment_errors/%s'%(model_name+'.pkl'),'rb'))
fit = model.sampling(data=data_dict['stan_data'], **fit_params['hmc_args'])
# fit = model.sampling(data=data_dict['stan_data'], iter=500, chains=4)
# fit_vals = {p:fit[p] for p in fit.model_pars}
#### Convert to Arviz and save ######
##########################################################
print('Converting to arviz ...')
model_params = {'observed_data':'y',
'log_likelihood':{'y':'log_lik'},
'posterior_predictive':'err_hat'}
fit_az = az.from_pystan(posterior=fit, **model_params)
loo = az.loo(fit_az, pointwise=True)
folds = hlp.folder_hierarchy(data_dict)
print('Saving everything to ' + SAVE_DIR+folds)
if not os.path.exists(SAVE_DIR+folds):
os.makedirs(SAVE_DIR+folds)
pkl.dump(fit_az, open(SAVE_DIR+folds+'/arviz_fit_%s.pkl'%model_name, 'wb'))
# pkl.dump(fit_vals, open(SAVE_DIR+folds+'/fitted_params_%s.pkl'%model_name, 'wb'))
pkl.dump(loo, open(SAVE_DIR+folds+'/cv_%s.pkl'%model_name, 'wb'))
if not os.path.exists(SAVE_DIR+folds+'/stan_data.pkl'):
pkl.dump(data_dict['stan_data'], open(SAVE_DIR+folds+'/stan_data.pkl', 'wb'))
print('Done !!!!!!!!!\n\n')
log_lik_incomp[j] = normal_lpdf(y_obs_incomp[j] | mu_i[j], sigma);
}
}
"""
stan_model = StanModel_cache(model_code=exp_code)
fit = stan_model.sampling(data=data, iter=4000, control={"adapt_delta": 0.9})
idata_exp = az.from_pystan(fit, dims=dims, log_likelihood=log_lik_dict)
log_lik_exp = idata_exp.log_likelihood
print("\n\nLeave one *observation* out cross validation (whole model)")
condition_dim = xr.DataArray(["compatible", "incompatible"], name="condition")
idata_exp.sample_stats["log_likelihood"] = xr.concat(
(log_lik_exp.y_obs_comp, log_lik_exp.y_obs_incomp), dim=condition_dim)
print(az.loo(idata_exp), "\n")
print("\n\nLeave one *subject* out cross validation (whole model)")
idata_exp.sample_stats["log_likelihood"] = log_lik_exp.to_array().sum(
"variable")
print(az.loo(idata_exp), "\n")
print("\n\nLeave one observation out cross validation (y_obs_comp only)")
idata_exp.sample_stats["log_likelihood"] = log_lik_exp.y_obs_comp
print(az.loo(idata_exp), "\n")
print("\n\nLeave one observation out cross validation (y_obs_incomp only)")
idata_exp.sample_stats["log_likelihood"] = log_lik_exp.y_obs_incomp
print(az.loo(idata_exp), "\n")
[Cross-validation](https://en.wikipedia.org/wiki/Cross-validation_(statistics)) (CV) is another method of estimating out-of-sample prediction accuracy. This method requires re-fitting a model many times, each time excluding a portion of the data, the excluded portion is then used to measure the accuracy of them model. This process is repeated many times and the estimated accuracy of the model will be the average of each run. Then the entire dataset is used to fit the model one more time and this is the model used for further analysis and/or predictions. Leave-one-out cross-validation (LOO-CV) is a particular type of cross-validation when the data excluded is a single data-point.
As CV can be quite time consuming (especially for Bayesian models) it is interesting to note that in theory it is possible to approximate LOO-CV. A practical and computational efficient way to do it requires using a combination of strategies that includes what is called [Pareto smoothed importance sampling](https://arxiv.org/abs/1507.02646). The resulting method is known as PSIS-LOO-CV which, while very useful, has a very complicated name, thus we just call it LOO.
While LOO and WAIC approximate two different quantities, asymptotically they converge to the same numerical value, and also in practice they generally agree. The main advantage of LOO is that it is more informative as it provides [useful diagnostics](https://arxiv.org/abs/1507.04544) and other goodies such as effective sample size and Monte Carlo standard error estimates.
Using ArviZ, both LOO and WAIC can be computed just by calling a function. Let's try on an arbitrary pre-loaded model:
# change this to some good example
model0 = az.load_arviz_data('regression1d')
model1 = az.load_arviz_data('regression1d')
az.waic(model0)
az.loo(model0)
As you can see both WAIC and LOO return similar values. ArviZ comes equipped with the `compare(.)` function. That is more convenient than using `loo(.)` or `waic(.)`
az.loo(model0)
## The compare function
This function takes a dictionary of names (keys) and models (values) as input and returns a DataFrame ordered (row-wise) from best to worst model.
cmp = az.compare({"m0":model0, "m1":model1,})
cmp
We have many columns, so let's check out their meaning one by one:
'rb') as fil:
data_dict = pkl.load(fil)
#### Load models ######
##########################################################
print('Comparing models')
folds = hlp.folder_hierarchy(data_dict)
model_files = [f for f in os.listdir(SAVE_DIR + folds) if 'arviz_fit_' in f]
these_models = [
re.findall('arviz_fit_(.+)_model.pkl', f)[0] for f in model_files
]
fits_az = {p: None for p in these_models}
for modfil, mod in zip(model_files, these_models):
with open(SAVE_DIR + folds + modfil, 'rb') as fil:
fits_az[mod] = pkl.load(fil)
loo = az.loo(fits_az[mod], pointwise=True)
with open(SAVE_DIR + folds + 'cv_%s_model.pkl' % mod, 'wb') as fil:
pkl.dump(loo, fil)
comp = az.compare(fits_az)
#### Save comparison ######
##########################################################
print('Saving ')
with open(SAVE_DIR + folds + 'model_comparisons.pkl', 'wb') as fil:
pkl.dump(comp, fil)
# }
# """
# stan_model = pystan.StanModel(model_code=the_separate_model)
data_for_stan = dict(N=accident_data.shape[0],
Y=accident_data.shape[1],
accidentData=accident_data,
years=np.arange(accident_data.shape[1]) + 1)
model_name = 'accident_hierarchical.stan'
stan_model = pystan.StanModel(file=model_path + '/' + model_name)
print(stan_model.model_code)
stan_results = stan_model.sampling(data=data_for_stan)
print(stan_results)
idata = az.from_pystan(stan_results, log_likelihood="log_lik")
loo = az.loo(idata)
print(loo)
# def result_statistics(stan_results, plot_ks =True,
# font_size=15,num_bins=20, fig_size = (12, 6)):
# log_lik = stan_results.extract()['log_lik']
# # According to the comments in psisloo, the first element in the result
# # is PSIS-LOO value, the third is ks
# psis_results = psisloo(log_lik)
# loo_value = psis_results[0]
# ks = psis_results[2]
# num_samples = log_lik.shape[0]
# # Equation 7.5 and 7.15 in BDA3
# computed_lppd = np.sum(np.log(np.sum(np.exp(log_lik), axis=0)/num_samples))
# p_loocv = computed_lppd - loo_value
"n": n_obs,
"age": age[mask],
"y_obs_comp": y_c[mask],
"y_obs_incomp": y_i[mask],
"mean_rt_c": y_c[mask].mean(),
"mean_rt_i": y_i[mask].mean(),
"n_ex": n_ex,
"age_ex": age[~mask],
"y_obs_comp_ex": y_c[~mask],
"y_obs_incomp_ex": y_i[~mask]
}
return observations, "log_lik_ex"
sys.stdout = backup
print("\n\n(PSIS) Leave one *subject* out cross validation (whole model)")
idata_exp.sample_stats["log_likelihood"] = idata_exp.log_likelihood.log_lik
loo_psis = az.loo(idata_exp, pointwise=True)
print(loo_psis, "\n")
print("\n\n(exact) Leave one *subject* out cross validation (whole model)")
sys.stdout = output
loo_psis.pareto_k[:] = 1.2
exp_wrapper = ExpWrapper(stan_model,
idata_orig=idata_exp,
sample_kwargs=fit_kwargs,
idata_kwargs=idata_kwargs)
loo_exact = az.reloo(exp_wrapper, loo_orig=loo_psis)
sys.stdout = backup
print(loo_exact, "\n")
