def get_waic_and_loo(fit): """Compute WAIC and LOO from a fit instance""" idata = az.from_pystan(fit, log_likelihood="llx") result = {} result.update(dict(az.loo(idata, scale='deviance'))) result.update(dict(az.waic(idata, scale='deviance'))) return result
def get_pareto_k(fit, k_thresh=.7): x = az.loo(fit, pointwise=True) k_val = x['pareto_k'] k_mask = k_val > k_thresh inds = np.where(k_mask)[0] trls = fit.observed_data.y[k_mask] return k_val, inds, trls
def plot_ppc_and_score(trace, data, ax=None, title='PPC', paras=None): # Sample PPC ppc_trace = pm.sample_posterior_predictive(trace=trace, var_names=['y']) # Calculate LOO score loo = az.loo(trace).loo loo_text = "LOO = %.2f"%loo # Aggregate binary responses new_trace = [] for soa in sorted(set((data.SOA_IN_FRAMES))): new_trace.append(ppc_trace['y'][:,(data.SOA_IN_FRAMES==soa) & (data.PROBE_SALIENT==0)].mean(axis=1)) new_trace.append(ppc_trace['y'][:,(data.SOA_IN_FRAMES==soa) & (data.PROBE_SALIENT==1)].mean(axis=1)) ppc_trace = {'y': np.array(new_trace).T} # Prepare axes if none provided if ax is None: f,ax= plt.subplots() # Get SOAs and condition mask from data SOAs = sorted(set(data['SOA_IN_MS'])) cond = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT'])['PROBE_SALIENT'].min().values # Plot az.plot_hdi(y=ppc_trace['y'][:,cond==0],x=SOAs, color='k', ax=ax, hdi_prob=0.95, fill_kwargs={'alpha' : 0.23}) az.plot_hdi(y=ppc_trace['y'][:,cond==1],x=SOAs, color='g', ax=ax, hdi_prob=0.95, fill_kwargs={'alpha' : 0.23}) ax.plot(SOAs, np.mean(ppc_trace['y'][:,cond==0],axis=0), color='k') ax.plot(SOAs, np.mean(ppc_trace['y'][:,cond==1],axis=0), color='g') pf_mean = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).mean().PROBE_FIRST_RESPONSE pf_count = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).sum().PROBE_FIRST_RESPONSE pf_obs = data.groupby(['SOA_IN_MS', 'PROBE_SALIENT']).count().PROBE_FIRST_RESPONSE pf_ci = abs(np.array(prop_ci(pf_count.values, pf_obs.values)) - pf_mean.values) ax.plot(SOAs, pf_mean.values[::2], 'k.') ax.errorbar(np.array(SOAs)-0.5, pf_mean.values[::2], pf_ci[:,::2], fmt='none', color='k', alpha=0.5) ax.plot(SOAs, pf_mean.values[1::2], 'g.') ax.errorbar(np.array(SOAs)+0.5, pf_mean.values[1::2], pf_ci[:,1::2], fmt='none', color='g', alpha=0.5) ax.axvline(0, linestyle='dashed') ax.axhline(0.5, linestyle='dashed') ax.text(-20,0, loo_text) if paras is not None: for i, varname in enumerate(paras): stats = az.summary(trace, var_names=[varname], hdi_prob=.95) for j, s in enumerate(stats['mean']): text = r'$' + varname + r'$: %.2f [%.2f, %.2f]' text = text%(s, stats['hdi_2.5%'][j], stats['hdi_97.5%'][j]) posx, posy = .1 + .5 - (1 - j) * .5, 0.95 - (.05*i) - ((1-j)*.5) ax.text(posx, posy, text, transform = ax.transAxes, color=['k','g'][j]) ax.set_title(title)
def generate_samples( study_name: str, measurements: pd.DataFrame, model_configurations: List[ModelConfiguration], ) -> None: """Run cmdstanpy.CmdStanModel.sample, do diagnostics and save results. :param study_name: a string """ infds = {} for model_config in model_configurations: fit_name = f"{study_name}-{model_config.name}" print(f"Fitting model {fit_name}...") loo_file = os.path.join(LOO_DIR, f"loo_{fit_name}.pkl") infd_file = os.path.join(INFD_DIR, f"infd_{fit_name}.ncdf") json_file = os.path.join(JSON_DIR, f"input_data_{fit_name}.json") stan_input = model_config.stan_input_function(measurements) print(f"Writing input data to {json_file}") jsondump(json_file, stan_input) model = CmdStanModel( model_name=fit_name, stan_file=model_config.stan_file ) print(f"Writing csv files to {SAMPLES_DIR}...") mcmc = model.sample( data=stan_input, output_dir=SAMPLES_DIR, **model_config.sample_kwargs, ) print(mcmc.diagnose().replace("\n\n", "\n")) infd = az.from_cmdstanpy( mcmc, **model_config.infd_kwargs_function(measurements) ) print(az.summary(infd)) infds[fit_name] = infd print(f"Writing inference data to {infd_file}") infd.to_netcdf(infd_file) print(f"Writing psis-loo results to {loo_file}\n") az.loo(infd, pointwise=True).to_pickle(loo_file) if len(infds) > 1: comparison = az.compare(infds) print(f"Loo comparison:\n{comparison}") comparison.to_csv(os.path.join(LOO_DIR, "loo_comparison.csv"))
def compare_models(dir_, p1, p2, include_sus=True, exclude=('divergence', )): md1 = read_models(dir_, p1) md2 = read_models(dir_, p2) common_mods = set(md1.keys()).intersection(set(md2.keys())) print(md2.keys()) print(common_mods) comparisons = {} for k in common_mods: m1, _, d1 = md1[k] m2, _, d2 = md2[k] list(d1.pop(e) for e in exclude) list(d2.pop(e) for e in exclude) if include_sus or (np.all(list(d1.values())) and np.all(list(d2.values()))): l1 = av.loo(m1.arviz) l2 = av.loo(m2.arviz) d = {'p1': m1.arviz, 'p2': m2.arviz} c = av.compare(d) comparisons[k] = (l1, l2, c) return comparisons
def run_validate_arviz(inferred): az.plot_ppc(inferred, data_pairs={'y': 'y_hat'}) loo = az.loo(inferred, pointwise=True) az.plot_khat(loo) az.plot_loo_pit(inferred, y='y', y_hat='y_hat') loo_pit = az.loo_pit(inferred, y='y', y_hat='y_hat') bfmi = az.bfmi(inferred) if any(bfmi < 0.5): print("BFMI warning:", bfmi < 0.5) print("LOO analysis:\n", loo) return loo, loo_pit, bfmi
def plot_k_trials(fit_az, k_thresh=.7, dim_red=None, ax=None, k_color=(1, 0, 0), reg_color=(.5, .5, .5), plot_nk=False): if ax is None: f, ax = plt.subplots(1, 1) x = az.loo(fit_az, pointwise=True) k_vals = x['pareto_k'] k_mask = k_vals > k_thresh k_trls = fit_az.observed_data.y[k_mask] nk_trls = fit_az.observed_data.y[np.logical_not(k_mask)] if dim_red is not None: k_trls = dim_red(k_trls.T) nk_trls = dim_red(nk_trls.T) ax.plot(*k_trls, 'o', color=k_color) if plot_nk: ax.plot(*nk_trls, 'o', color=reg_color)
def get_fit_quality(fit) -> dict: """Compute Widely-Available Information Criterion (WAIC) and Leave One Out (LOO) from a fit instance using Arviz. Args: fit: A PyStan4model instance (i.e. a PyStan fit). Returns: dict: WAIC and LOO statistics (and se's) for this fit. """ result = {} try: idata = az.from_pystan(fit, log_likelihood="llx") except KeyError as e: warn("'%s' not found; waic and loo will not be computed" % str(e), stacklevel=2) result.update({'waic': 0, 'loo': 0}) else: result.update(dict(az.loo(idata, scale='deviance'))) result.update(dict(az.waic(idata, scale='deviance'))) result.update({'lp__rhat': get_rhat(fit)}) return result
def extract_ic(model_data: TaskModel, ic: str = 'both', ncore: int = 2) \ -> Dict: """Extract model comparison estimates. Parameters ---------- model_data hBayesDM output objects from running model functions. ic Information criterion. 'looic', 'waic', or 'both'. Defaults to 'both'. ncore Number of cores to use when computing LOOIC. Defaults to 2. Returns ------- Dict Leave-One-Out and/or Watanabe-Akaike information criterion estimates. """ ic_options = ('looic', 'waic', 'both') if ic not in ic_options: raise RuntimeError('Information Criterion (ic) must be one of ' + repr(ic_options)) dat = az.from_pystan(model_data.fit, log_likelihood='log_lik') ret = {} if ic in ['looic', 'both']: ret['looic'] = az.loo(dat)['loo'] if ic in ['waic', 'both']: ret['waic'] = az.waic(dat)['waic'] return ret
import arviz as az idata = az.load_arviz_data("radon") log_lik = idata.sample_stats[["log_likelihood"]] loo_obs = az.loo(idata, pointwise=True) print(loo_obs) idata.sample_stats = log_lik.groupby("observed_county").sum() loo_county = az.loo(idata, pointwise=True) print(loo_county)
""" Pareto Shape Plot ================= _thumb: .7, .5 _example_title: k̂ plot """ import matplotlib.pyplot as plt import arviz as az az.style.use("arviz-darkgrid") idata = az.load_arviz_data("radon") loo = az.loo(idata, pointwise=True) az.plot_khat(loo, show_bins=True) plt.show()
sigma = pm.Exponential("sigma", 1.0) #Likelihood y = pm.Normal("y", mu, sigma=sigma, observed=log_electricity, dims="obs_id") #Fitting with partial_pooling_2: approx = pm.fit(n=50000, method='fullrank_advi', callbacks=[CheckParametersConvergence(tolerance=0.01)]) partial_pooled_loo = az.loo(partial_pooling_trace, partial_pooling_2) partial_pooled_waic = az.waic(partial_pooling_trace, partial_pooling_2) with pm.Model(coords=coords_2) as no_pooling_2: profile_cluster_idx = pm.Data("profile_cluster_idx", clusters, dims="obs_id") heat_temp_cluster_idx = pm.Data("heat_temp_cluster_idx", heat_clusters, dims="obs_id") cool_temp_cluster_idx = pm.Data("cool_temp_cluster_idx", cool_clusters, dims="obs_id") daypart = pm.Data("daypart", dayparts, dims="obs_id") fs_sin_1 = pm.Data("fs_sin_1", daypart_fs_sin_1, dims="obs_id")
def run(n=1000): if n == "short": n = 50 with schools: tr = sample(n) loo(tr)
model = TwTi_RoRi(parameters, hold_order=1) reg = Regressor(model) # Dynamic Hamiltonian Monte Carlo with multinomial sampling fit = reg.fit(df=df, inputs=inputs, outputs='T_int') # Compute the posterior predictive distribution ym = reg.posterior_predictive(trace=fit.posterior, df=df, inputs=inputs)[0] sns.set_style('darkgrid') sns.set_context('talk') percentile_plot( df.index, ym, n=10, percentile_min=0, percentile_max=100, plot_median=True, plot_mean=False, color='darkblue', line_color='navy', ) plt.plot(df.index, df['T_int'], color='darkred') plt.tight_layout() sns.despine() # Compute Leave-one out cross validation with pareto smoothing importance sampling loglik = reg.pointwise_log_likelihood(trace=fit.posterior, df=df, inputs=inputs, outputs='T_int') az_data = az.from_dict(posterior=fit.posterior, sample_stats=loglik) psis = az.loo(az_data, pointwise=True)
model = pkl.load(open(CODE_DIR+'assignment_errors/%s'%(model_name+'.pkl'),'rb')) fit = model.sampling(data=data_dict['stan_data'], **fit_params['hmc_args']) # fit = model.sampling(data=data_dict['stan_data'], iter=500, chains=4) # fit_vals = {p:fit[p] for p in fit.model_pars} #### Convert to Arviz and save ###### ########################################################## print('Converting to arviz ...') model_params = {'observed_data':'y', 'log_likelihood':{'y':'log_lik'}, 'posterior_predictive':'err_hat'} fit_az = az.from_pystan(posterior=fit, **model_params) loo = az.loo(fit_az, pointwise=True) folds = hlp.folder_hierarchy(data_dict) print('Saving everything to ' + SAVE_DIR+folds) if not os.path.exists(SAVE_DIR+folds): os.makedirs(SAVE_DIR+folds) pkl.dump(fit_az, open(SAVE_DIR+folds+'/arviz_fit_%s.pkl'%model_name, 'wb')) # pkl.dump(fit_vals, open(SAVE_DIR+folds+'/fitted_params_%s.pkl'%model_name, 'wb')) pkl.dump(loo, open(SAVE_DIR+folds+'/cv_%s.pkl'%model_name, 'wb')) if not os.path.exists(SAVE_DIR+folds+'/stan_data.pkl'): pkl.dump(data_dict['stan_data'], open(SAVE_DIR+folds+'/stan_data.pkl', 'wb')) print('Done !!!!!!!!!\n\n')
log_lik_incomp[j] = normal_lpdf(y_obs_incomp[j] | mu_i[j], sigma); } } """ stan_model = StanModel_cache(model_code=exp_code) fit = stan_model.sampling(data=data, iter=4000, control={"adapt_delta": 0.9}) idata_exp = az.from_pystan(fit, dims=dims, log_likelihood=log_lik_dict) log_lik_exp = idata_exp.log_likelihood print("\n\nLeave one *observation* out cross validation (whole model)") condition_dim = xr.DataArray(["compatible", "incompatible"], name="condition") idata_exp.sample_stats["log_likelihood"] = xr.concat( (log_lik_exp.y_obs_comp, log_lik_exp.y_obs_incomp), dim=condition_dim) print(az.loo(idata_exp), "\n") print("\n\nLeave one *subject* out cross validation (whole model)") idata_exp.sample_stats["log_likelihood"] = log_lik_exp.to_array().sum( "variable") print(az.loo(idata_exp), "\n") print("\n\nLeave one observation out cross validation (y_obs_comp only)") idata_exp.sample_stats["log_likelihood"] = log_lik_exp.y_obs_comp print(az.loo(idata_exp), "\n") print("\n\nLeave one observation out cross validation (y_obs_incomp only)") idata_exp.sample_stats["log_likelihood"] = log_lik_exp.y_obs_incomp print(az.loo(idata_exp), "\n")
[Cross-validation](https://en.wikipedia.org/wiki/Cross-validation_(statistics)) (CV) is another method of estimating out-of-sample prediction accuracy. This method requires re-fitting a model many times, each time excluding a portion of the data, the excluded portion is then used to measure the accuracy of them model. This process is repeated many times and the estimated accuracy of the model will be the average of each run. Then the entire dataset is used to fit the model one more time and this is the model used for further analysis and/or predictions. Leave-one-out cross-validation (LOO-CV) is a particular type of cross-validation when the data excluded is a single data-point. As CV can be quite time consuming (especially for Bayesian models) it is interesting to note that in theory it is possible to approximate LOO-CV. A practical and computational efficient way to do it requires using a combination of strategies that includes what is called [Pareto smoothed importance sampling](https://arxiv.org/abs/1507.02646). The resulting method is known as PSIS-LOO-CV which, while very useful, has a very complicated name, thus we just call it LOO. While LOO and WAIC approximate two different quantities, asymptotically they converge to the same numerical value, and also in practice they generally agree. The main advantage of LOO is that it is more informative as it provides [useful diagnostics](https://arxiv.org/abs/1507.04544) and other goodies such as effective sample size and Monte Carlo standard error estimates. Using ArviZ, both LOO and WAIC can be computed just by calling a function. Let's try on an arbitrary pre-loaded model: # change this to some good example model0 = az.load_arviz_data('regression1d') model1 = az.load_arviz_data('regression1d') az.waic(model0) az.loo(model0) As you can see both WAIC and LOO return similar values. ArviZ comes equipped with the `compare(.)` function. That is more convenient than using `loo(.)` or `waic(.)` az.loo(model0) ## The compare function This function takes a dictionary of names (keys) and models (values) as input and returns a DataFrame ordered (row-wise) from best to worst model. cmp = az.compare({"m0":model0, "m1":model1,}) cmp We have many columns, so let's check out their meaning one by one:
'rb') as fil: data_dict = pkl.load(fil) #### Load models ###### ########################################################## print('Comparing models') folds = hlp.folder_hierarchy(data_dict) model_files = [f for f in os.listdir(SAVE_DIR + folds) if 'arviz_fit_' in f] these_models = [ re.findall('arviz_fit_(.+)_model.pkl', f)[0] for f in model_files ] fits_az = {p: None for p in these_models} for modfil, mod in zip(model_files, these_models): with open(SAVE_DIR + folds + modfil, 'rb') as fil: fits_az[mod] = pkl.load(fil) loo = az.loo(fits_az[mod], pointwise=True) with open(SAVE_DIR + folds + 'cv_%s_model.pkl' % mod, 'wb') as fil: pkl.dump(loo, fil) comp = az.compare(fits_az) #### Save comparison ###### ########################################################## print('Saving ') with open(SAVE_DIR + folds + 'model_comparisons.pkl', 'wb') as fil: pkl.dump(comp, fil)
# } # """ # stan_model = pystan.StanModel(model_code=the_separate_model) data_for_stan = dict(N=accident_data.shape[0], Y=accident_data.shape[1], accidentData=accident_data, years=np.arange(accident_data.shape[1]) + 1) model_name = 'accident_hierarchical.stan' stan_model = pystan.StanModel(file=model_path + '/' + model_name) print(stan_model.model_code) stan_results = stan_model.sampling(data=data_for_stan) print(stan_results) idata = az.from_pystan(stan_results, log_likelihood="log_lik") loo = az.loo(idata) print(loo) # def result_statistics(stan_results, plot_ks =True, # font_size=15,num_bins=20, fig_size = (12, 6)): # log_lik = stan_results.extract()['log_lik'] # # According to the comments in psisloo, the first element in the result # # is PSIS-LOO value, the third is ks # psis_results = psisloo(log_lik) # loo_value = psis_results[0] # ks = psis_results[2] # num_samples = log_lik.shape[0] # # Equation 7.5 and 7.15 in BDA3 # computed_lppd = np.sum(np.log(np.sum(np.exp(log_lik), axis=0)/num_samples)) # p_loocv = computed_lppd - loo_value
"n": n_obs, "age": age[mask], "y_obs_comp": y_c[mask], "y_obs_incomp": y_i[mask], "mean_rt_c": y_c[mask].mean(), "mean_rt_i": y_i[mask].mean(), "n_ex": n_ex, "age_ex": age[~mask], "y_obs_comp_ex": y_c[~mask], "y_obs_incomp_ex": y_i[~mask] } return observations, "log_lik_ex" sys.stdout = backup print("\n\n(PSIS) Leave one *subject* out cross validation (whole model)") idata_exp.sample_stats["log_likelihood"] = idata_exp.log_likelihood.log_lik loo_psis = az.loo(idata_exp, pointwise=True) print(loo_psis, "\n") print("\n\n(exact) Leave one *subject* out cross validation (whole model)") sys.stdout = output loo_psis.pareto_k[:] = 1.2 exp_wrapper = ExpWrapper(stan_model, idata_orig=idata_exp, sample_kwargs=fit_kwargs, idata_kwargs=idata_kwargs) loo_exact = az.reloo(exp_wrapper, loo_orig=loo_psis) sys.stdout = backup print(loo_exact, "\n")