def play_components(self, X_hat=None, x=None, orig_mix=0.1): """ :: play individual separated feature spectrograms as audio separated spectrograms are stored in a list """ X_hat = self.X_hat if X_hat is None else X_hat for Xh in X_hat: x_hat = self.inverse(Xh) # invert to audio to F.x_hat x = x_hat if x is None else x x = x.mean(1) if len(x.shape) > 1 else x xn = min(len(x), len(x_hat)) x_orig = orig_mix * P.atleast_1d(x[:xn]).T x_hat = P.atleast_1d(x_hat[:xn] / (x_hat[:xn].max() + 0.01)) play(P.c_[x_orig, x_hat].T, self.feature_params['sample_rate'])
def play_components(self, X_hat=None, x=None, orig_mix=0.1): """ :: play individual separated feature spectrograms as audio separated spectrograms are stored in a list """ X_hat = self.X_hat if X_hat is None else X_hat for Xh in X_hat: x_hat = self.inverse(Xh) # invert to audio to F.x_hat x = x_hat if x is None else x x = x.mean(1) if len(x.shape)>1 else x xn = min(len(x), len(x_hat)) x_orig = orig_mix*P.atleast_1d(x[:xn]).T x_hat = P.atleast_1d(x_hat[:xn] / (x_hat[:xn].max() + 0.01)) play(P.c_[x_orig, x_hat].T, self.feature_params['sample_rate'])
def ImageWithColorbar(ax, image, **kwargs): """ Plot image with colorbar on the right Parameters ------------ ax : matplotlib Axes instance image : 2-d array Additional kwargs are passed to imshow Returns -------- ax : image axes cb : colorbar axes """ from mpl_toolkits.axes_grid1 import make_axes_locatable divider = make_axes_locatable(ax) ax_cb = divider.append_axes("right", size="5%", pad=0.05) fig1 = ax.get_figure() fig1.add_axes(ax_cb) dim = pl.atleast_1d(image.shape).astype('float') lim = dim - dim/2. extent = (-lim[0], lim[0], -lim[1], lim[1]) im = ax.imshow( image, interpolation='nearest', origin='lower', extent=extent, **kwargs) ax.minorticks_on() cb = pl.colorbar(im, cax=ax_cb, orientation='vertical') ax_cb.xaxis.tick_bottom() return ax, cb
def plot_viz_of_stochs(vars, viz_func, figsize=(8,6)): """ Plot autocorrelation for all stochs in a dict or dict of dicts :Parameters: - `vars` : dictionary - `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist`` - `figsize` : tuple, size of figure """ pl.figure(figsize=figsize) cells, stochs = tally_stochs(vars) # for each stoch, make an autocorrelation plot for each dimension rows = pl.floor(pl.sqrt(cells)) cols = pl.ceil(cells/rows) tile = 1 for s in sorted(stochs, key=lambda s: s.__name__): trace = s.trace() if len(trace.shape) == 1: trace = trace.reshape((len(trace), 1)) for d in range(len(pl.atleast_1d(s.value))): pl.subplot(rows, cols, tile) viz_func(pl.atleast_2d(trace)[:, d]) pl.title('\n\n%s[%d]'%(s.__name__, d), va='top', ha='center', fontsize=8) tile += 1
def plot_viz_of_stochs(vars, viz_func, figsize=(8, 6)): """ Plot autocorrelation for all stochs in a dict or dict of dicts :Parameters: - `vars` : dictionary - `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist`` - `figsize` : tuple, size of figure """ pl.figure(figsize=figsize) cells, stochs = tally_stochs(vars) # for each stoch, make an autocorrelation plot for each dimension rows = pl.floor(pl.sqrt(cells)) cols = pl.ceil(cells / rows) tile = 1 for s in sorted(stochs, key=lambda s: s.__name__): trace = s.trace() if len(trace.shape) == 1: trace = trace.reshape((len(trace), 1)) for d in range(len(pl.atleast_1d(s.value))): pl.subplot(rows, cols, tile) viz_func(pl.atleast_2d(trace)[:, d]) pl.title('\n\n%s[%d]' % (s.__name__, d), va='top', ha='center', fontsize=8) tile += 1
def tally_stochs(vars): """ Count number of stochastics in model :Parameters: - `vars` : dictionary """ cells = 0 stochs = [] for k in vars.keys(): # handle dicts and dicts of dicts by making a list of nodes if isinstance(vars[k], dict): nodes = vars[k].values() else: nodes = [vars[k]] # handle lists of stochs for n in nodes: if isinstance(n, list): nodes += n for n in nodes: if isinstance(n, mc.Stochastic) and not n.observed: trace = n.trace() if len(trace) > 0: stochs.append(n) cells += len(pl.atleast_1d(n.value)) return cells, stochs
def obs_lb(value=value, N=N, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], delta=delta, age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = pl.exp(pl.dot(Xa, alpha) + pl.dot(Xb, pl.atleast_1d(beta))) exp_gamma = pl.exp(gamma) mu_i = [pl.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed rate_param = mu_i*N violated_bounds = pl.nonzero(rate_param < value) logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta) return logp
def check_convergence(vars): """ Apply a simple test of convergence to the model: compare autocorrelation at 25 lags to zero lags. warn about convergence if it exceeds 10% for any stoch """ import dismod3 cells, stochs = dismod_mr.plot.tally_stochs(vars) for s in sorted(stochs, key=lambda s: s.__name__): tr = s.trace() if len(tr.shape) == 1: tr = tr.reshape((len(tr), 1)) for d in range(len(pl.atleast_1d(s.value))): for k in range(50,100): acorr = pl.dot(tr[:-k,d]-tr[:k,d].mean(), tr[k:,d]-tr[k:,d].mean()) / pl.dot(tr[k:,d]-tr[k:,d].mean(), tr[k:,d]-tr[k:,d].mean()) if abs(acorr) > .5: print 'potential non-convergence', s, acorr return False return True
def rates(S=data_sample, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = pl.exp(pl.dot(Xa[S], alpha) + pl.dot(Xb[S], pl.atleast_1d(beta))) exp_gamma = pl.exp(gamma) mu = pl.zeros_like(shifts) for i,s in enumerate(S): mu[i] = pl.dot(age_weights[s], bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s])) # TODO: evaluate speed increase and accuracy decrease of the following: #midpoint = age_indices[s][len(age_indices[s])/2] #mu[i] = bounds_func(shifts[i] * exp_gamma[midpoint], midpoint) # TODO: evaluate speed increase and accuracy decrease of the following: (to see speed increase, need to code this up using difference of running sums #mu[i] = pl.dot(pl.ones_like(age_weights[s]) / float(len(age_weights[s])), # bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s])) return mu
def check_convergence(vars): """ Apply a simple test of convergence to the model: compare autocorrelation at 25 lags to zero lags. warn about convergence if it exceeds 10% for any stoch """ import dismod_mr cells, stochs = dismod_mr.plot.tally_stochs(vars) for s in sorted(stochs, key=lambda s: s.__name__): tr = s.trace() if len(tr.shape) == 1: tr = tr.reshape((len(tr), 1)) for d in range(len(pl.atleast_1d(s.value))): for k in range(50,100): acorr = pl.dot(tr[:-k,d]-tr[:k,d].mean(), tr[k:,d]-tr[k:,d].mean()) / pl.dot(tr[k:,d]-tr[k:,d].mean(), tr[k:,d]-tr[k:,d].mean()) if abs(acorr) > .5: print('potential non-convergence', s, acorr) return False return True
def plot_one_effects(model, data_type): """ Plot random effects and fixed effects. :Parameters: - `model` : data.ModelData - `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf' """ vars = model.vars[data_type] hierarchy = model.hierarchy pl.figure(figsize=(22, 17)) for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]): if covariate not in vars: continue cov_name = list(vars[covariate].columns) if isinstance(vars.get(effect), mc.Stochastic): pl.subplot(1, 2, i + 1) pl.title('%s_%s' % (effect, data_type)) stats = vars[effect].stats() if stats: if effect == 'alpha': index = sorted( pl.arange(len(cov_name)), key=lambda i: str(cov_name[ i] in hierarchy and nx.shortest_path( hierarchy, 'all', cov_name[i]) or cov_name[i])) elif effect == 'beta': index = pl.arange(len(cov_name)) x = pl.atleast_1d(stats['mean']) y = pl.arange(len(x)) xerr = pl.array([ x - pl.atleast_2d(stats['95% HPD interval'])[:, 0], pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x ]) pl.errorbar(x[index], y[index], xerr=xerr[:, index], fmt='bs', mec='w') l, r, b, t = pl.axis() pl.vlines([0], b - .5, t + .5) pl.hlines(y, l, r, linestyle='dotted') pl.xticks([l, 0, r]) pl.yticks([]) for i in index: spaces = cov_name[i] in hierarchy and len( nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0 pl.text(l, y[i], '%s%s' % (' * ' * spaces, cov_name[i]), va='center', ha='left') pl.axis([l, r, -.5, t + .5]) if isinstance(vars.get(effect), list): pl.subplot(1, 2, i + 1) pl.title('%s_%s' % (effect, data_type)) index = sorted(pl.arange(len(cov_name)), key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path( hierarchy, 'all', cov_name[i]) or cov_name[i])) for y, i in enumerate(index): n = vars[effect][i] if isinstance(n, mc.Stochastic) or isinstance( n, mc.Deterministic): stats = n.stats() if stats: x = pl.atleast_1d(stats['mean']) xerr = pl.array([ x - pl.atleast_2d(stats['95% HPD interval'])[:, 0], pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x ]) pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w') l, r, b, t = pl.axis() pl.vlines([0], b - .5, t + .5) pl.hlines(y, l, r, linestyle='dotted') pl.xticks([l, 0, r]) pl.yticks([]) for y, i in enumerate(index): spaces = cov_name[i] in hierarchy and len( nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0 pl.text(l, y, '%s%s' % (' * ' * spaces, cov_name[i]), va='center', ha='left') pl.axis([l, r, -.5, t + .5]) if effect == 'alpha': effect_str = '' for sigma in vars['sigma_alpha']: stats = sigma.stats() if stats: effect_str += '%s = %.3f\n' % (sigma.__name__, stats['mean']) else: effect_str += '%s = %.3f\n' % (sigma.__name__, sigma.value) pl.text(r, t, effect_str, va='top', ha='right') elif effect == 'beta': effect_str = '' if 'eta' in vars: eta = vars['eta'] stats = eta.stats() if stats: effect_str += '%s = %.3f\n' % (eta.__name__, stats['mean']) else: effect_str += '%s = %.3f\n' % (eta.__name__, eta.value) pl.text(r, t, effect_str, va='top', ha='right')
def fit_emp_prior(dm, param_type, iter=100000, thin=50, burn=50000, dbname='/dev/null', map_only=False, store_results=True): """ Generate an empirical prior distribution for a single disease parameter Parameters ---------- dm : dismod3.DiseaseModel The object containing all the data, (hyper)-priors, and additional information (like input and output age-mesh). param_type : str, one of 'incidence', 'prevalence', 'remission', 'excess-mortality' The disease parameter to work with Notes ----- The results of this fit are stored in the disease model's params hash for use when fitting multiple paramter types together Example ------- $ python2.5 gbd_fit.py 231 -t incidence """ data = [d for d in dm.data if \ d['data_type'] == '%s data' % param_type \ and d.get('ignore') != -1] dm.clear_empirical_prior() dm.calc_effective_sample_size(data) dm.fit_initial_estimate(param_type, data) dm.vars = setup(dm, param_type, data) # don't do anything if there is no data for this parameter type if not dm.vars['data']: return debug('i: %s' % ', '.join(['%.2f' % x for x in dm.vars['rate_stoch'].value[::10]])) sys.stdout.flush() # fit the model def map_fit(stoch_names): print '\nfitting', ' '.join(stoch_names) map = mc.MAP([dm.vars[key] for key in stoch_names] + [dm.vars['observed_counts'], dm.vars['rate_potential'], dm.vars['priors']]) try: map.fit(method='fmin_powell', verbose=verbose) except KeyboardInterrupt: debug('User halted optimization routine before optimal value found') for key in stoch_names: print key, dm.vars[key].value.round(2) sys.stdout.flush() def mcmc_fit(stoch_names): print '\nfitting', ' '.join(stoch_names) mcmc = mc.MCMC([dm.vars[key] for key in stoch_names] + [dm.vars['observed_counts'], dm.vars['rate_potential'], dm.vars['priors']]) mcmc.use_step_method(mc.Metropolis, dm.vars['log_dispersion'], proposal_sd=dm.vars['dispersion_step_sd']) # TODO: make a wrapper function for handling this adaptive metropolis setup stoch_list = [dm.vars['study_coeffs'], dm.vars['region_coeffs'], dm.vars['age_coeffs_mesh']] d1 = len(dm.vars['study_coeffs'].value) d2 = len(dm.vars['region_coeffs_step_cov']) d3 = len(dm.vars['age_coeffs_mesh_step_cov']) C = pl.eye(d1+d2+d3) C[d1:(d1+d2), d1:(d1+d2)] = dm.vars['region_coeffs_step_cov'] C[(d1+d2):(d1+d2+d3), (d1+d2):(d1+d2+d3)] = dm.vars['age_coeffs_mesh_step_cov'] C *= .01 mcmc.use_step_method(mc.AdaptiveMetropolis, stoch_list, cov=C) # more step methods mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['study_coeffs']) mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['region_coeffs'], cov=dm.vars['region_coeffs_step_cov']) mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['age_coeffs_mesh'], cov=dm.vars['age_coeffs_mesh_step_cov']) try: mcmc.sample(iter=10000, burn=5000, thin=5, verbose=verbose) except KeyboardInterrupt: debug('User halted optimization routine before optimal value found') sys.stdout.flush() # reset stoch values to sample mean for key in stoch_names: mean = dm.vars[key].stats()['mean'] if isinstance(dm.vars[key], mc.Stochastic): dm.vars[key].value = mean print key, mean.round(2) verbose = 1 stoch_names = 'region_coeffs age_coeffs_mesh study_coeffs'.split() ## start by optimizing parameters separately for key in stoch_names: map_fit([key]) ## then fit them all together map_fit(stoch_names) # now find the over-dispersion parameter that matches these values map_fit(['log_dispersion']) if map_only: return # make pymc warnings go to stdout mc.warnings.warn = sys.stdout.write mcmc_fit(['log_dispersion', 'dispersion', 'study_coeffs', 'region_coeffs', 'age_coeffs_mesh', 'age_coeffs', 'predicted_rates', 'expected_rates', 'rate_stoch']) alpha = dm.vars['region_coeffs'].stats()['mean'] beta = dm.vars['study_coeffs'].stats()['mean'] gamma_mesh = dm.vars['age_coeffs_mesh'].stats()['mean'] debug('a: %s' % ', '.join(['%.2f' % x for x in alpha])) debug('b: %s' % ', '.join(['%.2f' % x for x in pl.atleast_1d(beta)])) debug('g: %s' % ', '.join(['%.2f' % x for x in gamma_mesh])) debug('d: %.2f' % dm.vars['dispersion'].stats()['mean']) covariates_dict = dm.get_covariates() derived_covariate = dm.get_derived_covariate_values() X = covariates(data[0], covariates_dict) debug('p: %s' % ', '.join(['%.2f' % x for x in predict_rate(X, alpha, beta, gamma_mesh, dm.vars['bounds_func'], dm.get_param_age_mesh())])) if not store_results: return # save the results in the param_hash prior_vals = dict( alpha=list(dm.vars['region_coeffs'].stats()['mean']), beta=list(pl.atleast_1d(dm.vars['study_coeffs'].stats()['mean'])), gamma=list(dm.vars['age_coeffs'].stats()['mean']), delta=float(dm.vars['dispersion'].stats()['mean'])) prior_vals.update( sigma_alpha=list(dm.vars['region_coeffs'].stats()['standard deviation']), sigma_beta=list(pl.atleast_1d(dm.vars['study_coeffs'].stats()['standard deviation'])), sigma_gamma=list(dm.vars['age_coeffs'].stats()['standard deviation']), sigma_delta=float(dm.vars['dispersion'].stats()['standard deviation'])) dm.set_empirical_prior(param_type, prior_vals) dispersion = prior_vals['delta'] median_sample_size = pl.median([values_from(dm, d)[3] for d in dm.vars['data']] + [1000]) debug('median effective sample size: %.1f' % median_sample_size) param_mesh = dm.get_param_age_mesh() age_mesh = dm.get_estimate_age_mesh() trace = zip(dm.vars['region_coeffs'].trace(), dm.vars['study_coeffs'].trace(), dm.vars['age_coeffs'].trace())[::5] for r in dismod3.gbd_regions: debug('predicting rates for %s' % r) for y in dismod3.gbd_years: for s in dismod3.gbd_sexes: key = dismod3.utils.gbd_key_for(param_type, r, y, s) rate_trace = [] for a, b, g in trace: rate_trace.append(predict_region_rate(key, alpha=a, beta=b, gamma=g, covariates_dict=covariates_dict, derived_covariate=derived_covariate, bounds_func=dm.vars['bounds_func'], ages=dm.get_estimate_age_mesh())) mu = dismod3.utils.interpolate(param_mesh, pl.mean(rate_trace, axis=0)[param_mesh], age_mesh) dm.set_initial_value(key, mu) dm.set_mcmc('emp_prior_mean', key, mu) # similar to saving upper_ui and lower_ui in function store_mcmc_fit below rate_trace = pl.sort(rate_trace, axis=0) dm.set_mcmc('emp_prior_upper_ui', key, dismod3.utils.interpolate(param_mesh, rate_trace[.975 * len(rate_trace), :][param_mesh], age_mesh)) dm.set_mcmc('emp_prior_lower_ui', key, dismod3.utils.interpolate(param_mesh, rate_trace[.025 * len(rate_trace), :][param_mesh], age_mesh))
continue print 'saving tables for', t if 'data' in dm.vars[t] and 'p_pred' in dm.vars[t]: stats = dm.vars[t]['p_pred'].stats(batches=5) dm.vars[t]['data']['mu_pred'] = stats['mean'] dm.vars[t]['data']['sigma_pred'] = stats['standard deviation'] stats = dm.vars[t]['pi'].stats(batches=5) dm.vars[t]['data']['mc_error'] = stats['mc error'] dm.vars[t]['data']['residual'] = dm.vars[t]['data'][ 'value'] - dm.vars[t]['data']['mu_pred'] dm.vars[t]['data']['abs_residual'] = pl.absolute( dm.vars[t]['data']['residual']) if 'delta' in dm.vars[t]: if len(pl.atleast_1d(dm.vars[t]['delta'].value)) == 1: d = pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']) dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred \ in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'])] else: dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \ in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'], pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']))] try: dm.vars[t]['data'].to_csv( dir + '/posterior/data-%s-%s+%s+%s.csv' % (t, predict_area, predict_sex, predict_year)) except IOError, e: print 'WARNING: could not save file' print e if 'U' in dm.vars[t]: re = dm.vars[t]['U'].T
def print_mare(vars): if 'p_obs' in vars: are = pl.atleast_1d(pl.absolute((vars['p_obs'].value - vars['pi'].value)/vars['pi'].value)) print 'mare:', pl.round_(pl.median(are), 2)
def plot_one_effects(model, data_type): """ Plot random effects and fixed effects. :Parameters: - `model` : data.ModelData - `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf' """ vars = model.vars[data_type] hierarchy = model.hierarchy pl.figure(figsize=(22, 17)) for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]): if covariate not in vars: continue cov_name = list(vars[covariate].columns) if isinstance(vars.get(effect), mc.Stochastic): pl.subplot(1, 2, i+1) pl.title('%s_%s' % (effect, data_type)) stats = vars[effect].stats() if stats: if effect == 'alpha': index = sorted(pl.arange(len(cov_name)), key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i])) elif effect == 'beta': index = pl.arange(len(cov_name)) x = pl.atleast_1d(stats['mean']) y = pl.arange(len(x)) xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0], pl.atleast_2d(stats['95% HPD interval'])[:,1] - x]) pl.errorbar(x[index], y[index], xerr=xerr[:, index], fmt='bs', mec='w') l,r,b,t = pl.axis() pl.vlines([0], b-.5, t+.5) pl.hlines(y, l, r, linestyle='dotted') pl.xticks([l, 0, r]) pl.yticks([]) for i in index: spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0 pl.text(l, y[i], '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left') pl.axis([l, r, -.5, t+.5]) if isinstance(vars.get(effect), list): pl.subplot(1, 2, i+1) pl.title('%s_%s' % (effect, data_type)) index = sorted(pl.arange(len(cov_name)), key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i])) for y, i in enumerate(index): n = vars[effect][i] if isinstance(n, mc.Stochastic) or isinstance(n, mc.Deterministic): stats = n.stats() if stats: x = pl.atleast_1d(stats['mean']) xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0], pl.atleast_2d(stats['95% HPD interval'])[:,1] - x]) pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w') l,r,b,t = pl.axis() pl.vlines([0], b-.5, t+.5) pl.hlines(y, l, r, linestyle='dotted') pl.xticks([l, 0, r]) pl.yticks([]) for y, i in enumerate(index): spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0 pl.text(l, y, '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left') pl.axis([l, r, -.5, t+.5]) if effect == 'alpha': effect_str = '' for sigma in vars['sigma_alpha']: stats = sigma.stats() if stats: effect_str += '%s = %.3f\n' % (sigma.__name__, stats['mean']) else: effect_str += '%s = %.3f\n' % (sigma.__name__, sigma.value) pl.text(r, t, effect_str, va='top', ha='right') elif effect == 'beta': effect_str = '' if 'eta' in vars: eta = vars['eta'] stats = eta.stats() if stats: effect_str += '%s = %.3f\n' % (eta.__name__, stats['mean']) else: effect_str += '%s = %.3f\n' % (eta.__name__, eta.value) pl.text(r, t, effect_str, va='top', ha='right')
for t in 'i r f p rr pf X m_with smr'.split(): if t not in dm.vars: continue print 'saving tables for', t if 'data' in dm.vars[t] and 'p_pred' in dm.vars[t]: stats = dm.vars[t]['p_pred'].stats(batches=5) dm.vars[t]['data']['mu_pred'] = stats['mean'] dm.vars[t]['data']['sigma_pred'] = stats['standard deviation'] stats = dm.vars[t]['pi'].stats(batches=5) dm.vars[t]['data']['mc_error'] = stats['mc error'] dm.vars[t]['data']['residual'] = dm.vars[t]['data']['value'] - dm.vars[t]['data']['mu_pred'] dm.vars[t]['data']['abs_residual'] = pl.absolute(dm.vars[t]['data']['residual']) if 'delta' in dm.vars[t]: if len(pl.atleast_1d(dm.vars[t]['delta'].value)) == 1: d = pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']) dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred \ in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'])] else: dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \ in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'], pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']))] try: dm.vars[t]['data'].to_csv(dir + '/posterior/data-%s-%s+%s+%s.csv'%(t, predict_area, predict_sex, predict_year)) except IOError, e: print 'WARNING: could not save file' print e if 'U' in dm.vars[t]: re = dm.vars[t]['U'].T columns = list(re.columns) mu = []
def sigma_explained(W=W, gamma=gamma): """ sigma_explained_i,r,c,t,a = gamma * W_i,r,c,t,a""" return pl.dot(pl.atleast_1d(gamma), pl.atleast_2d(W))
def print_mare(vars): if 'p_obs' in vars: are = pl.atleast_1d( pl.absolute( (vars['p_obs'].value - vars['pi'].value) / vars['pi'].value)) print 'mare:', pl.round_(pl.median(are), 2)
def normalized(a, axis=0, order=0.1): l2 = plb.atleast_1d(plb.linalg.norm(a, order, axis)) l2[l2 == 0] = 1 return a / np.expand_dims(l2, axis)