def play_components(self, X_hat=None, x=None, orig_mix=0.1):
""" ::
play individual separated feature spectrograms as audio
separated spectrograms are stored in a list
"""
X_hat = self.X_hat if X_hat is None else X_hat
for Xh in X_hat:
x_hat = self.inverse(Xh) # invert to audio to F.x_hat
x = x_hat if x is None else x
x = x.mean(1) if len(x.shape) > 1 else x
xn = min(len(x), len(x_hat))
x_orig = orig_mix * P.atleast_1d(x[:xn]).T
x_hat = P.atleast_1d(x_hat[:xn] / (x_hat[:xn].max() + 0.01))
play(P.c_[x_orig, x_hat].T, self.feature_params['sample_rate'])
def play_components(self, X_hat=None, x=None, orig_mix=0.1):
""" ::
play individual separated feature spectrograms as audio
separated spectrograms are stored in a list
"""
X_hat = self.X_hat if X_hat is None else X_hat
for Xh in X_hat:
x_hat = self.inverse(Xh) # invert to audio to F.x_hat
x = x_hat if x is None else x
x = x.mean(1) if len(x.shape)>1 else x
xn = min(len(x), len(x_hat))
x_orig = orig_mix*P.atleast_1d(x[:xn]).T
x_hat = P.atleast_1d(x_hat[:xn] / (x_hat[:xn].max() + 0.01))
play(P.c_[x_orig, x_hat].T, self.feature_params['sample_rate'])
def ImageWithColorbar(ax, image, **kwargs):
"""
Plot image with colorbar on the right
Parameters
------------
ax : matplotlib Axes instance
image : 2-d array
Additional kwargs are passed to imshow
Returns
--------
ax : image axes
cb : colorbar axes
"""
from mpl_toolkits.axes_grid1 import make_axes_locatable
divider = make_axes_locatable(ax)
ax_cb = divider.append_axes("right", size="5%", pad=0.05)
fig1 = ax.get_figure()
fig1.add_axes(ax_cb)
dim = pl.atleast_1d(image.shape).astype('float')
lim = dim - dim/2.
extent = (-lim[0], lim[0], -lim[1], lim[1])
im = ax.imshow(
image,
interpolation='nearest', origin='lower', extent=extent, **kwargs)
ax.minorticks_on()
cb = pl.colorbar(im, cax=ax_cb, orientation='vertical')
ax_cb.xaxis.tick_bottom()
return ax, cb
def plot_viz_of_stochs(vars, viz_func, figsize=(8,6)):
""" Plot autocorrelation for all stochs in a dict or dict of dicts
:Parameters:
- `vars` : dictionary
- `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist``
- `figsize` : tuple, size of figure
"""
pl.figure(figsize=figsize)
cells, stochs = tally_stochs(vars)
# for each stoch, make an autocorrelation plot for each dimension
rows = pl.floor(pl.sqrt(cells))
cols = pl.ceil(cells/rows)
tile = 1
for s in sorted(stochs, key=lambda s: s.__name__):
trace = s.trace()
if len(trace.shape) == 1:
trace = trace.reshape((len(trace), 1))
for d in range(len(pl.atleast_1d(s.value))):
pl.subplot(rows, cols, tile)
viz_func(pl.atleast_2d(trace)[:, d])
pl.title('\n\n%s[%d]'%(s.__name__, d), va='top', ha='center', fontsize=8)
tile += 1
def plot_viz_of_stochs(vars, viz_func, figsize=(8, 6)):
""" Plot autocorrelation for all stochs in a dict or dict of dicts
:Parameters:
- `vars` : dictionary
- `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist``
- `figsize` : tuple, size of figure
"""
pl.figure(figsize=figsize)
cells, stochs = tally_stochs(vars)
# for each stoch, make an autocorrelation plot for each dimension
rows = pl.floor(pl.sqrt(cells))
cols = pl.ceil(cells / rows)
tile = 1
for s in sorted(stochs, key=lambda s: s.__name__):
trace = s.trace()
if len(trace.shape) == 1:
trace = trace.reshape((len(trace), 1))
for d in range(len(pl.atleast_1d(s.value))):
pl.subplot(rows, cols, tile)
viz_func(pl.atleast_2d(trace)[:, d])
pl.title('\n\n%s[%d]' % (s.__name__, d),
va='top',
ha='center',
fontsize=8)
tile += 1
def tally_stochs(vars):
""" Count number of stochastics in model
:Parameters:
- `vars` : dictionary
"""
cells = 0
stochs = []
for k in vars.keys():
# handle dicts and dicts of dicts by making a list of nodes
if isinstance(vars[k], dict):
nodes = vars[k].values()
else:
nodes = [vars[k]]
# handle lists of stochs
for n in nodes:
if isinstance(n, list):
nodes += n
for n in nodes:
if isinstance(n, mc.Stochastic) and not n.observed:
trace = n.trace()
if len(trace) > 0:
stochs.append(n)
cells += len(pl.atleast_1d(n.value))
return cells, stochs
def tally_stochs(vars):
""" Count number of stochastics in model
:Parameters:
- `vars` : dictionary
"""
cells = 0
stochs = []
for k in vars.keys():
# handle dicts and dicts of dicts by making a list of nodes
if isinstance(vars[k], dict):
nodes = vars[k].values()
else:
nodes = [vars[k]]
# handle lists of stochs
for n in nodes:
if isinstance(n, list):
nodes += n
for n in nodes:
if isinstance(n, mc.Stochastic) and not n.observed:
trace = n.trace()
if len(trace) > 0:
stochs.append(n)
cells += len(pl.atleast_1d(n.value))
return cells, stochs
def obs_lb(value=value, N=N,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
delta=delta,
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa, alpha) + pl.dot(Xb, pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu_i = [pl.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed
rate_param = mu_i*N
violated_bounds = pl.nonzero(rate_param < value)
logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta)
return logp
def check_convergence(vars):
""" Apply a simple test of convergence to the model: compare
autocorrelation at 25 lags to zero lags. warn about convergence if it exceeds
10% for any stoch """
import dismod3
cells, stochs = dismod_mr.plot.tally_stochs(vars)
for s in sorted(stochs, key=lambda s: s.__name__):
tr = s.trace()
if len(tr.shape) == 1:
tr = tr.reshape((len(tr), 1))
for d in range(len(pl.atleast_1d(s.value))):
for k in range(50,100):
acorr = pl.dot(tr[:-k,d]-tr[:k,d].mean(), tr[k:,d]-tr[k:,d].mean()) / pl.dot(tr[k:,d]-tr[k:,d].mean(), tr[k:,d]-tr[k:,d].mean())
if abs(acorr) > .5:
print 'potential non-convergence', s, acorr
return False
return True
def rates(S=data_sample,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa[S], alpha) + pl.dot(Xb[S], pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu = pl.zeros_like(shifts)
for i,s in enumerate(S):
mu[i] = pl.dot(age_weights[s], bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
# TODO: evaluate speed increase and accuracy decrease of the following:
#midpoint = age_indices[s][len(age_indices[s])/2]
#mu[i] = bounds_func(shifts[i] * exp_gamma[midpoint], midpoint)
# TODO: evaluate speed increase and accuracy decrease of the following: (to see speed increase, need to code this up using difference of running sums
#mu[i] = pl.dot(pl.ones_like(age_weights[s]) / float(len(age_weights[s])),
# bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
return mu
def check_convergence(vars):
""" Apply a simple test of convergence to the model: compare
autocorrelation at 25 lags to zero lags. warn about convergence if it exceeds
10% for any stoch """
import dismod_mr
cells, stochs = dismod_mr.plot.tally_stochs(vars)
for s in sorted(stochs, key=lambda s: s.__name__):
tr = s.trace()
if len(tr.shape) == 1:
tr = tr.reshape((len(tr), 1))
for d in range(len(pl.atleast_1d(s.value))):
for k in range(50,100):
acorr = pl.dot(tr[:-k,d]-tr[:k,d].mean(), tr[k:,d]-tr[k:,d].mean()) / pl.dot(tr[k:,d]-tr[k:,d].mean(), tr[k:,d]-tr[k:,d].mean())
if abs(acorr) > .5:
print('potential non-convergence', s, acorr)
return False
return True
def plot_one_effects(model, data_type):
""" Plot random effects and fixed effects.
:Parameters:
- `model` : data.ModelData
- `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf'
"""
vars = model.vars[data_type]
hierarchy = model.hierarchy
pl.figure(figsize=(22, 17))
for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]):
if covariate not in vars:
continue
cov_name = list(vars[covariate].columns)
if isinstance(vars.get(effect), mc.Stochastic):
pl.subplot(1, 2, i + 1)
pl.title('%s_%s' % (effect, data_type))
stats = vars[effect].stats()
if stats:
if effect == 'alpha':
index = sorted(
pl.arange(len(cov_name)),
key=lambda i: str(cov_name[
i] in hierarchy and nx.shortest_path(
hierarchy, 'all', cov_name[i]) or cov_name[i]))
elif effect == 'beta':
index = pl.arange(len(cov_name))
x = pl.atleast_1d(stats['mean'])
y = pl.arange(len(x))
xerr = pl.array([
x - pl.atleast_2d(stats['95% HPD interval'])[:, 0],
pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x
])
pl.errorbar(x[index],
y[index],
xerr=xerr[:, index],
fmt='bs',
mec='w')
l, r, b, t = pl.axis()
pl.vlines([0], b - .5, t + .5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for i in index:
spaces = cov_name[i] in hierarchy and len(
nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l,
y[i],
'%s%s' % (' * ' * spaces, cov_name[i]),
va='center',
ha='left')
pl.axis([l, r, -.5, t + .5])
if isinstance(vars.get(effect), list):
pl.subplot(1, 2, i + 1)
pl.title('%s_%s' % (effect, data_type))
index = sorted(pl.arange(len(cov_name)),
key=lambda i:
str(cov_name[i] in hierarchy and nx.shortest_path(
hierarchy, 'all', cov_name[i]) or cov_name[i]))
for y, i in enumerate(index):
n = vars[effect][i]
if isinstance(n, mc.Stochastic) or isinstance(
n, mc.Deterministic):
stats = n.stats()
if stats:
x = pl.atleast_1d(stats['mean'])
xerr = pl.array([
x - pl.atleast_2d(stats['95% HPD interval'])[:, 0],
pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x
])
pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w')
l, r, b, t = pl.axis()
pl.vlines([0], b - .5, t + .5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for y, i in enumerate(index):
spaces = cov_name[i] in hierarchy and len(
nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l,
y,
'%s%s' % (' * ' * spaces, cov_name[i]),
va='center',
ha='left')
pl.axis([l, r, -.5, t + .5])
if effect == 'alpha':
effect_str = ''
for sigma in vars['sigma_alpha']:
stats = sigma.stats()
if stats:
effect_str += '%s = %.3f\n' % (sigma.__name__,
stats['mean'])
else:
effect_str += '%s = %.3f\n' % (sigma.__name__,
sigma.value)
pl.text(r, t, effect_str, va='top', ha='right')
elif effect == 'beta':
effect_str = ''
if 'eta' in vars:
eta = vars['eta']
stats = eta.stats()
if stats:
effect_str += '%s = %.3f\n' % (eta.__name__,
stats['mean'])
else:
effect_str += '%s = %.3f\n' % (eta.__name__, eta.value)
pl.text(r, t, effect_str, va='top', ha='right')
def fit_emp_prior(dm, param_type, iter=100000, thin=50, burn=50000, dbname='/dev/null', map_only=False, store_results=True):
""" Generate an empirical prior distribution for a single disease parameter
Parameters
----------
dm : dismod3.DiseaseModel
The object containing all the data, (hyper)-priors, and additional
information (like input and output age-mesh).
param_type : str, one of 'incidence', 'prevalence', 'remission', 'excess-mortality'
The disease parameter to work with
Notes
-----
The results of this fit are stored in the disease model's params
hash for use when fitting multiple paramter types together
Example
-------
$ python2.5 gbd_fit.py 231 -t incidence
"""
data = [d for d in dm.data if \
d['data_type'] == '%s data' % param_type \
and d.get('ignore') != -1]
dm.clear_empirical_prior()
dm.calc_effective_sample_size(data)
dm.fit_initial_estimate(param_type, data)
dm.vars = setup(dm, param_type, data)
# don't do anything if there is no data for this parameter type
if not dm.vars['data']:
return
debug('i: %s' % ', '.join(['%.2f' % x for x in dm.vars['rate_stoch'].value[::10]]))
sys.stdout.flush()
# fit the model
def map_fit(stoch_names):
print '\nfitting', ' '.join(stoch_names)
map = mc.MAP([dm.vars[key] for key in stoch_names] + [dm.vars['observed_counts'], dm.vars['rate_potential'], dm.vars['priors']])
try:
map.fit(method='fmin_powell', verbose=verbose)
except KeyboardInterrupt:
debug('User halted optimization routine before optimal value found')
for key in stoch_names:
print key, dm.vars[key].value.round(2)
sys.stdout.flush()
def mcmc_fit(stoch_names):
print '\nfitting', ' '.join(stoch_names)
mcmc = mc.MCMC([dm.vars[key] for key in stoch_names] + [dm.vars['observed_counts'], dm.vars['rate_potential'], dm.vars['priors']])
mcmc.use_step_method(mc.Metropolis, dm.vars['log_dispersion'],
proposal_sd=dm.vars['dispersion_step_sd'])
# TODO: make a wrapper function for handling this adaptive metropolis setup
stoch_list = [dm.vars['study_coeffs'], dm.vars['region_coeffs'], dm.vars['age_coeffs_mesh']]
d1 = len(dm.vars['study_coeffs'].value)
d2 = len(dm.vars['region_coeffs_step_cov'])
d3 = len(dm.vars['age_coeffs_mesh_step_cov'])
C = pl.eye(d1+d2+d3)
C[d1:(d1+d2), d1:(d1+d2)] = dm.vars['region_coeffs_step_cov']
C[(d1+d2):(d1+d2+d3), (d1+d2):(d1+d2+d3)] = dm.vars['age_coeffs_mesh_step_cov']
C *= .01
mcmc.use_step_method(mc.AdaptiveMetropolis, stoch_list, cov=C)
# more step methods
mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['study_coeffs'])
mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['region_coeffs'], cov=dm.vars['region_coeffs_step_cov'])
mcmc.use_step_method(mc.AdaptiveMetropolis, dm.vars['age_coeffs_mesh'], cov=dm.vars['age_coeffs_mesh_step_cov'])
try:
mcmc.sample(iter=10000, burn=5000, thin=5, verbose=verbose)
except KeyboardInterrupt:
debug('User halted optimization routine before optimal value found')
sys.stdout.flush()
# reset stoch values to sample mean
for key in stoch_names:
mean = dm.vars[key].stats()['mean']
if isinstance(dm.vars[key], mc.Stochastic):
dm.vars[key].value = mean
print key, mean.round(2)
verbose = 1
stoch_names = 'region_coeffs age_coeffs_mesh study_coeffs'.split()
## start by optimizing parameters separately
for key in stoch_names:
map_fit([key])
## then fit them all together
map_fit(stoch_names)
# now find the over-dispersion parameter that matches these values
map_fit(['log_dispersion'])
if map_only:
return
# make pymc warnings go to stdout
mc.warnings.warn = sys.stdout.write
mcmc_fit(['log_dispersion', 'dispersion', 'study_coeffs', 'region_coeffs',
'age_coeffs_mesh', 'age_coeffs',
'predicted_rates', 'expected_rates', 'rate_stoch'])
alpha = dm.vars['region_coeffs'].stats()['mean']
beta = dm.vars['study_coeffs'].stats()['mean']
gamma_mesh = dm.vars['age_coeffs_mesh'].stats()['mean']
debug('a: %s' % ', '.join(['%.2f' % x for x in alpha]))
debug('b: %s' % ', '.join(['%.2f' % x for x in pl.atleast_1d(beta)]))
debug('g: %s' % ', '.join(['%.2f' % x for x in gamma_mesh]))
debug('d: %.2f' % dm.vars['dispersion'].stats()['mean'])
covariates_dict = dm.get_covariates()
derived_covariate = dm.get_derived_covariate_values()
X = covariates(data[0], covariates_dict)
debug('p: %s' % ', '.join(['%.2f' % x for x in predict_rate(X, alpha, beta, gamma_mesh, dm.vars['bounds_func'], dm.get_param_age_mesh())]))
if not store_results:
return
# save the results in the param_hash
prior_vals = dict(
alpha=list(dm.vars['region_coeffs'].stats()['mean']),
beta=list(pl.atleast_1d(dm.vars['study_coeffs'].stats()['mean'])),
gamma=list(dm.vars['age_coeffs'].stats()['mean']),
delta=float(dm.vars['dispersion'].stats()['mean']))
prior_vals.update(
sigma_alpha=list(dm.vars['region_coeffs'].stats()['standard deviation']),
sigma_beta=list(pl.atleast_1d(dm.vars['study_coeffs'].stats()['standard deviation'])),
sigma_gamma=list(dm.vars['age_coeffs'].stats()['standard deviation']),
sigma_delta=float(dm.vars['dispersion'].stats()['standard deviation']))
dm.set_empirical_prior(param_type, prior_vals)
dispersion = prior_vals['delta']
median_sample_size = pl.median([values_from(dm, d)[3] for d in dm.vars['data']] + [1000])
debug('median effective sample size: %.1f' % median_sample_size)
param_mesh = dm.get_param_age_mesh()
age_mesh = dm.get_estimate_age_mesh()
trace = zip(dm.vars['region_coeffs'].trace(), dm.vars['study_coeffs'].trace(), dm.vars['age_coeffs'].trace())[::5]
for r in dismod3.gbd_regions:
debug('predicting rates for %s' % r)
for y in dismod3.gbd_years:
for s in dismod3.gbd_sexes:
key = dismod3.utils.gbd_key_for(param_type, r, y, s)
rate_trace = []
for a, b, g in trace:
rate_trace.append(predict_region_rate(key,
alpha=a,
beta=b,
gamma=g,
covariates_dict=covariates_dict,
derived_covariate=derived_covariate,
bounds_func=dm.vars['bounds_func'],
ages=dm.get_estimate_age_mesh()))
mu = dismod3.utils.interpolate(param_mesh, pl.mean(rate_trace, axis=0)[param_mesh], age_mesh)
dm.set_initial_value(key, mu)
dm.set_mcmc('emp_prior_mean', key, mu)
# similar to saving upper_ui and lower_ui in function store_mcmc_fit below
rate_trace = pl.sort(rate_trace, axis=0)
dm.set_mcmc('emp_prior_upper_ui', key, dismod3.utils.interpolate(param_mesh, rate_trace[.975 * len(rate_trace), :][param_mesh], age_mesh))
dm.set_mcmc('emp_prior_lower_ui', key, dismod3.utils.interpolate(param_mesh, rate_trace[.025 * len(rate_trace), :][param_mesh], age_mesh))
continue
print 'saving tables for', t
if 'data' in dm.vars[t] and 'p_pred' in dm.vars[t]:
stats = dm.vars[t]['p_pred'].stats(batches=5)
dm.vars[t]['data']['mu_pred'] = stats['mean']
dm.vars[t]['data']['sigma_pred'] = stats['standard deviation']
stats = dm.vars[t]['pi'].stats(batches=5)
dm.vars[t]['data']['mc_error'] = stats['mc error']
dm.vars[t]['data']['residual'] = dm.vars[t]['data'][
'value'] - dm.vars[t]['data']['mu_pred']
dm.vars[t]['data']['abs_residual'] = pl.absolute(
dm.vars[t]['data']['residual'])
if 'delta' in dm.vars[t]:
if len(pl.atleast_1d(dm.vars[t]['delta'].value)) == 1:
d = pl.atleast_1d(dm.vars[t]['delta'].stats()['mean'])
dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred \
in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'])]
else:
dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \
in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'], pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']))]
try:
dm.vars[t]['data'].to_csv(
dir + '/posterior/data-%s-%s+%s+%s.csv' %
(t, predict_area, predict_sex, predict_year))
except IOError, e:
print 'WARNING: could not save file'
print e
if 'U' in dm.vars[t]:
re = dm.vars[t]['U'].T
def print_mare(vars):
if 'p_obs' in vars:
are = pl.atleast_1d(pl.absolute((vars['p_obs'].value - vars['pi'].value)/vars['pi'].value))
print 'mare:', pl.round_(pl.median(are), 2)
def plot_one_effects(model, data_type):
""" Plot random effects and fixed effects.
:Parameters:
- `model` : data.ModelData
- `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf'
"""
vars = model.vars[data_type]
hierarchy = model.hierarchy
pl.figure(figsize=(22, 17))
for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]):
if covariate not in vars:
continue
cov_name = list(vars[covariate].columns)
if isinstance(vars.get(effect), mc.Stochastic):
pl.subplot(1, 2, i+1)
pl.title('%s_%s' % (effect, data_type))
stats = vars[effect].stats()
if stats:
if effect == 'alpha':
index = sorted(pl.arange(len(cov_name)),
key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i]))
elif effect == 'beta':
index = pl.arange(len(cov_name))
x = pl.atleast_1d(stats['mean'])
y = pl.arange(len(x))
xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - x])
pl.errorbar(x[index], y[index], xerr=xerr[:, index], fmt='bs', mec='w')
l,r,b,t = pl.axis()
pl.vlines([0], b-.5, t+.5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for i in index:
spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l, y[i], '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left')
pl.axis([l, r, -.5, t+.5])
if isinstance(vars.get(effect), list):
pl.subplot(1, 2, i+1)
pl.title('%s_%s' % (effect, data_type))
index = sorted(pl.arange(len(cov_name)),
key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i]))
for y, i in enumerate(index):
n = vars[effect][i]
if isinstance(n, mc.Stochastic) or isinstance(n, mc.Deterministic):
stats = n.stats()
if stats:
x = pl.atleast_1d(stats['mean'])
xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - x])
pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w')
l,r,b,t = pl.axis()
pl.vlines([0], b-.5, t+.5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for y, i in enumerate(index):
spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l, y, '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left')
pl.axis([l, r, -.5, t+.5])
if effect == 'alpha':
effect_str = ''
for sigma in vars['sigma_alpha']:
stats = sigma.stats()
if stats:
effect_str += '%s = %.3f\n' % (sigma.__name__, stats['mean'])
else:
effect_str += '%s = %.3f\n' % (sigma.__name__, sigma.value)
pl.text(r, t, effect_str, va='top', ha='right')
elif effect == 'beta':
effect_str = ''
if 'eta' in vars:
eta = vars['eta']
stats = eta.stats()
if stats:
effect_str += '%s = %.3f\n' % (eta.__name__, stats['mean'])
else:
effect_str += '%s = %.3f\n' % (eta.__name__, eta.value)
pl.text(r, t, effect_str, va='top', ha='right')
for t in 'i r f p rr pf X m_with smr'.split():
if t not in dm.vars:
continue
print 'saving tables for', t
if 'data' in dm.vars[t] and 'p_pred' in dm.vars[t]:
stats = dm.vars[t]['p_pred'].stats(batches=5)
dm.vars[t]['data']['mu_pred'] = stats['mean']
dm.vars[t]['data']['sigma_pred'] = stats['standard deviation']
stats = dm.vars[t]['pi'].stats(batches=5)
dm.vars[t]['data']['mc_error'] = stats['mc error']
dm.vars[t]['data']['residual'] = dm.vars[t]['data']['value'] - dm.vars[t]['data']['mu_pred']
dm.vars[t]['data']['abs_residual'] = pl.absolute(dm.vars[t]['data']['residual'])
if 'delta' in dm.vars[t]:
if len(pl.atleast_1d(dm.vars[t]['delta'].value)) == 1:
d = pl.atleast_1d(dm.vars[t]['delta'].stats()['mean'])
dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred \
in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'])]
else:
dm.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \
in zip(dm.vars[t]['data']['effective_sample_size'], dm.vars[t]['data']['value'], dm.vars[t]['data']['mu_pred'], pl.atleast_1d(dm.vars[t]['delta'].stats()['mean']))]
try:
dm.vars[t]['data'].to_csv(dir + '/posterior/data-%s-%s+%s+%s.csv'%(t, predict_area, predict_sex, predict_year))
except IOError, e:
print 'WARNING: could not save file'
print e
if 'U' in dm.vars[t]:
re = dm.vars[t]['U'].T
columns = list(re.columns)
mu = []
def sigma_explained(W=W, gamma=gamma):
""" sigma_explained_i,r,c,t,a = gamma * W_i,r,c,t,a"""
return pl.dot(pl.atleast_1d(gamma), pl.atleast_2d(W))
def print_mare(vars):
if 'p_obs' in vars:
are = pl.atleast_1d(
pl.absolute(
(vars['p_obs'].value - vars['pi'].value) / vars['pi'].value))
print 'mare:', pl.round_(pl.median(are), 2)
def normalized(a, axis=0, order=0.1):
l2 = plb.atleast_1d(plb.linalg.norm(a, order, axis))
l2[l2 == 0] = 1
return a / np.expand_dims(l2, axis)
