def plot_one_ppc(model, t):
""" plot data and posterior predictive check
:Parameters:
- `model` : data.ModelData
- `t` : str, data type of 'i', 'r', 'f', 'p', 'rr', 'm', 'X', 'pf', 'csmr'
"""
stats = model.vars[t]['p_pred'].stats()
if stats == None:
return
pl.figure()
pl.title(t)
x = model.vars[t]['p_obs'].value.__array__()
y = x - stats['quantiles'][50]
yerr = [stats['quantiles'][50] - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - stats['quantiles'][50]]
pl.errorbar(x, y, yerr=yerr, fmt='ko', mec='w', capsize=0,
label='Obs vs Residual (Obs - Pred)')
pl.xlabel('Observation')
pl.ylabel('Residual (observation-prediction)')
pl.grid()
l,r,b,t = pl.axis()
pl.hlines([0], l, r)
pl.axis([l, r, y.min()*1.1 - y.max()*.1, -y.min()*.1 + y.max()*1.1])
def plot_one_ppc(model, t):
""" plot data and posterior predictive check
:Parameters:
- `model` : data.ModelData
- `t` : str, data type of 'i', 'r', 'f', 'p', 'rr', 'm', 'X', 'pf', 'csmr'
"""
stats = model.vars[t]['p_pred'].stats()
if stats == None:
return
pl.figure()
pl.title(t)
x = model.vars[t]['p_obs'].value.__array__()
y = x - stats['quantiles'][50]
yerr = [stats['quantiles'][50] - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - stats['quantiles'][50]]
pl.errorbar(x, y, yerr=yerr, fmt='ko', mec='w', capsize=0,
label='Obs vs Residual (Obs - Pred)')
pl.xlabel('Observation')
pl.ylabel('Residual (observation-prediction)')
pl.grid()
l,r,b,t = pl.axis()
pl.hlines([0], l, r)
pl.axis([l, r, y.min()*1.1 - y.max()*.1, -y.min()*.1 + y.max()*1.1])
def invert_component(self, cls, w, z, h):
"""
Invert a single PLCA component to separated features in the original feature space.
"""
w = P.atleast_2d(w)
if cls == plca.PLCA: w = w.T
h = P.atleast_2d(h)
return cls.reconstruct(w, z, h)
def invert_component(self, cls, w, z, h):
"""
Invert a single PLCA component to separated features in the original feature space.
"""
w = P.atleast_2d(w)
if cls==plca.PLCA: w = w.T
h = P.atleast_2d(h)
return cls.reconstruct(w,z,h)
def new_bad_model(F):
""" Results in a matrix with shape matching X, but all rows sum to 1"""
N, T, J = F.shape
pi = pl.zeros_like(F)
for t in range(T):
u = F[:,t,:].var(axis=0)
u /= pl.sqrt(pl.dot(u,u))
F_t_par = pl.dot(pl.atleast_2d(pl.dot(F[:,t,:], u)).T, pl.atleast_2d(u))
F_t_perp = F[:,t,:] - F_t_par
for n in range(N):
alpha = (1 - F_t_perp[n].sum()) / F_t_par[n].sum()
pi[n,t,:] = F_t_perp[n,:] + alpha*F_t_par[n,:]
return pi
def new_bad_model(F):
""" Results in a matrix with shape matching X, but all rows sum to 1"""
N, T, J = F.shape
pi = pl.zeros_like(F)
for t in range(T):
u = F[:, t, :].var(axis=0)
u /= pl.sqrt(pl.dot(u, u))
F_t_par = pl.dot(
pl.atleast_2d(pl.dot(F[:, t, :], u)).T, pl.atleast_2d(u))
F_t_perp = F[:, t, :] - F_t_par
for n in range(N):
alpha = (1 - F_t_perp[n].sum()) / F_t_par[n].sum()
pi[n, t, :] = F_t_perp[n, :] + alpha * F_t_par[n, :]
return pi
def _phase_map(self):
self.dphi = (2*P.pi * self.nhop * P.arange(self.nfft/2+1)) / self.nfft
A = P.diff(P.angle(self.STFT),1) # Complete Phase Map
U = P.c_[P.angle(self.STFT[:,0]), A - P.atleast_2d(self.dphi).T ]
U = U - P.np.round(U/(2*P.pi))*2*P.pi
self.dPhi = U
return U
def plot_viz_of_stochs(vars, viz_func, figsize=(8,6)):
""" Plot autocorrelation for all stochs in a dict or dict of dicts
:Parameters:
- `vars` : dictionary
- `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist``
- `figsize` : tuple, size of figure
"""
pl.figure(figsize=figsize)
cells, stochs = tally_stochs(vars)
# for each stoch, make an autocorrelation plot for each dimension
rows = pl.floor(pl.sqrt(cells))
cols = pl.ceil(cells/rows)
tile = 1
for s in sorted(stochs, key=lambda s: s.__name__):
trace = s.trace()
if len(trace.shape) == 1:
trace = trace.reshape((len(trace), 1))
for d in range(len(pl.atleast_1d(s.value))):
pl.subplot(rows, cols, tile)
viz_func(pl.atleast_2d(trace)[:, d])
pl.title('\n\n%s[%d]'%(s.__name__, d), va='top', ha='center', fontsize=8)
tile += 1
def _pvoc2(self, X_hat, Phi_hat=None, R=None):
"""
::
alternate (batch) implementation of phase vocoder - time-stretch
inputs:
X_hat - estimate of signal magnitude
[Phi_hat] - estimate of signal phase
[R] - resynthesis hop ratio
output:
updates self.X_hat with modified complex spectrum
"""
N, W, H = self.nfft, self.wfft, self.nhop
R = 1.0 if R is None else R
dphi = P.atleast_2d((2*P.pi * H * P.arange(N/2+1)) / N).T
print "Phase Vocoder Resynthesis...", N, W, H, R
A = P.angle(self.STFT) if Phi_hat is None else Phi_hat
U = P.diff(A,1) - dphi
U = U - P.np.round(U/(2*P.pi))*2*P.pi
t = P.arange(0,n_cols,R)
tf = t - P.floor(t)
phs = P.c_[A[:,0], U]
phs += U[:,idx[1]] + dphi # Problem, what is idx ?
Xh = (1-tf)*Xh[:-1] + tf*Xh[1:]
Xh *= P.exp( 1j * phs)
self.X_hat = Xh
def _pvoc2(self, X_hat, Phi_hat=None, R=None):
"""
::
alternate (batch) implementation of phase vocoder - time-stretch
inputs:
X_hat - estimate of signal magnitude
[Phi_hat] - estimate of signal phase
[R] - resynthesis hop ratio
output:
updates self.X_hat with modified complex spectrum
"""
N, W, H = self.nfft, self.wfft, self.nhop
R = 1.0 if R is None else R
dphi = P.atleast_2d((2 * P.pi * H * P.arange(N / 2 + 1)) / N).T
print("Phase Vocoder Resynthesis...", N, W, H, R)
A = P.angle(self.STFT) if Phi_hat is None else Phi_hat
U = P.diff(A, 1) - dphi
U = U - P.np.round(U / (2 * P.pi)) * 2 * P.pi
t = P.arange(0, n_cols, R)
tf = t - P.floor(t)
phs = P.c_[A[:, 0], U]
phs += U[:, idx[1]] + dphi # Problem, what is idx ?
Xh = (1 - tf) * Xh[:-1] + tf * Xh[1:]
Xh *= P.exp(1j * phs)
self.X_hat = Xh
def plot_viz_of_stochs(vars, viz_func, figsize=(8, 6)):
""" Plot autocorrelation for all stochs in a dict or dict of dicts
:Parameters:
- `vars` : dictionary
- `viz_func` : visualazation function such as ``acorr``, ``show_trace``, or ``hist``
- `figsize` : tuple, size of figure
"""
pl.figure(figsize=figsize)
cells, stochs = tally_stochs(vars)
# for each stoch, make an autocorrelation plot for each dimension
rows = pl.floor(pl.sqrt(cells))
cols = pl.ceil(cells / rows)
tile = 1
for s in sorted(stochs, key=lambda s: s.__name__):
trace = s.trace()
if len(trace.shape) == 1:
trace = trace.reshape((len(trace), 1))
for d in range(len(pl.atleast_1d(s.value))):
pl.subplot(rows, cols, tile)
viz_func(pl.atleast_2d(trace)[:, d])
pl.title('\n\n%s[%d]' % (s.__name__, d),
va='top',
ha='center',
fontsize=8)
tile += 1
def stack_vectors(data, win=1, hop=1, zero_pad=True):
"""
::
create an overlapping stacked vector sequence from a series of vectors
data - row-wise multidimensional data to stack
win - number of consecutive vectors to stack [1]
hop - number of vectors to advance per stack [1]
zero_pad - zero pad if incomplete stack at end
"""
data = pylab.atleast_2d(data)
nrows, dim = data.shape
hop = min(hop, nrows)
nvecs = nrows / int(hop) if not zero_pad else int(
pylab.ceil(nrows / float(hop)))
features = pylab.zeros((nvecs, win * dim))
i = 0
while i < nrows - win + 1:
features[i / hop, :] = data[i:i + win, :].reshape(1, -1)
i += hop
if i / hop < nvecs:
x = data[i::, :].reshape(1, -1)
features[i / hop, :] = pylab.c_[x,
pylab.zeros(
(1, win * dim - x.shape[1]))]
return features
def _phase_map(self):
self.dphi = (2*P.pi * self.nhop * P.arange(self.nfft/2+1)) / self.nfft
A = P.diff(P.angle(self.STFT),1) # Complete Phase Map
U = P.c_[P.angle(self.STFT[:,0]), A - P.atleast_2d(self.dphi).T ]
U = U - P.np.round(U/(2*P.pi))*2*P.pi
self.dPhi = U
return U
def imagesc2(self,
data,
newfig=True,
str='',
ax=1,
cbar=1,
txt=False,
txtprec=2,
txtsz=18,
txtnz=0,
txtrev=0,
labels=None,
**kwargs):
kwargs = self.check_kwargs(kwargs, DEFAULT_IMAGESC_KWARGS)
if newfig:
fig = P.figure()
data = P.copy(data)
if len(data.shape) < 2:
data = P.atleast_2d(data)
# if txtnz:
# kwargs['vmin'] = data[where(data>txtnz)].min()
P.imshow(data, **kwargs)
if cbar: P.colorbar()
if labels is not None:
P.xticks(np.arange(len(labels)), labels)
P.yticks(np.arange(len(labels)), labels)
if txt:
thr = data.min() + (data.max() - data.min()) / 2.0
for a in range(data.shape[0]):
for b in range(data.shape[1]):
if data[a, b] < thr:
col = P.array([1, 1, 1])
else:
col = P.array([0, 0, 0])
if txtrev:
col = 1 - col
d = data[a, b].round(txtprec)
if txtprec == 0:
d = int(d)
if not txtnz or txtnz and d > txtnz: # scale only non-zero values
P.text(b - 0.125,
a + 0.125,
d,
color=col,
fontsize=txtsz)
plt.title(str, fontsize=12)
return fig
def _ichroma(self, V, **kwargs):
"""
::
Inverse chromagram transform. Make a signal from a folded constant-Q transform.
"""
if not (self._have_hcqft or self._have_cqft):
return None
a,b = self.HCQFT.shape if self._have_hcqft else self.CQFT.shape
complete_octaves = a/self.nbpo # integer division, number of complete octaves
if P.remainder(a,self.nbpo):
complete_octaves += 1
X = P.repeat(V, complete_octaves, 0)[:a,:] # truncate if necessary
X /= X.max()
X *= P.atleast_2d(P.linspace(1,0,X.shape[0])).T # weight the spectrum
self.x_hat = self._icqft(X, **kwargs)
return self.x_hat
def _ichroma(self, V, **kwargs):
"""
::
Inverse chromagram transform. Make a signal from a folded constant-Q transform.
"""
if not (self._have_hcqft or self._have_cqft):
return None
a,b = self.HCQFT.shape if self._have_hcqft else self.CQFT.shape
complete_octaves = a/self.nbpo # integer division, number of complete octaves
if P.remainder(a,self.nbpo):
complete_octaves += 1
X = P.repeat(V, complete_octaves, 0)[:a,:] # truncate if necessary
X /= X.max()
X *= P.atleast_2d(P.linspace(1,0,X.shape[0])).T # weight the spectrum
self.x_hat = self._icqft(X, **kwargs)
return self.x_hat
def stack_vectors(data, win=1, hop=1, zero_pad=True):
"""
::
create an overlapping stacked vector sequence from a series of vectors
data - row-wise multidimensional data to stack
win - number of consecutive vectors to stack [1]
hop - number of vectors to advance per stack [1]
zero_pad - zero pad if incomplete stack at end
"""
data = pylab.atleast_2d(data)
nrows, dim = data.shape
hop = min(hop, nrows)
nvecs = nrows / int(hop) if not zero_pad else int(pylab.ceil(nrows / float(hop)))
features = pylab.zeros((nvecs, win * dim))
i = 0
while i < nrows - win + 1:
features[i / hop, :] = data[i : i + win, :].reshape(1, -1)
i += hop
if i / hop < nvecs:
x = data[i::, :].reshape(1, -1)
features[i / hop, :] = pylab.c_[x, pylab.zeros((1, win * dim - x.shape[1]))]
return features
def plot_one_effects(model, data_type):
""" Plot random effects and fixed effects.
:Parameters:
- `model` : data.ModelData
- `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf'
"""
vars = model.vars[data_type]
hierarchy = model.hierarchy
pl.figure(figsize=(22, 17))
for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]):
if covariate not in vars:
continue
cov_name = list(vars[covariate].columns)
if isinstance(vars.get(effect), mc.Stochastic):
pl.subplot(1, 2, i + 1)
pl.title('%s_%s' % (effect, data_type))
stats = vars[effect].stats()
if stats:
if effect == 'alpha':
index = sorted(
pl.arange(len(cov_name)),
key=lambda i: str(cov_name[
i] in hierarchy and nx.shortest_path(
hierarchy, 'all', cov_name[i]) or cov_name[i]))
elif effect == 'beta':
index = pl.arange(len(cov_name))
x = pl.atleast_1d(stats['mean'])
y = pl.arange(len(x))
xerr = pl.array([
x - pl.atleast_2d(stats['95% HPD interval'])[:, 0],
pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x
])
pl.errorbar(x[index],
y[index],
xerr=xerr[:, index],
fmt='bs',
mec='w')
l, r, b, t = pl.axis()
pl.vlines([0], b - .5, t + .5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for i in index:
spaces = cov_name[i] in hierarchy and len(
nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l,
y[i],
'%s%s' % (' * ' * spaces, cov_name[i]),
va='center',
ha='left')
pl.axis([l, r, -.5, t + .5])
if isinstance(vars.get(effect), list):
pl.subplot(1, 2, i + 1)
pl.title('%s_%s' % (effect, data_type))
index = sorted(pl.arange(len(cov_name)),
key=lambda i:
str(cov_name[i] in hierarchy and nx.shortest_path(
hierarchy, 'all', cov_name[i]) or cov_name[i]))
for y, i in enumerate(index):
n = vars[effect][i]
if isinstance(n, mc.Stochastic) or isinstance(
n, mc.Deterministic):
stats = n.stats()
if stats:
x = pl.atleast_1d(stats['mean'])
xerr = pl.array([
x - pl.atleast_2d(stats['95% HPD interval'])[:, 0],
pl.atleast_2d(stats['95% HPD interval'])[:, 1] - x
])
pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w')
l, r, b, t = pl.axis()
pl.vlines([0], b - .5, t + .5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for y, i in enumerate(index):
spaces = cov_name[i] in hierarchy and len(
nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l,
y,
'%s%s' % (' * ' * spaces, cov_name[i]),
va='center',
ha='left')
pl.axis([l, r, -.5, t + .5])
if effect == 'alpha':
effect_str = ''
for sigma in vars['sigma_alpha']:
stats = sigma.stats()
if stats:
effect_str += '%s = %.3f\n' % (sigma.__name__,
stats['mean'])
else:
effect_str += '%s = %.3f\n' % (sigma.__name__,
sigma.value)
pl.text(r, t, effect_str, va='top', ha='right')
elif effect == 'beta':
effect_str = ''
if 'eta' in vars:
eta = vars['eta']
stats = eta.stats()
if stats:
effect_str += '%s = %.3f\n' % (eta.__name__,
stats['mean'])
else:
effect_str += '%s = %.3f\n' % (eta.__name__, eta.value)
pl.text(r, t, effect_str, va='top', ha='right')
def generate_prior_potentials(rate_vars, prior_str, age_mesh):
"""
augment the rate_vars dict to include a list of potentials that model priors on rate_vars['rate_stoch']
prior_str may have entries in the following format:
smooth <tau> [<age_start> <age_end>]
zero <age_start> <age_end>
confidence <mean> <tau>
increasing <age_start> <age_end>
decreasing <age_start> <age_end>
convex_up <age_start> <age_end>
convex_down <age_start> <age_end>
unimodal <age_start> <age_end>
value <mean> <tau> [<age_start> <age_end>]
at_least <value>
at_most <value>
max_at_most <value>
for example: 'smooth .1, zero 0 5, zero 95 100'
age_mesh[i] indicates what age the value of rate[i] corresponds to
"""
def derivative_sign_prior(rate, prior, deriv, sign):
age_start = int(prior[1])
age_end = int(prior[2])
age_indices = indices_for_range(age_mesh, age_start, age_end)
@mc.potential(name='deriv_sign_{%d,%d,%d,%d}^%s' %
(deriv, sign, age_start, age_end, str(rate)))
def deriv_sign_rate(f=rate,
age_indices=age_indices,
tau=1.e14,
deriv=deriv,
sign=sign):
df = pl.diff(f[age_indices], deriv)
return mc.normal_like(pl.absolute(df) * (sign * df < 0), 0., tau)
return [deriv_sign_rate]
priors = []
rate = rate_vars['rate_stoch']
rate_vars['bounds_func'] = lambda f, age: f
for line in prior_str.split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'smooth':
pass # handle this after applying all level bounds
elif prior[0] == 'heterogeneity':
# prior affects dispersion term of model; handle as a special case
continue
elif prior[0] == 'increasing':
priors += derivative_sign_prior(rate, prior, deriv=1, sign=1)
elif prior[0] == 'decreasing':
priors += derivative_sign_prior(rate, prior, deriv=1, sign=-1)
elif prior[0] == 'convex_down':
priors += derivative_sign_prior(rate, prior, deriv=2, sign=-1)
elif prior[0] == 'convex_up':
priors += derivative_sign_prior(rate, prior, deriv=2, sign=1)
elif prior[0] == 'unimodal':
age_start = int(prior[1])
age_end = int(prior[2])
age_indices = indices_for_range(age_mesh, age_start, age_end)
@mc.potential(name='unimodal_{%d,%d}^%s' %
(age_start, age_end, str(rate)))
def unimodal_rate(f=rate, age_indices=age_indices, tau=1.e5):
df = pl.diff(f[age_indices])
sign_changes = pl.find((df[:-1] > NEARLY_ZERO)
& (df[1:] < -NEARLY_ZERO))
sign = pl.ones(len(age_indices) - 2)
if len(sign_changes) > 0:
change_age = sign_changes[len(sign_changes) / 2]
sign[change_age:] = -1.
return -tau * pl.dot(pl.absolute(df[:-1]),
(sign * df[:-1] < 0))
priors += [unimodal_rate]
elif prior[0] == 'max_at_least':
val = float(prior[1])
@mc.potential(name='max_at_least_{%f}^{%s}' % (val, str(rate)))
def max_at_least(cur_max=rate, at_least=val, tau=(.001 * val)**-2):
return -tau * (cur_max - at_least)**2 * (cur_max < at_least)
priors += [max_at_least]
elif prior[0] == 'level_value':
val = float(prior[1]) + 1.e-9
if len(prior) == 4:
age_start = int(prior[2])
age_end = int(prior[3])
else:
age_start = 0
age_end = MAX_AGE
age_indices = indices_for_range(age_mesh, age_start, age_end)
def new_bounds_func(f,
age,
val=val,
age_start=age_start,
age_end=age_end,
prev_bounds_func=rate_vars['bounds_func']):
age = pl.array(age)
return pl.where((age >= age_start) & (age <= age_end), val,
prev_bounds_func(f, age))
rate_vars['bounds_func'] = new_bounds_func
elif prior[0] == 'at_most':
val = float(prior[1])
def new_bounds_func(f,
age,
val=val,
prev_bounds_func=rate_vars['bounds_func']):
return pl.minimum(prev_bounds_func(f, age), val)
rate_vars['bounds_func'] = new_bounds_func
elif prior[0] == 'at_least':
val = float(prior[1])
def new_bounds_func(f,
age,
val=val,
prev_bounds_func=rate_vars['bounds_func']):
return pl.maximum(prev_bounds_func(f, age), val)
rate_vars['bounds_func'] = new_bounds_func
else:
raise KeyError, 'Unrecognized prior: %s' % prior_str
# update rate stoch with the bounds func from the priors
# TODO: create this before smoothing, so that smoothing takes levels into account
@mc.deterministic(name='%s_w_bounds' % rate_vars['rate_stoch'].__name__)
def mu_bounded(mu=rate_vars['rate_stoch'],
bounds_func=rate_vars['bounds_func']):
return bounds_func(mu,
pl.arange(101)) # FIXME: don't hardcode age range
rate_vars['unbounded_rate'] = rate_vars['rate_stoch']
rate_vars['rate_stoch'] = mu_bounded
rate = rate_vars['rate_stoch']
# add potential to encourage rate to look like level bounds
@mc.potential(name='%s_potential' % rate_vars['rate_stoch'].__name__)
def mu_potential(mu1=rate_vars['unbounded_rate'],
mu2=rate_vars['rate_stoch']):
return mc.normal_like(mu1, mu2, .0001**-2)
rate_vars['rate_potential'] = mu_potential
# add smoothing prior to the rate with level bounds
for line in prior_str.split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'smooth':
scale = float(prior[1])
if len(prior) == 4:
age_start = int(prior[2])
age_end = int(prior[3])
else:
age_start = 0
age_end = MAX_AGE
age_indices = indices_for_range(pl.arange(MAX_AGE), age_start,
age_end)
from pymc.gp.cov_funs import matern
a = pl.atleast_2d(age_indices).T
C = matern.euclidean(a, a, diff_degree=2, amp=10., scale=scale)
@mc.potential(name='smooth_{%d,%d}^%s' %
(age_start, age_end, str(rate)))
def smooth_rate(f=rate, age_indices=age_indices, C=C):
log_rate = pl.log(pl.maximum(f, NEARLY_ZERO))
return mc.mv_normal_cov_like(log_rate[age_indices] -
log_rate[age_indices].mean(),
pl.zeros_like(age_indices),
C=C)
priors += [smooth_rate]
print 'added smoothing potential for %s' % smooth_rate
rate_vars['priors'] = priors
def setup(dm, key, data_list=[], rate_stoch=None, emp_prior={}, lower_bound_data=[]):
""" Generate the PyMC variables for a negative-binomial model of
a single rate function
Parameters
----------
dm : dismod3.DiseaseModel
the object containing all the data, priors, and additional
information (like input and output age-mesh)
key : str
the name of the key for everything about this model (priors,
initial values, estimations)
data_list : list of data dicts
the observed data to use in the negative binomial liklihood function
rate_stoch : pymc.Stochastic, optional
a PyMC stochastic (or deterministic) object, with
len(rate_stoch.value) == len(dm.get_estimation_age_mesh()).
This is used to link rate stochs into a larger model,
for example.
emp_prior : dict, optional
the empirical prior dictionary, retrieved from the disease model
if appropriate by::
>>> t, r, y, s = dismod3.utils.type_region_year_sex_from_key(key)
>>> emp_prior = dm.get_empirical_prior(t)
Results
-------
vars : dict
Return a dictionary of all the relevant PyMC objects for the
rate model. vars['rate_stoch'] is of particular
relevance; this is what is used to link the rate model
into more complicated models, like the generic disease model.
"""
vars = {}
est_mesh = dm.get_estimate_age_mesh()
param_mesh = dm.get_param_age_mesh()
if pl.any(pl.diff(est_mesh) != 1):
raise ValueError, 'ERROR: Gaps in estimation age mesh must all equal 1'
# calculate effective sample size for all data and lower bound data
dm.calc_effective_sample_size(data_list)
dm.calc_effective_sample_size(lower_bound_data)
# generate regional covariates
covariate_dict = dm.get_covariates()
derived_covariate = dm.get_derived_covariate_values()
X_region, X_study = regional_covariates(key, covariate_dict, derived_covariate)
# use confidence prior from prior_str (only for posterior estimate, this is overridden below for empirical prior estimate)
mu_delta = 1000.
sigma_delta = 10.
mu_log_delta = 3.
sigma_log_delta = .25
from dismod3.settings import PRIOR_SEP_STR
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'heterogeneity':
# originally designed for this:
mu_delta = float(prior[1])
sigma_delta = float(prior[2])
# HACK: override design to set sigma_log_delta,
# .25 = very, .025 = moderately, .0025 = slightly
if float(prior[2]) > 0:
sigma_log_delta = .025 / float(prior[2])
# use the empirical prior mean if it is available
if len(set(emp_prior.keys()) & set(['alpha', 'beta', 'gamma'])) == 3:
mu_alpha = pl.array(emp_prior['alpha'])
sigma_alpha = pl.array(emp_prior['sigma_alpha'])
alpha = pl.array(emp_prior['alpha']) # TODO: make this stochastic
vars.update(region_coeffs=alpha)
beta = pl.array(emp_prior['beta']) # TODO: make this stochastic
sigma_beta = pl.array(emp_prior['sigma_beta'])
vars.update(study_coeffs=beta)
mu_gamma = pl.array(emp_prior['gamma'])
sigma_gamma = pl.array(emp_prior['sigma_gamma'])
# Do not inform dispersion parameter from empirical prior stage
# if 'delta' in emp_prior:
# mu_delta = emp_prior['delta']
# if 'sigma_delta' in emp_prior:
# sigma_delta = emp_prior['sigma_delta']
else:
import dismod3.regional_similarity_matrices as similarity_matrices
n = len(X_region)
mu_alpha = pl.zeros(n)
sigma_alpha = .025 # TODO: make this a hyperparameter, with a traditional prior, like inverse gamma
C_alpha = similarity_matrices.regions_nested_in_superregions(n, sigma_alpha)
# use alternative region effect covariance structure if requested
region_prior_key = 'region_effects'
if region_prior_key in dm.params:
if dm.params[region_prior_key] == 'uninformative':
C_alpha = similarity_matrices.uninformative(n, sigma_alpha)
region_prior_key = 'region_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0]
if region_prior_key in dm.params:
if dm.params[region_prior_key] == 'uninformative':
C_alpha = similarity_matrices.regions_nested_in_superregions(n, dm.params[region_prior_key]['std'])
# add informative prior for sex effect if requested
sex_prior_key = 'sex_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0]
if sex_prior_key in dm.params:
print 'adjusting prior on sex effect coefficient for %s' % key
mu_alpha[n-1] = pl.log(dm.params[sex_prior_key]['mean'])
sigma_sex = (pl.log(dm.params[sex_prior_key]['upper_ci']) - pl.log(dm.params[sex_prior_key]['lower_ci'])) / (2*1.96)
C_alpha[n-1, n-1]= sigma_sex**2.
# add informative prior for time effect if requested
time_prior_key = 'time_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if time_prior_key in dm.params:
print 'adjusting prior on time effect coefficient for %s' % key
mu_alpha[n-2] = pl.log(dm.params[time_prior_key]['mean'])
sigma_time = (pl.log(dm.params[time_prior_key]['upper_ci']) - pl.log(dm.params[time_prior_key]['lower_ci'])) / (2*1.96)
C_alpha[n-2, n-2]= sigma_time**2.
#C_alpha = similarity_matrices.all_related_equally(n, sigma_alpha)
alpha = mc.MvNormalCov('region_coeffs_%s' % key, mu=mu_alpha,
C=C_alpha,
value=mu_alpha)
vars.update(region_coeffs=alpha, region_coeffs_step_cov=.005*C_alpha)
mu_beta = pl.zeros(len(X_study))
sigma_beta = .1
# add informative prior for beta effect if requested
prior_key = 'beta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on beta effect coefficients for %s' % key
mu_beta = pl.array(dm.params[prior_key]['mean'])
sigma_beta = pl.array(dm.params[prior_key]['std'])
beta = mc.Normal('study_coeffs_%s' % key, mu=mu_beta, tau=sigma_beta**-2., value=mu_beta)
vars.update(study_coeffs=beta)
mu_gamma = 0.*pl.ones(len(est_mesh))
sigma_gamma = 2.*pl.ones(len(est_mesh))
# add informative prior for gamma effect if requested
prior_key = 'gamma_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on gamma effect coefficients for %s' % key
mu_gamma = pl.array(dm.params[prior_key]['mean'])
sigma_gamma = pl.array(dm.params[prior_key]['std'])
# always use dispersed prior on delta for empirical prior phase
mu_log_delta = 3.
sigma_log_delta = .25
# add informative prior for delta effect if requested
prior_key = 'delta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on delta effect coefficients for %s' % key
mu_log_delta = dm.params[prior_key]['mean']
sigma_log_delta = dm.params[prior_key]['std']
mu_zeta = 0.
sigma_zeta = .25
# add informative prior for zeta effect if requested
prior_key = 'zeta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on zeta effect coefficients for %s' % key
mu_zeta = dm.params[prior_key]['mean']
sigma_zeta = dm.params[prior_key]['std']
if mu_delta != 0.:
if sigma_delta != 0.:
log_delta = mc.Normal('log_dispersion_%s' % key, mu=mu_log_delta, tau=sigma_log_delta**-2, value=3.)
zeta = mc.Normal('zeta_%s'%key, mu=mu_zeta, tau=sigma_zeta**-2, value=mu_zeta)
delta = mc.Lambda('dispersion_%s' % key, lambda x=log_delta: 50. + 10.**x)
vars.update(dispersion=delta, log_dispersion=log_delta, zeta=zeta, dispersion_step_sd=.1*log_delta.parents['tau']**-.5)
else:
delta = mc.Lambda('dispersion_%s' % key, lambda x=mu_delta: mu_delta)
vars.update(dispersion=delta)
else:
delta = mc.Lambda('dispersion_%s' % key, lambda mu=mu_delta: 0)
vars.update(dispersion=delta)
if len(sigma_gamma) == 1:
sigma_gamma = sigma_gamma[0]*pl.ones(len(est_mesh))
# create varible for interpolated rate;
# also create variable for age-specific rate function, if it does not yet exist
if rate_stoch:
# if the rate_stoch already exists, for example prevalence in the generic model,
# we use it to back-calculate mu and eventually gamma
mu = rate_stoch
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(mu=mu, Xa=X_region, Xb=X_study, alpha=alpha, beta=beta):
return pl.log(pl.maximum(dismod3.settings.NEARLY_ZERO, mu)) - pl.dot(alpha, Xa) - pl.dot(beta, Xb)
@mc.potential(name='age_coeffs_potential_%s' % key)
def gamma_potential(gamma=gamma, mu_gamma=mu_gamma, tau_gamma=1./sigma_gamma[param_mesh]**2, param_mesh=param_mesh):
return mc.normal_like(gamma[param_mesh], mu_gamma[param_mesh], tau_gamma)
vars.update(rate_stoch=mu, age_coeffs=gamma, age_coeffs_potential=gamma_potential)
else:
# if the rate_stoch does not yet exists, we make gamma a stoch, and use it to calculate mu
# for computational efficiency, gamma is a linearly interpolated version of gamma_mesh
initial_gamma = pl.log(dismod3.settings.NEARLY_ZERO + dm.get_initial_value(key))
gamma_mesh = mc.Normal('age_coeffs_mesh_%s' % key, mu=mu_gamma[param_mesh], tau=sigma_gamma[param_mesh]**-2, value=initial_gamma[param_mesh])
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(gamma_mesh=gamma_mesh, param_mesh=param_mesh, est_mesh=est_mesh):
return dismod3.utils.interpolate(param_mesh, gamma_mesh, est_mesh)
@mc.deterministic(name=key)
def mu(Xa=X_region, Xb=X_study, alpha=alpha, beta=beta, gamma=gamma):
return predict_rate([Xa, Xb], alpha, beta, gamma, lambda f, age: f, est_mesh)
# Create a guess at the covariance matrix for MCMC proposals to update gamma_mesh
from pymc.gp.cov_funs import matern
a = pl.atleast_2d(param_mesh).T
C = matern.euclidean(a, a, diff_degree = 2, amp = 1.**2, scale = 10.)
vars.update(age_coeffs_mesh=gamma_mesh, age_coeffs=gamma, rate_stoch=mu, age_coeffs_mesh_step_cov=.005*pl.array(C))
# adjust value of gamma_mesh based on priors, if necessary
# TODO: implement more adjustments, currently only adjusted based on at_least priors
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'at_least':
delta_gamma = pl.log(pl.maximum(mu.value, float(prior[1]))) - pl.log(mu.value)
gamma_mesh.value = gamma_mesh.value + delta_gamma[param_mesh]
# create potentials for priors
dismod3.utils.generate_prior_potentials(vars, dm.get_priors(key), est_mesh)
# create observed stochastics for data
vars['data'] = []
if mu_delta != 0.:
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
Xz = []
for d in data_list:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
Xz.append(float(d.get('bias') or 0.))
ai.append(age_indices)
aw.append(age_weights)
vars['data'].append(d)
N = pl.array(N)
Xa = pl.array(Xa)
Xb = pl.array(Xb)
Xz = pl.array(Xz)
value = pl.array(value)
vars['effective_sample_size'] = list(N)
if len(vars['data']) > 0:
# TODO: consider using only a subset of the rates at each step of the fit to speed computation; say 100 of them
k = 50000
if len(vars['data']) < k:
data_sample = range(len(vars['data']))
else:
import random
@mc.deterministic(name='data_sample_%s' % key)
def data_sample(n=len(vars['data']), k=k):
return random.sample(range(n), k)
@mc.deterministic(name='rate_%s' % key)
def rates(S=data_sample,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa[S], alpha) + pl.dot(Xb[S], pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu = pl.zeros_like(shifts)
for i,s in enumerate(S):
mu[i] = pl.dot(age_weights[s], bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
# TODO: evaluate speed increase and accuracy decrease of the following:
#midpoint = age_indices[s][len(age_indices[s])/2]
#mu[i] = bounds_func(shifts[i] * exp_gamma[midpoint], midpoint)
# TODO: evaluate speed increase and accuracy decrease of the following: (to see speed increase, need to code this up using difference of running sums
#mu[i] = pl.dot(pl.ones_like(age_weights[s]) / float(len(age_weights[s])),
# bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
return mu
vars['expected_rates'] = rates
@mc.observed
@mc.stochastic(name='data_%s' % key)
def obs(value=value,
S=data_sample,
N=N,
mu_i=rates,
Xz=Xz,
zeta=zeta,
delta=delta):
#zeta_i = .001
#residual = pl.log(value[S] + zeta_i) - pl.log(mu_i*N[S] + zeta_i)
#return mc.normal_like(residual, 0, 100. + delta)
logp = mc.negative_binomial_like(value[S], N[S]*mu_i, delta*pl.exp(Xz*zeta))
return logp
vars['observed_counts'] = obs
@mc.deterministic(name='predicted_data_%s' % key)
def predictions(value=value,
N=N,
S=data_sample,
mu=rates,
delta=delta):
r_S = mc.rnegative_binomial(N[S]*mu, delta)/N[S]
r = pl.zeros(len(vars['data']))
r[S] = r_S
return r
vars['predicted_rates'] = predictions
debug('likelihood of %s contains %d rates' % (key, len(vars['data'])))
# now do the same thing for the lower bound data
# TODO: refactor to remove duplicated code
vars['lower_bound_data'] = []
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
for d in lower_bound_data:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
ai.append(age_indices)
aw.append(age_weights)
vars['lower_bound_data'].append(d)
N = pl.array(N)
value = pl.array(value)
if len(vars['lower_bound_data']) > 0:
@mc.observed
@mc.stochastic(name='lower_bound_data_%s' % key)
def obs_lb(value=value, N=N,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
delta=delta,
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa, alpha) + pl.dot(Xb, pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu_i = [pl.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed
rate_param = mu_i*N
violated_bounds = pl.nonzero(rate_param < value)
logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta)
return logp
vars['observed_lower_bounds'] = obs_lb
debug('likelihood of %s contains %d lowerbounds' % (key, len(vars['lower_bound_data'])))
return vars
for i in range(len(plot_list)):
plot_list_ix.append(cov_name.index(plot_list[i]))
# correctly ordering geographic regions
plot_list_ix_order = []
for i in range(len(index)):
if (index[i] in plot_list_ix) == 1:
plot_list_ix_order.append(index[i]) # if true, append
for y, i in enumerate(plot_list_ix_order):
x1 = data.ix[cov_name[i], 'moderately']
x2 = data.ix[cov_name[i], 'slightly']
x3 = data.ix[cov_name[i], 'very']
xerr1 = pl.array([
x1 - pl.atleast_2d(data.ix[cov_name[i], 'moderately_l']),
pl.atleast_2d(data.ix[cov_name[i], 'moderately_u']) - x1
])
xerr2 = pl.array([
x2 - pl.atleast_2d(data.ix[cov_name[i], 'slightly_l']),
pl.atleast_2d(data.ix[cov_name[i], 'slightly_u']) - x2
])
xerr3 = pl.array([
x3 - pl.atleast_2d(data.ix[cov_name[i], 'very_l']),
pl.atleast_2d(data.ix[cov_name[i], 'very_u']) - x3
])
pl.errorbar(x2, 2.75 * y + .45, xerr=xerr2, fmt='ko',
mec='w') #, label = 'Slightly'
pl.errorbar(x1, 2.75 * y, xerr=xerr1, fmt='k^',
mec='w') #, label = 'Moderately'
def sigma_explained(W=W, gamma=gamma):
""" sigma_explained_i,r,c,t,a = gamma * W_i,r,c,t,a"""
return pl.dot(pl.atleast_1d(gamma), pl.atleast_2d(W))
udata_12 = data["VSHS_OP"][:-1].values[e:int_end[k]:int_step]
udata_13 = data["VSHS_CL"][:-1].values[e:int_end[k]:int_step]
udata = ca.horzcat([udata_0, udata_1, udata_2, udata_3, udata_4, udata_5, udata_6, udata_7, udata_8, udata_9, \
udata_10, udata_11, udata_12, udata_13])[:-1,:]
x0_init = data["TSH0"].values[e:int_end[k]:int_step]
x1_init = data["TSH2"].values[e:int_end[k]:int_step]
x2_init = data["TSH3"].values[e:int_end[k]:int_step]
x3_init = data["TSH1"].values[e:int_end[k]:int_step]
xinit = ca.horzcat([pl.atleast_2d(x0_init).T, pl.atleast_2d(x1_init).T, pl.atleast_2d(x2_init).T, pl.atleast_2d(x3_init).T,])
ydata_0 = data["TSH0"].values[e:int_end[k]:int_step]
ydata_1 = data["TSH2"].values[e:int_end[k]:int_step]
ydata_2 = data["TSH3"].values[e:int_end[k]:int_step]
ydata_3 = data["TSH1"].values[e:int_end[k]:int_step]
ydata = ca.horzcat([pl.atleast_2d(ydata_0).T, pl.atleast_2d(ydata_1).T, pl.atleast_2d(ydata_2).T, pl.atleast_2d(ydata_3).T,]) #ca.repmat(y1_5_init, (1, ydata.shape[0])).T])
pe_setups.append(cp.pe.LSq(system = system, time_points = time_points, \
udata = udata, \
pinit = pinit, \
ydata = ydata, \
xinit = xinit)) #, \
udata_13 = data["VSHS_CL"][:-1].values[e:int_end[k]:int_step]
udata = ca.horzcat([udata_0, udata_1, udata_2, udata_3, udata_4, udata_5, udata_6, udata_7, udata_8, udata_9, \
udata_10, udata_11, udata_12, udata_13])[:-1,:]
x0_init = data["TSH0"].values[e:int_end[k]:int_step]
x1_init = data["TSH2"].values[e:int_end[k]:int_step]
x2_init = data["TSH3"].values[e:int_end[k]:int_step]
x3_init = data["TSH1"].values[e:int_end[k]:int_step]
x4_init = data["TSH0_5"].values[e:int_end[k]:int_step]
x5_init = data["TSH2_5"].values[e:int_end[k]:int_step]
x6_init = data["TSH3_5"].values[e:int_end[k]:int_step]
xinit = ca.horzcat([pl.atleast_2d(x0_init).T, pl.atleast_2d(x1_init).T, pl.atleast_2d(x2_init).T, pl.atleast_2d(x3_init).T, \
pl.atleast_2d(x4_init).T, pl.atleast_2d(x5_init).T, pl.atleast_2d(x6_init).T,])
ydata_0 = data["TSH0"].values[e:int_end[k]:int_step]
ydata_1 = data["TSH2"].values[e:int_end[k]:int_step]
ydata_2 = data["TSH3"].values[e:int_end[k]:int_step]
ydata_3 = data["TSH1"].values[e:int_end[k]:int_step]
ydata_4 = data["TSH0_5"].values[e:int_end[k]:int_step]
ydata_5 = data["TSH2_5"].values[e:int_end[k]:int_step]
ydata_6 = data["TSH3_5"].values[e:int_end[k]:int_step]
ydata = ca.horzcat([pl.atleast_2d(ydata_0).T, pl.atleast_2d(ydata_1).T, pl.atleast_2d(ydata_2).T, pl.atleast_2d(ydata_3).T,\
pl.atleast_2d(ydata_4).T, pl.atleast_2d(ydata_5).T, pl.atleast_2d(ydata_6).T,])
# wv = pl.ones(ydata.shape[0])
def extract(self):
Features.extract(self)
mf = (self.X.T * self._logfrqs).sum(1) / self.X.T.sum(1)
self.X = (((self.X / self.X.T.sum(1)).T * ((P.atleast_2d(self._logfrqs).T - mf)).T)**2).sum(1)
def predict_for(model, parameters,
root_area, root_sex, root_year,
area, sex, year,
population_weighted,
vars,
lower, upper):
""" Generate draws from posterior predicted distribution for a
specific (area, sex, year)
:Parameters:
- `model` : data.DataModel
- `root_area` : str, area for which this model was fit consistently
- `root_sex` : str, area for which this model was fit consistently
- `root_year` : str, area for which this model was fit consistently
- `area` : str, area to predict for
- `sex` : str, sex to predict for
- `year` : str, year to predict for
- `population_weighted` : bool, should prediction be population weighted if it is the aggregation of units area RE hierarchy?
- `vars` : dict, including entries for alpha, beta, mu_age, U, and X
- `lower, upper` : float, bounds on predictions from expert priors
:Results:
- Returns array of draws from posterior predicted distribution
"""
area_hierarchy = model.hierarchy
output_template = model.output_template.copy()
# find number of samples from posterior
len_trace = len(vars['mu_age'].trace())
# compile array of draws from posterior distribution of alpha (random effect covariate values)
# a row for each draw from the posterior distribution
# a column for each random effect (e.g. countries with data, regions with countries with data, etc)
#
# there are several cases to handle, or at least at one time there were:
# vars['alpha'] is a pymc Stochastic with an array for its value (no longer used?)
# vars['alpha'] is a list of pymc Nodes
# vars['alpha'] is a list of floats
# vars['alpha'] is a list of some floats and some pymc Nodes
# 'alpha' is not in vars
#
# when vars['alpha'][i] is a float, there is also information on the uncertainty in this value, stored in
# vars['const_alpha_sigma'][i], which is not used when fitting the model, but should be incorporated in
# the prediction
if 'alpha' in vars and isinstance(vars['alpha'], mc.Node):
assert 0, 'No longer used'
alpha_trace = vars['alpha'].trace()
elif 'alpha' in vars and isinstance(vars['alpha'], list):
alpha_trace = []
for n, sigma in zip(vars['alpha'], vars['const_alpha_sigma']):
if isinstance(n, mc.Node):
alpha_trace.append(n.trace())
else:
# uncertainty of constant alpha incorporated here
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
assert not pl.isnan(sigma)
alpha_trace.append(mc.rnormal(float(n), sigma**-2, size=len_trace))
alpha_trace = pl.vstack(alpha_trace).T
else:
alpha_trace = pl.array([])
# compile array of draws from posterior distribution of beta (fixed effect covariate values)
# a row for each draw from the posterior distribution
# a column for each fixed effect
#
# there are several cases to handle, or at least at one time there were:
# vars['beta'] is a pymc Stochastic with an array for its value (no longer used?)
# vars['beta'] is a list of pymc Nodes
# vars['beta'] is a list of floats
# vars['beta'] is a list of some floats and some pymc Nodes
# 'beta' is not in vars
#
# when vars['beta'][i] is a float, there is also information on the uncertainty in this value, stored in
# vars['const_beta_sigma'][i], which is not used when fitting the model, but should be incorporated in
# the prediction
#
# TODO: refactor to reduce duplicate code (this is very similar to code for alpha above)
if 'beta' in vars and isinstance(vars['beta'], mc.Node):
assert 0, 'No longer used'
beta_trace = vars['beta'].trace()
elif 'beta' in vars and isinstance(vars['beta'], list):
beta_trace = []
for n, sigma in zip(vars['beta'], vars['const_beta_sigma']):
if isinstance(n, mc.Node):
beta_trace.append(n.trace())
else:
# uncertainty of constant beta incorporated here
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
assert not pl.isnan(sigma)
beta_trace.append(mc.rnormal(float(n), sigma**-2., size=len_trace))
beta_trace = pl.vstack(beta_trace).T
else:
beta_trace = pl.array([])
# the prediction for the requested area is produced by aggregating predictions for all of the childred
# of that area in the area_hierarchy (a networkx.DiGraph)
leaves = [n for n in nx.traversal.bfs_tree(area_hierarchy, area) if area_hierarchy.successors(n) == []]
if len(leaves) == 0:
# networkx returns an empty list when the bfs tree is a single node
leaves = [area]
# initialize covariate_shift and total_population
covariate_shift = pl.zeros(len_trace)
total_population = 0.
# group output_template for easy access
output_template = output_template.groupby(['area', 'sex', 'year']).mean()
# if there are fixed effects, the effect coefficients are stored as an array in vars['X']
# use this to put together a covariate matrix for the predictions, according to the output_template
# covariate values
#
# the resulting array is covs
if 'X' in vars:
covs = output_template.filter(vars['X'].columns)
if 'x_sex' in vars['X'].columns:
covs['x_sex'] = sex_value[sex]
assert pl.all(covs.columns == vars['X_shift'].index), 'covariate columns and unshift index should match up'
for x_i in vars['X_shift'].index:
covs[x_i] -= vars['X_shift'][x_i] # shift covariates so that the root node has X_ar,sr,yr == 0
else:
covs = pandas.DataFrame(index=output_template.index)
# if there are random effects, put together an indicator based on
# their hierarchical relationships
#
if 'U' in vars:
p_U = area_hierarchy.number_of_nodes() # random effects for area
U_l = pandas.DataFrame(pl.zeros((1, p_U)), columns=area_hierarchy.nodes())
U_l = U_l.filter(vars['U'].columns)
else:
U_l = pandas.DataFrame(index=[0])
# loop through leaves of area_hierarchy subtree rooted at 'area',
# make prediction for each using appropriate random
# effects and appropriate fixed effect covariates
#
for l in leaves:
log_shift_l = pl.zeros(len_trace)
U_l.ix[0,:] = 0.
root_to_leaf = nx.shortest_path(area_hierarchy, root_area, l)
for node in root_to_leaf[1:]:
if node not in U_l.columns:
## Add a columns U_l[node] = rnormal(0, appropriate_tau)
level = len(nx.shortest_path(area_hierarchy, 'all', node))-1
if 'sigma_alpha' in vars:
tau_l = vars['sigma_alpha'][level].trace()**-2
U_l[node] = 0.
# if this node was not already included in the alpha_trace array, add it
# there are several cases for adding:
# if the random effect has a distribution of Constant
# add it, using a sigma as well
# otherwise, sample from a normal with mean zero and standard deviation tau_l
if parameters.get('random_effects', {}).get(node, {}).get('dist') == 'Constant':
mu = parameters['random_effects'][node]['mu']
sigma = parameters['random_effects'][node]['sigma']
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
alpha_node = mc.rnormal(mu,
sigma**-2,
size=len_trace)
else:
if 'sigma_alpha' in vars:
alpha_node = mc.rnormal(0., tau_l)
else:
alpha_node = pl.zeros(len_trace)
if len(alpha_trace) > 0:
alpha_trace = pl.vstack((alpha_trace.T, alpha_node)).T
else:
alpha_trace = pl.atleast_2d(alpha_node).T
# TODO: implement a more robust way to align alpha_trace and U_l
U_l.ix[0, node] = 1.
# 'shift' the random effects matrix to have the intended
# level of the hierarchy as the reference value
if 'U_shift' in vars:
for node in vars['U_shift']:
U_l -= vars['U_shift'][node]
# add the random effect intercept shift (len_trace draws)
log_shift_l += pl.dot(alpha_trace, U_l.T).flatten()
# make X_l
if len(beta_trace) > 0:
X_l = covs.ix[l, sex, year]
log_shift_l += pl.dot(beta_trace, X_l.T).flatten()
if population_weighted:
# combine in linear-space with population weights
shift_l = pl.exp(log_shift_l)
covariate_shift += shift_l * output_template['pop'][l,sex,year]
total_population += output_template['pop'][l,sex,year]
else:
# combine in log-space without weights
covariate_shift += log_shift_l
total_population += 1.
if population_weighted:
covariate_shift /= total_population
else:
covariate_shift = pl.exp(covariate_shift / total_population)
parameter_prediction = (vars['mu_age'].trace().T * covariate_shift).T
# clip predictions to bounds from expert priors
parameter_prediction = parameter_prediction.clip(lower, upper)
return parameter_prediction
udata = ca.horzcat([udata_0, udata_1, udata_2, udata_3, udata_4, udata_5, udata_6, udata_7, udata_8, udata_9, \
udata_10, udata_11, udata_12, udata_13])
x0_init = data["TSH0"].values[int_start]
x1_init = data["TSH2"].values[int_start]
x2_init = data["TSH3"].values[int_start]
x3_init = data["TSH1"].values[int_start]
x4_init = data["TSH0_1"].values[int_start]
x5_init = data["TSH0_2"].values[int_start]
x6_init = data["TSH2_1"].values[int_start]
x7_init = data["TSH2_2"].values[int_start]
x8_init = data["TSH3_1"].values[int_start]
x9_init = data["TSH3_2"].values[int_start]
xinit = ca.horzcat([pl.atleast_2d(x0_init).T, pl.atleast_2d(x1_init).T, pl.atleast_2d(x2_init).T, pl.atleast_2d(x3_init).T, \
pl.atleast_2d(x4_init).T, pl.atleast_2d(x5_init).T, pl.atleast_2d(x6_init).T, pl.atleast_2d(x7_init).T, pl.atleast_2d(x8_init).T, \
pl.atleast_2d(x9_init).T,])
# mpe = cp.pe.MultiLSq(pe_setups)
# # # mpe.run_parameter_estimation({"linear_solver": "ma57"})
# mpe.run_parameter_estimation()
sim_est = cp.sim.Simulation(system=system, pdata=0.0)
# sim_est = cp.sim.Simulation(system = system, pdata = mpe.estimated_parameters)
sim_est.run_system_simulation(time_points = time_points, \
x0 = xinit[0,:], udata = udata)
pl.close("all")
# # # Plot
def extract(self):
Features.extract(self)
mf = (self.X.T * self._logfrqs).sum(1) / self.X.T.sum(1)
self.X = (((self.X / self.X.T.sum(1)).T *
((P.atleast_2d(self._logfrqs).T - mf)).T)**2).sum(1)
def predict_for(model, parameters, root_area, root_sex, root_year, area, sex,
year, population_weighted, vars, lower, upper):
""" Generate draws from posterior predicted distribution for a
specific (area, sex, year)
:Parameters:
- `model` : data.DataModel
- `root_area` : str, area for which this model was fit consistently
- `root_sex` : str, area for which this model was fit consistently
- `root_year` : str, area for which this model was fit consistently
- `area` : str, area to predict for
- `sex` : str, sex to predict for
- `year` : str, year to predict for
- `population_weighted` : bool, should prediction be population weighted if it is the aggregation of units area RE hierarchy?
- `vars` : dict, including entries for alpha, beta, mu_age, U, and X
- `lower, upper` : float, bounds on predictions from expert priors
:Results:
- Returns array of draws from posterior predicted distribution
"""
area_hierarchy = model.hierarchy
output_template = model.output_template.copy()
# find number of samples from posterior
len_trace = len(vars['mu_age'].trace())
# compile array of draws from posterior distribution of alpha (random effect covariate values)
# a row for each draw from the posterior distribution
# a column for each random effect (e.g. countries with data, regions with countries with data, etc)
#
# there are several cases to handle, or at least at one time there were:
# vars['alpha'] is a pymc Stochastic with an array for its value (no longer used?)
# vars['alpha'] is a list of pymc Nodes
# vars['alpha'] is a list of floats
# vars['alpha'] is a list of some floats and some pymc Nodes
# 'alpha' is not in vars
#
# when vars['alpha'][i] is a float, there is also information on the uncertainty in this value, stored in
# vars['const_alpha_sigma'][i], which is not used when fitting the model, but should be incorporated in
# the prediction
if 'alpha' in vars and isinstance(vars['alpha'], mc.Node):
assert 0, 'No longer used'
alpha_trace = vars['alpha'].trace()
elif 'alpha' in vars and isinstance(vars['alpha'], list):
alpha_trace = []
for n, sigma in zip(vars['alpha'], vars['const_alpha_sigma']):
if isinstance(n, mc.Node):
alpha_trace.append(n.trace())
else:
# uncertainty of constant alpha incorporated here
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
assert not pl.isnan(sigma)
alpha_trace.append(
mc.rnormal(float(n), sigma**-2, size=len_trace))
alpha_trace = pl.vstack(alpha_trace).T
else:
alpha_trace = pl.array([])
# compile array of draws from posterior distribution of beta (fixed effect covariate values)
# a row for each draw from the posterior distribution
# a column for each fixed effect
#
# there are several cases to handle, or at least at one time there were:
# vars['beta'] is a pymc Stochastic with an array for its value (no longer used?)
# vars['beta'] is a list of pymc Nodes
# vars['beta'] is a list of floats
# vars['beta'] is a list of some floats and some pymc Nodes
# 'beta' is not in vars
#
# when vars['beta'][i] is a float, there is also information on the uncertainty in this value, stored in
# vars['const_beta_sigma'][i], which is not used when fitting the model, but should be incorporated in
# the prediction
#
# TODO: refactor to reduce duplicate code (this is very similar to code for alpha above)
if 'beta' in vars and isinstance(vars['beta'], mc.Node):
assert 0, 'No longer used'
beta_trace = vars['beta'].trace()
elif 'beta' in vars and isinstance(vars['beta'], list):
beta_trace = []
for n, sigma in zip(vars['beta'], vars['const_beta_sigma']):
if isinstance(n, mc.Node):
beta_trace.append(n.trace())
else:
# uncertainty of constant beta incorporated here
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
assert not pl.isnan(sigma)
beta_trace.append(
mc.rnormal(float(n), sigma**-2., size=len_trace))
beta_trace = pl.vstack(beta_trace).T
else:
beta_trace = pl.array([])
# the prediction for the requested area is produced by aggregating predictions for all of the childred
# of that area in the area_hierarchy (a networkx.DiGraph)
leaves = [
n for n in nx.traversal.bfs_tree(area_hierarchy, area)
if area_hierarchy.successors(n) == []
]
if len(leaves) == 0:
# networkx returns an empty list when the bfs tree is a single node
leaves = [area]
# initialize covariate_shift and total_population
covariate_shift = pl.zeros(len_trace)
total_population = 0.
# group output_template for easy access
output_template = output_template.groupby(['area', 'sex', 'year']).mean()
# if there are fixed effects, the effect coefficients are stored as an array in vars['X']
# use this to put together a covariate matrix for the predictions, according to the output_template
# covariate values
#
# the resulting array is covs
if 'X' in vars:
covs = output_template.filter(vars['X'].columns)
if 'x_sex' in vars['X'].columns:
covs['x_sex'] = sex_value[sex]
assert pl.all(covs.columns == vars['X_shift'].index
), 'covariate columns and unshift index should match up'
for x_i in vars['X_shift'].index:
covs[x_i] -= vars['X_shift'][
x_i] # shift covariates so that the root node has X_ar,sr,yr == 0
else:
covs = pandas.DataFrame(index=output_template.index)
# if there are random effects, put together an indicator based on
# their hierarchical relationships
#
if 'U' in vars:
p_U = area_hierarchy.number_of_nodes() # random effects for area
U_l = pandas.DataFrame(pl.zeros((1, p_U)),
columns=area_hierarchy.nodes())
U_l = U_l.filter(vars['U'].columns)
else:
U_l = pandas.DataFrame(index=[0])
# loop through leaves of area_hierarchy subtree rooted at 'area',
# make prediction for each using appropriate random
# effects and appropriate fixed effect covariates
#
for l in leaves:
log_shift_l = pl.zeros(len_trace)
U_l.ix[0, :] = 0.
root_to_leaf = nx.shortest_path(area_hierarchy, root_area, l)
for node in root_to_leaf[1:]:
if node not in U_l.columns:
## Add a columns U_l[node] = rnormal(0, appropriate_tau)
level = len(nx.shortest_path(area_hierarchy, 'all', node)) - 1
if 'sigma_alpha' in vars:
tau_l = vars['sigma_alpha'][level].trace()**-2
U_l[node] = 0.
# if this node was not already included in the alpha_trace array, add it
# there are several cases for adding:
# if the random effect has a distribution of Constant
# add it, using a sigma as well
# otherwise, sample from a normal with mean zero and standard deviation tau_l
if parameters.get('random_effects',
{}).get(node, {}).get('dist') == 'Constant':
mu = parameters['random_effects'][node]['mu']
sigma = parameters['random_effects'][node]['sigma']
sigma = max(sigma, 1.e-9) # make sure sigma is non-zero
alpha_node = mc.rnormal(mu, sigma**-2, size=len_trace)
else:
if 'sigma_alpha' in vars:
alpha_node = mc.rnormal(0., tau_l)
else:
alpha_node = pl.zeros(len_trace)
if len(alpha_trace) > 0:
alpha_trace = pl.vstack((alpha_trace.T, alpha_node)).T
else:
alpha_trace = pl.atleast_2d(alpha_node).T
# TODO: implement a more robust way to align alpha_trace and U_l
U_l.ix[0, node] = 1.
# 'shift' the random effects matrix to have the intended
# level of the hierarchy as the reference value
if 'U_shift' in vars:
for node in vars['U_shift']:
U_l -= vars['U_shift'][node]
# add the random effect intercept shift (len_trace draws)
log_shift_l += pl.dot(alpha_trace, U_l.T).flatten()
# make X_l
if len(beta_trace) > 0:
X_l = covs.ix[l, sex, year]
log_shift_l += pl.dot(beta_trace, X_l.T).flatten()
if population_weighted:
# combine in linear-space with population weights
shift_l = pl.exp(log_shift_l)
covariate_shift += shift_l * output_template['pop'][l, sex, year]
total_population += output_template['pop'][l, sex, year]
else:
# combine in log-space without weights
covariate_shift += log_shift_l
total_population += 1.
if population_weighted:
covariate_shift /= total_population
else:
covariate_shift = pl.exp(covariate_shift / total_population)
parameter_prediction = (vars['mu_age'].trace().T * covariate_shift).T
# clip predictions to bounds from expert priors
parameter_prediction = parameter_prediction.clip(lower, upper)
return parameter_prediction
# index of area of interest in covariate list
plot_list_ix = []
for i in range(len(plot_list)):
plot_list_ix.append(cov_name.index(plot_list[i]))
# correctly ordering geographic regions
plot_list_ix_order = []
for i in range(len(index)):
if (index[i] in plot_list_ix) == 1: plot_list_ix_order.append(index[i]) # if true, append
for y, i in enumerate(plot_list_ix_order):
x1 = data.ix[cov_name[i],'moderately']
x2 = data.ix[cov_name[i],'slightly']
x3 = data.ix[cov_name[i],'very']
xerr1 = pl.array([x1 - pl.atleast_2d(data.ix[cov_name[i],'moderately_l']),
pl.atleast_2d(data.ix[cov_name[i],'moderately_u']) - x1])
xerr2 = pl.array([x2 - pl.atleast_2d(data.ix[cov_name[i],'slightly_l']),
pl.atleast_2d(data.ix[cov_name[i],'slightly_u']) - x2])
xerr3 = pl.array([x3 - pl.atleast_2d(data.ix[cov_name[i],'very_l']),
pl.atleast_2d(data.ix[cov_name[i],'very_u']) - x3])
pl.errorbar(x2, 2.75*y+.45, xerr=xerr2, fmt='ko', mec='w')#, label = 'Slightly'
pl.errorbar(x1, 2.75*y, xerr=xerr1, fmt='k^', mec='w')#, label = 'Moderately'
pl.errorbar(x3, 2.75*y-.45, xerr=xerr3, fmt='ks', mec='w')#, label = 'Very'
pl.plot([-10],[-10], 'ko-', mec='w', label = '$\delta \sim \\mathrm{Uniform}(9,$ $81)$')
pl.plot([-10],[-10], 'k^-', mec='w', label = '$\delta \sim \\mathrm{Uniform}(3,$ $27)$')
pl.plot([-10],[-10], 'ks-', mec='w', label = '$\delta \sim \mathrm{Uniform}(1,$ $9)$')
l,r,b,t = pl.axis()
r = 3.0
def generate_prior_potentials(rate_vars, prior_str, age_mesh):
"""
augment the rate_vars dict to include a list of potentials that model priors on rate_vars['rate_stoch']
prior_str may have entries in the following format:
smooth <tau> [<age_start> <age_end>]
zero <age_start> <age_end>
confidence <mean> <tau>
increasing <age_start> <age_end>
decreasing <age_start> <age_end>
convex_up <age_start> <age_end>
convex_down <age_start> <age_end>
unimodal <age_start> <age_end>
value <mean> <tau> [<age_start> <age_end>]
at_least <value>
at_most <value>
max_at_most <value>
for example: 'smooth .1, zero 0 5, zero 95 100'
age_mesh[i] indicates what age the value of rate[i] corresponds to
"""
def derivative_sign_prior(rate, prior, deriv, sign):
age_start = int(prior[1])
age_end = int(prior[2])
age_indices = indices_for_range(age_mesh, age_start, age_end)
@mc.potential(name='deriv_sign_{%d,%d,%d,%d}^%s' % (deriv, sign, age_start, age_end, str(rate)))
def deriv_sign_rate(f=rate,
age_indices=age_indices,
tau=1.e14,
deriv=deriv, sign=sign):
df = pl.diff(f[age_indices], deriv)
return mc.normal_like(pl.absolute(df) * (sign * df < 0), 0., tau)
return [deriv_sign_rate]
priors = []
rate = rate_vars['rate_stoch']
rate_vars['bounds_func'] = lambda f, age: f
for line in prior_str.split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'smooth':
pass # handle this after applying all level bounds
elif prior[0] == 'heterogeneity':
# prior affects dispersion term of model; handle as a special case
continue
elif prior[0] == 'increasing':
priors += derivative_sign_prior(rate, prior, deriv=1, sign=1)
elif prior[0] == 'decreasing':
priors += derivative_sign_prior(rate, prior, deriv=1, sign=-1)
elif prior[0] == 'convex_down':
priors += derivative_sign_prior(rate, prior, deriv=2, sign=-1)
elif prior[0] == 'convex_up':
priors += derivative_sign_prior(rate, prior, deriv=2, sign=1)
elif prior[0] == 'unimodal':
age_start = int(prior[1])
age_end = int(prior[2])
age_indices = indices_for_range(age_mesh, age_start, age_end)
@mc.potential(name='unimodal_{%d,%d}^%s' % (age_start, age_end, str(rate)))
def unimodal_rate(f=rate, age_indices=age_indices, tau=1.e5):
df = pl.diff(f[age_indices])
sign_changes = pl.find((df[:-1] > NEARLY_ZERO) & (df[1:] < -NEARLY_ZERO))
sign = pl.ones(len(age_indices)-2)
if len(sign_changes) > 0:
change_age = sign_changes[len(sign_changes)/2]
sign[change_age:] = -1.
return -tau*pl.dot(pl.absolute(df[:-1]), (sign * df[:-1] < 0))
priors += [unimodal_rate]
elif prior[0] == 'max_at_least':
val = float(prior[1])
@mc.potential(name='max_at_least_{%f}^{%s}' % (val, str(rate)))
def max_at_least(cur_max=rate, at_least=val, tau=(.001*val)**-2):
return -tau * (cur_max - at_least)**2 * (cur_max < at_least)
priors += [max_at_least]
elif prior[0] == 'level_value':
val = float(prior[1]) + 1.e-9
if len(prior) == 4:
age_start = int(prior[2])
age_end = int(prior[3])
else:
age_start = 0
age_end = MAX_AGE
age_indices = indices_for_range(age_mesh, age_start, age_end)
def new_bounds_func(f, age, val=val, age_start=age_start, age_end=age_end, prev_bounds_func=rate_vars['bounds_func']):
age = pl.array(age)
return pl.where((age >= age_start) & (age <= age_end), val, prev_bounds_func(f, age))
rate_vars['bounds_func'] = new_bounds_func
elif prior[0] == 'at_most':
val = float(prior[1])
def new_bounds_func(f, age, val=val, prev_bounds_func=rate_vars['bounds_func']):
return pl.minimum(prev_bounds_func(f, age), val)
rate_vars['bounds_func'] = new_bounds_func
elif prior[0] == 'at_least':
val = float(prior[1])
def new_bounds_func(f, age, val=val, prev_bounds_func=rate_vars['bounds_func']):
return pl.maximum(prev_bounds_func(f, age), val)
rate_vars['bounds_func'] = new_bounds_func
else:
raise KeyError, 'Unrecognized prior: %s' % prior_str
# update rate stoch with the bounds func from the priors
# TODO: create this before smoothing, so that smoothing takes levels into account
@mc.deterministic(name='%s_w_bounds'%rate_vars['rate_stoch'].__name__)
def mu_bounded(mu=rate_vars['rate_stoch'], bounds_func=rate_vars['bounds_func']):
return bounds_func(mu, pl.arange(101)) # FIXME: don't hardcode age range
rate_vars['unbounded_rate'] = rate_vars['rate_stoch']
rate_vars['rate_stoch'] = mu_bounded
rate = rate_vars['rate_stoch']
# add potential to encourage rate to look like level bounds
@mc.potential(name='%s_potential'%rate_vars['rate_stoch'].__name__)
def mu_potential(mu1=rate_vars['unbounded_rate'], mu2=rate_vars['rate_stoch']):
return mc.normal_like(mu1, mu2, .0001**-2)
rate_vars['rate_potential'] = mu_potential
# add smoothing prior to the rate with level bounds
for line in prior_str.split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'smooth':
scale = float(prior[1])
if len(prior) == 4:
age_start = int(prior[2])
age_end = int(prior[3])
else:
age_start = 0
age_end = MAX_AGE
age_indices = indices_for_range(pl.arange(MAX_AGE), age_start, age_end)
from pymc.gp.cov_funs import matern
a = pl.atleast_2d(age_indices).T
C = matern.euclidean(a, a, diff_degree=2, amp=10., scale=scale)
@mc.potential(name='smooth_{%d,%d}^%s' % (age_start, age_end, str(rate)))
def smooth_rate(f=rate, age_indices=age_indices, C=C):
log_rate = pl.log(pl.maximum(f, NEARLY_ZERO))
return mc.mv_normal_cov_like(log_rate[age_indices] - log_rate[age_indices].mean(),
pl.zeros_like(age_indices),
C=C)
priors += [smooth_rate]
print 'added smoothing potential for %s' % smooth_rate
rate_vars['priors'] = priors
def plot_one_effects(model, data_type):
""" Plot random effects and fixed effects.
:Parameters:
- `model` : data.ModelData
- `data_types` : str, one of 'i', 'r', 'f', 'p', 'rr', 'pf'
"""
vars = model.vars[data_type]
hierarchy = model.hierarchy
pl.figure(figsize=(22, 17))
for i, (covariate, effect) in enumerate([['U', 'alpha'], ['X', 'beta']]):
if covariate not in vars:
continue
cov_name = list(vars[covariate].columns)
if isinstance(vars.get(effect), mc.Stochastic):
pl.subplot(1, 2, i+1)
pl.title('%s_%s' % (effect, data_type))
stats = vars[effect].stats()
if stats:
if effect == 'alpha':
index = sorted(pl.arange(len(cov_name)),
key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i]))
elif effect == 'beta':
index = pl.arange(len(cov_name))
x = pl.atleast_1d(stats['mean'])
y = pl.arange(len(x))
xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - x])
pl.errorbar(x[index], y[index], xerr=xerr[:, index], fmt='bs', mec='w')
l,r,b,t = pl.axis()
pl.vlines([0], b-.5, t+.5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for i in index:
spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l, y[i], '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left')
pl.axis([l, r, -.5, t+.5])
if isinstance(vars.get(effect), list):
pl.subplot(1, 2, i+1)
pl.title('%s_%s' % (effect, data_type))
index = sorted(pl.arange(len(cov_name)),
key=lambda i: str(cov_name[i] in hierarchy and nx.shortest_path(hierarchy, 'all', cov_name[i]) or cov_name[i]))
for y, i in enumerate(index):
n = vars[effect][i]
if isinstance(n, mc.Stochastic) or isinstance(n, mc.Deterministic):
stats = n.stats()
if stats:
x = pl.atleast_1d(stats['mean'])
xerr = pl.array([x - pl.atleast_2d(stats['95% HPD interval'])[:,0],
pl.atleast_2d(stats['95% HPD interval'])[:,1] - x])
pl.errorbar(x, y, xerr=xerr, fmt='bs', mec='w')
l,r,b,t = pl.axis()
pl.vlines([0], b-.5, t+.5)
pl.hlines(y, l, r, linestyle='dotted')
pl.xticks([l, 0, r])
pl.yticks([])
for y, i in enumerate(index):
spaces = cov_name[i] in hierarchy and len(nx.shortest_path(hierarchy, 'all', cov_name[i])) or 0
pl.text(l, y, '%s%s' % (' * '*spaces, cov_name[i]), va='center', ha='left')
pl.axis([l, r, -.5, t+.5])
if effect == 'alpha':
effect_str = ''
for sigma in vars['sigma_alpha']:
stats = sigma.stats()
if stats:
effect_str += '%s = %.3f\n' % (sigma.__name__, stats['mean'])
else:
effect_str += '%s = %.3f\n' % (sigma.__name__, sigma.value)
pl.text(r, t, effect_str, va='top', ha='right')
elif effect == 'beta':
effect_str = ''
if 'eta' in vars:
eta = vars['eta']
stats = eta.stats()
if stats:
effect_str += '%s = %.3f\n' % (eta.__name__, stats['mean'])
else:
effect_str += '%s = %.3f\n' % (eta.__name__, eta.value)
pl.text(r, t, effect_str, va='top', ha='right')
