def setup(dm, key, data_list=[], rate_stoch=None, emp_prior={}, lower_bound_data=[]):
""" Generate the PyMC variables for a negative-binomial model of
a single rate function
Parameters
----------
dm : dismod3.DiseaseModel
the object containing all the data, priors, and additional
information (like input and output age-mesh)
key : str
the name of the key for everything about this model (priors,
initial values, estimations)
data_list : list of data dicts
the observed data to use in the negative binomial liklihood function
rate_stoch : pymc.Stochastic, optional
a PyMC stochastic (or deterministic) object, with
len(rate_stoch.value) == len(dm.get_estimation_age_mesh()).
This is used to link rate stochs into a larger model,
for example.
emp_prior : dict, optional
the empirical prior dictionary, retrieved from the disease model
if appropriate by::
>>> t, r, y, s = type_region_year_sex_from_key(key)
>>> emp_prior = dm.get_empirical_prior(t)
Results
-------
vars : dict
Return a dictionary of all the relevant PyMC objects for the
rate model. vars['rate_stoch'] is of particular
relevance; this is what is used to link the rate model
into more complicated models, like the generic disease model.
"""
vars = {}
est_mesh = dm.get_estimate_age_mesh()
param_mesh = dm.get_param_age_mesh()
if np.any(np.diff(est_mesh) != 1):
raise ValueError, 'ERROR: Gaps in estimation age mesh must all equal 1'
# calculate effective sample size for all data and lower bound data
dm.calc_effective_sample_size(data_list)
dm.calc_effective_sample_size(lower_bound_data)
# generate regional covariates
covariate_dict = dm.get_covariates()
X_region, X_study = regional_covariates(key, covariate_dict)
# use confidence prior from prior_str
mu_delta = 100.
sigma_delta = 1.
from dismod3.settings import PRIOR_SEP_STR
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'heterogeneity':
mu_delta = float(prior[1])
sigma_delta = float(prior[2])
# use the empirical prior mean if it is available
if len(set(emp_prior.keys()) & set(['alpha', 'beta', 'gamma'])) == 3:
mu_alpha = np.array(emp_prior['alpha'])
sigma_alpha = np.maximum(.1, emp_prior['sigma_alpha'])
alpha = np.array(emp_prior['alpha'])
vars.update(region_coeffs=alpha)
beta = np.array(emp_prior['beta'])
sigma_beta = np.maximum(.1, emp_prior['sigma_beta'])
vars.update(study_coeffs=beta)
mu_gamma = np.array(emp_prior['gamma'])
sigma_gamma = np.maximum(.1, emp_prior['sigma_gamma'])
# leave mu_delta and sigma_delta as they were set in the expert prior
else:
import dismod3.regional_similarity_matrices as similarity_matrices
n = len(X_region)
mu_alpha = np.zeros(n)
sigma_alpha = .01
C_alpha = similarity_matrices.regions_nested_in_superregions(n, sigma_alpha)
#C_alpha = similarity_matrices.all_related_equally(n, sigma_alpha)
alpha = mc.MvNormalCov('region_coeffs_%s' % key, mu=mu_alpha,
C=C_alpha,
value=mu_alpha)
vars.update(region_coeffs=alpha)
mu_beta = np.zeros(len(X_study))
sigma_beta = .1
beta = mc.Normal('study_coeffs_%s' % key, mu=mu_beta, tau=sigma_beta**-2., value=mu_beta)
vars.update(study_coeffs=beta)
mu_gamma = -5.*np.ones(len(est_mesh))
sigma_gamma = 10.*np.ones(len(est_mesh))
if mu_delta != 0.:
log_delta = mc.Uninformative('log_dispersion_%s' % key, value=np.log(mu_delta-1))
delta = mc.Lambda('dispersion_%s' % key, lambda x=log_delta: 1. + np.exp(x))
@mc.potential(name='potential_dispersion_%s' % key)
def delta_pot(delta=delta, mu=mu_delta, tau=sigma_delta**-2):
return mc.normal_like(delta, mu, tau)
vars.update(dispersion=delta, log_dispersion=log_delta, dispersion_potential=delta_pot, dispersion_step_sd=.1*sigma_delta)
if len(sigma_gamma) == 1:
sigma_gamma = sigma_gamma[0]*np.ones(len(est_mesh))
# create varible for interpolated rate;
# also create variable for age-specific rate function, if it does not yet exist
if rate_stoch:
# if the rate_stoch already exists, for example prevalence in the generic model,
# we use it to back-calculate mu and eventually gamma
mu = rate_stoch
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(mu=mu, Xa=X_region, Xb=X_study, alpha=alpha, beta=beta):
return np.log(1.e-8 + mu) - np.dot(alpha, Xa) - np.dot(beta, Xb)
@mc.potential(name='age_coeffs_potential_%s' % key)
def gamma_potential(gamma=gamma, mu_gamma=mu_gamma, tau_gamma=1./sigma_gamma[param_mesh]**2, param_mesh=param_mesh):
return mc.normal_like(gamma[param_mesh], mu_gamma[param_mesh], tau_gamma)
vars.update(rate_stoch=mu, age_coeffs=gamma, age_coeffs_potential=gamma_potential)
else:
# if the rate_stoch does not yet exists, we make gamma a stoch, and use it to calculate mu
# for computational efficiency, gamma is a linearly interpolated version of gamma_mesh
initial_gamma = mu_gamma
# FOR TEST: use a linear age pattern for remission, since there is not sufficient data for more complicated fit
#if key.find('remission') == 0:
# param_mesh = [0., 100.]
#param_mesh = est_mesh # try full mesh; how much does this slow things down, really? answer: a lot
gamma_mesh = mc.Normal('age_coeffs_mesh_%s' % key, mu=mu_gamma[param_mesh], tau=sigma_gamma[param_mesh]**-2, value=initial_gamma[param_mesh])
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(gamma_mesh=gamma_mesh, param_mesh=param_mesh, est_mesh=est_mesh):
return interpolate(param_mesh, gamma_mesh, est_mesh)
@mc.deterministic(name=key)
def mu(Xa=X_region, Xb=X_study, alpha=alpha, beta=beta, gamma=gamma):
return predict_rate([Xa, Xb], alpha, beta, gamma, lambda f, age: f, est_mesh)
# Create a guess at the covariance matrix for MCMC proposals to update gamma_mesh
from pymc.gp.cov_funs import matern
a = np.atleast_2d(param_mesh).T
C = matern.euclidean(a, a, diff_degree = 2, amp = 1.**2, scale = 10.)
vars.update(age_coeffs_mesh=gamma_mesh, age_coeffs=gamma, rate_stoch=mu, age_coeffs_mesh_step_cov=.005*np.array(C))
# adjust value of gamma_mesh based on priors, if necessary
# TODO: implement more adjustments, currently only adjusted based on at_least priors
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'at_least':
delta_gamma = np.log(np.maximum(mu.value, float(prior[1]))) - np.log(mu.value)
gamma_mesh.value = gamma_mesh.value + delta_gamma[param_mesh]
# create potentials for priors
generate_prior_potentials(vars, dm.get_priors(key), est_mesh)
# create effect coefficients to explain overdispersion
eta = mc.Laplace('eta_%s' % key, mu=0., tau=1., value=0.)
vars['eta'] = eta
# create observed stochastics for data
vars['data'] = []
if mu_delta != 0.:
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
# overdispersion-explaining covariates
Z = []
for d in data_list:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
ai.append(age_indices)
aw.append(age_weights)
Z.append(float(d.get('bias', 0.)))
vars['data'].append(d)
N = np.array(N)
Z = np.array(Z)
vars['effective_sample_size'] = list(N)
if len(vars['data']) > 0:
@mc.deterministic(name='rate_%s' % key)
def rates(N=N,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = np.exp(np.dot(Xa, alpha) + np.dot(Xb, np.atleast_1d(beta)))
exp_gamma = np.exp(gamma)
mu_i = [np.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed
return mu_i
vars['expected_rates'] = rates
@mc.observed
@mc.stochastic(name='data_%s' % key)
def obs(value=value, N=N,
mu_i=rates,
delta=delta,
Z=Z, eta=0.):
logp = mc.negative_binomial_like(value, N*mu_i, delta + eta*Z)
return logp
vars['observed_counts'] = obs
@mc.deterministic(name='predicted_data_%s' % key)
def predictions(value=value, N=N,
mu_i=rates,
delta=delta,
Z=Z, eta=0.):
return mc.rnegative_binomial(N*mu_i, delta + eta*Z)/N
vars['predicted_rates'] = predictions
debug('likelihood of %s contains %d rates' % (key, len(vars['data'])))
# now do the same thing for the lower bound data
# TODO: refactor to remove duplicated code
vars['lower_bound_data'] = []
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
for d in lower_bound_data:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
ai.append(age_indices)
aw.append(age_weights)
vars['lower_bound_data'].append(d)
N = np.array(N)
value = np.array(value)
if len(vars['lower_bound_data']) > 0:
@mc.observed
@mc.stochastic(name='lower_bound_data_%s' % key)
def obs_lb(value=value, N=N,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
delta=delta,
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = np.exp(np.dot(Xa, alpha) + np.dot(Xb, np.atleast_1d(beta)))
exp_gamma = np.exp(gamma)
mu_i = [np.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed
rate_param = mu_i*N
violated_bounds = np.nonzero(rate_param < value)
logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta)
return logp
vars['observed_lower_bounds'] = obs_lb
debug('likelihood of %s contains %d lowerbounds' % (key, len(vars['lower_bound_data'])))
return vars
def setup(dm, key, data_list=[], rate_stoch=None, emp_prior={}, lower_bound_data=[]):
""" Generate the PyMC variables for a negative-binomial model of
a single rate function
Parameters
----------
dm : dismod3.DiseaseModel
the object containing all the data, priors, and additional
information (like input and output age-mesh)
key : str
the name of the key for everything about this model (priors,
initial values, estimations)
data_list : list of data dicts
the observed data to use in the negative binomial liklihood function
rate_stoch : pymc.Stochastic, optional
a PyMC stochastic (or deterministic) object, with
len(rate_stoch.value) == len(dm.get_estimation_age_mesh()).
This is used to link rate stochs into a larger model,
for example.
emp_prior : dict, optional
the empirical prior dictionary, retrieved from the disease model
if appropriate by::
>>> t, r, y, s = dismod3.utils.type_region_year_sex_from_key(key)
>>> emp_prior = dm.get_empirical_prior(t)
Results
-------
vars : dict
Return a dictionary of all the relevant PyMC objects for the
rate model. vars['rate_stoch'] is of particular
relevance; this is what is used to link the rate model
into more complicated models, like the generic disease model.
"""
vars = {}
est_mesh = dm.get_estimate_age_mesh()
param_mesh = dm.get_param_age_mesh()
if pl.any(pl.diff(est_mesh) != 1):
raise ValueError, 'ERROR: Gaps in estimation age mesh must all equal 1'
# calculate effective sample size for all data and lower bound data
dm.calc_effective_sample_size(data_list)
dm.calc_effective_sample_size(lower_bound_data)
# generate regional covariates
covariate_dict = dm.get_covariates()
derived_covariate = dm.get_derived_covariate_values()
X_region, X_study = regional_covariates(key, covariate_dict, derived_covariate)
# use confidence prior from prior_str (only for posterior estimate, this is overridden below for empirical prior estimate)
mu_delta = 1000.
sigma_delta = 10.
mu_log_delta = 3.
sigma_log_delta = .25
from dismod3.settings import PRIOR_SEP_STR
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'heterogeneity':
# originally designed for this:
mu_delta = float(prior[1])
sigma_delta = float(prior[2])
# HACK: override design to set sigma_log_delta,
# .25 = very, .025 = moderately, .0025 = slightly
if float(prior[2]) > 0:
sigma_log_delta = .025 / float(prior[2])
# use the empirical prior mean if it is available
if len(set(emp_prior.keys()) & set(['alpha', 'beta', 'gamma'])) == 3:
mu_alpha = pl.array(emp_prior['alpha'])
sigma_alpha = pl.array(emp_prior['sigma_alpha'])
alpha = pl.array(emp_prior['alpha']) # TODO: make this stochastic
vars.update(region_coeffs=alpha)
beta = pl.array(emp_prior['beta']) # TODO: make this stochastic
sigma_beta = pl.array(emp_prior['sigma_beta'])
vars.update(study_coeffs=beta)
mu_gamma = pl.array(emp_prior['gamma'])
sigma_gamma = pl.array(emp_prior['sigma_gamma'])
# Do not inform dispersion parameter from empirical prior stage
# if 'delta' in emp_prior:
# mu_delta = emp_prior['delta']
# if 'sigma_delta' in emp_prior:
# sigma_delta = emp_prior['sigma_delta']
else:
import dismod3.regional_similarity_matrices as similarity_matrices
n = len(X_region)
mu_alpha = pl.zeros(n)
sigma_alpha = .025 # TODO: make this a hyperparameter, with a traditional prior, like inverse gamma
C_alpha = similarity_matrices.regions_nested_in_superregions(n, sigma_alpha)
# use alternative region effect covariance structure if requested
region_prior_key = 'region_effects'
if region_prior_key in dm.params:
if dm.params[region_prior_key] == 'uninformative':
C_alpha = similarity_matrices.uninformative(n, sigma_alpha)
region_prior_key = 'region_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0]
if region_prior_key in dm.params:
if dm.params[region_prior_key] == 'uninformative':
C_alpha = similarity_matrices.regions_nested_in_superregions(n, dm.params[region_prior_key]['std'])
# add informative prior for sex effect if requested
sex_prior_key = 'sex_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0]
if sex_prior_key in dm.params:
print 'adjusting prior on sex effect coefficient for %s' % key
mu_alpha[n-1] = pl.log(dm.params[sex_prior_key]['mean'])
sigma_sex = (pl.log(dm.params[sex_prior_key]['upper_ci']) - pl.log(dm.params[sex_prior_key]['lower_ci'])) / (2*1.96)
C_alpha[n-1, n-1]= sigma_sex**2.
# add informative prior for time effect if requested
time_prior_key = 'time_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if time_prior_key in dm.params:
print 'adjusting prior on time effect coefficient for %s' % key
mu_alpha[n-2] = pl.log(dm.params[time_prior_key]['mean'])
sigma_time = (pl.log(dm.params[time_prior_key]['upper_ci']) - pl.log(dm.params[time_prior_key]['lower_ci'])) / (2*1.96)
C_alpha[n-2, n-2]= sigma_time**2.
#C_alpha = similarity_matrices.all_related_equally(n, sigma_alpha)
alpha = mc.MvNormalCov('region_coeffs_%s' % key, mu=mu_alpha,
C=C_alpha,
value=mu_alpha)
vars.update(region_coeffs=alpha, region_coeffs_step_cov=.005*C_alpha)
mu_beta = pl.zeros(len(X_study))
sigma_beta = .1
# add informative prior for beta effect if requested
prior_key = 'beta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on beta effect coefficients for %s' % key
mu_beta = pl.array(dm.params[prior_key]['mean'])
sigma_beta = pl.array(dm.params[prior_key]['std'])
beta = mc.Normal('study_coeffs_%s' % key, mu=mu_beta, tau=sigma_beta**-2., value=mu_beta)
vars.update(study_coeffs=beta)
mu_gamma = 0.*pl.ones(len(est_mesh))
sigma_gamma = 2.*pl.ones(len(est_mesh))
# add informative prior for gamma effect if requested
prior_key = 'gamma_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on gamma effect coefficients for %s' % key
mu_gamma = pl.array(dm.params[prior_key]['mean'])
sigma_gamma = pl.array(dm.params[prior_key]['std'])
# always use dispersed prior on delta for empirical prior phase
mu_log_delta = 3.
sigma_log_delta = .25
# add informative prior for delta effect if requested
prior_key = 'delta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on delta effect coefficients for %s' % key
mu_log_delta = dm.params[prior_key]['mean']
sigma_log_delta = dm.params[prior_key]['std']
mu_zeta = 0.
sigma_zeta = .25
# add informative prior for zeta effect if requested
prior_key = 'zeta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex
if prior_key in dm.params:
print 'adjusting prior on zeta effect coefficients for %s' % key
mu_zeta = dm.params[prior_key]['mean']
sigma_zeta = dm.params[prior_key]['std']
if mu_delta != 0.:
if sigma_delta != 0.:
log_delta = mc.Normal('log_dispersion_%s' % key, mu=mu_log_delta, tau=sigma_log_delta**-2, value=3.)
zeta = mc.Normal('zeta_%s'%key, mu=mu_zeta, tau=sigma_zeta**-2, value=mu_zeta)
delta = mc.Lambda('dispersion_%s' % key, lambda x=log_delta: 50. + 10.**x)
vars.update(dispersion=delta, log_dispersion=log_delta, zeta=zeta, dispersion_step_sd=.1*log_delta.parents['tau']**-.5)
else:
delta = mc.Lambda('dispersion_%s' % key, lambda x=mu_delta: mu_delta)
vars.update(dispersion=delta)
else:
delta = mc.Lambda('dispersion_%s' % key, lambda mu=mu_delta: 0)
vars.update(dispersion=delta)
if len(sigma_gamma) == 1:
sigma_gamma = sigma_gamma[0]*pl.ones(len(est_mesh))
# create varible for interpolated rate;
# also create variable for age-specific rate function, if it does not yet exist
if rate_stoch:
# if the rate_stoch already exists, for example prevalence in the generic model,
# we use it to back-calculate mu and eventually gamma
mu = rate_stoch
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(mu=mu, Xa=X_region, Xb=X_study, alpha=alpha, beta=beta):
return pl.log(pl.maximum(dismod3.settings.NEARLY_ZERO, mu)) - pl.dot(alpha, Xa) - pl.dot(beta, Xb)
@mc.potential(name='age_coeffs_potential_%s' % key)
def gamma_potential(gamma=gamma, mu_gamma=mu_gamma, tau_gamma=1./sigma_gamma[param_mesh]**2, param_mesh=param_mesh):
return mc.normal_like(gamma[param_mesh], mu_gamma[param_mesh], tau_gamma)
vars.update(rate_stoch=mu, age_coeffs=gamma, age_coeffs_potential=gamma_potential)
else:
# if the rate_stoch does not yet exists, we make gamma a stoch, and use it to calculate mu
# for computational efficiency, gamma is a linearly interpolated version of gamma_mesh
initial_gamma = pl.log(dismod3.settings.NEARLY_ZERO + dm.get_initial_value(key))
gamma_mesh = mc.Normal('age_coeffs_mesh_%s' % key, mu=mu_gamma[param_mesh], tau=sigma_gamma[param_mesh]**-2, value=initial_gamma[param_mesh])
@mc.deterministic(name='age_coeffs_%s' % key)
def gamma(gamma_mesh=gamma_mesh, param_mesh=param_mesh, est_mesh=est_mesh):
return dismod3.utils.interpolate(param_mesh, gamma_mesh, est_mesh)
@mc.deterministic(name=key)
def mu(Xa=X_region, Xb=X_study, alpha=alpha, beta=beta, gamma=gamma):
return predict_rate([Xa, Xb], alpha, beta, gamma, lambda f, age: f, est_mesh)
# Create a guess at the covariance matrix for MCMC proposals to update gamma_mesh
from pymc.gp.cov_funs import matern
a = pl.atleast_2d(param_mesh).T
C = matern.euclidean(a, a, diff_degree = 2, amp = 1.**2, scale = 10.)
vars.update(age_coeffs_mesh=gamma_mesh, age_coeffs=gamma, rate_stoch=mu, age_coeffs_mesh_step_cov=.005*pl.array(C))
# adjust value of gamma_mesh based on priors, if necessary
# TODO: implement more adjustments, currently only adjusted based on at_least priors
for line in dm.get_priors(key).split(PRIOR_SEP_STR):
prior = line.strip().split()
if len(prior) == 0:
continue
if prior[0] == 'at_least':
delta_gamma = pl.log(pl.maximum(mu.value, float(prior[1]))) - pl.log(mu.value)
gamma_mesh.value = gamma_mesh.value + delta_gamma[param_mesh]
# create potentials for priors
dismod3.utils.generate_prior_potentials(vars, dm.get_priors(key), est_mesh)
# create observed stochastics for data
vars['data'] = []
if mu_delta != 0.:
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
Xz = []
for d in data_list:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
Xz.append(float(d.get('bias') or 0.))
ai.append(age_indices)
aw.append(age_weights)
vars['data'].append(d)
N = pl.array(N)
Xa = pl.array(Xa)
Xb = pl.array(Xb)
Xz = pl.array(Xz)
value = pl.array(value)
vars['effective_sample_size'] = list(N)
if len(vars['data']) > 0:
# TODO: consider using only a subset of the rates at each step of the fit to speed computation; say 100 of them
k = 50000
if len(vars['data']) < k:
data_sample = range(len(vars['data']))
else:
import random
@mc.deterministic(name='data_sample_%s' % key)
def data_sample(n=len(vars['data']), k=k):
return random.sample(range(n), k)
@mc.deterministic(name='rate_%s' % key)
def rates(S=data_sample,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa[S], alpha) + pl.dot(Xb[S], pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu = pl.zeros_like(shifts)
for i,s in enumerate(S):
mu[i] = pl.dot(age_weights[s], bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
# TODO: evaluate speed increase and accuracy decrease of the following:
#midpoint = age_indices[s][len(age_indices[s])/2]
#mu[i] = bounds_func(shifts[i] * exp_gamma[midpoint], midpoint)
# TODO: evaluate speed increase and accuracy decrease of the following: (to see speed increase, need to code this up using difference of running sums
#mu[i] = pl.dot(pl.ones_like(age_weights[s]) / float(len(age_weights[s])),
# bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s]))
return mu
vars['expected_rates'] = rates
@mc.observed
@mc.stochastic(name='data_%s' % key)
def obs(value=value,
S=data_sample,
N=N,
mu_i=rates,
Xz=Xz,
zeta=zeta,
delta=delta):
#zeta_i = .001
#residual = pl.log(value[S] + zeta_i) - pl.log(mu_i*N[S] + zeta_i)
#return mc.normal_like(residual, 0, 100. + delta)
logp = mc.negative_binomial_like(value[S], N[S]*mu_i, delta*pl.exp(Xz*zeta))
return logp
vars['observed_counts'] = obs
@mc.deterministic(name='predicted_data_%s' % key)
def predictions(value=value,
N=N,
S=data_sample,
mu=rates,
delta=delta):
r_S = mc.rnegative_binomial(N[S]*mu, delta)/N[S]
r = pl.zeros(len(vars['data']))
r[S] = r_S
return r
vars['predicted_rates'] = predictions
debug('likelihood of %s contains %d rates' % (key, len(vars['data'])))
# now do the same thing for the lower bound data
# TODO: refactor to remove duplicated code
vars['lower_bound_data'] = []
value = []
N = []
Xa = []
Xb = []
ai = []
aw = []
for d in lower_bound_data:
try:
age_indices, age_weights, Y_i, N_i = values_from(dm, d)
except ValueError:
debug('WARNING: could not calculate likelihood for data %d' % d['id'])
continue
value.append(Y_i*N_i)
N.append(N_i)
Xa.append(covariates(d, covariate_dict)[0])
Xb.append(covariates(d, covariate_dict)[1])
ai.append(age_indices)
aw.append(age_weights)
vars['lower_bound_data'].append(d)
N = pl.array(N)
value = pl.array(value)
if len(vars['lower_bound_data']) > 0:
@mc.observed
@mc.stochastic(name='lower_bound_data_%s' % key)
def obs_lb(value=value, N=N,
Xa=Xa, Xb=Xb,
alpha=alpha, beta=beta, gamma=gamma,
bounds_func=vars['bounds_func'],
delta=delta,
age_indices=ai,
age_weights=aw):
# calculate study-specific rate function
shifts = pl.exp(pl.dot(Xa, alpha) + pl.dot(Xb, pl.atleast_1d(beta)))
exp_gamma = pl.exp(gamma)
mu_i = [pl.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed
rate_param = mu_i*N
violated_bounds = pl.nonzero(rate_param < value)
logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta)
return logp
vars['observed_lower_bounds'] = obs_lb
debug('likelihood of %s contains %d lowerbounds' % (key, len(vars['lower_bound_data'])))
return vars