def setup(dm, key, data_list=[], rate_stoch=None, emp_prior={}, lower_bound_data=[]): """ Generate the PyMC variables for a negative-binomial model of a single rate function Parameters ---------- dm : dismod3.DiseaseModel the object containing all the data, priors, and additional information (like input and output age-mesh) key : str the name of the key for everything about this model (priors, initial values, estimations) data_list : list of data dicts the observed data to use in the negative binomial liklihood function rate_stoch : pymc.Stochastic, optional a PyMC stochastic (or deterministic) object, with len(rate_stoch.value) == len(dm.get_estimation_age_mesh()). This is used to link rate stochs into a larger model, for example. emp_prior : dict, optional the empirical prior dictionary, retrieved from the disease model if appropriate by:: >>> t, r, y, s = type_region_year_sex_from_key(key) >>> emp_prior = dm.get_empirical_prior(t) Results ------- vars : dict Return a dictionary of all the relevant PyMC objects for the rate model. vars['rate_stoch'] is of particular relevance; this is what is used to link the rate model into more complicated models, like the generic disease model. """ vars = {} est_mesh = dm.get_estimate_age_mesh() param_mesh = dm.get_param_age_mesh() if np.any(np.diff(est_mesh) != 1): raise ValueError, 'ERROR: Gaps in estimation age mesh must all equal 1' # calculate effective sample size for all data and lower bound data dm.calc_effective_sample_size(data_list) dm.calc_effective_sample_size(lower_bound_data) # generate regional covariates covariate_dict = dm.get_covariates() X_region, X_study = regional_covariates(key, covariate_dict) # use confidence prior from prior_str mu_delta = 100. sigma_delta = 1. from dismod3.settings import PRIOR_SEP_STR for line in dm.get_priors(key).split(PRIOR_SEP_STR): prior = line.strip().split() if len(prior) == 0: continue if prior[0] == 'heterogeneity': mu_delta = float(prior[1]) sigma_delta = float(prior[2]) # use the empirical prior mean if it is available if len(set(emp_prior.keys()) & set(['alpha', 'beta', 'gamma'])) == 3: mu_alpha = np.array(emp_prior['alpha']) sigma_alpha = np.maximum(.1, emp_prior['sigma_alpha']) alpha = np.array(emp_prior['alpha']) vars.update(region_coeffs=alpha) beta = np.array(emp_prior['beta']) sigma_beta = np.maximum(.1, emp_prior['sigma_beta']) vars.update(study_coeffs=beta) mu_gamma = np.array(emp_prior['gamma']) sigma_gamma = np.maximum(.1, emp_prior['sigma_gamma']) # leave mu_delta and sigma_delta as they were set in the expert prior else: import dismod3.regional_similarity_matrices as similarity_matrices n = len(X_region) mu_alpha = np.zeros(n) sigma_alpha = .01 C_alpha = similarity_matrices.regions_nested_in_superregions(n, sigma_alpha) #C_alpha = similarity_matrices.all_related_equally(n, sigma_alpha) alpha = mc.MvNormalCov('region_coeffs_%s' % key, mu=mu_alpha, C=C_alpha, value=mu_alpha) vars.update(region_coeffs=alpha) mu_beta = np.zeros(len(X_study)) sigma_beta = .1 beta = mc.Normal('study_coeffs_%s' % key, mu=mu_beta, tau=sigma_beta**-2., value=mu_beta) vars.update(study_coeffs=beta) mu_gamma = -5.*np.ones(len(est_mesh)) sigma_gamma = 10.*np.ones(len(est_mesh)) if mu_delta != 0.: log_delta = mc.Uninformative('log_dispersion_%s' % key, value=np.log(mu_delta-1)) delta = mc.Lambda('dispersion_%s' % key, lambda x=log_delta: 1. + np.exp(x)) @mc.potential(name='potential_dispersion_%s' % key) def delta_pot(delta=delta, mu=mu_delta, tau=sigma_delta**-2): return mc.normal_like(delta, mu, tau) vars.update(dispersion=delta, log_dispersion=log_delta, dispersion_potential=delta_pot, dispersion_step_sd=.1*sigma_delta) if len(sigma_gamma) == 1: sigma_gamma = sigma_gamma[0]*np.ones(len(est_mesh)) # create varible for interpolated rate; # also create variable for age-specific rate function, if it does not yet exist if rate_stoch: # if the rate_stoch already exists, for example prevalence in the generic model, # we use it to back-calculate mu and eventually gamma mu = rate_stoch @mc.deterministic(name='age_coeffs_%s' % key) def gamma(mu=mu, Xa=X_region, Xb=X_study, alpha=alpha, beta=beta): return np.log(1.e-8 + mu) - np.dot(alpha, Xa) - np.dot(beta, Xb) @mc.potential(name='age_coeffs_potential_%s' % key) def gamma_potential(gamma=gamma, mu_gamma=mu_gamma, tau_gamma=1./sigma_gamma[param_mesh]**2, param_mesh=param_mesh): return mc.normal_like(gamma[param_mesh], mu_gamma[param_mesh], tau_gamma) vars.update(rate_stoch=mu, age_coeffs=gamma, age_coeffs_potential=gamma_potential) else: # if the rate_stoch does not yet exists, we make gamma a stoch, and use it to calculate mu # for computational efficiency, gamma is a linearly interpolated version of gamma_mesh initial_gamma = mu_gamma # FOR TEST: use a linear age pattern for remission, since there is not sufficient data for more complicated fit #if key.find('remission') == 0: # param_mesh = [0., 100.] #param_mesh = est_mesh # try full mesh; how much does this slow things down, really? answer: a lot gamma_mesh = mc.Normal('age_coeffs_mesh_%s' % key, mu=mu_gamma[param_mesh], tau=sigma_gamma[param_mesh]**-2, value=initial_gamma[param_mesh]) @mc.deterministic(name='age_coeffs_%s' % key) def gamma(gamma_mesh=gamma_mesh, param_mesh=param_mesh, est_mesh=est_mesh): return interpolate(param_mesh, gamma_mesh, est_mesh) @mc.deterministic(name=key) def mu(Xa=X_region, Xb=X_study, alpha=alpha, beta=beta, gamma=gamma): return predict_rate([Xa, Xb], alpha, beta, gamma, lambda f, age: f, est_mesh) # Create a guess at the covariance matrix for MCMC proposals to update gamma_mesh from pymc.gp.cov_funs import matern a = np.atleast_2d(param_mesh).T C = matern.euclidean(a, a, diff_degree = 2, amp = 1.**2, scale = 10.) vars.update(age_coeffs_mesh=gamma_mesh, age_coeffs=gamma, rate_stoch=mu, age_coeffs_mesh_step_cov=.005*np.array(C)) # adjust value of gamma_mesh based on priors, if necessary # TODO: implement more adjustments, currently only adjusted based on at_least priors for line in dm.get_priors(key).split(PRIOR_SEP_STR): prior = line.strip().split() if len(prior) == 0: continue if prior[0] == 'at_least': delta_gamma = np.log(np.maximum(mu.value, float(prior[1]))) - np.log(mu.value) gamma_mesh.value = gamma_mesh.value + delta_gamma[param_mesh] # create potentials for priors generate_prior_potentials(vars, dm.get_priors(key), est_mesh) # create effect coefficients to explain overdispersion eta = mc.Laplace('eta_%s' % key, mu=0., tau=1., value=0.) vars['eta'] = eta # create observed stochastics for data vars['data'] = [] if mu_delta != 0.: value = [] N = [] Xa = [] Xb = [] ai = [] aw = [] # overdispersion-explaining covariates Z = [] for d in data_list: try: age_indices, age_weights, Y_i, N_i = values_from(dm, d) except ValueError: debug('WARNING: could not calculate likelihood for data %d' % d['id']) continue value.append(Y_i*N_i) N.append(N_i) Xa.append(covariates(d, covariate_dict)[0]) Xb.append(covariates(d, covariate_dict)[1]) ai.append(age_indices) aw.append(age_weights) Z.append(float(d.get('bias', 0.))) vars['data'].append(d) N = np.array(N) Z = np.array(Z) vars['effective_sample_size'] = list(N) if len(vars['data']) > 0: @mc.deterministic(name='rate_%s' % key) def rates(N=N, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = np.exp(np.dot(Xa, alpha) + np.dot(Xb, np.atleast_1d(beta))) exp_gamma = np.exp(gamma) mu_i = [np.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed return mu_i vars['expected_rates'] = rates @mc.observed @mc.stochastic(name='data_%s' % key) def obs(value=value, N=N, mu_i=rates, delta=delta, Z=Z, eta=0.): logp = mc.negative_binomial_like(value, N*mu_i, delta + eta*Z) return logp vars['observed_counts'] = obs @mc.deterministic(name='predicted_data_%s' % key) def predictions(value=value, N=N, mu_i=rates, delta=delta, Z=Z, eta=0.): return mc.rnegative_binomial(N*mu_i, delta + eta*Z)/N vars['predicted_rates'] = predictions debug('likelihood of %s contains %d rates' % (key, len(vars['data']))) # now do the same thing for the lower bound data # TODO: refactor to remove duplicated code vars['lower_bound_data'] = [] value = [] N = [] Xa = [] Xb = [] ai = [] aw = [] for d in lower_bound_data: try: age_indices, age_weights, Y_i, N_i = values_from(dm, d) except ValueError: debug('WARNING: could not calculate likelihood for data %d' % d['id']) continue value.append(Y_i*N_i) N.append(N_i) Xa.append(covariates(d, covariate_dict)[0]) Xb.append(covariates(d, covariate_dict)[1]) ai.append(age_indices) aw.append(age_weights) vars['lower_bound_data'].append(d) N = np.array(N) value = np.array(value) if len(vars['lower_bound_data']) > 0: @mc.observed @mc.stochastic(name='lower_bound_data_%s' % key) def obs_lb(value=value, N=N, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], delta=delta, age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = np.exp(np.dot(Xa, alpha) + np.dot(Xb, np.atleast_1d(beta))) exp_gamma = np.exp(gamma) mu_i = [np.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed rate_param = mu_i*N violated_bounds = np.nonzero(rate_param < value) logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta) return logp vars['observed_lower_bounds'] = obs_lb debug('likelihood of %s contains %d lowerbounds' % (key, len(vars['lower_bound_data']))) return vars
def setup(dm, key, data_list=[], rate_stoch=None, emp_prior={}, lower_bound_data=[]): """ Generate the PyMC variables for a negative-binomial model of a single rate function Parameters ---------- dm : dismod3.DiseaseModel the object containing all the data, priors, and additional information (like input and output age-mesh) key : str the name of the key for everything about this model (priors, initial values, estimations) data_list : list of data dicts the observed data to use in the negative binomial liklihood function rate_stoch : pymc.Stochastic, optional a PyMC stochastic (or deterministic) object, with len(rate_stoch.value) == len(dm.get_estimation_age_mesh()). This is used to link rate stochs into a larger model, for example. emp_prior : dict, optional the empirical prior dictionary, retrieved from the disease model if appropriate by:: >>> t, r, y, s = dismod3.utils.type_region_year_sex_from_key(key) >>> emp_prior = dm.get_empirical_prior(t) Results ------- vars : dict Return a dictionary of all the relevant PyMC objects for the rate model. vars['rate_stoch'] is of particular relevance; this is what is used to link the rate model into more complicated models, like the generic disease model. """ vars = {} est_mesh = dm.get_estimate_age_mesh() param_mesh = dm.get_param_age_mesh() if pl.any(pl.diff(est_mesh) != 1): raise ValueError, 'ERROR: Gaps in estimation age mesh must all equal 1' # calculate effective sample size for all data and lower bound data dm.calc_effective_sample_size(data_list) dm.calc_effective_sample_size(lower_bound_data) # generate regional covariates covariate_dict = dm.get_covariates() derived_covariate = dm.get_derived_covariate_values() X_region, X_study = regional_covariates(key, covariate_dict, derived_covariate) # use confidence prior from prior_str (only for posterior estimate, this is overridden below for empirical prior estimate) mu_delta = 1000. sigma_delta = 10. mu_log_delta = 3. sigma_log_delta = .25 from dismod3.settings import PRIOR_SEP_STR for line in dm.get_priors(key).split(PRIOR_SEP_STR): prior = line.strip().split() if len(prior) == 0: continue if prior[0] == 'heterogeneity': # originally designed for this: mu_delta = float(prior[1]) sigma_delta = float(prior[2]) # HACK: override design to set sigma_log_delta, # .25 = very, .025 = moderately, .0025 = slightly if float(prior[2]) > 0: sigma_log_delta = .025 / float(prior[2]) # use the empirical prior mean if it is available if len(set(emp_prior.keys()) & set(['alpha', 'beta', 'gamma'])) == 3: mu_alpha = pl.array(emp_prior['alpha']) sigma_alpha = pl.array(emp_prior['sigma_alpha']) alpha = pl.array(emp_prior['alpha']) # TODO: make this stochastic vars.update(region_coeffs=alpha) beta = pl.array(emp_prior['beta']) # TODO: make this stochastic sigma_beta = pl.array(emp_prior['sigma_beta']) vars.update(study_coeffs=beta) mu_gamma = pl.array(emp_prior['gamma']) sigma_gamma = pl.array(emp_prior['sigma_gamma']) # Do not inform dispersion parameter from empirical prior stage # if 'delta' in emp_prior: # mu_delta = emp_prior['delta'] # if 'sigma_delta' in emp_prior: # sigma_delta = emp_prior['sigma_delta'] else: import dismod3.regional_similarity_matrices as similarity_matrices n = len(X_region) mu_alpha = pl.zeros(n) sigma_alpha = .025 # TODO: make this a hyperparameter, with a traditional prior, like inverse gamma C_alpha = similarity_matrices.regions_nested_in_superregions(n, sigma_alpha) # use alternative region effect covariance structure if requested region_prior_key = 'region_effects' if region_prior_key in dm.params: if dm.params[region_prior_key] == 'uninformative': C_alpha = similarity_matrices.uninformative(n, sigma_alpha) region_prior_key = 'region_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] if region_prior_key in dm.params: if dm.params[region_prior_key] == 'uninformative': C_alpha = similarity_matrices.regions_nested_in_superregions(n, dm.params[region_prior_key]['std']) # add informative prior for sex effect if requested sex_prior_key = 'sex_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] if sex_prior_key in dm.params: print 'adjusting prior on sex effect coefficient for %s' % key mu_alpha[n-1] = pl.log(dm.params[sex_prior_key]['mean']) sigma_sex = (pl.log(dm.params[sex_prior_key]['upper_ci']) - pl.log(dm.params[sex_prior_key]['lower_ci'])) / (2*1.96) C_alpha[n-1, n-1]= sigma_sex**2. # add informative prior for time effect if requested time_prior_key = 'time_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex if time_prior_key in dm.params: print 'adjusting prior on time effect coefficient for %s' % key mu_alpha[n-2] = pl.log(dm.params[time_prior_key]['mean']) sigma_time = (pl.log(dm.params[time_prior_key]['upper_ci']) - pl.log(dm.params[time_prior_key]['lower_ci'])) / (2*1.96) C_alpha[n-2, n-2]= sigma_time**2. #C_alpha = similarity_matrices.all_related_equally(n, sigma_alpha) alpha = mc.MvNormalCov('region_coeffs_%s' % key, mu=mu_alpha, C=C_alpha, value=mu_alpha) vars.update(region_coeffs=alpha, region_coeffs_step_cov=.005*C_alpha) mu_beta = pl.zeros(len(X_study)) sigma_beta = .1 # add informative prior for beta effect if requested prior_key = 'beta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex if prior_key in dm.params: print 'adjusting prior on beta effect coefficients for %s' % key mu_beta = pl.array(dm.params[prior_key]['mean']) sigma_beta = pl.array(dm.params[prior_key]['std']) beta = mc.Normal('study_coeffs_%s' % key, mu=mu_beta, tau=sigma_beta**-2., value=mu_beta) vars.update(study_coeffs=beta) mu_gamma = 0.*pl.ones(len(est_mesh)) sigma_gamma = 2.*pl.ones(len(est_mesh)) # add informative prior for gamma effect if requested prior_key = 'gamma_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex if prior_key in dm.params: print 'adjusting prior on gamma effect coefficients for %s' % key mu_gamma = pl.array(dm.params[prior_key]['mean']) sigma_gamma = pl.array(dm.params[prior_key]['std']) # always use dispersed prior on delta for empirical prior phase mu_log_delta = 3. sigma_log_delta = .25 # add informative prior for delta effect if requested prior_key = 'delta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex if prior_key in dm.params: print 'adjusting prior on delta effect coefficients for %s' % key mu_log_delta = dm.params[prior_key]['mean'] sigma_log_delta = dm.params[prior_key]['std'] mu_zeta = 0. sigma_zeta = .25 # add informative prior for zeta effect if requested prior_key = 'zeta_effect_%s'%key.split(dismod3.settings.KEY_DELIM_CHAR)[0] # HACK: sometimes key is just parameter type, sometimes it is type+region+year+sex if prior_key in dm.params: print 'adjusting prior on zeta effect coefficients for %s' % key mu_zeta = dm.params[prior_key]['mean'] sigma_zeta = dm.params[prior_key]['std'] if mu_delta != 0.: if sigma_delta != 0.: log_delta = mc.Normal('log_dispersion_%s' % key, mu=mu_log_delta, tau=sigma_log_delta**-2, value=3.) zeta = mc.Normal('zeta_%s'%key, mu=mu_zeta, tau=sigma_zeta**-2, value=mu_zeta) delta = mc.Lambda('dispersion_%s' % key, lambda x=log_delta: 50. + 10.**x) vars.update(dispersion=delta, log_dispersion=log_delta, zeta=zeta, dispersion_step_sd=.1*log_delta.parents['tau']**-.5) else: delta = mc.Lambda('dispersion_%s' % key, lambda x=mu_delta: mu_delta) vars.update(dispersion=delta) else: delta = mc.Lambda('dispersion_%s' % key, lambda mu=mu_delta: 0) vars.update(dispersion=delta) if len(sigma_gamma) == 1: sigma_gamma = sigma_gamma[0]*pl.ones(len(est_mesh)) # create varible for interpolated rate; # also create variable for age-specific rate function, if it does not yet exist if rate_stoch: # if the rate_stoch already exists, for example prevalence in the generic model, # we use it to back-calculate mu and eventually gamma mu = rate_stoch @mc.deterministic(name='age_coeffs_%s' % key) def gamma(mu=mu, Xa=X_region, Xb=X_study, alpha=alpha, beta=beta): return pl.log(pl.maximum(dismod3.settings.NEARLY_ZERO, mu)) - pl.dot(alpha, Xa) - pl.dot(beta, Xb) @mc.potential(name='age_coeffs_potential_%s' % key) def gamma_potential(gamma=gamma, mu_gamma=mu_gamma, tau_gamma=1./sigma_gamma[param_mesh]**2, param_mesh=param_mesh): return mc.normal_like(gamma[param_mesh], mu_gamma[param_mesh], tau_gamma) vars.update(rate_stoch=mu, age_coeffs=gamma, age_coeffs_potential=gamma_potential) else: # if the rate_stoch does not yet exists, we make gamma a stoch, and use it to calculate mu # for computational efficiency, gamma is a linearly interpolated version of gamma_mesh initial_gamma = pl.log(dismod3.settings.NEARLY_ZERO + dm.get_initial_value(key)) gamma_mesh = mc.Normal('age_coeffs_mesh_%s' % key, mu=mu_gamma[param_mesh], tau=sigma_gamma[param_mesh]**-2, value=initial_gamma[param_mesh]) @mc.deterministic(name='age_coeffs_%s' % key) def gamma(gamma_mesh=gamma_mesh, param_mesh=param_mesh, est_mesh=est_mesh): return dismod3.utils.interpolate(param_mesh, gamma_mesh, est_mesh) @mc.deterministic(name=key) def mu(Xa=X_region, Xb=X_study, alpha=alpha, beta=beta, gamma=gamma): return predict_rate([Xa, Xb], alpha, beta, gamma, lambda f, age: f, est_mesh) # Create a guess at the covariance matrix for MCMC proposals to update gamma_mesh from pymc.gp.cov_funs import matern a = pl.atleast_2d(param_mesh).T C = matern.euclidean(a, a, diff_degree = 2, amp = 1.**2, scale = 10.) vars.update(age_coeffs_mesh=gamma_mesh, age_coeffs=gamma, rate_stoch=mu, age_coeffs_mesh_step_cov=.005*pl.array(C)) # adjust value of gamma_mesh based on priors, if necessary # TODO: implement more adjustments, currently only adjusted based on at_least priors for line in dm.get_priors(key).split(PRIOR_SEP_STR): prior = line.strip().split() if len(prior) == 0: continue if prior[0] == 'at_least': delta_gamma = pl.log(pl.maximum(mu.value, float(prior[1]))) - pl.log(mu.value) gamma_mesh.value = gamma_mesh.value + delta_gamma[param_mesh] # create potentials for priors dismod3.utils.generate_prior_potentials(vars, dm.get_priors(key), est_mesh) # create observed stochastics for data vars['data'] = [] if mu_delta != 0.: value = [] N = [] Xa = [] Xb = [] ai = [] aw = [] Xz = [] for d in data_list: try: age_indices, age_weights, Y_i, N_i = values_from(dm, d) except ValueError: debug('WARNING: could not calculate likelihood for data %d' % d['id']) continue value.append(Y_i*N_i) N.append(N_i) Xa.append(covariates(d, covariate_dict)[0]) Xb.append(covariates(d, covariate_dict)[1]) Xz.append(float(d.get('bias') or 0.)) ai.append(age_indices) aw.append(age_weights) vars['data'].append(d) N = pl.array(N) Xa = pl.array(Xa) Xb = pl.array(Xb) Xz = pl.array(Xz) value = pl.array(value) vars['effective_sample_size'] = list(N) if len(vars['data']) > 0: # TODO: consider using only a subset of the rates at each step of the fit to speed computation; say 100 of them k = 50000 if len(vars['data']) < k: data_sample = range(len(vars['data'])) else: import random @mc.deterministic(name='data_sample_%s' % key) def data_sample(n=len(vars['data']), k=k): return random.sample(range(n), k) @mc.deterministic(name='rate_%s' % key) def rates(S=data_sample, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = pl.exp(pl.dot(Xa[S], alpha) + pl.dot(Xb[S], pl.atleast_1d(beta))) exp_gamma = pl.exp(gamma) mu = pl.zeros_like(shifts) for i,s in enumerate(S): mu[i] = pl.dot(age_weights[s], bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s])) # TODO: evaluate speed increase and accuracy decrease of the following: #midpoint = age_indices[s][len(age_indices[s])/2] #mu[i] = bounds_func(shifts[i] * exp_gamma[midpoint], midpoint) # TODO: evaluate speed increase and accuracy decrease of the following: (to see speed increase, need to code this up using difference of running sums #mu[i] = pl.dot(pl.ones_like(age_weights[s]) / float(len(age_weights[s])), # bounds_func(shifts[i] * exp_gamma[age_indices[s]], age_indices[s])) return mu vars['expected_rates'] = rates @mc.observed @mc.stochastic(name='data_%s' % key) def obs(value=value, S=data_sample, N=N, mu_i=rates, Xz=Xz, zeta=zeta, delta=delta): #zeta_i = .001 #residual = pl.log(value[S] + zeta_i) - pl.log(mu_i*N[S] + zeta_i) #return mc.normal_like(residual, 0, 100. + delta) logp = mc.negative_binomial_like(value[S], N[S]*mu_i, delta*pl.exp(Xz*zeta)) return logp vars['observed_counts'] = obs @mc.deterministic(name='predicted_data_%s' % key) def predictions(value=value, N=N, S=data_sample, mu=rates, delta=delta): r_S = mc.rnegative_binomial(N[S]*mu, delta)/N[S] r = pl.zeros(len(vars['data'])) r[S] = r_S return r vars['predicted_rates'] = predictions debug('likelihood of %s contains %d rates' % (key, len(vars['data']))) # now do the same thing for the lower bound data # TODO: refactor to remove duplicated code vars['lower_bound_data'] = [] value = [] N = [] Xa = [] Xb = [] ai = [] aw = [] for d in lower_bound_data: try: age_indices, age_weights, Y_i, N_i = values_from(dm, d) except ValueError: debug('WARNING: could not calculate likelihood for data %d' % d['id']) continue value.append(Y_i*N_i) N.append(N_i) Xa.append(covariates(d, covariate_dict)[0]) Xb.append(covariates(d, covariate_dict)[1]) ai.append(age_indices) aw.append(age_weights) vars['lower_bound_data'].append(d) N = pl.array(N) value = pl.array(value) if len(vars['lower_bound_data']) > 0: @mc.observed @mc.stochastic(name='lower_bound_data_%s' % key) def obs_lb(value=value, N=N, Xa=Xa, Xb=Xb, alpha=alpha, beta=beta, gamma=gamma, bounds_func=vars['bounds_func'], delta=delta, age_indices=ai, age_weights=aw): # calculate study-specific rate function shifts = pl.exp(pl.dot(Xa, alpha) + pl.dot(Xb, pl.atleast_1d(beta))) exp_gamma = pl.exp(gamma) mu_i = [pl.dot(weights, bounds_func(s_i * exp_gamma[ages], ages)) for s_i, ages, weights in zip(shifts, age_indices, age_weights)] # TODO: try vectorizing this loop to increase speed rate_param = mu_i*N violated_bounds = pl.nonzero(rate_param < value) logp = mc.negative_binomial_like(value[violated_bounds], rate_param[violated_bounds], delta) return logp vars['observed_lower_bounds'] = obs_lb debug('likelihood of %s contains %d lowerbounds' % (key, len(vars['lower_bound_data']))) return vars