def find_consistent_spline_initial_vals(vars, method, tol, verbose):
## generate initial value by fitting knots sequentially
vars_to_fit = [vars['logit_C0']]
for t in param_types:
vars_to_fit += [vars[t].get('covariate_constraint'),
vars[t].get('mu_age_derivative_potential'), vars[t].get('mu_sim'),
vars[t].get('p_obs'), vars[t].get('parent_similarity'), vars[t].get('smooth_gamma'),]
max_knots = max([len(vars[t]['gamma']) for t in 'irf'])
for i in [max_knots]: #range(1, max_knots+1):
if verbose:
print 'fitting first %d knots of %d' % (i, max_knots)
vars_to_fit += [vars[t]['gamma'][:i] for t in 'irf']
mc.MAP(vars_to_fit).fit(method=method, tol=tol, verbose=verbose)
if verbose:
from fit_posterior import inspect_vars
print inspect_vars({}, vars)[-10:]
else:
logger.info('.')
def find_consistent_spline_initial_vals(vars, method, tol, verbose):
## generate initial value by fitting knots sequentially
vars_to_fit = [vars['logit_C0']]
for t in param_types:
vars_to_fit += [
vars[t].get('covariate_constraint'),
vars[t].get('mu_age_derivative_potential'),
vars[t].get('mu_sim'),
vars[t].get('p_obs'),
vars[t].get('parent_similarity'),
vars[t].get('smooth_gamma'),
]
max_knots = max([len(vars[t]['gamma']) for t in 'irf'])
for i in [max_knots]: #range(1, max_knots+1):
if verbose:
print 'fitting first %d knots of %d' % (i, max_knots)
vars_to_fit += [vars[t]['gamma'][:i] for t in 'irf']
mc.MAP(vars_to_fit).fit(method=method, tol=tol, verbose=verbose)
if verbose:
from fit_posterior import inspect_vars
print inspect_vars({}, vars)[-10:]
else:
logger.info('.')
def consistent(model,
iter=2000,
burn=1000,
thin=1,
tune_interval=100,
verbose=False):
"""Fit data model for all epidemiologic parameters using MCMC
:Parameters:
- `model` : data.ModelData
- `iter` : int, number of posterior samples fit
- `burn` : int, number of posterior samples to discard as burn-in
- `thin` : int, samples thinned by this number
- `tune_interval` : int
- `verbose` : boolean
:Results:
- returns a pymc.MCMC object created from vars, that has been fit with MCMC
.. note::
- `burn` must be less than `iter`
- `thin` must be less than `iter` minus `burn`
"""
assert burn < iter, 'burn must be less than iter'
assert thin < iter - burn, 'thin must be less than iter-burn'
param_types = 'i r f p pf rr smr m_with X'.split()
vars = model.vars
start_time = time.time()
map = mc.MAP(vars)
m = mc.MCMC(vars)
## use MAP to generate good initial conditions
try:
method = 'fmin_powell'
tol = .001
logger.info('fitting submodels')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
for t in param_types:
find_re_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
for t in param_types:
find_fe_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
for t in param_types:
find_dispersion_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
logger.info('\nfitting all stochs\n')
map.fit(method=method, tol=tol, verbose=verbose)
if verbose:
from fit_posterior import inspect_vars
print inspect_vars({}, vars)
except KeyboardInterrupt:
logger.warning('Initial condition calculation interrupted')
## use MCMC to fit the model
try:
logger.info('finding step covariances')
vars_to_fit = [[
vars[t].get('p_obs'), vars[t].get('pi_sim'),
vars[t].get('smooth_gamma'), vars[t].get('parent_similarity'),
vars[t].get('mu_sim'), vars[t].get('mu_age_derivative_potential'),
vars[t].get('covariate_constraint')
] for t in param_types]
max_knots = max([len(vars[t]['gamma']) for t in 'irf'])
for i in range(max_knots):
stoch = [
vars[t]['gamma'][i] for t in 'ifr' if i < len(vars[t]['gamma'])
]
if verbose:
print 'finding Normal Approx for', [n.__name__ for n in stoch]
try:
na = mc.NormApprox(vars_to_fit + stoch)
na.fit(method='fmin_powell', verbose=verbose)
cov = np.array(np.linalg.inv(-na.hess), order='F')
if np.all(np.linalg.eigvals(cov) >= 0):
m.use_step_method(mc.AdaptiveMetropolis, stoch, cov=cov)
else:
raise ValueError
except ValueError:
if verbose:
print 'cov matrix is not positive semi-definite'
m.use_step_method(mc.AdaptiveMetropolis, stoch)
logger.info('.')
for t in param_types:
setup_asr_step_methods(m, vars[t], vars_to_fit)
# reset values to MAP
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
map.fit(method=method, tol=tol, verbose=verbose)
logger.info('.')
except KeyboardInterrupt:
logger.warning('Initial condition calculation interrupted')
logger.info('\nsampling from posterior distribution\n')
m.iter = iter
m.burn = burn
m.thin = thin
if verbose:
try:
m.sample(m.iter,
m.burn,
m.thin,
tune_interval=tune_interval,
progress_bar=True,
progress_bar_fd=sys.stdout)
except TypeError:
m.sample(m.iter,
m.burn,
m.thin,
tune_interval=tune_interval,
progress_bar=False,
verbose=verbose)
else:
m.sample(m.iter,
m.burn,
m.thin,
tune_interval=tune_interval,
progress_bar=False)
m.wall_time = time.time() - start_time
model.map = map
model.mcmc = m
return model.map, model.mcmc
def consistent(model, iter=2000, burn=1000, thin=1, tune_interval=100, verbose=False):
"""Fit data model for all epidemiologic parameters using MCMC
:Parameters:
- `model` : data.ModelData
- `iter` : int, number of posterior samples fit
- `burn` : int, number of posterior samples to discard as burn-in
- `thin` : int, samples thinned by this number
- `tune_interval` : int
- `verbose` : boolean
:Results:
- returns a pymc.MCMC object created from vars, that has been fit with MCMC
.. note::
- `burn` must be less than `iter`
- `thin` must be less than `iter` minus `burn`
"""
assert burn < iter, 'burn must be less than iter'
assert thin < iter - burn, 'thin must be less than iter-burn'
param_types = 'i r f p pf rr smr m_with X'.split()
vars = model.vars
start_time = time.time()
map = mc.MAP(vars)
m = mc.MCMC(vars)
## use MAP to generate good initial conditions
try:
method='fmin_powell'
tol=.001
logger.info('fitting submodels')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
for t in param_types:
find_re_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
for t in param_types:
find_fe_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
for t in param_types:
find_dispersion_initial_vals(vars[t], method, tol, verbose)
logger.info('.')
logger.info('\nfitting all stochs\n')
map.fit(method=method, tol=tol, verbose=verbose)
if verbose:
from fit_posterior import inspect_vars
print inspect_vars({}, vars)
except KeyboardInterrupt:
logger.warning('Initial condition calculation interrupted')
## use MCMC to fit the model
try:
logger.info('finding step covariances')
vars_to_fit = [[vars[t].get('p_obs'), vars[t].get('pi_sim'), vars[t].get('smooth_gamma'), vars[t].get('parent_similarity'),
vars[t].get('mu_sim'), vars[t].get('mu_age_derivative_potential'), vars[t].get('covariate_constraint')] for t in param_types]
max_knots = max([len(vars[t]['gamma']) for t in 'irf'])
for i in range(max_knots):
stoch = [vars[t]['gamma'][i] for t in 'ifr' if i < len(vars[t]['gamma'])]
if verbose:
print 'finding Normal Approx for', [n.__name__ for n in stoch]
try:
na = mc.NormApprox(vars_to_fit + stoch)
na.fit(method='fmin_powell', verbose=verbose)
cov = np.array(np.linalg.inv(-na.hess), order='F')
if np.all(np.linalg.eigvals(cov) >= 0):
m.use_step_method(mc.AdaptiveMetropolis, stoch, cov=cov)
else:
raise ValueError
except ValueError:
if verbose:
print 'cov matrix is not positive semi-definite'
m.use_step_method(mc.AdaptiveMetropolis, stoch)
logger.info('.')
for t in param_types:
setup_asr_step_methods(m, vars[t], vars_to_fit)
# reset values to MAP
find_consistent_spline_initial_vals(vars, method, tol, verbose)
logger.info('.')
map.fit(method=method, tol=tol, verbose=verbose)
logger.info('.')
except KeyboardInterrupt:
logger.warning('Initial condition calculation interrupted')
logger.info('\nsampling from posterior distribution\n')
m.iter=iter
m.burn=burn
m.thin=thin
if verbose:
try:
m.sample(m.iter, m.burn, m.thin, tune_interval=tune_interval, progress_bar=True, progress_bar_fd=sys.stdout)
except TypeError:
m.sample(m.iter, m.burn, m.thin, tune_interval=tune_interval, progress_bar=False, verbose=verbose)
else:
m.sample(m.iter, m.burn, m.thin, tune_interval=tune_interval, progress_bar=False)
m.wall_time = time.time() - start_time
model.map = map
model.mcmc = m
return model.map, model.mcmc
