def main():
usage = 'usage: %prog [options] disease_model_id'
parser = optparse.OptionParser(usage)
# flags for parameters to modify
parser.add_option('-y', '--ymax',
help='set the maximum y value for summary plots')
parser.add_option('-t', '--condition',
help='set the condition of the model')
parser.add_option('-n', '--notes',
help='set the notes of the model')
# boolean flags
parser.add_option('-c', '--clone',
action='store_true', dest='clone',
help='create a clone of the model (leave specified model unchanged)')
(opts, args) = parser.parse_args()
# check that args are correct
if len(args) == 1:
try:
id = int(args[0])
except ValueError:
parser.error('disease_model_id must be an integer')
return
else:
parser.error('incorrect number of arguments')
return
# fetch requested model
dm = dismod3.get_disease_model(id)
# change values specified
if opts.ymax:
dm.set_ymax(float(opts.ymax))
# TODO: get condition to actually change
if opts.condition:
dm.set_condition(opts.condition)
if opts.notes:
dm.set_notes(opts.notes)
# clone if requested
if opts.clone:
dm.params.pop('id') # dismod_data_server creates new model if it doesn't find an id
# post results to dismod_data_server
url = dismod3.post_disease_model(dm)
# announce url to view results
print url
def hep_c_fit(regions, prediction_years, data_year_start=-inf, data_year_end=inf, egypt_flag=False):
""" Fit prevalence for regions and years specified """
print "\n***************************\nfitting %s for %s (using data from years %f to %f)" % (
regions,
prediction_years,
data_year_start,
data_year_end,
)
## load model to fit
# dm = DiseaseJson(file('tests/hep_c.json').read())
id = 8788
dismod3.disease_json.create_disease_model_dir(id)
dm = dismod3.fetch_disease_model(id)
## adjust the expert priors
dm.params["global_priors"]["heterogeneity"]["prevalence"] = "Very"
dm.params["global_priors"]["smoothness"]["prevalence"]["amount"] = "Slightly"
# TODO: construct examples of adjusting other covariates
# ipdb> dm.params['global_priors'].keys()
# [u'increasing', u'unimodal', u'level_bounds', u'y_maximum', u'note', u'level_value', u'decreasing', u'parameter_age_mesh', u'heterogeneity', u'smoothness']
# ipdb> dm.params['global_priors']['smoothness']['prevalence']
# {u'age_start': 0, u'amount': u'Moderately', u'age_end': 100}
# include a study-level covariate for 'bias'
covariates_dict = dm.get_covariates()
covariates_dict["Study_level"]["bias"]["rate"]["value"] = 1
# TODO: construct additional examples of adjusting covariates
## select relevant prevalence data
# TODO: streamline data selection functions
if egypt_flag:
dm.data = [d for d in dm.data if d["country_iso3_code"] == "EGY"]
else:
dm.data = [
d
for d in dm.data
if dismod3.utils.clean(d["gbd_region"]) in regions
and float(d["year_end"]) >= data_year_start
and float(d["year_start"]) <= data_year_end
and d["country_iso3_code"] != "EGY"
]
## create, fit, and save rate model
dm.vars = {}
keys = dismod3.utils.gbd_keys(type_list=["prevalence"], region_list=regions, year_list=prediction_years)
# TODO: consider how to do this for models that use the complete disease model
# TODO: consider adding hierarchical similarity priors for the male and female models
k0 = keys[0] # looks like k0='prevalence+asia_south+1990+male'
dm.vars[k0] = neg_binom_model.setup(dm, k0, dm.data)
dm.mcmc = mc.MCMC(dm.vars)
dm.mcmc.sample(iter=50000, burn=25000, thin=50, verbose=1)
# make map object so we can compute AIC and BIC
dm.map = mc.MAP(dm.vars)
dm.map.fit()
for k in keys:
# save the results in the disease model
dm.vars[k] = dm.vars[k0]
neg_binom_model.store_mcmc_fit(dm, k, dm.vars[k])
# check autocorrelation to confirm chain has mixed
test_model.summarize_acorr(dm.vars[k]["rate_stoch"].trace())
# generate plots of results
dismod3.tile_plot_disease_model(dm, [k], defaults={"ymax": 0.15, "alpha": 0.5})
dm.savefig("dm-%d-posterior-%s.%f.png" % (dm.id, k, random()))
# summarize fit quality graphically, as well as parameter posteriors
dismod3.plotting.plot_posterior_predicted_checks(dm, k0)
dm.savefig("dm-%d-check-%s.%f.png" % (dm.id, k0, random()))
dismod3.post_disease_model(dm)
return dm
pop_1 = neg_binom_model.population_by_age[("EGY", str(y), s)]
key = "prevalence+north_africa_middle_east+%d+%s" % (y, s)
prev_0 = neg_binom_model.calc_rate_trace(dm_na_me, key, dm_na_me.vars[key])
pop_0 = neg_binom_model.regional_population(key)
# generate population weighted average
prev = (prev_0 * (pop_0 - pop_1) + prev_1 * pop_1) / pop_0
neg_binom_model.store_mcmc_fit(dm_na_me, key, None, prev)
# generate plots of results
dismod3.tile_plot_disease_model(dm_na_me, [key], defaults={"ymax": 0.15, "alpha": 0.5})
dm_na_me.savefig("dm-%d-posterior-na_me_w_egypt.%f.png" % (dm_na_me.id, random()))
# save results
dismod3.post_disease_model(dm_na_me)
dm = hep_c_fit(
"caribbean latin_america_tropical latin_america_andean latin_america_central latin_america_southern".split(),
[1990, 2005],
)
dm = hep_c_fit(
"sub-saharan_africa_central sub-saharan_africa_southern sub-saharan_africa_west".split(), [1990, 2005]
)
for (
r
) in "europe_eastern europe_central asia_central asia_east asia_south asia_southeast australasia oceania sub-saharan_africa_east asia_pacific_high_income".split():
dm = hep_c_fit([r], [1990, 2005])
for r in "north_america_high_income europe_western ".split():
def add_region_effect_prior(id, val):
dm = dismod3.disease_json.DiseaseJson(json.dumps({'params': {}, 'data': [], 'id': id}))
dm.params['region_effects'] = val
dismod3.post_disease_model(dm)
def add_sex_effect_prior(id, type, mean, lower_ci, upper_ci):
dm = dismod3.disease_json.DiseaseJson(json.dumps({'params': {}, 'data': [], 'id': id}))
dm.params['sex_effect_%s'%type] = dict(mean=mean, upper_ci=upper_ci, lower_ci=lower_ci)
print dm.to_json()
dismod3.post_disease_model(dm)
dm.data = [d for d in dm.data if \
d['gbd_region'] == k.replace('prevalence data+', '')]
dm.set_priors('prevalence data', ' smooth 10\n zero 0 15\n zero 99 100\n confidence 1000 .0001\n')
import dismod3.beta_binomial_model as model
print 'Processing %s (%d data points)' % (k, len(dm.data))
model.fit(dm, 'map')
model.fit(dm, 'mcmc')
model.fit(dm, 'map')
model.fit(dm, 'mcmc')
model.fit(dm, 'map')
else:
keys = dm.params['priors'].keys()
for k in keys:
dm.set_priors(k, ' smooth 10\n zero 0 15\n zero 99 100\n confidence 1000 .0001\n')
import dismod3.multiregion_model as model
print 'Processing all regions (%d data points)' % len(dm.data)
model.fit(dm, 'map')
model.fit(dm, 'mcmc')
model.fit(dm, 'map')
model.fit(dm, 'mcmc')
model.fit(dm, 'map')
print dismod3.post_disease_model(dm)
'Year Start': d['year_start'],
'Year End': d['year_end'],
'Self Reported': d.get('self-reported', '')
}
return c
f_file = open('simulated_data.tsv', 'w')
csv_f = csv.writer(f_file, dialect=csv.excel_tab)
col_names = sorted(data_dict_for_csv(data[0]).keys())
csv_f.writerow(col_names)
for d in data:
dd = data_dict_for_csv(d)
csv_f.writerow([dd[c] for c in col_names])
f_file.close()
# upload a new disease model which knows ground truth (but needs to
# have the data from the csv loaded separately)
dm.data = []
dm.params.pop('id')
dm.id = -1
for key in truth:
dm.set_truth(key, truth[key])
url = dismod3.post_disease_model(dm)
print url
def fit(id, opts):
fit_str = '(%d) %s %s %s' % (id, opts.region or '', opts.sex or '', opts.year or '')
#tweet('fitting disease model %s' % fit_str)
sys.stdout.flush()
# update job status file
if opts.log:
if opts.type and not (opts.region and opts.sex and opts.year):
dismod3.log_job_status(id, 'empirical_priors', opts.type, 'Running')
elif opts.region and opts.sex and opts.year and not opts.type:
dismod3.log_job_status(id, 'posterior', '%s--%s--%s' % (opts.region, opts.sex, opts.year), 'Running')
dm = dismod3.get_disease_model(id)
fit_str = '%s %s' % (dm.params['condition'], fit_str)
sex_list = opts.sex and [ opts.sex ] or dismod3.gbd_sexes
year_list = opts.year and [ opts.year ] or dismod3.gbd_years
region_list = opts.region and [ opts.region ] or dismod3.gbd_regions
keys = gbd_keys(region_list=region_list, year_list=year_list, sex_list=sex_list)
# fit empirical priors, if type is specified
if opts.type:
fit_str += ' emp prior for %s' % opts.type
#print 'beginning ', fit_str
import dismod3.neg_binom_model as model
dir = dismod3.settings.JOB_WORKING_DIR % id
model.fit_emp_prior(dm, opts.type, dbname='%s/empirical_priors/pickle/dm-%d-emp_prior-%s.pickle' % (dir, id, opts.type))
# if type is not specified, find consistient fit of all parameters
else:
import dismod3.gbd_disease_model as model
# get the all-cause mortality data, and merge it into the model
mort = dismod3.get_disease_model('all-cause_mortality')
dm.data += mort.data
# fit individually, if sex, year, and region are specified
if opts.sex and opts.year and opts.region:
dm.params['estimate_type'] = 'fit individually'
# fit the model
#print 'beginning ', fit_str
dir = dismod3.settings.JOB_WORKING_DIR % id
model.fit(dm, method='map', keys=keys, verbose=1)
model.fit(dm, method='mcmc', keys=keys, iter=10000, thin=5, burn=5000, verbose=1,
dbname='%s/posterior/pickle/dm-%d-posterior-%s-%s-%s.pickle' % (dir, id, opts.region, opts.sex, opts.year))
#model.fit(dm, method='mcmc', keys=keys, iter=1, thin=1, burn=0, verbose=1)
# remove all keys that have not been changed by running this model
for k in dm.params.keys():
if type(dm.params[k]) == dict:
for j in dm.params[k].keys():
if not j in keys:
dm.params[k].pop(j)
# post results to dismod_data_server
# "dumb" error handling, in case post fails (try: except: sleep random time, try again, stop after 4 tries)
from twill.errors import TwillAssertionError
from urllib2 import URLError
import random
PossibleExceptions = [TwillAssertionError, URLError]
try:
url = dismod3.post_disease_model(dm)
except PossibleExceptions:
time.sleep(random.random()*30)
try:
url = dismod3.post_disease_model(dm)
except PossibleExceptions:
time.sleep(random.random()*30)
try:
url = dismod3.post_disease_model(dm)
except PossibleExceptions:
time.sleep(random.random()*30)
url = dismod3.post_disease_model(dm)
# form url to view results
#if opts.sex and opts.year and opts.region:
# url += '/%s/%s/%s' % (opts.region, opts.year, opts.sex)
#elif opts.region:
# url += '/%s' % opts.region
# announce completion, and url to view results
#tweet('%s fit complete %s' % (fit_str, url))
sys.stdout.flush()
# update job status file
if opts.log:
if opts.type and not (opts.region and opts.sex and opts.year):
dismod3.log_job_status(id, 'empirical_priors', opts.type, 'Completed')
elif opts.region and opts.sex and opts.year and not opts.type:
dismod3.log_job_status(id, 'posterior', '%s--%s--%s' % (opts.region, opts.sex, opts.year), 'Completed')
