def download_model(id): """ Copy model from web to j drive Parameters ---------- id : int The model id number to copy """ dir = dismod3.settings.JOB_WORKING_DIR % id # TODO: refactor into a function try: model = dismod3.data.ModelData.load(dir) print 'model already on j drive in %s' % dir except (IOError, AssertionError): print 'downloading disease model' dm = dismod3.load_disease_model(id) import simplejson as json try: model = dismod3.data.ModelData.from_gbd_jsons(json.loads(dm.to_json())) except Exception as e: print e print 'attempting to use old covariate format' import old_cov_data model = old_cov_data.from_gbd_jsons(json.loads(dm.to_json())) model.save(dir) print 'loaded data from json, saved in new format for next time in %s' % dir
def fit_all(id, consistent_empirical_prior=True, consistent_posterior=True, posteriors_only=False, posterior_types='p i r', fast=False, zero_re=True, alt_prior=True, global_heterogeneity='Slightly'): """ Enqueues all jobs necessary to fit specified model to the cluster Parameters ---------- id : int The model id number for the job to fit Example ------- >>> import fit_all >>> fit_all.fit_all(2552) """ dir = dismod3.settings.JOB_WORKING_DIR % id # TODO: refactor into a function try: model = data.ModelData.load(dir) print 'loaded data from new format from %s' % dir # if we make it here, this model has already been run, so clean out the stdout/stderr dirs to make room for fresh messages call_str = 'rm -rf %s/empirical_priors/stdout/* %s/empirical_priors/stderr/* %s/posterior/stdout/* %s/posterior/stderr/* %s/json/dm-*-*.json' % (dir, dir, dir, dir, dir) print call_str subprocess.call(call_str, shell=True) # now load just the model, all previous fits are deleted dm = dismod3.load_disease_model(id) except (IOError, AssertionError): print 'downloading disease model' dm = dismod3.load_disease_model(id) import simplejson as json try: model = data.ModelData.from_gbd_jsons(json.loads(dm.to_json())) except Exception as e: print e print 'attempting to use old covariate format' import old_cov_data model = old_cov_data.from_gbd_jsons(json.loads(dm.to_json())) model.save(dir) print 'loaded data from json, saved in new format for next time in %s' % dir def options(fast, zero_re, alt_prior, global_heterogeneity): call_str = '' call_str += ' --fast=%s'%fast call_str += ' --zerore=%s'%zero_re call_str += ' --altprior=%s'%alt_prior call_str += ' --globalheterogeneity=%s'%global_heterogeneity return call_str o = '%s/empirical_priors/stdout/%d_running.txt' % (dir, id) f = open(o, 'w') import time f.write('./run_on_cluster.sh fit_all.py --priorconsistent=%s --posteriorconsistent=%s %s\n' % (consistent_empirical_prior, consistent_posterior, options(fast, zero_re, alt_prior, global_heterogeneity))) f.write('Enqueued model %d on cluster at %s' % (id, time.strftime('%c'))) f.close() # fit empirical priors (by pooling data from all regions) emp_names = [] if not posteriors_only: if consistent_empirical_prior: t = 'all' o = '%s/empirical_priors/stdout/dismod_log_%s' % (dir, t) e = '%s/empirical_priors/stderr/dismod_log_%s' % (dir, t) name_str = '%s-%d' %(t[0], id) emp_names.append(name_str) if dismod3.settings.ON_SGE: call_str = 'qsub -cwd -o %s -e %s ' % (o, e) \ + '-N %s ' % name_str \ + 'run_on_cluster.sh ' else: call_str = 'python ' call_str += 'fit_world.py %d' % id call_str += options(fast, zero_re, alt_prior, global_heterogeneity) subprocess.call(call_str, shell=True) else: for t in ['excess-mortality', 'remission', 'incidence', 'prevalence']: o = '%s/empirical_priors/stdout/dismod_log_%s' % (dir, t) e = '%s/empirical_priors/stderr/dismod_log_%s' % (dir, t) name_str = '%s-%d' %(t[0], id) emp_names.append(name_str) if dismod3.settings.ON_SGE: call_str = 'qsub -cwd -o %s -e %s ' % (o, e) \ + '-N %s ' % name_str \ + 'run_on_cluster.sh ' else: call_str = 'python ' call_str += 'fit_emp_prior.py %d -t %s' % (id, t) call_str += options(fast, zero_re, alt_prior, global_heterogeneity) subprocess.call(call_str, shell=True) # directory to save the country level posterior csv files temp_dir = dir + '/posterior/country_level_posterior_dm-' + str(id) + '/' #fit each region/year/sex individually for this model hold_str = '-hold_jid %s ' % ','.join(emp_names) if posteriors_only: hold_str = '' post_names = [] for ii, r in enumerate(dismod3.gbd_regions): for s in dismod3.gbd_sexes: for y in dismod3.gbd_years: k = '%s+%s+%s' % (dismod3.utils.clean(r), dismod3.utils.clean(s), y) o = '%s/posterior/stdout/dismod_log_%s' % (dir, k) e = '%s/posterior/stderr/dismod_log_%s' % (dir, k) name_str = '%s%d%s%s%d' % (r[0], ii+1, s[0], str(y)[-1], id) post_names.append(name_str) if dismod3.settings.ON_SGE: call_str = 'qsub -cwd -o %s -e %s ' % (o,e) \ + hold_str \ + '-N %s ' % name_str \ + 'run_on_cluster.sh ' else: call_str = 'python ' call_str += 'fit_posterior.py %d -r %s -s %s -y %s' % (id, dismod3.utils.clean(r), dismod3.utils.clean(s), y) if not consistent_posterior: call_str += ' --inconsistent=True --types="%s"' % posterior_types if posteriors_only: call_str += ' --onlyposterior=True' if fast: call_str += ' --fast=true' call_str += ' --zerore=%s'%zero_re subprocess.call(call_str, shell=True) # after all posteriors have finished running, upload disease model json hold_str = '-hold_jid %s ' % ','.join(post_names) o = '%s/empirical_priors/stdout/%d_upload.txt' % (dir, id) e = '%s/empirical_priors/stderr/%d_upload.txt' % (dir, id) if dismod3.settings.ON_SGE: call_str = 'qsub -cwd -o %s -e %s ' % (o,e) \ + hold_str \ + '-N upld-%s ' % id \ + 'run_on_cluster.sh ' else: call_str = 'python ' call_str += 'upload_fits.py %d' % id subprocess.call(call_str, shell=True) return dm