Esempio n. 1
0
def daemon_loop():
    while True:
        try:
            job_queue = dismod3.get_job_queue()
        except:
            job_queue = []
        
        for param_id in job_queue:
            print 'processing job %d' % param_id
            job_params = dismod3.remove_from_job_queue(param_id)
            id = int(job_params['dm_id'])
            fit_all.fit_all(id)
            
        time.sleep(dismod3.settings.SLEEP_SECS)
Esempio n. 2
0
def daemon_loop():
    on_sge = dismod3.settings.ON_SGE
    while True:
        try:
            job_queue = dismod3.get_job_queue()
        except:
            job_queue = []
        
        for param_id in job_queue:
            #tweet('processing job %d' % id)
            log('processing job %d' % param_id)
            job_params = dismod3.remove_from_job_queue(param_id)
            id = int(job_params['dm_id'])
            dm = dismod3.get_disease_model(id)

            # make a working directory for the id
            dir = dismod3.settings.JOB_WORKING_DIR % id
            if not os.path.exists(dir):
                os.makedirs(dir)

            estimate_type = dm.params.get('run_status', {}).get('estimate_type', 'fit all individually')

            if estimate_type.find('posterior') != -1:
                #fit each region/year/sex individually for this model
                regions_to_fit = dm.params.get('run_status', {}).get('regions_to_fit', [])
                if regions_to_fit[0] == 'all_regions':
                    regions_to_fit = dismod3.gbd_regions
                d = '%s/posterior' % dir
                if os.path.exists(d):
                    rmtree(d)
                os.mkdir(d)
                os.mkdir('%s/stdout' % d)
                os.mkdir('%s/stderr' % d)
                dismod3.init_job_log(id, 'posterior', param_id)
                for r in regions_to_fit:
                    for s in dismod3.gbd_sexes:
                        for y in dismod3.gbd_years:
                            # fit only one region, for the time being...
                            # TODO: make region selection a user-settable option from the gui
                            #if clean(r) != 'asia_southeast':
                            #    continue
                            k = '%s+%s+%s' % (clean(r), s, y)
                            o = '%s/stdout/%s' % (d, k)
                            e = '%s/stderr/%s' % (d, k)
                            if on_sge:
                                call_str = dismod3.settings.GBD_FIT_STR % (o, e, '-l -r %s -s %s -y %s' % (clean(r), s, y), id)
                                subprocess.call(call_str, shell=True)
                            else:
                                call_str = dismod3.settings.GBD_FIT_STR % ('-l -r %s -s %s -y %s' % (clean(r), s, y), id, o, e)
                                subprocess.call(call_str, shell=True)
                            time.sleep(1.)

            elif estimate_type.find('empirical priors') != -1:
                # fit empirical priors (by pooling data from all regions
                d = '%s/empirical_priors' % dir
                if os.path.exists(d):
                    rmtree(d)
                os.mkdir(d)
                os.mkdir('%s/stdout' % d)
                os.mkdir('%s/stderr' % d)
                dismod3.init_job_log(id, 'empirical_priors', param_id)
                for t in ['excess-mortality', 'remission', 'incidence', 'prevalence']:
                    o = '%s/stdout/%s' % (d, t)
                    e = '%s/stderr/%s' % (d, t)
                    if on_sge:
                        subprocess.call(dismod3.settings.GBD_FIT_STR % (o, e, '-l -t %s' % t, id), shell=True)
                    else:
                        subprocess.call(dismod3.settings.GBD_FIT_STR % ('-l -t %s' % t, id, o, e), shell=True)

            else:
                #tweet('unrecognized estimate type: %s' % estimate_type)
                log('unrecognized estimate type: %s' % estimate_type)
            
        time.sleep(dismod3.settings.SLEEP_SECS)
Esempio n. 3
0
def daemon_loop():
    on_sge = dismod3.settings.ON_SGE
    while True:
        try:
            job_queue = dismod3.get_job_queue()
        except:
            job_queue = []
        
        for param_id in job_queue:
            #tweet('processing job %d' % id)
            log('processing job %d' % param_id)
            job_params = dismod3.remove_from_job_queue(param_id)
            id = int(job_params['dm_id'])
            dm = dismod3.get_disease_model(id)

            # make a working directory for the id
            dir = dismod3.settings.JOB_WORKING_DIR % id
            if os.path.exists(dir):
                dismod3.disease_json.random_rename(dir)
            os.makedirs(dir)

            estimate_type = dm.params.get('run_status', {}).get('estimate_type', 'fit all individually')

            # sort the regions so that the data rich regions are fit first
            #data_hash = GBDDataHash(dm.data)
            #sorted_regions = sorted(dismod3.gbd_regions, reverse=True,
                                    #key=lambda r: len(data_hash.get(region=r)))

            if estimate_type == 'Fit continuous single parameter model':
                #dismod3.disease_json.create_disease_model_dir(id)
                o = '%s/continuous_spm.stdout' % dir
                e = '%s/continuous_spm.stderr' % dir
                if on_sge:
                    print o
                    print e
                    call_str = 'qsub -cwd -o %s -e %s ' % (o, e) \
                               + 'run_on_cluster.sh /home/OUTPOST/abie/gbd_dev/gbd/fit_continuous_spm.py %d' % id
                else:
                    call_str = 'python -u /home/abie/gbd/fit_continuous_spm.py %d 2>%s |tee %s' % (id, e, o)
                subprocess.call(call_str, shell=True)
                continue
            
            if estimate_type.find('posterior') != -1:
                #fit each region/year/sex individually for this model
                regions_to_fit = dm.params.get('run_status', {}).get('regions_to_fit', [])
                if regions_to_fit[0] == 'all_regions':
                    regions_to_fit = dismod3.gbd_regions
                d = '%s/posterior' % dir
                if os.path.exists(d):
                    rmtree(d)
                os.mkdir(d)
                os.mkdir('%s/stdout' % d)
                os.mkdir('%s/stderr' % d)
                os.mkdir('%s/pickle' % d)
                dismod3.init_job_log(id, 'posterior', param_id)
                for r in regions_to_fit:
                    for s in dismod3.gbd_sexes:
                        for y in dismod3.gbd_years:
                            # fit only one region, for the time being...
                            # TODO: make region selection a user-settable option from the gui
                            #if clean(r) != 'asia_southeast':
                            #    continue
                            k = '%s+%s+%s' % (clean(r), s, y)
                            o = '%s/stdout/%s' % (d, k)
                            e = '%s/stderr/%s' % (d, k)
                            if on_sge:
                                call_str = dismod3.settings.GBD_FIT_STR % (o, e, '-l -r %s -s %s -y %s' % (clean(r), s, y), id)
                                subprocess.call(call_str, shell=True)
                            else:
                                call_str = dismod3.settings.GBD_FIT_STR % ('-l -r %s -s %s -y %s' % (clean(r), s, y), id, o, e)
                                subprocess.call(call_str, shell=True)
                            #time.sleep(1.)

            elif estimate_type.find('empirical priors') != -1:
                # fit empirical priors (by pooling data from all regions
                d = '%s/empirical_priors' % dir
                if os.path.exists(d):
                    rmtree(d)
                os.mkdir(d)
                os.mkdir('%s/stdout' % d)
                os.mkdir('%s/stderr' % d)
                os.mkdir('%s/pickle' % d)
                dismod3.init_job_log(id, 'empirical_priors', param_id)
                for t in ['excess-mortality', 'remission', 'incidence', 'prevalence']:
                    o = '%s/stdout/%s' % (d, t)
                    e = '%s/stderr/%s' % (d, t)
                    if on_sge:
                        subprocess.call(dismod3.settings.GBD_FIT_STR % (o, e, '-l -t %s' % t, id), shell=True)
                    else:
                        subprocess.call(dismod3.settings.GBD_FIT_STR % ('-l -t %s' % t, id, o, e), shell=True)

            else:
                #tweet('unrecognized estimate type: %s' % estimate_type)
                log('unrecognized estimate type: %s' % estimate_type)
            
        time.sleep(dismod3.settings.SLEEP_SECS)