def execute(self): # compile submission arguments kwargs = self.build_args[1] me_map = kwargs.pop("me_map") # get qmaster q = self.get_qmaster() # command line args for adjuster.py # parallelize by year for i in [1990, 1995, 2000, 2005, 2010, 2016]: ex_params = ["--me_map", json.dumps(me_map), "--out_dir", data_dir, "--year_id", str(i)] remote_job = job.Job( mon_dir=log_dir, runfile=os.path.join(code_dir, "scripts", "run_ex_adjust.py"), name="{proc}_{year}".format(proc=self.identity, year=i), job_args=ex_params ) q.queue_job( remote_job, slots=20, memory=40, project="proj_epic") q.block_till_done(poll_interval=60) outputs_tuples = self.builder.get_process_outputs(self.identity) result_meids = [task_tuple[0] for task_tuple in outputs_tuples] for meid in result_meids: save_params = [ meid, description, os.path.join(data_dir, str(meid)), "--best", "--file_pattern", "{year_id}.h5", "--h5_tablename", "draws"] remote_job = job.Job( mon_dir=log_dir, name="save_" + str(meid), runfile=sge.true_path(executable="save_custom_results"), job_args=save_params) q.queue_job( remote_job, slots=40, memory=80, project="proj_epic") q.block_till_done(poll_interval=60)
def saver(q, dirs, save_func, log_dir): runfile = sge.true_path(executable=save_func) for d in dirs: meid = os.path.basename(d) params = [ meid, "super-squeeze result", d, '--env', 'prod', '--file_pattern', '{location_id}_{year_id}_{sex_id}.h5', '--h5_tablename', 'draws', '--best' ] remote_job = job.Job(mon_dir=log_dir, runfile=runfile, name='ss_save_%s' % meid, job_args=params) q.queue_job(remote_job, slots=20, memory=40, project='proj_epic', stderr='/FILEPATH', stdout='/FILEPATH') q.block_till_done(poll_interval=60)
def execute(self): # get args kwargs = self.build_args[1] parent_meid = kwargs["parent_meid"] env = "prod" # get qmaster q = self.get_qmaster() # submit split job remote_job = job.Job( mon_dir=log_dir, name="split_" + (str(parent_meid)), runfile=os.path.join(code_dir, "scripts", "FILEPATH.py"), job_args=[str(parent_meid), env]) q.queue_job( remote_job, slots=49, memory=98, project="proj_epic") q.block_till_done(poll_interval=60) # submit aggregation/save jobs outputs_tuples = self.builder.get_process_outputs(self.identity) children_meids = [task_tuple[0] for task_tuple in outputs_tuples] for meid in children_meids: mvid = self._get_latest_mvid(meid) remote_job = job.Job( mon_dir=log_dir, name="save_" + str(mvid), runfile=sge.true_path(executable="aggregate_mvid"), job_args=[str(mvid), '--env', env, '--mark_best']) q.queue_job( remote_job, slots=40, memory=80, project="proj_epic") q.block_till_done(poll_interval=60)
def run_pipeline_como( root_dir, gbd_round_id=4, location_id=[], year_id=[], sex_id=[], age_group_id=[], measure_id=[], n_draws=1000, n_simulants=20000, components=["sequela", "cause", "impairment", "injuries"]): cv = ComoVersion.new(root_dir, gbd_round_id, location_id, year_id, sex_id, age_group_id, measure_id, n_draws, components) try: cjm = CentralJobMonitor(cv.como_dir, persistent=False) time.sleep(5) except Exception as e: raise e else: executor_params = {"request_timeout": 10000} jobq = JobQueue(cv.como_dir, scheduler=RetryScheduler, executor=SGEExecutor, executor_params=executor_params) # run nonfatal pipeline by location/year/sex parallelism = ["location_id", "sex_id"] for slices in cv.dimensions.index_slices(parallelism): jobname = "como_e_sim_{location_id}_{sex_id}".format( location_id=slices[0], sex_id=slices[1]) job = jobq.create_job( jobname=jobname, runfile=true_path(executable="compute_nonfatal"), parameters=[ "--como_dir", cv.como_dir, "--location_id", str(slices[0]), "--sex_id", str(slices[1]), "--n_processes", "23", "--n_simulants", str(n_simulants) ]) jobq.queue_job(job, slots=50, memory=400, project="proj_como", process_timeout=(60 * 180)) jobq.block_till_done(stop_scheduler_when_done=False) # run aggregation by year/sex/measure parallelism = ["year_id", "sex_id", "measure_id"] for slices in cv.dimensions.index_slices(parallelism): for component in cv.components: if component != "sequela": loc_sets = [35, 40] else: loc_sets = [35] for location_set_id in loc_sets: jobname = ("como_e_agg_{component}_{year_id}_{sex_id}" "_{measure_id}_{location_set_id}").format( component=component, year_id=slices[0], sex_id=slices[1], measure_id=slices[2], location_set_id=location_set_id) job = jobq.create_job( jobname=jobname, runfile=true_path(executable="aggregate_nonfatal"), parameters=[ "--como_dir", cv.como_dir, "--component", component, "--year_id", str(slices[0]), "--sex_id", str(slices[1]), "--measure_id", str(slices[2]), "--location_set_id", str(location_set_id) ]) jobq.queue_job(job, slots=25, memory=200, project="proj_como", process_timeout=(60 * 600)) jobq.block_till_done(stop_scheduler_when_done=False) # run summaries by component/location lt = dbtrees.loctree(None, 35) sdi_lts = dbtrees.loctree(None, 40, return_many=True) locs = [l.id for l in lt.nodes] sdi_locs = [l.root.id for l in sdi_lts] for component in cv.components: if component != "sequela": summ_locs = locs + sdi_locs else: summ_locs = locs[:] for location_id in summ_locs: jobname = "como_e_summ_{component}_{location_id}".format( component=component, location_id=location_id) job = jobq.create_job( jobname=jobname, runfile=true_path(executable="summarize_nonfatal"), parameters=[ "--como_dir", cv.como_dir, "--component", component, "--location_id", str(location_id) ]) jobq.queue_job(job, slots=48, memory=96, project="proj_como", process_timeout=(60 * 240)) jobq.block_till_done(stop_scheduler_when_done=False) for component in cv.components: jobname = "como_e_upload_{component}".format(component=component) job = jobq.create_job( jobname=jobname, runfile=true_path(executable="upload_nonfatal"), parameters=[ "--como_dir", cv.como_dir, "--component", component, "--location_id", " ".join([str(l) for l in locs + sdi_locs]) ]) jobq.queue_job(job, slots=20, memory=40, project="proj_como", process_timeout=(60 * 720)) jobq.block_till_done() finally: cjm.generate_report() cjm.stop_responder() cjm.stop_publisher()