class SquareImport(object): _idx_dmnsns = { "year_id": [{YEAR IDS}], "age_group_id": getset.get_age_group_set({AGE GROUP SET ID})["age_group_id"], "sex_id": [{SEX IDS}], "location_id": getset.get_most_detailed_location_ids(), "measure_id": [{MEASURE IDS}] } _draw_cols = ["draw_{i}".format(i=i) for i in range(0, 1000)] def __init__(self, idx_dmnsns=None, draw_cols=None): if idx_dmnsns is None: self.idx_dmnsns = collections.OrderedDict( sorted(self.dUSERt_idx_dmnsns.items())) else: self.idx_dmnsns = collections.OrderedDict( sorted(idx_dmnsns.items())) if draw_cols is None: self.draw_cols = self.dUSERt_draw_cols else: self.draw_cols = draw_cols # expected index self.index_df = self.get_index_df() @ClassProperty @classmethod def dUSERt_idx_dmnsns(cls): return cls._idx_dmnsns.copy() @ClassProperty @classmethod def dUSERt_draw_cols(cls): return cls._draw_cols[:] def get_index_df(self): """create template index for square dataset""" idx = pd.MultiIndex.from_product( self.idx_dmnsns.values(), names=self.idx_dmnsns.keys()) return pd.DataFrame(index=idx) def import_square(self, gopher_what, source, filler=None, **kwargs): """get draws for the specified modelable entity by dimensions""" if not kwargs: kwargs = self.idx_dmnsns.copy() # replace keys with their plural form for gopher.draws for k in kwargs.keys(): kwargs[k + "s"] = kwargs.pop(k) if filler is None: filler = 0 df = gopher.draws(gopher_what, source=source, verbose=False, **kwargs) for c in self.idx_dmnsns.keys(): df[c] = pd.to_numeric(df[c]) df = df.set_index(self.idx_dmnsns.keys()) df = df[self.draw_cols] df = pd.concat([self.index_df, df], axis=1) df.fillna(value=filler, inplace=True) return df
def execute(self): # compile submission arguments me_map = self.build_args[0][0] # make server directory directory = "{root}/{proc}".format(root=root, proc=self.identity) if not os.path.exists(directory): os.makedirs(directory) else: shutil.rmtree(directory) os.makedirs(directory) # make output directories save_ids = [] for mapper in me_map.values(): outputs = mapper.get("trgs", {}) for me_id in outputs.values(): os.makedirs(os.path.join(directory, str(me_id))) save_ids.append(me_id) attr_params = [ "--me_map", json.dumps(me_map), "--out_dir", directory, "--location_id" ] q = qmaster.MonitoredQ(directory, request_timeout=120000) # monitor try: # attribution jobs by location_id for i in getset.get_most_detailed_location_ids(): q.qsub(runfile="{root}/infertility/female_attr.py".format( root=root), jobname="{proc}_{loc}".format(proc=self.identity, loc=i), parameters=attr_params + [i], slots=4, memory=8, project="proj_custom_models") time.sleep(1.5) q.qblock(poll_interval=60) # monitor them fail = q.manager.query( "select count(*) as num from job where current_status != 5;" )[1]["num"].item() if fail > 0: raise Exception("Not all attr jobs finished") # save the results for save_id in save_ids: save_params = [ save_id, female_attr_description, os.path.join(directory, str(save_id)), "--best", "--sexes", "2", "--file_pattern", "{location_id}.h5", "--h5_tablename", "data" ] q.qsub(runfile=("/home/j/WORK/10_gbd/00_library/adding_machine" "/bin/save_custom_results"), jobname="save_" + str(save_id), parameters=save_params, slots=20, memory=40, project="proj_custom_models") q.qblock(poll_interval=60) # monitor them fail = q.manager.query( "select count(*) as num from job where current_status != 5;" )[1]["num"].item() if fail > 0: raise Exception("Not all save jobs finished") finally: q.stop_monitor() # stop monitor
def execute(self): # compile submission arguments me_map = { "env": { "srcs": { "prim": "3339", "sec": "3341" } }, "idio": { "trgs": { "prim": "2061", "sec": "2062" } }, "kline": { "srcs": { "bord": "2218", "mild": "2659", "asym": "3069" } }, "cong_uro": { "srcs": { "inf_only": "11033", "ag": "11037", "uti": "11039", "imp": "11041", "ag_uti": "11042", "ag_imp": "11044", "imp_uti": "11045", "ag_imp_uti": "11046" } } } # make server directory directory = "{root}/{proc}".format(root=root, proc='male_cause_attribution') if os.path.exists(directory): shutil.rmtree(directory) os.makedirs(directory) else: os.makedirs(directory) # make output directories save_ids = [] for mapper in me_map.values(): outputs = mapper.get("trgs", {}) for me_id in outputs.values(): if not os.path.exists(os.path.join(directory, str(me_id))): os.makedirs(os.path.join(directory, str(me_id))) save_ids.append(me_id) attr_params = [ "--me_map", "\'{}\'".format(json.dumps(me_map)), "--out_dir", directory, "--location_id" ] # attribution jobs by year_id male_attr_string = '' for i in getset.get_most_detailed_location_ids(): male_attr_job = "male_attr_{}".format(i) male_attr_string = male_attr_string + ',' + male_attr_job call = ('qsub -hold_jid {hj}' ' -pe multi_slot 4' ' -cwd -P proj_custom_models' ' -o {o}' ' -e {e}' ' -N {jn}' ' cluster_shell.sh' ' male_attr.py' ' {arg1}'.format(hj=self.hold, o=output_path, e=error_path, jn=male_attr_job, arg1=' '.join(attr_params + [str(i)]))) subprocess.call(call, shell=True) # save the results hold = male_attr_string for save_id in save_ids: save_params = [ str(save_id), "--description", "\'{}\'".format(male_attr_description), "--input_dir", os.path.join(directory, str(save_id)), "--best", "--sexes", "1", "--meas_ids", "5", "--file_pattern", "{location_id}.h5" ] call = ('qsub -hold_jid {hj} -pe multi_slot 15' ' -cwd -P proj_custom_models' ' -o {o}' ' -e {e}' ' -N {jn}' ' cluster_shell.sh' ' save.py' ' {arg1}'.format(hj=hold, o=output_path, e=error_path, jn='male_attr_save_{}'.format(save_id), arg1=' '.join(save_params))) subprocess.call(call, shell=True)
def execute(self): # compile submission arguments me_map = { "env": { "type": "envelope", "srcs": { "prim": "3340", "sec": "3342" } }, "resid": { "type": "residual", "trgs": { "prim": "2071", "sec": "2072" } }, "excess": { "type": "excess", "trgs": { "endo": "9748", "pcos": "9743" } }, "cong_uro": { "type": "locked", "srcs": { "inf_only": "11033", "ag": "11037", "uti": "11039", "imp": "11041", "ag_uti": "11042", "ag_imp": "11044", "imp_uti": "11045", "ag_imp_uti": "11046" } }, "turner": { "type": "locked", "srcs": { "nohf": "2208", "hf": "2653" } }, "sepsis": { "type": "sub_group", "srcs": { "tot": "2624" }, "trgs": { "sec": "9678" } }, "pcos_asymp": { "type": "sub_group", "excess": "pcos", "srcs": { "tot": "9675" }, "trgs": { "prim": "2069", "sec": "3088" } }, "pcos_disfig": { "type": "sub_group", "excess": "pcos", "srcs": { "tot": "9676" }, "trgs": { "prim": "2938", "sec": "3087" } }, "endo_asymp": { "type": "sub_group", "excess": "endo", "srcs": { "tot": "9671" }, "trgs": { "prim": "2076", "sec": "2077" } }, "endo_mild": { "type": "sub_group", "excess": "endo", "srcs": { "tot": "9672" }, "trgs": { "prim": "2959", "sec": "2962" } }, "endo_mod": { "type": "sub_group", "excess": "endo", "srcs": { "tot": "9673" }, "trgs": { "prim": "2960", "sec": "2963" } }, "endo_sev": { "type": "sub_group", "excess": "endo", "srcs": { "tot": "9674" }, "trgs": { "prim": "2961", "sec": "2964" } }, "gono": { "type": "sub_group", "srcs": { "tot": "3023" }, "trgs": { "prim": "1639", "sec": "1640" } }, "chlam": { "type": "sub_group", "srcs": { "tot": "3022" }, "trgs": { "prim": "1633", "sec": "1634" } }, "otherstd": { "type": "sub_group", "srcs": { "tot": "3024" }, "trgs": { "prim": "1645", "sec": "1646" } } } # make server directory directory = "{root}/{proc}".format(root=root, proc='female_cause_attribution') if os.path.exists(directory): shutil.rmtree(directory) os.makedirs(directory) else: os.makedirs(directory) # make output directories save_ids = [] for mapper in me_map.values(): outputs = mapper.get("trgs", {}) for me_id in outputs.values(): if not os.path.exists(os.path.join(directory, str(me_id))): os.makedirs(os.path.join(directory, str(me_id))) save_ids.append(me_id) attr_params = [ "--me_map", "\'{}\'".format(json.dumps(me_map)), "--out_dir", directory, "--location_id" ] # parallelize by location fem_attr_string = '' for i in getset.get_most_detailed_location_ids(): fem_attr_job = "fem_attr_{}".format(i) fem_attr_string = fem_attr_string + ',' + fem_attr_job call = ('qsub -hold_jid {hj}' ' -pe multi_slot 4' ' -cwd -P proj_custom_models' ' -o {o}' ' -e {e}' ' -N {jn}' ' cluster_shell.sh' ' female_attr.py' ' {arg1}'.format(hj=self.hold, o=output_path, e=error_path, jn=fem_attr_job, arg1=' '.join(attr_params + [str(i)]))) subprocess.call(call, shell=True) # save the results hold = fem_attr_string for save_id in save_ids: save_params = [ str(save_id), "--description", "\'{}\'".format(female_attr_description), "--input_dir", os.path.join(directory, str(save_id)), "--best", "--sexes", "2", "--meas_ids", "5", "--file_pattern", "{location_id}.h5" ] job_name = 'fem_attr_save_{}'.format(save_id) self.save_hold = self.save_hold + ',' + job_name call = ('qsub -hold_jid {hj} -pe multi_slot 15' ' -cwd -P proj_custom_models' ' -o {o}' ' -e {e}' ' -N {jn}' ' cluster_shell.sh' ' save.py' ' {arg1}'.format(hj=hold, o=output_path, e=error_path, jn=job_name, arg1=' '.join(save_params))) subprocess.call(call, shell=True) return self.save_hold
class SquareImport(object): _idx_dmnsns = { "year_id": [1990, 1995, 2000, 2005, 2010, 2015, 2017, 2019], #"age_group_id": getset.get_age_group_set(12)["age_group_id"], "age_group_id": [i for i in range(2, 21, 1)] + [30, 31, 32, 235, 164], "sex_id": [1, 2], "location_id": getset.get_most_detailed_location_ids(), "measure_id": [5, 6] } _draw_cols = ["draw_{i}".format(i=i) for i in range(0, 1000)] def __init__(self, idx_dmnsns=None, draw_cols=None): if idx_dmnsns is None: self.idx_dmnsns = collections.OrderedDict( sorted(self.default_idx_dmnsns.items())) else: self.idx_dmnsns = collections.OrderedDict( sorted(idx_dmnsns.items())) if draw_cols is None: self.draw_cols = self.default_draw_cols else: self.draw_cols = draw_cols # expected index self.index_df = self.get_index_df() @ClassProperty @classmethod def default_idx_dmnsns(cls): return cls._idx_dmnsns.copy() @ClassProperty @classmethod def default_draw_cols(cls): return cls._draw_cols[:] def get_index_df(self): """create template index for square dataset""" idx = pd.MultiIndex.from_product(self.idx_dmnsns.values(), names=self.idx_dmnsns.keys()) return pd.DataFrame(index=idx) def import_square(self, meid, source, filler=None, **kwargs): """get draws for the specified modelable entity by dimensions""" if not kwargs: kwargs = self.idx_dmnsns.copy() # spillover from needing to replace keys with their plural form for gopher.draws for k in kwargs.keys(): kwargs[k] = kwargs.pop(k) if filler is None: filler = 0 df = get_draws('modelable_entity_id', meid, source=source, status='latest', decomp_step='step1', **kwargs) for c in self.idx_dmnsns.keys(): df[c] = pd.to_numeric(df[c]) df = df.set_index(self.idx_dmnsns.keys()) df = df[self.draw_cols] df = pd.concat([self.index_df, df], axis=1) df.fillna(value=filler, inplace=True) return df
def execute(self): # compile submission arguments me_map = self.build_args[0][0] # make server directory directory = "{root}/{proc}".format(root=root, proc=self.identity) if not os.path.exists(directory): os.makedirs(directory) else: shutil.rmtree(directory) os.makedirs(directory) # make output directories save_ids = [] for mapper in me_map.values(): outputs = mapper.get("trgs", {}) for me_id in outputs.values(): os.makedirs(os.path.join(directory, str(me_id))) save_ids.append(me_id) attr_params = ["--me_map", json.dumps(me_map), "--out_dir", directory, "--location_id"] q = self.get_qmaster() # attribution jobs by location_id for i in getset.get_most_detailed_location_ids(): remote_job=job.Job( mon_dir=log_dir, name="{proc}_{loc}".format(proc=self.identity, loc=i), runfile="{root}/infertility/female_attr.py".format( root=root), job_args=attr_params + [str(i)]) q.queue_job( remote_job, slots=4, memory=8, project={PROJECT}, stderr=log_dir, stdout=log_dir) time.sleep(1.5) q.block_till_done(poll_interval=60) # monitor them # save the results for save_id in save_ids: save_params = [ str(save_id), female_attr_description, os.path.join(directory, str(save_id)), "--best", "--sexes", "{SEX ID}", "--file_pattern", "{location_id}.h5", "--h5_tablename", "data"] remote_job=job.Job( mon_dir=log_dir, name="save_" + str(save_id), runfile=("{FILEPATH}/save_custom_results"), job_args=save_params) q.queue_job( remote_job, slots=20, memory=40, project={PROJECT}, stderr=log_dir, stdout=log_dir) q.block_till_done(poll_interval=60) # monitor them