def validate_decomp_step(obj_name, decomp_step, gbd_round_id): # validate decomp_step decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id) rules_manager = RulesManager(ResearchAreas.EPI, Tools.SAVE_RESULTS, decomp_step_id=decomp_step_id) validate_step_viewable(obj_name, decomp_step, rules_manager)
def __init__( self, decomp_step: str = None, gbd_round_id: int = gbd.GBD_ROUND_ID, include_como: bool = False): _validate_decomp_step( decomp_step, gbd_round_id, ResearchAreas.EPI, Tools.SAVE_RESULTS, mark_best=True ) self.include_como = include_como self._loaded_maps = {} self.all_processes = {} self.decomp_step = decomp_step self.gbd_round_id = gbd_round_id self.decomp_step_id = decomp_step_id_from_decomp_step( step=self.decomp_step, gbd_round_id=self.gbd_round_id ) self.G = nx.DiGraph() # meid graph self.P = nx.DiGraph() # process graph self.start_node = Params.EPIC_START_NODE self.this_file = os.path.realpath(__file__) self.this_dir = os.path.dirname(self.this_file) self.load_maps(self.this_dir) self.build_graphs(self.this_dir) self._best_models = None self._inputs = None self._outputs = None
def get_regional_scalars( gbd_round_id: int, decomp_step: str, ) -> pd.DataFrame: """ Previous iterations of scalars produced sex- and age-specific results. Starting GBD2017 scalars are produced with location/year-specific detail, at the both-sex and all-age level. """ q = """ SELECT * FROM mortality.upload_population_scalar_estimate WHERE run_id = ( SELECT run_id FROM mortality.vw_decomp_process_version WHERE process_id = 23 AND gbd_round_id = :gbd_round_id AND decomp_step_id = :decomp_step_id AND year_id IN :year_ids AND is_best = 1) """ params = { "gbd_round_id": gbd_round_id, "decomp_step_id": gbd_decomp_step.decomp_step_id_from_decomp_step( decomp_step, gbd_round_id), "year_ids": mmr_constants.OUTPUT_YEARS } scalars = ezfuncs.query(q, parameters=params, conn_def='mortality') scalars = scalars.rename(columns={'mean': 'scaling_factor'}) scalars = scalars[['location_id', 'year_id', 'scaling_factor']] return scalars
def new_mvid_list(self): mvid_list = ezfuncs.query(""" SELECT cause.cause_id, mv.modelable_entity_id, mv.model_version_id, mv.decomp_step_id FROM epi.model_version mv LEFT JOIN epi.modelable_entity_cause cause ON mv.modelable_entity_id = cause.modelable_entity_id WHERE mv.model_version_status_id = :best and mv.gbd_round_id = :gbd_round_id and mv.decomp_step_id IN (:decomp_step_id, :iterative) """, conn_def="epi", parameters={ "best": 1, "gbd_round_id": self.gbd_round_id, "decomp_step_id": self.decomp_step_id, "iterative": decomp_step_id_from_decomp_step( step="iterative", gbd_round_id=self.gbd_round_id) }) # Determine which decomp step to find model version in all_ntd_cause = [ 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 405, 843, 935, 936 ] ntd_decomp_me = [ 1500, 1503, 10402, 1513, 1514, 1515, 2999, 3109, 20265, 1516, 1517, 1518, 3001, 3110, 20266, 1519, 1520, 1521, 3000, 3139, 3111, 20009, 2797, 1474, 1469, 2965, 1475, 10524, 10525, 1476, 2966, 1470, 1471, 10480, 1477, 10537, 1472, 1468, 1466, 1473, 1465, 16393, 1478 ] new_gbd_2019_cause = [ 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 628 ] has_decomp_version = mvid_list.loc[ mvid_list.decomp_step_id == self.decomp_step_id, ].modelable_entity_id.unique().tolist() use_iterative = mvid_list.loc[ ~mvid_list.modelable_entity_id.isin(ntd_decomp_me) & mvid_list.cause_id.isin(all_ntd_cause + new_gbd_2019_cause) & ~mvid_list.modelable_entity_id.isin( has_decomp_version)].modelable_entity_id.unique().tolist() mvid_list = mvid_list.loc[ (mvid_list.decomp_step_id == self.decomp_step_id) | (mvid_list.modelable_entity_id.isin(use_iterative))] self.mvid_list = mvid_list[[ 'modelable_entity_id', 'model_version_id', 'decomp_step_id' ]]
def new(cls, year_ids: List[int], location_set_ids: List[int], sex_ids: List[int] = [gbd.sex.MALE, gbd.sex.FEMALE], measure_ids: List[int] = [gbd.measures.DEATH, gbd.measures.YLL], gbd_round_id: int = gbd.GBD_ROUND_ID, decomp_step: str = gbd.decomp_step.ONE, n_draws: int = constants.Draws.N_DRAWS, process: str = GBD_PROCESS, databases: List[str] = [constants.DataBases.GBD], year_start_ids: Optional[List[int]] = None, year_end_ids: Optional[List[int]] = None): version_id: int = cls.get_new_version_id() inst = cls(version_id) inst.year_ids: Optional[List[int]] = year_ids inst.year_start_ids: Optional[List[int]] = year_start_ids inst.year_end_ids: Optional[List[int]] = year_end_ids inst.location_set_ids: Optional[List[int]] = location_set_ids inst.sex_ids: Optional[List[int]] = sex_ids inst.measure_ids: Optional[List[int]] = measure_ids inst.gbd_round_id: Optional[int] = gbd_round_id inst.decomp_step: Optional[str] = decomp_step inst.decomp_step_id: Optional[int] = decomp_step_id_from_decomp_step( step=decomp_step, gbd_round_id=gbd_round_id) inst.n_draws: Optional[int] = n_draws inst.process: Optional[str] = process # scalar_version_id will be retrieved for FauxcCorrect AND CoDCorrect # runs but will only be used in FauxCorrect runs inst.scalar_version_id: Optional[int] = ( fauxcorrect_version.get_scalar_version_id(gbd_round_id)) inst.databases: Optional[List[str]] = databases # Need to instantiate envelope, population, and life table parameter # objs before creating the process version row. inst._create_envelope_parameter() inst._create_population_parameter() inst._create_life_table_parameter() metadata, version_note = inst._create_process_version_info(process) # Create version row in the appropriate table for the run inst.create_new_version_row(version_id, inst.DESCRIPTION, inst.decomp_step_id, inst.gbd_round_id) # Create gbd process version id for the run inst.gbd_process_version_id = (inst.create_gbd_process_version( metadata, version_note)) # Create parameter objects inst._create_cause_parameters() inst._create_location_parameters() inst._create_model_version_parameter() # Create eligible metadata inst._create_eligible_metadata() return inst
def __init__(self, conn_def, codcorrect_vers=None, decomp_step=None, gbd_round_id=None): self.conn_def = conn_def self.codcorrect_vers = codcorrect_vers self.decomp_step = decomp_step self.gbd_round_id = gbd_round_id if decomp_step: self.decomp_step_id = decomp_step_id_from_decomp_step( self.decomp_step, self.gbd_round_id)
def get_best_models(self) -> pd.DataFrame: mvid_list = ezfuncs.query( """ SELECT cause.cause_id, mv.modelable_entity_id, mv.model_version_id, mv.decomp_step_id, mv.best_start FROM epi.model_version mv LEFT JOIN epi.modelable_entity_cause cause ON mv.modelable_entity_id = cause.modelable_entity_id WHERE mv.model_version_status_id = :best and mv.gbd_round_id = :gbd_round_id and mv.decomp_step_id IN (:decomp_step_id, :iterative) """, conn_def="epi", parameters={ "best": 1, "gbd_round_id": self.gbd_round_id, "decomp_step_id": self.decomp_step_id, "iterative": decomp_step_id_from_decomp_step( step=gbd.decomp_step.ITERATIVE, gbd_round_id=self.gbd_round_id ) } ) # Determine what version (decomp/iterative) to use: all_ntd_cause = [346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 405, 843, 935, 936] ntd_decomp_me = [1500, 1503, 10402, 1513, 1514, 1515, 2999, 3109, 20265, 1516, 1517, 1518, 3001, 3110, 20266, 1519, 1520, 1521, 3000, 3139, 3111, 20009, 2797, 1474, 1469, 2965, 1475, 10524, 10525, 1476, 2966, 1470, 1471, 10480, 1477, 10537, 1472, 1468, 1466, 1473, 1465, 16393, 1478] new_gbd_2019_cause = [1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017] has_decomp_version = mvid_list.loc[ mvid_list.decomp_step_id == self.decomp_step_id, ].modelable_entity_id.unique().tolist() use_iterative = mvid_list.loc[ ~mvid_list.modelable_entity_id.isin(ntd_decomp_me) & mvid_list.cause_id.isin(all_ntd_cause + new_gbd_2019_cause) & ~mvid_list.modelable_entity_id.isin(has_decomp_version) ].modelable_entity_id.unique().tolist() mvid_list = mvid_list.loc[ (mvid_list.decomp_step_id == self.decomp_step_id) | (mvid_list.modelable_entity_id.isin(use_iterative))] mvid_list["decomp_step"] = mvid_list["decomp_step_id"].apply( decomp_step_from_decomp_step_id) return mvid_list
def get_best_codcorrect_vers(decomp_step, gbd_round_id): decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id) params = {"decomp_step_id": decomp_step_id, "gbd_round_id": gbd_round_id} q = """ SELECT MAX(CAST(output_version_id AS UNSIGNED)) as id FROM cod.output_version WHERE status = 1 AND is_best = 1 AND decomp_step_id = :decomp_step_id AND code_version = :gbd_round_id """ result = query(q, parameters=params, conn_def='cod') return int(result.at[0, 'id'])
def _validate_in_sync( bundle_id: int, gbd_round_id: int, decomp_step: str, crosswalk_version_id: Optional[int], session: orm.Session ) -> None: """ Thin wrapper for elmo's validate_in_sync. We only want to call the function when the crosswalk version exists. """ if not crosswalk_version_id: return try: elmo_validate.validate_in_sync( bundle_id=bundle_id, gbd_round_id=gbd_round_id, decomp_step=decomp_step, session=session, crosswalk_version_id=crosswalk_version_id ) except ValueError as e: # No previous best model found if decomp_step != ds.TWO: previous_step = decomp_step_from_decomp_step_id( helpers.get_previous_decomp_step_id( decomp_step_id_from_decomp_step( decomp_step, gbd_round_id))) previous_step_str = ' AND the previous decomp step (GBD round ' \ f'ID {gbd_round_id}, {previous_step})' else: previous_step_str = '' raise ValueError( str(e) + '\n\nYou must have a best model with an ' f'associated crosswalk version from the previous round ' f'(GBD round ID {gbd_round_id - 1}){previous_step_str}. ' 'If this is a custom bundle, don\'t pass in a crosswalk version.')
def best_versions(cause_id, gbd_round_id, sex_id, decomp_step): decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id) q = """ SELECT model_version_id FROM cod.model_version WHERE cause_id={cid} AND sex_id = {sex} AND gbd_round_id={gbd} AND is_best = 1 AND model_version_type_id IN (3, 4) AND decomp_step_id = {decomp} """.format(cid=cause_id, sex=sex_id, gbd=gbd_round_id, decomp=decomp_step_id) res = query(q, conn_def='cod') mvids = res.model_version_id.tolist() bad_mvids_msg = ("Error: Returned more than one model_version_id: " "{mvids} for cause_id: {cid}. Cannot split model.".format( mvids=mvids, cid=cause_id)) if len(mvids) != 1: raise RuntimeError(bad_mvids_msg) return mvids[0]
def best_version(input_machine, gbd_round_id, decomp_step): """Find the 'best' model versions for como and cod from the database. Used as defaults """ if input_machine == 'como': gbd_process_id = c.gbd_process['EPI'] metadata_type_id = c.gbd_metadata_type['COMO'] elif input_machine == 'codcorrect': gbd_process_id = c.gbd_process['COD'] metadata_type_id = c.gbd_metadata_type['CODCORRECT'] elif input_machine == 'fauxcorrect': gbd_process_id = c.gbd_process['FAUXCORRECT'] metadata_type_id = c.gbd_metadata_type['FAUXCORRECT'] else: raise ValueError("app_common.best_version accepts 'como', " "'codcorrect' or 'fauxcorrect' as " "input_machine. Got {}".format(input_machine)) decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id) q = """ SELECT pvm.val as version_id from gbd.gbd_process_version gpv JOIN gbd.gbd_process_version_metadata pvm using (gbd_process_version_id) WHERE gbd_process_id = {process_id} and metadata_type_id = {metadata_type_id} and gbd_round_id = {gbd_round_id} and decomp_step_id = {decomp_step_id} and gbd_process_version_status_id = 1 order by gpv.date_inserted desc limit 1 """.format(process_id=gbd_process_id, metadata_type_id=metadata_type_id, gbd_round_id=gbd_round_id, decomp_step_id=decomp_step_id) return int(ezfuncs.query(q, conn_def='gbd').squeeze())
def generate_maps(decomp_step: int, gbd_round_id: int) -> None: generate_como_config( decomp_step_id_from_decomp_step(decomp_step, gbd_round_id), gbd_round_id) generate_severity_split_config(gbd_round_id) generate_super_squeeze_config()