def inspect_submodels(paths):
rate_vars = import_json(paths.COVARIATE_FILES['ln_rate'])
cf_vars = import_json(paths.COVARIATE_FILES['lt_cf'])
return {
'num_submodels':
int(len(rate_vars['ln_rate_vars'] + cf_vars['lt_cf_vars']))
}
def reload_data(self, gbd_round_id):
"""
Reload a codem model after it has been completed.
Do not re-save model outliers.
"""
sexn = "male" if self.mod_inputs.sex_id == 1 else "female"
st_model_file = os.path.join(self.model_dir, "space_time_json.txt")
json_dict = space.import_json(st_model_file)
draw_data = pd.read_hdf(
self.model_dir + "/draws/deaths_{}.h5".format(sexn), "data")
self.mod_inputs.data_frame, self.mod_inputs.covariates, self.mod_inputs.priors = \
Q.getCodemInputData(self.model_version_id, self.db_connection, gbd_round_id, outlier_save = False)
self.adjust_input_data(save=False)
st_models_temp = \
All_Models(pd.concat([self.mod_inputs.data_frame,
self.mod_inputs.covariates], axis=1),
self.mod_inputs.knockouts,
self.mod_inputs.linear_floor_rate, json_dict,
self.db_connection, make_preds=False)
self.st_models.all_models[-1].models = st_models_temp.all_models[
-1].models
self.mod_inputs.bare_necessities()
draw_data = draw_data.rename(columns={
"age_group_id": "age",
"year_id": "year",
"sex_id": "sex"
})
self.mod_inputs.data_frame = pd.merge(self.mod_inputs.data_frame,
draw_data,
how='inner')
self.mod_inputs.data_frame.drop_duplicates(
["year", "location_id", "age"], inplace=True)
def save_response_list(self):
logger.info(
"Generating the response (outcome variable) list that will be used throughout the pipeline."
)
json_dict = import_json(
os.path.join(self.step_dir, 'space_time_json.txt'))
response_list = [
x.rsplit("_", 1)[0] for x in sorted(list(json_dict.keys()))
]
self.pickled_outputs['response_list'] = response_list
def read_models(self, input_json_path, output_object_name):
logger.info("Reading the result of linear model builds.")
self.json_dict = import_json(input_json_path)
self.pickled_outputs[output_object_name] = All_Models(
df=pd.concat([
self.model_metadata.data_frame, self.model_metadata.covariates
],
axis=1),
knockouts=self.model_metadata.ko_data,
linear_floor_rate=self.model_metadata.
model_parameters['linear_floor_rate'],
json_dict=self.json_dict,
db_connection=self.db_connection)
def read_linear_models(self):
"""
Read linear models created in R
"""
logger.info("Reading linear models.")
json_dict = space.import_json("linear_model_json.txt")
self.linear_models = All_Models(
pd.concat([self.mod_inputs.data_frame, self.mod_inputs.covariates],
axis=1), self.mod_inputs.knockouts,
self.mod_inputs.linear_floor_rate, json_dict, self.db_connection)
if self.debug_mode:
self.linear_models.pred_mat_lin = self.linear_models.all_models[
-1].pred_mat.copy()
def __init__(self, **kwargs):
"""
Create diagnostic plots and tables for modelers to vet.
Attach all information + predictive validity metrics to the modeler email
and send to the relevant modelers.
:param kwargs:
"""
super().__init__(**kwargs, step_id=STEP_IDS['Diagnostics'])
self.linear_models_json = import_json(
self.model_paths.JSON_FILES['linear'])
self.st_models_json = import_json(
self.model_paths.JSON_FILES['spacetime'])
self.submodel_summary_df = None
self.covariate_summary_df = None
self.model_type_df = None
self.submodel_betas_df = None
self.merged_submodels_df = None
self.submodel_weighted_betas_df = None
self.covariate_table = None
self.beta_draws = None
def run_linear_model_builds(self):
"""
Run the linear model portions of all codem models.
"""
logger.info("Running linear model buids.")
if self.log_results:
self.warnings.time_stamp("Running Linear Model Builds")
# run linear models and linear portion of space time models
self.submit_R("lm_model_prototype")
# check to make sure covariate selection picked something
if not os.path.isfile("space_time_json.txt"):
sys.stderr.write("No covariates seem to have been selected.")
# import json file from R build and save the model list portion
self.json_dict = space.import_json("space_time_json.txt")
self.mod_inputs.response_list = [
x.rsplit("_", 1)[0] for x in sorted(list(self.json_dict.keys()))
]