def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['InputData'])
logger.info("Initiating the knockout generation.")
t = ModelTask(model_version_id=args.model_version_id,
db_connection=args.db_connection,
old_covariates_mvid=args.old_covariates_mvid,
debug_mode=args.debug_mode,
cores=args.cores,
step_id=STEP_IDS['InputData'],
make_inputs=False,
make_ko=False)
logger.info("Finished with input creation.")
t.alerts.alert("Done creating inputs.")
logger.info("Updating job metadata parameters.")
with open(t.model_paths.JOB_METADATA, 'r') as json_file:
logger.info("Reading inputs json.")
inputs_info = json.load(json_file)
logger.info(f"{inputs_info}")
inputs_info.update(inspect_all_inputs(t.model_metadata))
with open(t.model_paths.JOB_METADATA, 'w') as outfile:
logger.info("Writing inputs json.")
logger.info(f"{inputs_info}")
json.dump(inputs_info, outfile)
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['OptimalPSI'])
logger.info("Initiating Optimal Psi")
t = OptimalPSI(
model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores,
cutoff=100
)
t.alerts.alert("Beginning the optimal psi process")
t.database_wipe()
t.get_best_psi()
t.calculate_weights()
t.make_temp_ensemble_preds()
t.calculate_in_sample_rmse()
t.calculate_in_sample_trend()
t.determine_submodel_ids()
t.delete_predictions()
t.return_submodel_labels()
t.submodel_pv()
t.write_submodel_covariates()
t.save_outputs()
logger.info(f"Finished finding optimal psi: {t.pickled_outputs['model_pv']['best_psi']}")
t.alerts.alert("Finished with optimal psi")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['ApplySpacetimeSmoothing'])
logger.info("Initiating spacetime smoothing.")
t = ApplySpacetimeSmoothing(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.alerts.alert("Initiating spacetime smoothing")
t.apply_spacetime_smoothing()
t.save_outputs()
t.alerts.alert("Done with spacetime smoothing")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['GPRDraws'])
ne.set_num_threads(1)
logger.info("Initiating GPR draws.")
t = GPRDraws(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.alerts.alert("Creating GPR draws.")
t.make_draws()
t.save_outputs()
t.alerts.alert("Done creating GPR draws.")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['ApplyGPSmoothing'])
logger.info("Initiating GPR smoothing.")
ne.set_num_threads(1)
t = ApplyGPSmoothing(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.alerts.alert("Applying GP Smoothing")
t.apply_gp_smoothing()
t.save_outputs()
t.alerts.alert("Done with GP Smoothing")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=str(STEP_IDS['CovariateSelection']) +
f"_{args.outcome}")
logger.info("Initiating Covariate Selection")
t = CovariateSelection(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores,
outcome=args.outcome)
t.launch_covariate_selection()
logger.info(f"Finished with Covariate Selection for {args.outcome}")
t.alerts.alert(f"Finished with covariate selection for {args.outcome}")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['LinearModelBuilds'])
logger.info("Initiating linear model builds")
t = LinearModelBuilds(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.launch_linear_model_builds()
t.save_response_list()
t.save_outputs()
logger.info("Finished with Linear Model Builds")
t.alerts.alert("Finished with linear model builds")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['Diagnostics'])
logger.info("Initiating model diagnostics.")
t = Diagnostics(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.alerts.alert("Making diagnostics for modeler")
t.get_table_diagnostics()
t.save_tables()
t.get_plot_diagnostics()
t.alerts.alert("Done making diagnostics for modeler")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['GenerateKnockouts'])
logger.info("Initiating the knockout generation.")
t = ModelTask(model_version_id=args.model_version_id,
db_connection=args.db_connection,
old_covariates_mvid=args.old_covariates_mvid,
debug_mode=args.debug_mode,
cores=args.cores,
step_id=STEP_IDS['GenerateKnockouts'],
make_inputs=False,
make_ko=True)
t.alerts.alert("Done creating knockouts.")
logger.info("Finished with knockout creation.")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['WriteResults'])
logger.info("Initiating writing results.")
t = WriteResults(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores)
t.database_wipe()
t.read_draws()
t.aggregate_draws()
t.get_full_envelope()
t.write_submodel_means()
t.write_model_mean()
t.create_global_table()
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['Email'])
logger.info("Initiating modeler email.")
t = Email(
model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores
)
t.alerts.alert("Creating the email to send to modelers")
t.create_email_body()
t.create_email_text()
t.send_email()
t.alerts.alert("Check your email! :)")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['ReadSpacetimeModels'])
logger.info("Initiating reading spacetime models.")
t = ReadModels(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores,
step_id=STEP_IDS['ReadSpacetimeModels'])
t.alerts.alert("Reading spacetime models")
t.read_models(input_json_path=t.model_paths.JSON_FILES['spacetime'],
output_object_name='st_models_linear')
t.pickled_outputs['st_models_linear'].delete_mixed_model_parameters()
t.save_outputs()
t.alerts.alert("Done reading spacetime models")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['LinearPV'])
logger.info("Initiating Linear Model PV.")
t = PV(model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores,
step_id=STEP_IDS['LinearPV'])
t.alerts.alert(
"Calculating submodel predictive validity for linear models.")
t.calculate_submodel_pv(input_object_name='linear_models_linear',
output_object_prefix='linear_models')
t.save_outputs()
t.alerts.alert(
"Done calculating submodel predictive validity for linear models.")
def main():
args = get_step_args()
setup_logging(model_version_id=args.model_version_id,
step_id=STEP_IDS['EnsemblePredictions'])
ne.set_num_threads(1)
logger.info("Initiating calculate coverage.")
t = EnsemblePredictions(
model_version_id=args.model_version_id,
db_connection=args.db_connection,
debug_mode=args.debug_mode,
old_covariates_mvid=args.old_covariates_mvid,
cores=args.cores
)
t.alerts.alert("Ensemble-ing all the predictions!")
t.calculate_oos_coverage()
t.calculate_is_coverage()
t.write_pv()
t.add_draws_to_df()
t.write_draws()
t.save_outputs()
t.alerts.alert("Done with ensemble-ing the predictions.")
