def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_COD, args) logging.info(f"Summarizing location_id: {args.location_id}, " f"and year_id: {args.year_id}.") codcorrect = master.CoDCorrectParameters.recreate_from_version_id( args.version_id) summarize_cod(args.parent_dir, args.gbd_round_id, args.location_id, args.year_id, codcorrect) logging.info("Summarization completed.")
def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_PCT_CHANGE, args) logging.info( f"Summarizing machine_process: {args.machine_process}, location_id: " f"{args.location_id}, measure_id: {args.measure_id}, for year_start_id: " f"{args.year_start_id} to year_end_id {args.year_end_id}.") summarize_pct_change(args.machine_process, args.parent_dir, args.gbd_round_id, args.location_id, args.measure_id, args.year_start_id, args.year_end_id) logging.info("Summarization completed.")
def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Type.CAUSE_AGG, args) codcorrect = master.CoDCorrectParameters.recreate_from_version_id( args.version_id) logging.info( f"Aggregating causes for location_id: {args.location_id}, sex_id: " f"{args.sex_id}") ac = AggregateCauses(args.parent_dir, args.location_id, args.sex_id, codcorrect) ac.run() logging.info("Cause aggregation completed.")
def main() -> None: args = parse_arguments() fauxcorrect = master.FauxCorrectParameters.recreate_from_version_id( args.version_id ) setup_logging(args.parent_dir, DAG.Tasks.Type.SCALE, args) logging.info( f"Applying CoDCorrect scalars to location_id: {args.location_id} and " f"sex_id: {args.sex_id} using CoDCorrect version: " f"{args.scalar_version_id}." ) apply_scalars( args.parent_dir, args.location_id, args.sex_id, fauxcorrect.year_ids, args.scalar_version_id ) logging.info("Applying scalars completed.")
def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Type.CORRECT, args) if args.action == Action.CACHE: logging.info("Caching spacetime restrictions") cache_spacetime_restrictions(args.parent_dir, args.gbd_round_id) logging.info("Completed.") elif args.action == Action.CORRECT: codcorrect = master.CoDCorrectParameters.recreate_from_version_id( args.version_id) logging.info( f"Applying deaths correction to location_id: {args.location_id} " f"and sex_id: {args.sex_id} using all-cause mortality envelope " f"run_id: {args.env_version_id}.") apply_correction(args.parent_dir, args.location_id, args.sex_id, codcorrect) logging.info("Deaths correction completed.")
def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_GBD, args) logging.info( f"Summarizing machine_process: {args.machine_process}, location_id: " f"{args.location_id}, measure_id: {args.measure_id}, and year_id: " f"{args.year_id}." ) summarize( args.machine_process, args.parent_dir, args.gbd_round_id, args.location_id, args.measure_id, args.year_id ) logging.info("Summarization completed.")
def main(): args = parse_args() if args.resume and not args.version_id: raise RuntimeError( "Cannot resume a run without a fauxcorrect version id.") if not args.resume: fauxcorrect = FauxCorrectParameters.new( year_ids=args.year_ids, location_set_ids=args.location_set_ids, measure_ids=args.measure_ids, gbd_round_id=gbd.GBD_ROUND_ID, decomp_step=args.decomp_step, process=constants.GBD.Process.Name.FAUXCORRECT, databases=args.databases) setup_logging(fauxcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH, args) logging.info( f"Beginning Fauxcorrect version {fauxcorrect.version_id} run.\n" f"Arguments: decomp_step: {args.decomp_step}, location_set_ids: " f"{args.location_set_ids}, year_ids: {args.year_ids}, measure_ids: " f"{args.measure_ids}. Uploading to the following databases: " f"{args.databases}.") logging.info("Generating filesystem assets.") fauxcorrect.generate_project_directories() logging.info("Caching fauxcorrect parameters.") fauxcorrect.cache_parameters() logging.info("Validating models") fauxcorrect.validate_model_versions(constants.CauseSetId.FAUXCORRECT) else: fauxcorrect = FauxCorrectParameters.recreate_from_version_id( args.version_id) setup_logging(fauxcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH, args) logging.info( f"Resuming FauxCorrect version {fauxcorrect.version_id} run.") swarm = FauxCorrectSwarm(parameters=fauxcorrect, resume=args.resume) logging.info("Constructing workflow.") swarm.construct_workflow() logging.info("Running workflow.") swarm.run()
def main() -> None: args = parse_arguments() if args.machine_process == constants.GBD.Process.Name.CODCORRECT: params = CoDCorrectParameters.recreate_from_version_id( version_id=args.version_id ) elif args.machine_process == constants.GBD.Process.Name.FAUXCORRECT: params = FauxCorrectParameters.recreate_from_version_id( version_id=args.version_id ) else: raise ValueError( f'--machine_process argument must be ' f'"{constants.GBD.Process.Name.CODCORRECT}" or ' f'"{constants.GBD.Process.Name.FAUXCORRECT}". ' f'Recieved "{args.machine_process}".' ) setup_logging( params.parent_dir, constants.DAG.Tasks.Type.VALIDATE, args ) logging.info( f"Validating input draws for model_version_id {args.model_version_id}." ) logging.info("Reading input draws.") draws = read_input_draws( params, model_version_id=args.model_version_id ) logging.info("Validating input draws.") draws = validate_draws( params, args.model_version_id, draws ) logging.info("Saving validated draws to filesystem.") save_validated_draws( params, args.model_version_id, draws ) logging.info("Validation complete.")
def main() -> None: args = parse_arguments() setup_logging(args.parent_dir, DAG.Tasks.Type.LOC_AGG, args) validate_arguments(args.action, args.measure_id, args.location_set_id, args.aggregation_type, args.year_id) if args.action == Action.CACHE: logging.info("Caching regional scalars.") cache_regional_scalars(args.parent_dir, args.gbd_round_id) logging.info("Completed.") elif args.action == Action.LOC_AGG: logging.info(f"Beginning location aggregation for aggregation_type: " f"{args.aggregation_type}, location_set_id: " f"{args.location_set_id}, measure_id: {args.measure_id}, " f"and year_id: {args.year_id}.") aggregate_locations(args.aggregation_type, args.parent_dir, args.measure_id, args.gbd_round_id, args.location_set_id, args.year_id) logging.info("Location aggregation completed.")
def main() -> None: args = parse_arguments() setup_logging( args.parent_dir, constants.DAG.Tasks.Type.APPEND, args ) logging.info( f"Appending shocks for location_id: {args.location_id}, " f"sex_id: {args.sex_id}, machine_process: {args.machine_process}, " f"measure_ids: {args.measure_ids}." ) append_shocks( args.parent_dir, args.machine_process, args.measure_ids, args.location_id, args.most_detailed_location, args.sex_id ) logging.info("Appending shocks completed.")
def main() -> None: args = parse_arguments() if args.machine_process == constants.GBD.Process.Name.CODCORRECT: version = CoDCorrectParameters.recreate_from_version_id( version_id=args.version_id) elif args.machine_process == constants.GBD.Process.Name.FAUXCORRECT: version = FauxCorrectParameters.recreate_from_version_id( version_id=args.version_id) else: raise ValueError(f'--machine_process argument must be ' f'"{constants.GBD.Process.Name.CODCORRECT}" or ' f'"{constants.GBD.Process.Name.FAUXCORRECT}". ' f'Recieved "{args.machine_process}".') setup_logging(version.parent_dir, constants.DAG.Tasks.Type.CACHE, args) logging.info("Validating input arguments") mortality_inputs.validate_mortality_process_argument(args.mort_process) logging.info(f"Caching {args.mort_process}.") mortality_inputs.cache_data(args.mort_process, version=version) logging.info("Completed.")
def main() -> None: args: argparse.Namespace = parse_arguments() version: MachineParameters = MachineParameters.recreate_from_version_id( version_id=args.version_id, process_name=args.machine_process ) setup_logging( version.parent_dir, constants.DAG.Tasks.Type.UPLOAD, args ) logging.info( f"Beginning upload for {args.machine_process} to {args.database}" f"database for measure {args.measure_id}, upload_type " f"{args.upload_type}." ) upload_summaries( database=args.database, version=version, measure_id=args.measure_id, upload_type=args.upload_type ) logging.info("Completed.")
def main() -> None: args = parse_arguments() setup_logging( args.parent_dir, constants.DAG.Tasks.Type.VALIDATE, args ) validate_arguments( args.action, args.location_id, args.sex_id, args.machine_process ) if args.action == Action.CACHE: cache_pred_ex(args.parent_dir, args.gbd_round_id) elif args.action == Action.CALC: logging.info( f"Calculating YLLs for location_id: {args.location_id}, " f"sex_id: {args.sex_id}, machine_process: {args.machine_process}." ) calculate_ylls( args.machine_process, args.parent_dir, args.location_id, args.sex_id ) logging.info("Calculating YLLs completed.")
return diag def format_for_upload_and_save(df: pd.DataFrame, version: MachineParameters): # Format for upload df['output_version_id'] = version.version_id df = df[Diagnostics.DataBase.COLUMNS] # Save logging.info("Saving single diagnostic file") file_path = os.path.join(version.parent_dir, FilePaths.DIAGNOSTICS_DIR, FilePaths.DIAGNOSTICS_UPLOAD_FILE) df.to_csv(file_path, index=False) permissions_change = ['chmod', '775', file_path] subprocess.check_output(permissions_change) logging.info('All done!') if __name__ == '__main__': args = parse_args() version = CoDCorrectParameters.recreate_from_version_id( version_id=args.version_id) setup_logging(version.parent_dir, DAG.Tasks.Type.DIAGNOSTIC, args) df = compile_and_compute_diagnostics(version) format_for_upload_and_save(df, version)
def main(): args = parse_args() if args.resume and not args.version_id: raise RuntimeError( "Cannot resume a run without a CoDCcorrect version id." ) validate_pct_change_years(args.year_start_ids, args.year_end_ids) if not args.resume: codcorrect = CoDCorrectParameters.new( year_ids=args.year_ids, year_start_ids=args.year_start_ids, year_end_ids=args.year_end_ids, location_set_ids=args.location_set_ids, measure_ids=args.measure_ids, gbd_round_id=args.gbd_round_id, decomp_step=args.decomp_step, process=constants.GBD.Process.Name.CODCORRECT, databases=args.databases ) setup_logging( codcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH, args ) logging.info( f"Beginning CoDCcorrect version {codcorrect.version_id} run.\n" f"Arguments: decomp_step: {args.decomp_step}, location_set_ids: " f"{args.location_set_ids}, year_ids: {args.year_ids}, measure_ids: " f"{args.measure_ids}. Uploading to the following databases: " f"{args.databases}." ) logging.info("Generating filesystem assets.") codcorrect.generate_project_directories() logging.info("Caching CoDCorrect parameters.") codcorrect.cache_parameters() logging.info("Validating models") codcorrect.validate_model_versions(constants.CauseSetId.COMPUTATION) codcorrect.create_input_model_tracking_report() else: codcorrect = CoDCorrectParameters.recreate_from_version_id( args.version_id ) setup_logging( codcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH, args ) logging.info( f"Resuming CoDCorrect version {codcorrect.version_id} run." ) swarm = CoDCorrectSwarm(parameters=codcorrect, resume=args.resume) logging.info("Constructing workflow.") swarm.construct_workflow() logging.info("Running workflow.") swarm.run()