def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_COD, args)
logging.info(f"Summarizing location_id: {args.location_id}, "
f"and year_id: {args.year_id}.")
codcorrect = master.CoDCorrectParameters.recreate_from_version_id(
args.version_id)
summarize_cod(args.parent_dir, args.gbd_round_id, args.location_id,
args.year_id, codcorrect)
logging.info("Summarization completed.")
def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_PCT_CHANGE, args)
logging.info(
f"Summarizing machine_process: {args.machine_process}, location_id: "
f"{args.location_id}, measure_id: {args.measure_id}, for year_start_id: "
f"{args.year_start_id} to year_end_id {args.year_end_id}.")
summarize_pct_change(args.machine_process, args.parent_dir,
args.gbd_round_id, args.location_id, args.measure_id,
args.year_start_id, args.year_end_id)
logging.info("Summarization completed.")
def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Type.CAUSE_AGG, args)
codcorrect = master.CoDCorrectParameters.recreate_from_version_id(
args.version_id)
logging.info(
f"Aggregating causes for location_id: {args.location_id}, sex_id: "
f"{args.sex_id}")
ac = AggregateCauses(args.parent_dir, args.location_id, args.sex_id,
codcorrect)
ac.run()
logging.info("Cause aggregation completed.")
def main() -> None:
args = parse_arguments()
fauxcorrect = master.FauxCorrectParameters.recreate_from_version_id(
args.version_id
)
setup_logging(args.parent_dir, DAG.Tasks.Type.SCALE, args)
logging.info(
f"Applying CoDCorrect scalars to location_id: {args.location_id} and "
f"sex_id: {args.sex_id} using CoDCorrect version: "
f"{args.scalar_version_id}."
)
apply_scalars(
args.parent_dir, args.location_id, args.sex_id, fauxcorrect.year_ids,
args.scalar_version_id
)
logging.info("Applying scalars completed.")
def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Type.CORRECT, args)
if args.action == Action.CACHE:
logging.info("Caching spacetime restrictions")
cache_spacetime_restrictions(args.parent_dir, args.gbd_round_id)
logging.info("Completed.")
elif args.action == Action.CORRECT:
codcorrect = master.CoDCorrectParameters.recreate_from_version_id(
args.version_id)
logging.info(
f"Applying deaths correction to location_id: {args.location_id} "
f"and sex_id: {args.sex_id} using all-cause mortality envelope "
f"run_id: {args.env_version_id}.")
apply_correction(args.parent_dir, args.location_id, args.sex_id,
codcorrect)
logging.info("Deaths correction completed.")
def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Name.SUMMARIZE_GBD, args)
logging.info(
f"Summarizing machine_process: {args.machine_process}, location_id: "
f"{args.location_id}, measure_id: {args.measure_id}, and year_id: "
f"{args.year_id}."
)
summarize(
args.machine_process,
args.parent_dir,
args.gbd_round_id,
args.location_id,
args.measure_id,
args.year_id
)
logging.info("Summarization completed.")
def main():
args = parse_args()
if args.resume and not args.version_id:
raise RuntimeError(
"Cannot resume a run without a fauxcorrect version id.")
if not args.resume:
fauxcorrect = FauxCorrectParameters.new(
year_ids=args.year_ids,
location_set_ids=args.location_set_ids,
measure_ids=args.measure_ids,
gbd_round_id=gbd.GBD_ROUND_ID,
decomp_step=args.decomp_step,
process=constants.GBD.Process.Name.FAUXCORRECT,
databases=args.databases)
setup_logging(fauxcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH,
args)
logging.info(
f"Beginning Fauxcorrect version {fauxcorrect.version_id} run.\n"
f"Arguments: decomp_step: {args.decomp_step}, location_set_ids: "
f"{args.location_set_ids}, year_ids: {args.year_ids}, measure_ids: "
f"{args.measure_ids}. Uploading to the following databases: "
f"{args.databases}.")
logging.info("Generating filesystem assets.")
fauxcorrect.generate_project_directories()
logging.info("Caching fauxcorrect parameters.")
fauxcorrect.cache_parameters()
logging.info("Validating models")
fauxcorrect.validate_model_versions(constants.CauseSetId.FAUXCORRECT)
else:
fauxcorrect = FauxCorrectParameters.recreate_from_version_id(
args.version_id)
setup_logging(fauxcorrect.parent_dir, constants.DAG.Tasks.Type.LAUNCH,
args)
logging.info(
f"Resuming FauxCorrect version {fauxcorrect.version_id} run.")
swarm = FauxCorrectSwarm(parameters=fauxcorrect, resume=args.resume)
logging.info("Constructing workflow.")
swarm.construct_workflow()
logging.info("Running workflow.")
swarm.run()
def main() -> None:
args = parse_arguments()
if args.machine_process == constants.GBD.Process.Name.CODCORRECT:
params = CoDCorrectParameters.recreate_from_version_id(
version_id=args.version_id
)
elif args.machine_process == constants.GBD.Process.Name.FAUXCORRECT:
params = FauxCorrectParameters.recreate_from_version_id(
version_id=args.version_id
)
else:
raise ValueError(
f'--machine_process argument must be '
f'"{constants.GBD.Process.Name.CODCORRECT}" or '
f'"{constants.GBD.Process.Name.FAUXCORRECT}". '
f'Recieved "{args.machine_process}".'
)
setup_logging(
params.parent_dir,
constants.DAG.Tasks.Type.VALIDATE,
args
)
logging.info(
f"Validating input draws for model_version_id {args.model_version_id}."
)
logging.info("Reading input draws.")
draws = read_input_draws(
params,
model_version_id=args.model_version_id
)
logging.info("Validating input draws.")
draws = validate_draws(
params,
args.model_version_id,
draws
)
logging.info("Saving validated draws to filesystem.")
save_validated_draws(
params,
args.model_version_id,
draws
)
logging.info("Validation complete.")
def main() -> None:
args = parse_arguments()
setup_logging(args.parent_dir, DAG.Tasks.Type.LOC_AGG, args)
validate_arguments(args.action, args.measure_id, args.location_set_id,
args.aggregation_type, args.year_id)
if args.action == Action.CACHE:
logging.info("Caching regional scalars.")
cache_regional_scalars(args.parent_dir, args.gbd_round_id)
logging.info("Completed.")
elif args.action == Action.LOC_AGG:
logging.info(f"Beginning location aggregation for aggregation_type: "
f"{args.aggregation_type}, location_set_id: "
f"{args.location_set_id}, measure_id: {args.measure_id}, "
f"and year_id: {args.year_id}.")
aggregate_locations(args.aggregation_type, args.parent_dir,
args.measure_id, args.gbd_round_id,
args.location_set_id, args.year_id)
logging.info("Location aggregation completed.")
def main() -> None:
args = parse_arguments()
setup_logging(
args.parent_dir,
constants.DAG.Tasks.Type.APPEND,
args
)
logging.info(
f"Appending shocks for location_id: {args.location_id}, "
f"sex_id: {args.sex_id}, machine_process: {args.machine_process}, "
f"measure_ids: {args.measure_ids}."
)
append_shocks(
args.parent_dir,
args.machine_process,
args.measure_ids,
args.location_id,
args.most_detailed_location,
args.sex_id
)
logging.info("Appending shocks completed.")
def main() -> None:
args = parse_arguments()
if args.machine_process == constants.GBD.Process.Name.CODCORRECT:
version = CoDCorrectParameters.recreate_from_version_id(
version_id=args.version_id)
elif args.machine_process == constants.GBD.Process.Name.FAUXCORRECT:
version = FauxCorrectParameters.recreate_from_version_id(
version_id=args.version_id)
else:
raise ValueError(f'--machine_process argument must be '
f'"{constants.GBD.Process.Name.CODCORRECT}" or '
f'"{constants.GBD.Process.Name.FAUXCORRECT}". '
f'Recieved "{args.machine_process}".')
setup_logging(version.parent_dir, constants.DAG.Tasks.Type.CACHE, args)
logging.info("Validating input arguments")
mortality_inputs.validate_mortality_process_argument(args.mort_process)
logging.info(f"Caching {args.mort_process}.")
mortality_inputs.cache_data(args.mort_process, version=version)
logging.info("Completed.")
def main() -> None:
args: argparse.Namespace = parse_arguments()
version: MachineParameters = MachineParameters.recreate_from_version_id(
version_id=args.version_id, process_name=args.machine_process
)
setup_logging(
version.parent_dir,
constants.DAG.Tasks.Type.UPLOAD,
args
)
logging.info(
f"Beginning upload for {args.machine_process} to {args.database}"
f"database for measure {args.measure_id}, upload_type "
f"{args.upload_type}."
)
upload_summaries(
database=args.database,
version=version,
measure_id=args.measure_id,
upload_type=args.upload_type
)
logging.info("Completed.")
def main() -> None:
args = parse_arguments()
setup_logging(
args.parent_dir,
constants.DAG.Tasks.Type.VALIDATE,
args
)
validate_arguments(
args.action, args.location_id, args.sex_id, args.machine_process
)
if args.action == Action.CACHE:
cache_pred_ex(args.parent_dir, args.gbd_round_id)
elif args.action == Action.CALC:
logging.info(
f"Calculating YLLs for location_id: {args.location_id}, "
f"sex_id: {args.sex_id}, machine_process: {args.machine_process}."
)
calculate_ylls(
args.machine_process,
args.parent_dir,
args.location_id,
args.sex_id
)
logging.info("Calculating YLLs completed.")
return diag
def format_for_upload_and_save(df: pd.DataFrame, version: MachineParameters):
# Format for upload
df['output_version_id'] = version.version_id
df = df[Diagnostics.DataBase.COLUMNS]
# Save
logging.info("Saving single diagnostic file")
file_path = os.path.join(version.parent_dir, FilePaths.DIAGNOSTICS_DIR,
FilePaths.DIAGNOSTICS_UPLOAD_FILE)
df.to_csv(file_path, index=False)
permissions_change = ['chmod', '775', file_path]
subprocess.check_output(permissions_change)
logging.info('All done!')
if __name__ == '__main__':
args = parse_args()
version = CoDCorrectParameters.recreate_from_version_id(
version_id=args.version_id)
setup_logging(version.parent_dir, DAG.Tasks.Type.DIAGNOSTIC, args)
df = compile_and_compute_diagnostics(version)
format_for_upload_and_save(df, version)
def main():
args = parse_args()
if args.resume and not args.version_id:
raise RuntimeError(
"Cannot resume a run without a CoDCcorrect version id."
)
validate_pct_change_years(args.year_start_ids, args.year_end_ids)
if not args.resume:
codcorrect = CoDCorrectParameters.new(
year_ids=args.year_ids,
year_start_ids=args.year_start_ids,
year_end_ids=args.year_end_ids,
location_set_ids=args.location_set_ids,
measure_ids=args.measure_ids,
gbd_round_id=args.gbd_round_id,
decomp_step=args.decomp_step,
process=constants.GBD.Process.Name.CODCORRECT,
databases=args.databases
)
setup_logging(
codcorrect.parent_dir,
constants.DAG.Tasks.Type.LAUNCH,
args
)
logging.info(
f"Beginning CoDCcorrect version {codcorrect.version_id} run.\n"
f"Arguments: decomp_step: {args.decomp_step}, location_set_ids: "
f"{args.location_set_ids}, year_ids: {args.year_ids}, measure_ids: "
f"{args.measure_ids}. Uploading to the following databases: "
f"{args.databases}."
)
logging.info("Generating filesystem assets.")
codcorrect.generate_project_directories()
logging.info("Caching CoDCorrect parameters.")
codcorrect.cache_parameters()
logging.info("Validating models")
codcorrect.validate_model_versions(constants.CauseSetId.COMPUTATION)
codcorrect.create_input_model_tracking_report()
else:
codcorrect = CoDCorrectParameters.recreate_from_version_id(
args.version_id
)
setup_logging(
codcorrect.parent_dir,
constants.DAG.Tasks.Type.LAUNCH,
args
)
logging.info(
f"Resuming CoDCorrect version {codcorrect.version_id} run."
)
swarm = CoDCorrectSwarm(parameters=codcorrect, resume=args.resume)
logging.info("Constructing workflow.")
swarm.construct_workflow()
logging.info("Running workflow.")
swarm.run()