def construct_args_burdenator_cleanup(parser, cli_args=None):
"""Creates arguments from parser for rearranging the draw files at the end
of the burdenator run"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.tool_name = 'burdenator'
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
# Create log directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log_cleanup', str(args.year_id),
str(args.measure_id))
log_filename = "{}_{}_{}.log".format(args.measure_id, args.location_id,
args.year_id)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory("dalynator", log_level,
args.log_dir + "/" + log_filename)
# Get cod/epi env directories
if args.cod_version is None:
args.cod_version = ac.best_version('cod', args.gbd_round_id)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id)
args.cod_dir = (args.input_data_root + "/codcorrect/" +
str(args.cod_version) + "/draws")
args.epi_dir = get_como_folder_structure(
os.path.join(args.input_data_root, 'como', str(args.epi_version)))
return args
def get_args_pct_change(parser, cli_args=None):
"""Creates arguments from parser for pct change calculation"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
args.log_dir = os.path.join(args.out_dir, 'log_pct_change',
str(args.location_id))
makedirs_safely(args.log_dir)
logfn = "FILEPATH".format(args.start_year, args.end_year)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory("dalynator", log_level,
"FILEPATH".format(args.log_dir, logfn),
['aggregator.aggregators', 'jobmon'])
# Get cod/epi env directories
if args.codcorrect_version == 'best':
args.codcorrect_version = ac.best_version(
'codcorrect', args.gbd_round_id, args.decomp_step)
if args.fauxcorrect_version == 'best':
args.fauxcorrect_version = ac.best_version(
'fauxcorrect', args.gbd_round_id, args.decomp_step)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id,
args.decomp_step)
args.epi_dir = get_como_folder_structure(os.path.join(
args.input_data_root, 'como', str(args.epi_version)))
cod_object = to.cod_or_faux_correct(
args.input_data_root,
codcorrect_version=args.codcorrect_version,
fauxcorrect_version=args.fauxcorrect_version)
args.cod_dir = cod_object.abs_path_to_draws
args.cod_pattern = cod_object.file_pattern
return args
def from_parser(cls, cli_args=None):
"""
Factory method to create a DalynatorMostDetailed instance from an
argument string.
Uses the argparse package.
Args:
cli_args: If none then reads from sysargs
Returns:
"""
parser = get_input_args.construct_parser_dalynator()
args = parser.parse_args(cli_args)
if args.codcorrect_version == 'best':
args.codcorrect_version = best_version('codcorrect',
args.gbd_round_id,
args.decomp_step)
if args.fauxcorrect_version == 'best':
args.fauxcorrect_version = best_version('fauxcorrect',
args.gbd_round_id,
args.decomp_step)
if args.epi_version is None:
args.epi_version = best_version('como', args.gbd_round_id,
args.decomp_step)
args.output_draws_dir, args.log_dir, args.cache_dir, args.cod_dir, \
args.cod_pattern, args.epi_dir, args.output_file_name = \
DalynatorMostDetailed._construct_extra_paths(
input_data_root=args.input_data_root,
out_dir=args.out_dir,
codcorrect_version=args.codcorrect_version,
fauxcorrect_version=args.fauxcorrect_version,
epi_version=args.epi_version,
location_id=args.location_id,
year_id=args.year_id)
return cls(
output_draws_dir=args.output_draws_dir,
output_file_name=args.output_file_name,
log_dir=args.log_dir,
cache_dir=args.cache_dir,
cod_dir=args.cod_dir,
cod_pattern=args.cod_pattern,
epi_dir=args.epi_dir,
gbd_round_id=args.gbd_round_id,
decomp_step=args.decomp_step,
location_id=args.location_id,
year_id=args.year_id,
n_draws=args.n_draws,
output_version=args.output_version,
turn_off_null_and_nan_check=args.turn_off_null_and_nan_check,
verbose=args.verbose,
dual_upload=args.dual_upload)
def get_args_and_create_dirs(parser, cli_args=None):
"""Parses the command line using the parser and creates output directory
and logger. Called by run_pipeline_*. Not used by run_all."""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
# resolve defaults for cod and epi versions
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
if args.cod_version is None:
args.cod_version = ac.best_version('cod', args.gbd_round_id)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id)
# Store all years for each location in one directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log', str(args.location_id))
args.out_dir = os.path.join(top_out_dir, 'draws', str(args.location_id))
makedirs_safely(args.out_dir)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory(
"dalynator", log_level, args.log_dir +
"/daly_{}_{}.log".format(args.location_id, args.year_id))
args.cod_dir = "{}/codcorrect/{}/draws/".format(args.input_data_root,
args.cod_version)
if hasattr(args, 'daly_version'):
args.daly_dir = "{}/dalynator/{}/draws/".format(
args.input_data_root, args.daly_version)
else:
args.daly_dir = None
args.epi_dir = get_como_folder_structure(
os.path.join(args.input_data_root, 'como', str(args.epi_version)))
if hasattr(args, 'paf_version'):
args.paf_dir = "{}/pafs/{}".format(args.input_data_root,
args.paf_version)
else:
args.paf_dir = None
return args
def construct_args_burdenator_cleanup(parser, cli_args=None):
"""Creates arguments from parser for rearranging the draw files at the end
of the burdenator run"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.tool_name = 'burdenator'
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
# Create log directory
top_out_dir = args.out_dir
args.cache_dir = 'FILEPATH'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log_cleanup',
str(args.year_id), str(args.measure_id))
log_filename = "FILEPATH".format(
args.measure_id, args.location_id, args.year_id)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory(
"dalynator", log_level, args.log_dir + "/" + log_filename,
['aggregator.aggregators', 'jobmon'])
# Get cod/epi env directories
if args.codcorrect_version == 'best':
args.codcorrect_version = ac.best_version(
'codcorrect', args.gbd_round_id, args.decomp_step)
if args.fauxcorrect_version == 'best':
args.fauxcorrect_version = ac.best_version(
'fauxcorrect', args.gbd_round_id, args.decomp_step)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id,
args.decomp_step)
args.epi_dir = get_como_folder_structure(os.path.join(
args.input_data_root, 'como', str(args.epi_version)))
cod_object = to.cod_or_faux_correct(
args.input_data_root,
codcorrect_version=args.codcorrect_version,
fauxcorrect_version=args.fauxcorrect_version)
args.cod_dir = cod_object.abs_path_to_draws
args.cod_pattern = cod_object.file_pattern
return args
def parse(self, tool_name="dalynator", cli_args=None):
"""
Perform ALL argument parsing - argparse parsing, setting of defaults,
checking of argument consistency. NO side effects in the file system.
Args:
tool_name: 'dalynator' or 'burdenator'
cli_args: None or a list of strings. If None then read use sys.argv
"""
self._construct_base_parser(tool_name)
args = self.parser.parse_known_args(cli_args)
if args[0].resume:
print("Nator run in resume mode. Reading args from file")
args = self.load_args_from_file(args, tool_name)
args = self.set_phase_defaults(args)
return args
self._add_non_resume_args_to_parser()
args = self.parser.parse_args(cli_args)
args = self.set_phase_defaults(args)
args.tool_name = tool_name
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
if args.cod_version is None:
args.cod_version = ac.best_version('cod', args.gbd_round_id)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id)
args.location_set_version_id = ac.location_set_to_location_set_version(
args.location_set_id, args.location_set_version_id, args.gbd_round)
args.sge_project = ac.create_sge_project(args.sge_project,
args.tool_name)
year_n_draws_map = ac.construct_year_n_draws_map(
args.year_ids_1, args.n_draws_1, args.year_ids_2, args.n_draws_2)
ac.validate_multi_mode_years(year_n_draws_map, args.year_ids_1,
args.n_draws_1, args.year_ids_2,
args.n_draws_2, args.start_year_ids,
args.end_year_ids)
args.out_dir, args.log_dir, args.cache_dir = (
self._construct_extra_paths(args.out_dir_without_version,
args.log_dir, args.tool_name,
args.version))
return args
def from_parser(cls, cli_args=None):
"""
Factory method to create a DalynatorMostDetailed instance from an
argument string.
Uses the argparse package.
Args:
cli_args: If none then reads from sysargs
Returns:
"""
parser = DalynatorMostDetailed._construct_parser()
args = parser.parse_args(cli_args)
if args.cod_version is None:
args.cod_version = best_version('cod', args.gbd_round_id)
if args.epi_version is None:
args.epi_version = best_version('como', args.gbd_round_id)
args.output_draws_dir, args.log_dir, args.cache_dir, args.cod_dir, \
args.epi_dir, args.output_file_name = \
DalynatorMostDetailed._construct_extra_paths(
input_data_root=args.input_data_root,
out_dir=args.out_dir,
cod_version=args.cod_version,
epi_version=args.epi_version,
location_id=args.location_id,
year_id=args.year_id)
return cls(
output_draws_dir=args.output_draws_dir,
output_file_name=args.output_file_name,
log_dir=args.log_dir,
cache_dir=args.cache_dir,
cod_dir=args.cod_dir,
epi_dir=args.epi_dir,
gbd_round_id=args.gbd_round_id,
location_id=args.location_id,
year_id=args.year_id,
n_draws=args.n_draws,
version=args.version,
no_sex_aggr=args.no_sex_aggr,
no_age_aggr=args.no_age_aggr,
turn_off_null_and_nan_check=args.turn_off_null_and_nan_check,
verbose=args.verbose)
def parse(self, tool_name="burdenator", cli_args=None):
parser = get_input_args.construct_parser_run_all_tool(tool_name)
args = parser.parse_known_args(cli_args)
if args[0].resume:
print("Nator run in resume mode. Reading args from file")
args = get_input_args.load_args_from_file(args, tool_name)
args = get_input_args.set_phase_defaults(args)
return args
parser = get_input_args.add_non_resume_args_burdenator(
parser, tool_name)
args = parser.parse_args(cli_args)
args = get_input_args.set_phase_defaults(args)
args.tool_name = tool_name
if args.codcorrect_version == 'best':
args.codcorrect_version = ac.best_version('codcorrect',
args.gbd_round_id,
args.decomp_step)
if args.fauxcorrect_version == 'best':
args.fauxcorrect_version = ac.best_version('fauxcorrect',
args.gbd_round_id,
args.decomp_step)
if not args.epi_version:
args.epi_version = ac.best_version('como', args.gbd_round_id,
args.decomp_step)
args.sge_project = ac.create_sge_project(args.sge_project,
args.tool_name)
# checking to make sure that mixed_draws contains the right info
if args.mixed_draw_years is None:
n_draws_years_dict = {args.n_draws: args.years}
else:
ac.check_mixed_draw_years_format(args.mixed_draw_years)
n_draws_years_dict = args.mixed_draw_years
year_n_draws_map = ac.construct_year_n_draws_map(n_draws_years_dict)
ac.validate_multi_mode_years(year_n_draws_map, n_draws_years_dict,
args.start_year_ids, args.end_year_ids)
args.out_dir, args.log_dir, args.cache_dir = (
get_input_args.construct_extra_paths(args.out_dir_without_version,
args.log_dir, args.tool_name,
args.output_version))
if args.measures == 'daly' or args.measures == 'DALY':
raise ValueError("must expand measures to include yll, yld, "
"or daly")
if 'yld' in args.measures or 'YLD' in args.measures:
if not args.epi_version:
raise ValueError("An epi_version is needed if yld's are being "
"burdenated")
if not args.location_set_ids:
raise ValueError("To aggregate risk-attributable burden "
"up the location hierarchy, "
"--location_set_ids or must be provided")
if args.dual_upload:
raise NotImplementedError(
"'--dual_upload' flag is not valid at this time. ")
return args
def get_args_and_create_dirs(parser, cli_args=None):
"""Parses the command line using the parser and creates output directory
and logger. Called by run_pipeline_*. Not used by run_all.
"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
cod_object = to.cod_or_faux_correct(args.input_data_root,
args.codcorrect_version,
args.fauxcorrect_version)
# resolve defaults for cod and epi versions
if args.codcorrect_version == 'best':
args.codcorrect_version = ac.best_version(
'codcorrect', args.gbd_round_id, args.decomp_step)
if args.fauxcorrect_version == 'best':
args.fauxcorrect_version = ac.best_version(
'fauxcorrect', args.gbd_round_id, args.decomp_step)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id,
args.decomp_step)
# Store all years for each location in one directory
top_out_dir = args.out_dir
args.cache_dir = 'FILEPATH'.format(args.out_dir)
makedirs_safely(os.path.join(top_out_dir, 'log_most_detailed'))
args.log_dir = os.path.join(top_out_dir, 'log_most_detailed',
str(args.location_id))
args.out_dir = os.path.join(top_out_dir, 'draws', str(args.location_id))
makedirs_safely(args.out_dir)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory(
"dalynator", log_level,
args.log_dir + "FILEPATH".format(args.location_id,
args.year_id),
['aggregator.aggregators', 'jobmon'])
args.cod_dir = cod_object.abs_path_to_draws
args.cod_pattern = cod_object.file_pattern
# this had daly_version before but I think we still want this
# differentiated file path
if hasattr(args, 'tool_name') and args.tool_name == "dalynator":
args.daly_dir = "FILEPATH".format(args.input_data_root,
args.output_version)
else:
args.daly_dir = None
# our customers want the flag to be named "epi" not como"
args.epi_dir = get_como_folder_structure(os.path.join(
args.input_data_root, 'como', str(args.epi_version)))
if hasattr(args, 'paf_version'):
# PAF directory structure has no "draws" sub-folder
args.paf_dir = "FILEPATH".format(args.input_data_root,
args.paf_version)
else:
args.paf_dir = None
return args