def _create_logger(self, out_dir, log_dir, verbose, resume):
"""
Create the logger object, and rotate the logs
:param out_dir: The root directory WITH the version number
:param log_dir: The path to the log directory
:param verbose: The verbose flag. If True, run the logger at
DEBUG level
:param resume: True if this is running in resume mode
:return:
"""
log_level = logging.DEBUG if verbose else logging.INFO
create_logger_in_memory("dalynator", log_level,
log_dir + "/daly_run_all.log")
def construct_args_burdenator_cleanup(parser, cli_args=None):
"""Creates arguments from parser for rearranging the draw files at the end of the burdenator run"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.tool_name = 'burdenator'
# Create log directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log_cleanup', str(args.year_id),
str(args.measure_id))
log_filename = "{}_{}_{}.log".format(args.measure_id, args.location_id,
args.year_id)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory("dalynator", log_level,
args.log_dir + "/" + log_filename)
# Get cod/epi env directories
args.cod_dir = (args.input_data_root + "/codcorrect/" +
str(args.cod_version) + "/draws")
args.epi_dir = get_folder_structure(
os.path.join(args.input_data_root, 'como', str(args.epi_version)))
return args
def create_run_all_directories(args):
"""Create the output directory and the run_all logger. Used by both burdenator and dalynator."""
add_run_all_directories_to_args(args)
# Check that both directories are empty. If they are not-empty then only continue if we are in resume mode
if os.path.isdir(args.out_dir):
if os.listdir(args.out_dir) and not args.resume:
raise ValueError(
"Output directory {} contains files and NOT running in resume mode"
.format(args.out_dir))
if os.path.isdir(args.log_dir):
if os.listdir(args.log_dir) and not args.resume:
raise ValueError(
"Log directory {} contains files and NOT running in resume mode"
.format(args.log_dir))
makedirs_safely(args.log_dir)
makedirs_safely(args.out_dir)
makedirs_safely(args.cache_dir)
if args.resume:
# If resuming then rotate (rename) the main log, daly_run_all.log
rotate_logs(args.out_dir, args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
logger = create_logger_in_memory("dalynator", log_level,
args.log_dir + "/daly_run_all.log")
def get_args_pct_change(parser, cli_args=None):
"""Creates arguments from parser for pct change calculation"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.log_dir = os.path.join(args.out_dir, 'log_pct_change',
str(args.location_id))
makedirs_safely(args.log_dir)
logfn = "pc_{}_{}.log".format(args.start_year, args.end_year)
args.logger = create_logger_in_memory("dalynator", logging.DEBUG,
"{}/{}".format(args.log_dir, logfn))
return args
def _prepare_with_external_side_effects(self):
"""
Creates output directories, loggers.
Returns:
Nothing
"""
makedirs_safely(self.output_draws_dir)
makedirs_safely(self.log_dir)
log_level = logging.DEBUG if self.verbose else logging.INFO
_ = create_logger_in_memory(
"dalynator", log_level, self.log_dir +
"/daly_{}_{}.log".format(self.location_id, self.year_id))
def get_args_and_create_dirs(parser, cli_args=None):
"""Parses the command line using the parser and creates output directory
and logger. Called by run_pipeline_*. Not used by run_all."""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
# resolve defaults for cod and epi versions
args.gbd_round, args.gbd_round_id = ac.populate_gbd_round_args(
args.gbd_round, args.gbd_round_id)
if args.cod_version is None:
args.cod_version = ac.best_version('cod', args.gbd_round_id)
if args.epi_version is None:
args.epi_version = ac.best_version('como', args.gbd_round_id)
# Store all years for each location in one directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log', str(args.location_id))
args.out_dir = os.path.join(top_out_dir, 'draws', str(args.location_id))
makedirs_safely(args.out_dir)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory(
"dalynator", log_level, args.log_dir +
"/daly_{}_{}.log".format(args.location_id, args.year_id))
args.cod_dir = "{}/codcorrect/{}/draws/".format(args.input_data_root,
args.cod_version)
if hasattr(args, 'daly_version'):
args.daly_dir = "{}/dalynator/{}/draws/".format(
args.input_data_root, args.daly_version)
else:
args.daly_dir = None
args.epi_dir = get_como_folder_structure(
os.path.join(args.input_data_root, 'como', str(args.epi_version)))
if hasattr(args, 'paf_version'):
args.paf_dir = "{}/pafs/{}".format(args.input_data_root,
args.paf_version)
else:
args.paf_dir = None
return args
def get_args_burdenator_loc_agg(parser, cli_args=None):
"""Creates arguments from parser for burdenator location aggregation"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
# Create log directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log_loc_agg', str(args.year_id),
str(args.measure_id))
log_filename = "{}_{}_{}_{}.log".format(args.measure_id, args.rei_id,
args.year_id, args.sex_id)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory("dalynator", log_level,
args.log_dir + "/" + log_filename)
return args
def construct_args_dalynator_upload(parser, cli_args=None):
"""Creates arguments from parser for uploading dalynator data"""
if cli_args is None:
cli_args = sys.argv[1:]
args = parser.parse_args(cli_args)
args.tool_name = 'dalynator'
# Create log directory
top_out_dir = args.out_dir
args.cache_dir = '{}/cache'.format(args.out_dir)
args.log_dir = os.path.join(top_out_dir, 'log_upload', args.table_type,
str(args.measure_id))
log_filename = "upload_{}_{}.log".format(args.table_type, args.measure_id)
makedirs_safely(args.log_dir)
log_level = logging.DEBUG if args.verbose else logging.INFO
args.logger = create_logger_in_memory("dalynator", log_level,
args.log_dir + "/" + log_filename)
return args
