def make_fout(name, year):
"""Make an appropriate file output from name and year.
Parameters
----------
name : str
Job name.
year : int | str
Analysis year.
Returns
-------
fout : str
.h5 output file based on name and year
"""
try:
match = parse_year(name)
except RuntimeError:
match = False
# if the year isn't in the name, add it before setting the file output
if match and year:
if str(year) != str(match):
raise ConfigError(
'Tried to submit gen job for {}, but found a '
'different year in the base job name: "{}". '
'Please remove the year from the job name.'.format(year, name))
if year:
fout = '{}{}.h5'.format(name, '_{}'.format(year) if not match else '')
else:
fout = '{}.h5'.format(name)
return fout
def from_config(ctx, config_file, verbose):
"""Run reV econ from a config file."""
name = ctx.obj['NAME']
verbose = any([verbose, ctx.obj['VERBOSE']])
# Instantiate the config object
config = EconConfig(config_file)
# take name from config if not default
if config.name.lower() != 'rev':
name = config.name
ctx.obj['NAME'] = name
# Enforce verbosity if logging level is specified in the config
if config.log_level == logging.DEBUG:
verbose = True
# make output directory if does not exist
if not os.path.exists(config.dirout):
os.makedirs(config.dirout)
# initialize loggers.
init_mult(name, config.logdir, modules=['reV', 'rex'], verbose=verbose)
cf_files = config.parse_cf_files()
# Initial log statements
logger.info('Running reV Econ from config file: "{}"'.format(config_file))
logger.info('Target output directory: "{}"'.format(config.dirout))
logger.info('Target logging directory: "{}"'.format(config.logdir))
logger.info('The following project points were specified: "{}"'.format(
config.get('project_points', None)))
logger.info(
'The following SAM configs are available to this run:\n{}'.format(
pprint.pformat(config.get('sam_files', None), indent=4)))
logger.debug(
'Submitting jobs for the following cf_files: {}'.format(cf_files))
logger.debug('The full configuration input is as follows:\n{}'.format(
pprint.pformat(config, indent=4)))
# set config objects to be passed through invoke to direct methods
ctx.obj['POINTS'] = config.project_points
ctx.obj['SAM_FILES'] = config.parse_sam_config()
ctx.obj['SITE_DATA'] = config.site_data
ctx.obj['DIROUT'] = config.dirout
ctx.obj['LOGDIR'] = config.logdir
ctx.obj['APPEND'] = config.append
ctx.obj['OUTPUT_REQUEST'] = config.output_request
ctx.obj['SITES_PER_WORKER'] = config.execution_control.sites_per_worker
ctx.obj['MAX_WORKERS'] = config.execution_control.max_workers
ctx.obj['TIMEOUT'] = config.timeout
if len(config.years) == len(cf_files):
for i, year in enumerate(config.years):
cf_file = cf_files[i]
submit_from_config(ctx, name, cf_file, year, config, verbose)
else:
for i, cf_file in enumerate(cf_files):
year = parse_year(cf_file)
if str(year) in [str(y) for y in config.years]:
submit_from_config(ctx, name, cf_file, year, config, verbose)
def _get_years(file_paths, years):
"""Reduce file path list to requested years and/or return list of
years corresponding to file paths
Parameters
----------
file_paths : list
List of filepaths for this handler to handle.
years : list | None
List of years of interest. Should be a subset of years in file_map.
Can also be None for all years found by the h5_path input.
Returns
-------
file_paths : list
List of filepaths for this handler to handle.
years : list
List of integer years corresponding to the file_paths list
"""
if years is None:
years = [
parse_year(os.path.basename(fp), option='raise')
for fp in file_paths
]
new_list = []
years = sorted([int(y) for y in years])
for year in years:
for fp in file_paths:
if str(year) in os.path.basename(fp):
new_list.append(fp)
break
if not new_list:
msg = (
'No files were found for the given years:\n{}'.format(years))
raise RuntimeError(msg)
if len(new_list) != len(years):
msg = ('Found a bad file listing for requested years. '
'Years requested: {}, files found: {}'.format(
years, new_list))
raise RuntimeError(msg)
file_paths = new_list
return file_paths, years
def _create_dset_name(source_h5, dset):
"""
Create output dataset name by parsing year from source_h5 and
appending to source dataset name.
Parameters
----------
source_h5 : str
Path to source .h5 file to copy data from
dset : str
Dataset to copy
Returns
-------
dset_out : str
Ouput dataset name
"""
f_name = os.path.basename(source_h5)
year = parse_year(f_name)
dset_out = "{}-{}".format(dset, year)
return dset_out
def submit_from_config(ctx, name, cf_file, year, config, verbose):
"""Function to submit one year from a config file.
Parameters
----------
ctx : cli.ctx
Click context object. Use case: data = ctx.obj['key']
cf_file : str
reV generation file with capacity factors to calculate econ for.
name : str
Job name.
year : int | str | NoneType
4 digit year or None.
config : reV.config.EconConfig
Econ config object.
"""
# set the year-specific variables
ctx.obj['CF_FILE'] = cf_file
ctx.obj['CF_YEAR'] = year
# check to make sure that the year matches the resource file
if str(year) not in cf_file:
warn('reV gen results file and year do not appear to match. '
'Expected the string representation of the year '
'to be in the generation results file name. '
'Year: {}, generation results file: {}'.format(year, cf_file))
# if the year isn't in the name, add it before setting the file output
ctx.obj['FOUT'] = make_fout(name, year)
if config.append:
ctx.obj['FOUT'] = os.path.basename(cf_file)
# invoke direct methods based on the config execution option
if config.execution_control.option == 'local':
name_year = make_fout(name, year).replace('.h5', '')
name_year = name_year.replace('gen', 'econ')
ctx.obj['NAME'] = name_year
status = Status.retrieve_job_status(config.dirout, 'econ', name_year)
if status != 'successful':
Status.add_job(config.dirout,
'econ',
name_year,
replace=True,
job_attrs={
'hardware': 'local',
'fout': ctx.obj['FOUT'],
'dirout': config.dirout
})
ctx.invoke(local,
max_workers=config.execution_control.max_workers,
timeout=config.timeout,
points_range=None,
verbose=verbose)
elif config.execution_control.option in ('eagle', 'slurm'):
if not parse_year(name, option='bool') and year:
# Add year to name before submitting
ctx.obj['NAME'] = '{}_{}'.format(name, str(year))
ctx.invoke(slurm,
nodes=config.execution_control.nodes,
alloc=config.execution_control.allocation,
walltime=config.execution_control.walltime,
memory=config.execution_control.memory,
feature=config.execution_control.feature,
module=config.execution_control.module,
conda_env=config.execution_control.conda_env,
stdout_path=os.path.join(config.logdir, 'stdout'),
verbose=verbose)
def submit_from_config(ctx, name, year, config, i, verbose=False):
"""Function to submit one year from a config file.
Parameters
----------
ctx : cli.ctx
Click context object. Use case: data = ctx.obj['key']
name : str
Job name.
year : int | str | NoneType
4 digit year or None.
config : reV.config.GenConfig
Generation config object.
i : int
Index variable associated with the index of the year in analysis years.
verbose : bool
Flag to turn on debug logging. Default is not verbose.
"""
res_files = config.parse_res_files()
# set the year-specific variables
ctx.obj['RES_FILE'] = res_files[i]
# check to make sure that the year matches the resource file
if str(year) not in res_files[i]:
warn('Resource file and year do not appear to match. '
'Expected the string representation of the year '
'to be in the resource file name. '
'Year: {}, Resource file: {}'.format(year, res_files[i]))
# if the year isn't in the name, add it before setting the file output
ctx.obj['FOUT'] = make_fout(name, year)
# invoke direct methods based on the config execution option
if config.execution_control.option == 'local':
name_year = make_fout(name, year).replace('.h5', '')
ctx.obj['NAME'] = name_year
status = Status.retrieve_job_status(config.dirout, 'generation',
name_year)
if status != 'successful':
Status.add_job(config.dirout,
'generation',
name_year,
replace=True,
job_attrs={
'hardware': 'local',
'fout': ctx.obj['FOUT'],
'dirout': config.dirout
})
ctx.invoke(local,
max_workers=config.execution_control.max_workers,
timeout=config.timeout,
points_range=None,
verbose=verbose)
elif config.execution_control.option in ('eagle', 'slurm'):
if not parse_year(name, option='bool') and year:
# Add year to name before submitting
ctx.obj['NAME'] = '{}_{}'.format(name, str(year))
ctx.invoke(slurm,
nodes=config.execution_control.nodes,
alloc=config.execution_control.allocation,
walltime=config.execution_control.walltime,
memory=config.execution_control.memory,
feature=config.execution_control.feature,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module,
stdout_path=os.path.join(config.logdir, 'stdout'),
verbose=verbose)
