def make_fout(name, year): """Make an appropriate file output from name and year. Parameters ---------- name : str Job name. year : int | str Analysis year. Returns ------- fout : str .h5 output file based on name and year """ try: match = parse_year(name) except RuntimeError: match = False # if the year isn't in the name, add it before setting the file output if match and year: if str(year) != str(match): raise ConfigError( 'Tried to submit gen job for {}, but found a ' 'different year in the base job name: "{}". ' 'Please remove the year from the job name.'.format(year, name)) if year: fout = '{}{}.h5'.format(name, '_{}'.format(year) if not match else '') else: fout = '{}.h5'.format(name) return fout
def from_config(ctx, config_file, verbose): """Run reV econ from a config file.""" name = ctx.obj['NAME'] verbose = any([verbose, ctx.obj['VERBOSE']]) # Instantiate the config object config = EconConfig(config_file) # take name from config if not default if config.name.lower() != 'rev': name = config.name ctx.obj['NAME'] = name # Enforce verbosity if logging level is specified in the config if config.log_level == logging.DEBUG: verbose = True # make output directory if does not exist if not os.path.exists(config.dirout): os.makedirs(config.dirout) # initialize loggers. init_mult(name, config.logdir, modules=['reV', 'rex'], verbose=verbose) cf_files = config.parse_cf_files() # Initial log statements logger.info('Running reV Econ from config file: "{}"'.format(config_file)) logger.info('Target output directory: "{}"'.format(config.dirout)) logger.info('Target logging directory: "{}"'.format(config.logdir)) logger.info('The following project points were specified: "{}"'.format( config.get('project_points', None))) logger.info( 'The following SAM configs are available to this run:\n{}'.format( pprint.pformat(config.get('sam_files', None), indent=4))) logger.debug( 'Submitting jobs for the following cf_files: {}'.format(cf_files)) logger.debug('The full configuration input is as follows:\n{}'.format( pprint.pformat(config, indent=4))) # set config objects to be passed through invoke to direct methods ctx.obj['POINTS'] = config.project_points ctx.obj['SAM_FILES'] = config.parse_sam_config() ctx.obj['SITE_DATA'] = config.site_data ctx.obj['DIROUT'] = config.dirout ctx.obj['LOGDIR'] = config.logdir ctx.obj['APPEND'] = config.append ctx.obj['OUTPUT_REQUEST'] = config.output_request ctx.obj['SITES_PER_WORKER'] = config.execution_control.sites_per_worker ctx.obj['MAX_WORKERS'] = config.execution_control.max_workers ctx.obj['TIMEOUT'] = config.timeout if len(config.years) == len(cf_files): for i, year in enumerate(config.years): cf_file = cf_files[i] submit_from_config(ctx, name, cf_file, year, config, verbose) else: for i, cf_file in enumerate(cf_files): year = parse_year(cf_file) if str(year) in [str(y) for y in config.years]: submit_from_config(ctx, name, cf_file, year, config, verbose)
def _get_years(file_paths, years): """Reduce file path list to requested years and/or return list of years corresponding to file paths Parameters ---------- file_paths : list List of filepaths for this handler to handle. years : list | None List of years of interest. Should be a subset of years in file_map. Can also be None for all years found by the h5_path input. Returns ------- file_paths : list List of filepaths for this handler to handle. years : list List of integer years corresponding to the file_paths list """ if years is None: years = [ parse_year(os.path.basename(fp), option='raise') for fp in file_paths ] new_list = [] years = sorted([int(y) for y in years]) for year in years: for fp in file_paths: if str(year) in os.path.basename(fp): new_list.append(fp) break if not new_list: msg = ( 'No files were found for the given years:\n{}'.format(years)) raise RuntimeError(msg) if len(new_list) != len(years): msg = ('Found a bad file listing for requested years. ' 'Years requested: {}, files found: {}'.format( years, new_list)) raise RuntimeError(msg) file_paths = new_list return file_paths, years
def _create_dset_name(source_h5, dset): """ Create output dataset name by parsing year from source_h5 and appending to source dataset name. Parameters ---------- source_h5 : str Path to source .h5 file to copy data from dset : str Dataset to copy Returns ------- dset_out : str Ouput dataset name """ f_name = os.path.basename(source_h5) year = parse_year(f_name) dset_out = "{}-{}".format(dset, year) return dset_out
def submit_from_config(ctx, name, cf_file, year, config, verbose): """Function to submit one year from a config file. Parameters ---------- ctx : cli.ctx Click context object. Use case: data = ctx.obj['key'] cf_file : str reV generation file with capacity factors to calculate econ for. name : str Job name. year : int | str | NoneType 4 digit year or None. config : reV.config.EconConfig Econ config object. """ # set the year-specific variables ctx.obj['CF_FILE'] = cf_file ctx.obj['CF_YEAR'] = year # check to make sure that the year matches the resource file if str(year) not in cf_file: warn('reV gen results file and year do not appear to match. ' 'Expected the string representation of the year ' 'to be in the generation results file name. ' 'Year: {}, generation results file: {}'.format(year, cf_file)) # if the year isn't in the name, add it before setting the file output ctx.obj['FOUT'] = make_fout(name, year) if config.append: ctx.obj['FOUT'] = os.path.basename(cf_file) # invoke direct methods based on the config execution option if config.execution_control.option == 'local': name_year = make_fout(name, year).replace('.h5', '') name_year = name_year.replace('gen', 'econ') ctx.obj['NAME'] = name_year status = Status.retrieve_job_status(config.dirout, 'econ', name_year) if status != 'successful': Status.add_job(config.dirout, 'econ', name_year, replace=True, job_attrs={ 'hardware': 'local', 'fout': ctx.obj['FOUT'], 'dirout': config.dirout }) ctx.invoke(local, max_workers=config.execution_control.max_workers, timeout=config.timeout, points_range=None, verbose=verbose) elif config.execution_control.option in ('eagle', 'slurm'): if not parse_year(name, option='bool') and year: # Add year to name before submitting ctx.obj['NAME'] = '{}_{}'.format(name, str(year)) ctx.invoke(slurm, nodes=config.execution_control.nodes, alloc=config.execution_control.allocation, walltime=config.execution_control.walltime, memory=config.execution_control.memory, feature=config.execution_control.feature, module=config.execution_control.module, conda_env=config.execution_control.conda_env, stdout_path=os.path.join(config.logdir, 'stdout'), verbose=verbose)
def submit_from_config(ctx, name, year, config, i, verbose=False): """Function to submit one year from a config file. Parameters ---------- ctx : cli.ctx Click context object. Use case: data = ctx.obj['key'] name : str Job name. year : int | str | NoneType 4 digit year or None. config : reV.config.GenConfig Generation config object. i : int Index variable associated with the index of the year in analysis years. verbose : bool Flag to turn on debug logging. Default is not verbose. """ res_files = config.parse_res_files() # set the year-specific variables ctx.obj['RES_FILE'] = res_files[i] # check to make sure that the year matches the resource file if str(year) not in res_files[i]: warn('Resource file and year do not appear to match. ' 'Expected the string representation of the year ' 'to be in the resource file name. ' 'Year: {}, Resource file: {}'.format(year, res_files[i])) # if the year isn't in the name, add it before setting the file output ctx.obj['FOUT'] = make_fout(name, year) # invoke direct methods based on the config execution option if config.execution_control.option == 'local': name_year = make_fout(name, year).replace('.h5', '') ctx.obj['NAME'] = name_year status = Status.retrieve_job_status(config.dirout, 'generation', name_year) if status != 'successful': Status.add_job(config.dirout, 'generation', name_year, replace=True, job_attrs={ 'hardware': 'local', 'fout': ctx.obj['FOUT'], 'dirout': config.dirout }) ctx.invoke(local, max_workers=config.execution_control.max_workers, timeout=config.timeout, points_range=None, verbose=verbose) elif config.execution_control.option in ('eagle', 'slurm'): if not parse_year(name, option='bool') and year: # Add year to name before submitting ctx.obj['NAME'] = '{}_{}'.format(name, str(year)) ctx.invoke(slurm, nodes=config.execution_control.nodes, alloc=config.execution_control.allocation, walltime=config.execution_control.walltime, memory=config.execution_control.memory, feature=config.execution_control.feature, conda_env=config.execution_control.conda_env, module=config.execution_control.module, stdout_path=os.path.join(config.logdir, 'stdout'), verbose=verbose)