def _cancel_all_jobs(self): """Cancel all jobs in this pipeline via SLURM scancel.""" status = self._get_status_obj() s = SLURM() for job_id in status.job_ids: s.scancel(job_id) logger.info('Pipeline job "{}" cancelled.'.format(self._config.name))
def multi_year_slurm(ctx, group_params, alloc, walltime, feature, memory, conda_env, module, stdout_path, verbose): """ Run multi year collection and means on HPC via SLURM job submission. """ name = ctx.obj['NAME'] my_file = ctx.obj['MY_FILE'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(os.path.dirname(my_file), 'multi-year', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, os.path.dirname(my_file))) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV multi-year collection on SLURM with node ' ' name "{}", collecting into "{}".'.format(name, my_file)) # create and submit the SLURM job slurm_cmd = get_slurm_cmd(name, my_file, group_params, verbose=verbose) out = slurm_manager.sbatch(slurm_cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV multi-year collection job "{}" ' '(SLURM jobid #{}).'.format(name, out)) # add job to reV status file. Status.add_job(os.path.dirname(my_file), 'multi-year', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': os.path.basename(my_file), 'dirout': os.path.dirname(my_file) }) click.echo(msg) logger.info(msg)
def launch_slurm(config, verbose): """ Launch slurm QA/QC job Parameters ---------- config : dict 'reV QA/QC configuration dictionary' """ out_dir = config.dirout log_file = os.path.join(config.logdir, config.name + '.log') stdout_path = os.path.join(config.logdir, 'stdout/') node_cmd = get_multiple_cmds(config, out_dir, log_file, verbose) slurm_manager = SLURM() status = Status.retrieve_job_status(out_dir, 'qa-qc', config.name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(config.name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( config.name, status)) else: logger.info('Running reV QA-QC on SLURM with ' 'node name "{}"'.format(config.name)) out = slurm_manager.sbatch( node_cmd, name=config.name, alloc=config.execution_control.allocation, memory=config.execution_control.memory, feature=config.execution_control.feature, walltime=config.execution_control.walltime, conda_env=config.execution_control.conda_env, module=config.execution_control.module, stdout_path=stdout_path)[0] if out: msg = ('Kicked off reV QA-QC job "{}" ' '(SLURM jobid #{}).'.format(config.name, out)) Status.add_job(out_dir, 'qa-qc', config.name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def get_sc_cmd(name, sc_table, out_dir, sub_dir, columns, plot_type, cmap, lcoe, log_file, verbose, terminal): """Build CLI call for supply_curve.""" args = ('-sct {sc_table} ' '-o {out_dir} ' '-sd {sub_dir} ' '-cols {columns} ' '-plt {plot_type} ' '-cmap {cmap} ' '-lcoe {lcoe} ' '-log {log_file} ') args = args.format( sc_table=SLURM.s(sc_table), out_dir=SLURM.s(out_dir), sub_dir=SLURM.s(sub_dir), columns=SLURM.s(columns), plot_type=SLURM.s(plot_type), cmap=SLURM.s(cmap), lcoe=SLURM.s(lcoe), log_file=SLURM.s(log_file), ) if verbose: args += '-v ' if terminal: args += '-t ' cmd = ('python -m reV.qa_qc.cli_qa_qc -n {} supply-curve {}'.format( SLURM.s(name), args)) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def get_slurm_cmd(name, my_file, group_params, verbose=False): """Make a reV multi-year collection local CLI call string. Parameters ---------- name : str reV collection jobname. my_file : str Path to .h5 file to use for multi-year collection. group_params : list List of groups and their parameters to collect verbose : bool Flag to turn on DEBUG logging Returns ------- cmd : str Argument to call the neccesary CLI calls on the node to collect desired groups """ # make a cli arg string for direct() in this module main_args = ['-n {}'.format(SLURM.s(name))] if verbose: main_args.append('-v') direct_args = '-f {}'.format(SLURM.s(my_file)) collect_args = '-gp {}'.format(SLURM.s(group_params)) # Python command that will be executed on a node # command strings after cli v7.0 use dashes instead of underscores cmd = ('python -m reV.handlers.cli_multi_year {} direct {} ' 'multi-year-groups {}'.format(' '.join(main_args), direct_args, collect_args)) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def slurm(resolution, allocation, time, memory, country, dstdir): """Submit this job to slurm. This isn't worked in yet. I have to make it so this won't submit itself. """ # Build command name = "transmission_{:03d}".format(resolution) template = "rrconnections -r {} -a {} -t {} -m {} -c {} -d {}" cmd = template.format(resolution, allocation, time, memory, country, dstdir) # Submit command to slurm - there redundancy because of aggregation step slurm = SLURM() slurm.sbatch(cmd=cmd, alloc=allocation, walltime=time, memory=memory, name=name, stdout_path='./stdout', keep_sh=False, conda_env="revruns", module=None, module_root=None)
def subprocess_manager(self): """Get the subprocess manager object based on the hardware spec.""" if self._subprocess_manager is None and self._hardware == 'eagle': self._subprocess_manager = SLURM() if self._subprocess_manager is None and self._hardware == 'local': self._subprocess_manager = SubprocessManager elif self._subprocess_manager is None: msg = ('Cannot recognize requested hardware: {}'.format( self._hardware)) logger.error(msg) raise ValueError(msg) return self._subprocess_manager
def get_node_cmd(name, gen_fpath, offshore_fpath, points, sam_files, log_dir, verbose): """Get a CLI call command for the offshore aggregation cli.""" args = [ '-gf {}'.format(SLURM.s(gen_fpath)), '-of {}'.format(SLURM.s(offshore_fpath)), '-pp {}'.format(SLURM.s(points)), '-sf {}'.format(SLURM.s(sam_files)), '-ld {}'.format(SLURM.s(log_dir)), ] if verbose: args.append('-v') cmd = ('python -m reV.offshore.cli_offshore -n {} direct {}'.format( SLURM.s(name), ' '.join(args))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def get_h5_cmd(name, h5_file, out_dir, sub_dir, dsets, group, process_size, max_workers, plot_type, cmap, log_file, verbose, terminal): """Build CLI call for reV_h5.""" args = ('-h5 {h5_file} ' '-o {out_dir} ' '-sd {sub_dir} ' '-ds {dsets} ' '-grp {group} ' '-ps {process_size} ' '-w {max_workers} ' '-plt {plot_type} ' '-cmap {cmap} ' '-log {log_file} ') args = args.format( h5_file=SLURM.s(h5_file), out_dir=SLURM.s(out_dir), sub_dir=SLURM.s(sub_dir), dsets=SLURM.s(dsets), group=SLURM.s(group), process_size=SLURM.s(process_size), max_workers=SLURM.s(max_workers), plot_type=SLURM.s(plot_type), cmap=SLURM.s(cmap), log_file=SLURM.s(log_file), ) if verbose: args += '-v ' if terminal: args += '-t ' cmd = ('python -m reV.qa_qc.cli_qa_qc -n {} rev-h5 {}'.format( SLURM.s(name), args)) return cmd
def get_node_cmd(name, sam_files, cf_file, cf_year=None, site_data=None, points=slice(0, 100), points_range=None, sites_per_worker=None, max_workers=None, timeout=1800, fout='reV.h5', dirout='./out/econ_out', logdir='./out/log_econ', output_request='lcoe_fcr', append=False, verbose=False): """Made a reV econ direct-local command line interface call string. Parameters ---------- name : str Name of the job to be submitted. sam_files : dict | str | list SAM input configuration ID(s) and file path(s). Keys are the SAM config ID(s), top level value is the SAM path. Can also be a single config file str. If it's a list, it is mapped to the sorted list of unique configs requested by points csv. cf_file : str reV generation results file name + path. cf_year : int | str reV generation year to calculate econ for. cf_year='my' will look for the multi-year mean generation results. site_data : str | None Site-specific data for econ calculation. points : slice | str | list | tuple Slice/list specifying project points, string pointing to a project points_range : list | None Optional range list to run a subset of sites sites_per_worker : int | None Number of sites to be analyzed in serial on a single local core. max_workers : int | None Number of workers to use on a node. None defaults to all available workers. timeout : int | float Number of seconds to wait for parallel run iteration to complete before returning zeros. Default is 1800 seconds. fout : str Target filename to dump econ outputs. dirout : str Target directory to dump econ fout. logdir : str Target directory to save log files. output_request : list | tuple Output variable requested from SAM. append : bool Flag to append econ datasets to source cf_file. This has priority over the fout and dirout inputs. verbose : bool Flag to turn on debug logging. Default is False. Returns ------- cmd : str Single line command line argument to call the following CLI with appropriately formatted arguments based on input args: python -m reV.econ.cli_econ [args] direct [args] local [args] """ # mark a cli arg string for main() in this module arg_main = '-n {}'.format(SLURM.s(name)) # make a cli arg string for direct() in this module arg_direct = [ '-p {}'.format(SLURM.s(points)), '-sf {}'.format(SLURM.s(sam_files)), '-cf {}'.format(SLURM.s(cf_file)), '-cfy {}'.format(SLURM.s(cf_year)), '-spw {}'.format(SLURM.s(sites_per_worker)), '-fo {}'.format(SLURM.s(fout)), '-do {}'.format(SLURM.s(dirout)), '-lo {}'.format(SLURM.s(logdir)), '-or {}'.format(SLURM.s(output_request))] if site_data: arg_direct.append('-sd {}'.format(SLURM.s(site_data))) if append: arg_direct.append('-ap') arg_loc = ['-mw {}'.format(SLURM.s(max_workers)), '-to {}'.format(SLURM.s(timeout)), '-pr {}'.format(SLURM.s(points_range))] if verbose: arg_loc.append('-v') # Python command that will be executed on a node # command strings after cli v7.0 use dashes instead of underscores cmd = ('python -m reV.econ.cli_econ ' '{arg_main} direct {arg_direct} local {arg_loc}' .format(arg_main=arg_main, arg_direct=' '.join(arg_direct), arg_loc=' '.join(arg_loc))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env, stdout_path, verbose): """Run econ on HPC via SLURM job submission.""" name = ctx.obj['NAME'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] cf_file = ctx.obj['CF_FILE'] cf_year = ctx.obj['CF_YEAR'] site_data = ctx.obj['SITE_DATA'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] max_workers = ctx.obj['MAX_WORKERS'] timeout = ctx.obj['TIMEOUT'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] append = ctx.obj['APPEND'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize a logger on the year level log_modules = [__name__, 'reV.econ.econ', 'reV.config', 'reV.utilities', 'reV.SAM', 'rex.utilities'] init_mult(name, logdir, modules=log_modules, verbose=verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager if append: pc = [None] else: pc = get_node_pc(points, sam_files, nodes) for i, split in enumerate(pc): node_name, fout_node = get_node_name_fout(name, fout, i, pc, hpc='slurm') node_name = node_name.replace('gen', 'econ') points_range = split.split_range if split is not None else None cmd = get_node_cmd(node_name, sam_files, cf_file, cf_year=cf_year, site_data=site_data, points=points, points_range=points_range, sites_per_worker=sites_per_worker, max_workers=max_workers, timeout=timeout, fout=fout_node, dirout=dirout, logdir=logdir, output_request=output_request, append=append, verbose=verbose) status = Status.retrieve_job_status(dirout, 'econ', node_name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.' .format(node_name, dirout)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting' .format(node_name, status)) else: logger.info('Running reV econ on SLURM with node name "{}" for ' '{} (points range: {}).' .format(node_name, pc, points_range)) # create and submit the SLURM job out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=node_name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV econ job "{}" (SLURM jobid #{}).' .format(node_name, out)) # add job to reV status file. Status.add_job( dirout, 'econ', node_name, replace=True, job_attrs={'job_id': out, 'hardware': 'eagle', 'fout': fout_node, 'dirout': dirout}) click.echo(msg) logger.info(msg)
def get_node_cmd(name, sc_points, trans_table, fixed_charge_rate, sc_features, transmission_costs, sort_on, offshore_trans_table, wind_dirs, n_dirs, downwind, offshore_compete, max_workers, out_dir, log_dir, simple, line_limited, verbose): """Get a CLI call command for the Supply Curve cli.""" args = [ '-sc {}'.format(SLURM.s(sc_points)), '-tt {}'.format(SLURM.s(trans_table)), '-fcr {}'.format(SLURM.s(fixed_charge_rate)), '-scf {}'.format(SLURM.s(sc_features)), '-tc {}'.format(SLURM.s(transmission_costs)), '-so {}'.format(SLURM.s(sort_on)), '-ott {}'.format(SLURM.s(offshore_trans_table)), '-dirs {}'.format(SLURM.s(n_dirs)), '-mw {}'.format(SLURM.s(max_workers)), '-o {}'.format(SLURM.s(out_dir)), '-ld {}'.format(SLURM.s(log_dir)), ] if wind_dirs is not None: args.append('-wd {}'.format(SLURM.s(wind_dirs))) if downwind: args.append('-dw') if offshore_compete: args.append('-oc') if simple: args.append('-s') elif line_limited: args.append('-ll') if verbose: args.append('-v') cmd = ( 'python -m reV.supply_curve.cli_supply_curve -n {} direct {}'.format( SLURM.s(name), ' '.join(args))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def collect_slurm(ctx, alloc, memory, walltime, feature, conda_env, module, stdout_path, verbose): """Run collection on HPC via SLURM job submission.""" name = ctx.obj['NAME'] h5_file = ctx.obj['H5_FILE'] h5_dir = ctx.obj['H5_DIR'] log_dir = ctx.obj['LOG_DIR'] project_points = ctx.obj['PROJECT_POINTS'] dsets = ctx.obj['DSETS'] file_prefix = ctx.obj['FILE_PREFIX'] purge_chunks = ctx.obj['PURGE_CHUNKS'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager cmd = get_node_cmd(name, h5_file, h5_dir, project_points, dsets, file_prefix=file_prefix, log_dir=log_dir, purge_chunks=purge_chunks, verbose=verbose) status = Status.retrieve_job_status(os.path.dirname(h5_file), 'collect', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, os.path.dirname(h5_file))) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info( 'Running reV collection on SLURM with node name "{}", ' 'collecting data to "{}" from "{}" with file prefix "{}".'.format( name, h5_file, h5_dir, file_prefix)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ( 'Kicked off reV collection job "{}" (SLURM jobid #{}).'.format( name, out)) # add job to reV status file. Status.add_job(os.path.dirname(h5_file), 'collect', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': os.path.basename(h5_file), 'dirout': os.path.dirname(h5_file) }) click.echo(msg) logger.info(msg)
def test_eagle(year): """Gen PV CF profiles with write to disk and compare against rev1.""" res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' rev2_out_dir = os.path.join(TESTDATADIR, 'ri_pv_reV2') rev2_out = 'gen_ri_pv_smart_{}.h5'.format(year) if not os.path.exists(rev2_out_dir): os.mkdir(rev2_out_dir) name = 'etest' points = slice(0, 100) verbose = True log_level = 'DEBUG' log_file = os.path.join(rev2_out_dir, '{}.log'.format(name)) modules = [__name__, 'reV.utilities', 'reV.generation'] for mod in modules: init_logger(mod, log_level=log_level, log_file=log_file) cmd = get_node_cmd(name=name, tech='pvwattsv5', points=points, points_range=None, sam_files=sam_files, res_file=res_file, sites_per_worker=None, max_workers=None, fout=rev2_out, dirout=rev2_out_dir, logdir=rev2_out_dir, output_request=('cf_profile', 'cf_mean'), verbose=verbose) # create and submit the SLURM job slurm = SLURM(cmd, alloc='rev', memory=96, walltime=0.1, name=name, stdout_path=rev2_out_dir) while True: status = slurm.check_status(name, var='name') if status == 'CG': break else: time.sleep(5) # get reV 2.0 generation profiles from disk flist = os.listdir(rev2_out_dir) for fname in flist: if '.h5' in fname: if rev2_out.strip('.h5') in fname: full_f = os.path.join(rev2_out_dir, fname) with Outputs(full_f, 'r') as cf: rev2_profiles = cf['cf_profile'] break # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) rev1_profiles = rev1_profiles[:, points] result = np.allclose(rev1_profiles, rev2_profiles, rtol=RTOL, atol=ATOL) if result: # remove output files if test passes. flist = os.listdir(rev2_out_dir) for fname in flist: os.remove(os.path.join(rev2_out_dir, fname)) assert result is True
def get_excl_cmd(name, excl_fpath, out_dir, sub_dir, excl_dict, area_filter_kernel, min_area, plot_type, cmap, plot_step, log_file, verbose, terminal): """Build CLI call for exclusions.""" args = [ '-excl {}'.format(SLURM.s(excl_fpath)), '-o {}'.format(SLURM.s(out_dir)), '-sd {}'.format(SLURM.s(sub_dir)), '-exd {}'.format(SLURM.s(excl_dict)), '-afk {}'.format(SLURM.s(area_filter_kernel)), '-ma {}'.format(SLURM.s(min_area)), '-plt {}'.format(SLURM.s(plot_type)), '-cmap {}'.format(SLURM.s(cmap)), '-step {}'.format(SLURM.s(plot_step)), '-log {}'.format(SLURM.s(log_file)), ] if verbose: args.append('-v') if terminal: args.append('-t') cmd = ('python -m reV.qa_qc.cli_qa_qc -n {} exclusions {}'.format( SLURM.s(name), ' '.join(args))) return cmd
def slurm(ctx, alloc, feature, memory, walltime, module, conda_env, stdout_path): """slurm (Eagle) submission tool for reV supply curve aggregation.""" name = ctx.obj['NAME'] gen_fpath = ctx.obj['GEN_FPATH'] offshore_fpath = ctx.obj['OFFSHORE_FPATH'] project_points = ctx.obj['PROJECT_POINTS'] sam_files = ctx.obj['SAM_FILES'] log_dir = ctx.obj['LOG_DIR'] out_dir = ctx.obj['OUT_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, gen_fpath, offshore_fpath, project_points, sam_files, log_dir, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'offshore', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV offshore aggregation on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV offshore job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'offshore', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, walltime, feature, memory, module, conda_env, stdout_path): """slurm (Eagle) submission tool for reV supply curve aggregation.""" name = ctx.obj['NAME'] excl_fpath = ctx.obj['EXCL_FPATH'] gen_fpath = ctx.obj['GEN_FPATH'] econ_fpath = ctx.obj['ECON_FPATH'] res_fpath = ctx.obj['RES_FPATH'] tm_dset = ctx.obj['TM_DSET'] excl_dict = ctx.obj['EXCL_DICT'] check_excl_layers = ctx.obj['CHECK_LAYERS'] res_class_dset = ctx.obj['RES_CLASS_DSET'] res_class_bins = ctx.obj['RES_CLASS_BINS'] cf_dset = ctx.obj['CF_DSET'] lcoe_dset = ctx.obj['LCOE_DSET'] h5_dsets = ctx.obj['H5_DSETS'] data_layers = ctx.obj['DATA_LAYERS'] resolution = ctx.obj['RESOLUTION'] excl_area = ctx.obj['EXCL_AREA'] power_density = ctx.obj['POWER_DENSITY'] area_filter_kernel = ctx.obj['AREA_FILTER_KERNEL'] min_area = ctx.obj['MIN_AREA'] friction_fpath = ctx.obj['FRICTION_FPATH'] friction_dset = ctx.obj['FRICTION_DSET'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, excl_dict, check_excl_layers, res_class_dset, res_class_bins, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, out_dir, log_dir, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'supply-curve-aggregation', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV SC aggregation on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV SC aggregation job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'supply-curve-aggregation', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, excl_dict, check_excl_layers, res_class_dset, res_class_bins, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, out_dir, log_dir, verbose): """Get a CLI call command for the SC aggregation cli.""" args = [ '-exf {}'.format(SLURM.s(excl_fpath)), '-gf {}'.format(SLURM.s(gen_fpath)), '-ef {}'.format(SLURM.s(econ_fpath)), '-rf {}'.format(SLURM.s(res_fpath)), '-tm {}'.format(SLURM.s(tm_dset)), '-exd {}'.format(SLURM.s(excl_dict)), '-cd {}'.format(SLURM.s(res_class_dset)), '-cb {}'.format(SLURM.s(res_class_bins)), '-cf {}'.format(SLURM.s(cf_dset)), '-lc {}'.format(SLURM.s(lcoe_dset)), '-hd {}'.format(SLURM.s(h5_dsets)), '-d {}'.format(SLURM.s(data_layers)), '-r {}'.format(SLURM.s(resolution)), '-ea {}'.format(SLURM.s(excl_area)), '-pd {}'.format(SLURM.s(power_density)), '-afk {}'.format(SLURM.s(area_filter_kernel)), '-ma {}'.format(SLURM.s(min_area)), '-ff {}'.format(SLURM.s(friction_fpath)), '-fd {}'.format(SLURM.s(friction_dset)), '-o {}'.format(SLURM.s(out_dir)), '-ld {}'.format(SLURM.s(log_dir)), ] if check_excl_layers: args.append('-cl') if verbose: args.append('-v') cmd = ( 'python -m reV.supply_curve.cli_sc_aggregation -n {} direct {}'.format( SLURM.s(name), ' '.join(args))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def slurm(ctx, alloc, nodes, memory, walltime, feature, conda_env, module, stdout_path, verbose): """Run generation on HPC via SLURM job submission.""" name = ctx.obj['NAME'] tech = ctx.obj['TECH'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] res_file = ctx.obj['RES_FILE'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] site_data = ctx.obj['SITE_DATA'] max_workers = ctx.obj['MAX_WORKERS'] mem_util_lim = ctx.obj['MEM_UTIL_LIM'] timeout = ctx.obj['TIMEOUT'] curtailment = ctx.obj['CURTAILMENT'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager pc = get_node_pc(points, sam_files, tech, res_file, nodes) for i, split in enumerate(pc): node_name, fout_node = get_node_name_fout(name, fout, i, pc, hpc='slurm') cmd = get_node_cmd(node_name, tech, sam_files, res_file, points=points, points_range=split.split_range, sites_per_worker=sites_per_worker, max_workers=max_workers, fout=fout_node, dirout=dirout, logdir=logdir, output_request=output_request, site_data=site_data, mem_util_lim=mem_util_lim, timeout=timeout, curtailment=curtailment, verbose=verbose) status = Status.retrieve_job_status(dirout, 'generation', node_name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(node_name, dirout)) elif 'fail' not in str(status).lower() and status is not None: msg = ( 'Job "{}" was found with status "{}", not resubmitting'.format( node_name, status)) else: logger.info('Running reV generation on SLURM with node name "{}" ' 'for {} (points range: {}).'.format( node_name, pc, split.split_range)) # create and submit the SLURM job out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=node_name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV generation job "{}" (SLURM jobid #{}).'. format(node_name, out)) # add job to reV status file. Status.add_job(dirout, 'generation', node_name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': fout_node, 'dirout': dirout }) click.echo(msg) logger.info(msg)
def get_node_cmd(name, tech, sam_files, res_file, points=slice(0, 100), points_range=None, sites_per_worker=None, max_workers=None, fout='reV.h5', dirout='./out/gen_out', logdir='./out/log_gen', output_request=('cf_mean', ), site_data=None, mem_util_lim=0.4, timeout=1800, curtailment=None, verbose=False): """Make a reV geneneration direct-local CLI call string. Parameters ---------- name : str Name of the job to be submitted. tech : str Name of the reV technology to be analyzed. (e.g. pv, csp, landbasedwind, offshorewind). sam_files : dict | str | list SAM input configuration ID(s) and file path(s). Keys are the SAM config ID(s), top level value is the SAM path. Can also be a single config file str. If it's a list, it is mapped to the sorted list of unique configs requested by points csv. res_file : str WTK or NSRDB resource file name + path. points : slice | str | list | tuple Slice/list specifying project points, string pointing to a project points_range : list | None Optional range list to run a subset of sites sites_per_worker : int | None Number of sites to be analyzed in serial on a single local core. max_workers : int | None Number of workers to use on a node. None defaults to all available workers. fout : str Target filename to dump generation outputs. dirout : str Target directory to dump generation fout. logdir : str Target directory to save log files. output_request : list | tuple Output variables requested from SAM. site_data : str | None Site-specific input data for SAM calculation. String should be a filepath that points to a csv, Rows match sites, columns are input keys. Need a "gid" column. Input as None if no site-specific data. mem_util_lim : float Memory utilization limit (fractional). timeout : int | float Number of seconds to wait for parallel run iteration to complete before returning zeros. Default is 1800 seconds. curtailment : NoneType | str Pointer to a file containing curtailment input parameters or None if no curtailment. verbose : bool Flag to turn on debug logging. Default is False. Returns ------- cmd : str Single line command line argument to call the following CLI with appropriately formatted arguments based on input args: python -m reV.generation.cli_gen [args] direct [args] local [args] """ # mark a cli arg string for main() in this module arg_main = '-n {}'.format(SLURM.s(name)) # make a cli arg string for direct() in this module arg_direct = [ '-t {}'.format(SLURM.s(tech)), '-p {}'.format(SLURM.s(points)), '-sf {}'.format(SLURM.s(sam_files)), '-rf {}'.format(SLURM.s(res_file)), '-spw {}'.format( SLURM.s(sites_per_worker)), '-fo {}'.format(SLURM.s(fout)), '-do {}'.format(SLURM.s(dirout)), '-lo {}'.format(SLURM.s(logdir)), '-or {}'.format(SLURM.s(output_request)), '-mem {}'.format(SLURM.s(mem_util_lim)) ] if site_data: arg_direct.append('-sd {}'.format(SLURM.s(site_data))) if curtailment: arg_direct.append('-curt {}'.format(SLURM.s(curtailment))) # make a cli arg string for local() in this module arg_loc = [ '-mw {}'.format(SLURM.s(max_workers)), '-to {}'.format(SLURM.s(timeout)), '-pr {}'.format(SLURM.s(points_range)) ] if verbose: arg_loc.append('-v') # Python command that will be executed on a node # command strings after cli v7.0 use dashes instead of underscores cmd = ('python -m reV.generation.cli_gen ' '{arg_main} direct {arg_direct} local {arg_loc}'.format( arg_main=arg_main, arg_direct=' '.join(arg_direct), arg_loc=' '.join(arg_loc))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method, err_method, weight, n_profiles, out_dir, log_dir, max_workers, aggregate_profiles, verbose): """Get a CLI call command for the rep profiles cli.""" args = [ '-g {}'.format(SLURM.s(gen_fpath)), '-r {}'.format(SLURM.s(rev_summary)), '-rc {}'.format(SLURM.s(reg_cols)), '-cf {}'.format(SLURM.s(cf_dset)), '-rm {}'.format(SLURM.s(rep_method)), '-em {}'.format(SLURM.s(err_method)), '-w {}'.format(SLURM.s(weight)), '-np {}'.format(SLURM.s(n_profiles)), '-od {}'.format(SLURM.s(out_dir)), '-ld {}'.format(SLURM.s(log_dir)), '-mw {}'.format(SLURM.s(max_workers)), ] if aggregate_profiles: args.append('-agg') if verbose: args.append('-v') cmd = ( 'python -m reV.rep_profiles.cli_rep_profiles -n {} direct {}'.format( SLURM.s(name), ' '.join(args))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd
def slurm(ctx, alloc, memory, walltime, feature, conda_env, module, stdout_path): """slurm (Eagle) submission tool for reV representative profiles.""" name = ctx.obj['NAME'] gen_fpath = ctx.obj['GEN_FPATH'] rev_summary = ctx.obj['REV_SUMMARY'] reg_cols = ctx.obj['REG_COLS'] cf_dset = ctx.obj['CF_DSET'] rep_method = ctx.obj['REP_METHOD'] err_method = ctx.obj['ERR_METHOD'] weight = ctx.obj['WEIGHT'] n_profiles = ctx.obj['N_PROFILES'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] max_workers = ctx.obj['MAX_WORKERS'] aggregate_profiles = ctx.obj['AGGREGATE_PROFILES'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method, err_method, weight, n_profiles, out_dir, log_dir, max_workers, aggregate_profiles, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'rep-profiles', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV SC rep profiles on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV rep profiles job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'rep-profiles', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.h5'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, memory, walltime, feature, module, conda_env, stdout_path): """slurm (eagle) submission tool for reV supply curve.""" name = ctx.obj['NAME'] sc_points = ctx.obj['SC_POINTS'] trans_table = ctx.obj['TRANS_TABLE'] fixed_charge_rate = ctx.obj['FIXED_CHARGE_RATE'] sc_features = ctx.obj['SC_FEATURES'] transmission_costs = ctx.obj['TRANSMISSION_COSTS'] simple = ctx.obj['SIMPLE'] line_limited = ctx.obj['LINE_LIMITED'] sort_on = ctx.obj['SORT_ON'] offshore_trans_table = ctx.obj['OFFSHORE_TRANS_TABLE'] wind_dirs = ctx.obj['WIND_DIRS'] n_dirs = ctx.obj['N_DIRS'] downwind = ctx.obj['DOWNWIND'] offshore_compete = ctx.obj['OFFSHORE_COMPETE'] max_workers = ctx.obj['MAX_WORKERS'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, sc_points, trans_table, fixed_charge_rate, sc_features, transmission_costs, sort_on, offshore_trans_table, wind_dirs, n_dirs, downwind, offshore_compete, max_workers, out_dir, log_dir, simple, line_limited, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'supply-curve', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV Supply Curve on SLURM with ' 'node name "{}"'.format(name)) logger.debug('\t{}'.format(cmd)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV SC job "{}" (SLURM jobid #{}).'.format( name, out)) Status.add_job(out_dir, 'supply-curve', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def get_node_cmd(name, h5_file, h5_dir, project_points, dsets, file_prefix=None, log_dir='./logs/', purge_chunks=False, verbose=False): """Make a reV collection local CLI call string. Parameters ---------- name : str reV collection jobname. h5_file : str Path to .h5 file into which data will be collected h5_dir : str Root directory containing .h5 files to combine project_points : str | slice | list | pandas.DataFrame Project points that correspond to the full collection of points contained in the .h5 files to be collected dsets : list List of datasets (strings) to be collected. file_prefix : str .h5 file prefix, if None collect all files on h5_dir log_dir : str Log directory. purge_chunks : bool Flag to delete the chunked files after collection. verbose : bool Flag to turn on DEBUG logging Returns ------- cmd : str Single line command line argument to call the following CLI with appropriately formatted arguments based on input args: python -m reV.handlers.cli_collect [args] collect """ # make a cli arg string for direct() in this module args = [ '-f {}'.format(SLURM.s(h5_file)), '-d {}'.format(SLURM.s(h5_dir)), '-pp {}'.format(SLURM.s(project_points)), '-ds {}'.format(SLURM.s(dsets)), '-fp {}'.format(SLURM.s(file_prefix)), '-ld {}'.format(SLURM.s(log_dir)), ] if purge_chunks: args.append('-p') if verbose: args.append('-v') # Python command that will be executed on a node # command strings after cli v7.0 use dashes instead of underscores cmd = ('python -m reV.handlers.cli_collect -n {} direct {} collect'.format( SLURM.s(name), ' '.join(args))) logger.debug('Creating the following command line call:\n\t{}'.format(cmd)) return cmd