def multi_year_slurm(ctx, group_params, alloc, walltime, feature, memory, conda_env, module, stdout_path, verbose): """ Run multi year collection and means on HPC via SLURM job submission. """ name = ctx.obj['NAME'] my_file = ctx.obj['MY_FILE'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(os.path.dirname(my_file), 'multi-year', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, os.path.dirname(my_file))) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV multi-year collection on SLURM with node ' ' name "{}", collecting into "{}".'.format(name, my_file)) # create and submit the SLURM job slurm_cmd = get_slurm_cmd(name, my_file, group_params, verbose=verbose) out = slurm_manager.sbatch(slurm_cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV multi-year collection job "{}" ' '(SLURM jobid #{}).'.format(name, out)) # add job to reV status file. Status.add_job(os.path.dirname(my_file), 'multi-year', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': os.path.basename(my_file), 'dirout': os.path.dirname(my_file) }) click.echo(msg) logger.info(msg)
def launch_slurm(config, verbose): """ Launch slurm QA/QC job Parameters ---------- config : dict 'reV QA/QC configuration dictionary' """ out_dir = config.dirout log_file = os.path.join(config.logdir, config.name + '.log') stdout_path = os.path.join(config.logdir, 'stdout/') node_cmd = get_multiple_cmds(config, out_dir, log_file, verbose) slurm_manager = SLURM() status = Status.retrieve_job_status(out_dir, 'qa-qc', config.name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(config.name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( config.name, status)) else: logger.info('Running reV QA-QC on SLURM with ' 'node name "{}"'.format(config.name)) out = slurm_manager.sbatch( node_cmd, name=config.name, alloc=config.execution_control.allocation, memory=config.execution_control.memory, feature=config.execution_control.feature, walltime=config.execution_control.walltime, conda_env=config.execution_control.conda_env, module=config.execution_control.module, stdout_path=stdout_path)[0] if out: msg = ('Kicked off reV QA-QC job "{}" ' '(SLURM jobid #{}).'.format(config.name, out)) Status.add_job(out_dir, 'qa-qc', config.name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(resolution, allocation, time, memory, country, dstdir): """Submit this job to slurm. This isn't worked in yet. I have to make it so this won't submit itself. """ # Build command name = "transmission_{:03d}".format(resolution) template = "rrconnections -r {} -a {} -t {} -m {} -c {} -d {}" cmd = template.format(resolution, allocation, time, memory, country, dstdir) # Submit command to slurm - there redundancy because of aggregation step slurm = SLURM() slurm.sbatch(cmd=cmd, alloc=allocation, walltime=time, memory=memory, name=name, stdout_path='./stdout', keep_sh=False, conda_env="revruns", module=None, module_root=None)
def slurm(ctx, alloc, memory, walltime, feature, module, conda_env, stdout_path): """slurm (eagle) submission tool for reV supply curve.""" name = ctx.obj['NAME'] sc_points = ctx.obj['SC_POINTS'] trans_table = ctx.obj['TRANS_TABLE'] fixed_charge_rate = ctx.obj['FIXED_CHARGE_RATE'] sc_features = ctx.obj['SC_FEATURES'] transmission_costs = ctx.obj['TRANSMISSION_COSTS'] simple = ctx.obj['SIMPLE'] line_limited = ctx.obj['LINE_LIMITED'] sort_on = ctx.obj['SORT_ON'] offshore_trans_table = ctx.obj['OFFSHORE_TRANS_TABLE'] wind_dirs = ctx.obj['WIND_DIRS'] n_dirs = ctx.obj['N_DIRS'] downwind = ctx.obj['DOWNWIND'] offshore_compete = ctx.obj['OFFSHORE_COMPETE'] max_workers = ctx.obj['MAX_WORKERS'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, sc_points, trans_table, fixed_charge_rate, sc_features, transmission_costs, sort_on, offshore_trans_table, wind_dirs, n_dirs, downwind, offshore_compete, max_workers, out_dir, log_dir, simple, line_limited, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'supply-curve', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV Supply Curve on SLURM with ' 'node name "{}"'.format(name)) logger.debug('\t{}'.format(cmd)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV SC job "{}" (SLURM jobid #{}).'.format( name, out)) Status.add_job(out_dir, 'supply-curve', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env, stdout_path, verbose): """Run econ on HPC via SLURM job submission.""" name = ctx.obj['NAME'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] cf_file = ctx.obj['CF_FILE'] cf_year = ctx.obj['CF_YEAR'] site_data = ctx.obj['SITE_DATA'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] max_workers = ctx.obj['MAX_WORKERS'] timeout = ctx.obj['TIMEOUT'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] append = ctx.obj['APPEND'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize a logger on the year level log_modules = [__name__, 'reV.econ.econ', 'reV.config', 'reV.utilities', 'reV.SAM', 'rex.utilities'] init_mult(name, logdir, modules=log_modules, verbose=verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager if append: pc = [None] else: pc = get_node_pc(points, sam_files, nodes) for i, split in enumerate(pc): node_name, fout_node = get_node_name_fout(name, fout, i, pc, hpc='slurm') node_name = node_name.replace('gen', 'econ') points_range = split.split_range if split is not None else None cmd = get_node_cmd(node_name, sam_files, cf_file, cf_year=cf_year, site_data=site_data, points=points, points_range=points_range, sites_per_worker=sites_per_worker, max_workers=max_workers, timeout=timeout, fout=fout_node, dirout=dirout, logdir=logdir, output_request=output_request, append=append, verbose=verbose) status = Status.retrieve_job_status(dirout, 'econ', node_name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.' .format(node_name, dirout)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting' .format(node_name, status)) else: logger.info('Running reV econ on SLURM with node name "{}" for ' '{} (points range: {}).' .format(node_name, pc, points_range)) # create and submit the SLURM job out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=node_name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV econ job "{}" (SLURM jobid #{}).' .format(node_name, out)) # add job to reV status file. Status.add_job( dirout, 'econ', node_name, replace=True, job_attrs={'job_id': out, 'hardware': 'eagle', 'fout': fout_node, 'dirout': dirout}) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, memory, walltime, feature, conda_env, module, stdout_path): """slurm (Eagle) submission tool for reV representative profiles.""" name = ctx.obj['NAME'] gen_fpath = ctx.obj['GEN_FPATH'] rev_summary = ctx.obj['REV_SUMMARY'] reg_cols = ctx.obj['REG_COLS'] cf_dset = ctx.obj['CF_DSET'] rep_method = ctx.obj['REP_METHOD'] err_method = ctx.obj['ERR_METHOD'] weight = ctx.obj['WEIGHT'] n_profiles = ctx.obj['N_PROFILES'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] max_workers = ctx.obj['MAX_WORKERS'] aggregate_profiles = ctx.obj['AGGREGATE_PROFILES'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method, err_method, weight, n_profiles, out_dir, log_dir, max_workers, aggregate_profiles, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'rep-profiles', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV SC rep profiles on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV rep profiles job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'rep-profiles', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.h5'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, walltime, feature, memory, module, conda_env, stdout_path): """slurm (Eagle) submission tool for reV supply curve aggregation.""" name = ctx.obj['NAME'] excl_fpath = ctx.obj['EXCL_FPATH'] gen_fpath = ctx.obj['GEN_FPATH'] econ_fpath = ctx.obj['ECON_FPATH'] res_fpath = ctx.obj['RES_FPATH'] tm_dset = ctx.obj['TM_DSET'] excl_dict = ctx.obj['EXCL_DICT'] check_excl_layers = ctx.obj['CHECK_LAYERS'] res_class_dset = ctx.obj['RES_CLASS_DSET'] res_class_bins = ctx.obj['RES_CLASS_BINS'] cf_dset = ctx.obj['CF_DSET'] lcoe_dset = ctx.obj['LCOE_DSET'] h5_dsets = ctx.obj['H5_DSETS'] data_layers = ctx.obj['DATA_LAYERS'] resolution = ctx.obj['RESOLUTION'] excl_area = ctx.obj['EXCL_AREA'] power_density = ctx.obj['POWER_DENSITY'] area_filter_kernel = ctx.obj['AREA_FILTER_KERNEL'] min_area = ctx.obj['MIN_AREA'] friction_fpath = ctx.obj['FRICTION_FPATH'] friction_dset = ctx.obj['FRICTION_DSET'] out_dir = ctx.obj['OUT_DIR'] log_dir = ctx.obj['LOG_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, excl_dict, check_excl_layers, res_class_dset, res_class_bins, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, out_dir, log_dir, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'supply-curve-aggregation', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV SC aggregation on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV SC aggregation job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'supply-curve-aggregation', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, nodes, memory, walltime, feature, conda_env, module, stdout_path, verbose): """Run generation on HPC via SLURM job submission.""" name = ctx.obj['NAME'] tech = ctx.obj['TECH'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] res_file = ctx.obj['RES_FILE'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] site_data = ctx.obj['SITE_DATA'] max_workers = ctx.obj['MAX_WORKERS'] mem_util_lim = ctx.obj['MEM_UTIL_LIM'] timeout = ctx.obj['TIMEOUT'] curtailment = ctx.obj['CURTAILMENT'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager pc = get_node_pc(points, sam_files, tech, res_file, nodes) for i, split in enumerate(pc): node_name, fout_node = get_node_name_fout(name, fout, i, pc, hpc='slurm') cmd = get_node_cmd(node_name, tech, sam_files, res_file, points=points, points_range=split.split_range, sites_per_worker=sites_per_worker, max_workers=max_workers, fout=fout_node, dirout=dirout, logdir=logdir, output_request=output_request, site_data=site_data, mem_util_lim=mem_util_lim, timeout=timeout, curtailment=curtailment, verbose=verbose) status = Status.retrieve_job_status(dirout, 'generation', node_name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(node_name, dirout)) elif 'fail' not in str(status).lower() and status is not None: msg = ( 'Job "{}" was found with status "{}", not resubmitting'.format( node_name, status)) else: logger.info('Running reV generation on SLURM with node name "{}" ' 'for {} (points range: {}).'.format( node_name, pc, split.split_range)) # create and submit the SLURM job out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=node_name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV generation job "{}" (SLURM jobid #{}).'. format(node_name, out)) # add job to reV status file. Status.add_job(dirout, 'generation', node_name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': fout_node, 'dirout': dirout }) click.echo(msg) logger.info(msg)
def slurm(ctx, alloc, feature, memory, walltime, module, conda_env, stdout_path): """slurm (Eagle) submission tool for reV supply curve aggregation.""" name = ctx.obj['NAME'] gen_fpath = ctx.obj['GEN_FPATH'] offshore_fpath = ctx.obj['OFFSHORE_FPATH'] project_points = ctx.obj['PROJECT_POINTS'] sam_files = ctx.obj['SAM_FILES'] log_dir = ctx.obj['LOG_DIR'] out_dir = ctx.obj['OUT_DIR'] verbose = ctx.obj['VERBOSE'] if stdout_path is None: stdout_path = os.path.join(log_dir, 'stdout/') cmd = get_node_cmd(name, gen_fpath, offshore_fpath, project_points, sam_files, log_dir, verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager status = Status.retrieve_job_status(out_dir, 'offshore', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, out_dir)) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info('Running reV offshore aggregation on SLURM with ' 'node name "{}"'.format(name)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ('Kicked off reV offshore job "{}" ' '(SLURM jobid #{}).'.format(name, out)) Status.add_job(out_dir, 'offshore', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': '{}.csv'.format(name), 'dirout': out_dir }) click.echo(msg) logger.info(msg)
def collect_slurm(ctx, alloc, memory, walltime, feature, conda_env, module, stdout_path, verbose): """Run collection on HPC via SLURM job submission.""" name = ctx.obj['NAME'] h5_file = ctx.obj['H5_FILE'] h5_dir = ctx.obj['H5_DIR'] log_dir = ctx.obj['LOG_DIR'] project_points = ctx.obj['PROJECT_POINTS'] dsets = ctx.obj['DSETS'] file_prefix = ctx.obj['FILE_PREFIX'] purge_chunks = ctx.obj['PURGE_CHUNKS'] verbose = any([verbose, ctx.obj['VERBOSE']]) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: slurm_manager = SLURM() ctx.obj['SLURM_MANAGER'] = slurm_manager cmd = get_node_cmd(name, h5_file, h5_dir, project_points, dsets, file_prefix=file_prefix, log_dir=log_dir, purge_chunks=purge_chunks, verbose=verbose) status = Status.retrieve_job_status(os.path.dirname(h5_file), 'collect', name, hardware='eagle', subprocess_manager=slurm_manager) if status == 'successful': msg = ('Job "{}" is successful in status json found in "{}", ' 'not re-running.'.format(name, os.path.dirname(h5_file))) elif 'fail' not in str(status).lower() and status is not None: msg = ('Job "{}" was found with status "{}", not resubmitting'.format( name, status)) else: logger.info( 'Running reV collection on SLURM with node name "{}", ' 'collecting data to "{}" from "{}" with file prefix "{}".'.format( name, h5_file, h5_dir, file_prefix)) out = slurm_manager.sbatch(cmd, alloc=alloc, memory=memory, walltime=walltime, feature=feature, name=name, stdout_path=stdout_path, conda_env=conda_env, module=module)[0] if out: msg = ( 'Kicked off reV collection job "{}" (SLURM jobid #{}).'.format( name, out)) # add job to reV status file. Status.add_job(os.path.dirname(h5_file), 'collect', name, replace=True, job_attrs={ 'job_id': out, 'hardware': 'eagle', 'fout': os.path.basename(h5_file), 'dirout': os.path.dirname(h5_file) }) click.echo(msg) logger.info(msg)