def multi_year_slurm(ctx, group_params, alloc, walltime, feature, memory,
conda_env, module, stdout_path, verbose):
"""
Run multi year collection and means on HPC via SLURM job submission.
"""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(os.path.dirname(my_file),
'multi-year',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, os.path.dirname(my_file)))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV multi-year collection on SLURM with node '
' name "{}", collecting into "{}".'.format(name, my_file))
# create and submit the SLURM job
slurm_cmd = get_slurm_cmd(name, my_file, group_params, verbose=verbose)
out = slurm_manager.sbatch(slurm_cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV multi-year collection job "{}" '
'(SLURM jobid #{}).'.format(name, out))
# add job to reV status file.
Status.add_job(os.path.dirname(my_file),
'multi-year',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': os.path.basename(my_file),
'dirout': os.path.dirname(my_file)
})
click.echo(msg)
logger.info(msg)
def launch_slurm(config, verbose):
"""
Launch slurm QA/QC job
Parameters
----------
config : dict
'reV QA/QC configuration dictionary'
"""
out_dir = config.dirout
log_file = os.path.join(config.logdir, config.name + '.log')
stdout_path = os.path.join(config.logdir, 'stdout/')
node_cmd = get_multiple_cmds(config, out_dir, log_file, verbose)
slurm_manager = SLURM()
status = Status.retrieve_job_status(out_dir,
'qa-qc',
config.name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(config.name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
config.name, status))
else:
logger.info('Running reV QA-QC on SLURM with '
'node name "{}"'.format(config.name))
out = slurm_manager.sbatch(
node_cmd,
name=config.name,
alloc=config.execution_control.allocation,
memory=config.execution_control.memory,
feature=config.execution_control.feature,
walltime=config.execution_control.walltime,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module,
stdout_path=stdout_path)[0]
if out:
msg = ('Kicked off reV QA-QC job "{}" '
'(SLURM jobid #{}).'.format(config.name, out))
Status.add_job(out_dir,
'qa-qc',
config.name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def slurm(resolution, allocation, time, memory, country, dstdir):
"""Submit this job to slurm.
This isn't worked in yet. I have to make it so this won't submit itself.
"""
# Build command
name = "transmission_{:03d}".format(resolution)
template = "rrconnections -r {} -a {} -t {} -m {} -c {} -d {}"
cmd = template.format(resolution, allocation, time, memory, country,
dstdir)
# Submit command to slurm - there redundancy because of aggregation step
slurm = SLURM()
slurm.sbatch(cmd=cmd,
alloc=allocation,
walltime=time,
memory=memory,
name=name,
stdout_path='./stdout',
keep_sh=False,
conda_env="revruns",
module=None,
module_root=None)
def slurm(ctx, alloc, memory, walltime, feature, module, conda_env,
stdout_path):
"""slurm (eagle) submission tool for reV supply curve."""
name = ctx.obj['NAME']
sc_points = ctx.obj['SC_POINTS']
trans_table = ctx.obj['TRANS_TABLE']
fixed_charge_rate = ctx.obj['FIXED_CHARGE_RATE']
sc_features = ctx.obj['SC_FEATURES']
transmission_costs = ctx.obj['TRANSMISSION_COSTS']
simple = ctx.obj['SIMPLE']
line_limited = ctx.obj['LINE_LIMITED']
sort_on = ctx.obj['SORT_ON']
offshore_trans_table = ctx.obj['OFFSHORE_TRANS_TABLE']
wind_dirs = ctx.obj['WIND_DIRS']
n_dirs = ctx.obj['N_DIRS']
downwind = ctx.obj['DOWNWIND']
offshore_compete = ctx.obj['OFFSHORE_COMPETE']
max_workers = ctx.obj['MAX_WORKERS']
out_dir = ctx.obj['OUT_DIR']
log_dir = ctx.obj['LOG_DIR']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, sc_points, trans_table, fixed_charge_rate,
sc_features, transmission_costs, sort_on,
offshore_trans_table, wind_dirs, n_dirs, downwind,
offshore_compete, max_workers, out_dir, log_dir, simple,
line_limited, verbose)
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(out_dir,
'supply-curve',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV Supply Curve on SLURM with '
'node name "{}"'.format(name))
logger.debug('\t{}'.format(cmd))
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV SC job "{}" (SLURM jobid #{}).'.format(
name, out))
Status.add_job(out_dir,
'supply-curve',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': '{}.csv'.format(name),
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env,
stdout_path, verbose):
"""Run econ on HPC via SLURM job submission."""
name = ctx.obj['NAME']
points = ctx.obj['POINTS']
sam_files = ctx.obj['SAM_FILES']
cf_file = ctx.obj['CF_FILE']
cf_year = ctx.obj['CF_YEAR']
site_data = ctx.obj['SITE_DATA']
sites_per_worker = ctx.obj['SITES_PER_WORKER']
max_workers = ctx.obj['MAX_WORKERS']
timeout = ctx.obj['TIMEOUT']
fout = ctx.obj['FOUT']
dirout = ctx.obj['DIROUT']
logdir = ctx.obj['LOGDIR']
output_request = ctx.obj['OUTPUT_REQUEST']
append = ctx.obj['APPEND']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize a logger on the year level
log_modules = [__name__, 'reV.econ.econ', 'reV.config', 'reV.utilities',
'reV.SAM', 'rex.utilities']
init_mult(name, logdir, modules=log_modules, verbose=verbose)
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
if append:
pc = [None]
else:
pc = get_node_pc(points, sam_files, nodes)
for i, split in enumerate(pc):
node_name, fout_node = get_node_name_fout(name, fout, i, pc,
hpc='slurm')
node_name = node_name.replace('gen', 'econ')
points_range = split.split_range if split is not None else None
cmd = get_node_cmd(node_name, sam_files, cf_file, cf_year=cf_year,
site_data=site_data, points=points,
points_range=points_range,
sites_per_worker=sites_per_worker,
max_workers=max_workers, timeout=timeout,
fout=fout_node,
dirout=dirout, logdir=logdir,
output_request=output_request, append=append,
verbose=verbose)
status = Status.retrieve_job_status(dirout, 'econ', node_name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'
.format(node_name, dirout))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'
.format(node_name, status))
else:
logger.info('Running reV econ on SLURM with node name "{}" for '
'{} (points range: {}).'
.format(node_name, pc, points_range))
# create and submit the SLURM job
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=node_name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV econ job "{}" (SLURM jobid #{}).'
.format(node_name, out))
# add job to reV status file.
Status.add_job(
dirout, 'econ', node_name, replace=True,
job_attrs={'job_id': out, 'hardware': 'eagle',
'fout': fout_node, 'dirout': dirout})
click.echo(msg)
logger.info(msg)
def slurm(ctx, alloc, memory, walltime, feature, conda_env, module,
stdout_path):
"""slurm (Eagle) submission tool for reV representative profiles."""
name = ctx.obj['NAME']
gen_fpath = ctx.obj['GEN_FPATH']
rev_summary = ctx.obj['REV_SUMMARY']
reg_cols = ctx.obj['REG_COLS']
cf_dset = ctx.obj['CF_DSET']
rep_method = ctx.obj['REP_METHOD']
err_method = ctx.obj['ERR_METHOD']
weight = ctx.obj['WEIGHT']
n_profiles = ctx.obj['N_PROFILES']
out_dir = ctx.obj['OUT_DIR']
log_dir = ctx.obj['LOG_DIR']
max_workers = ctx.obj['MAX_WORKERS']
aggregate_profiles = ctx.obj['AGGREGATE_PROFILES']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset,
rep_method, err_method, weight, n_profiles, out_dir,
log_dir, max_workers, aggregate_profiles, verbose)
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(out_dir,
'rep-profiles',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV SC rep profiles on SLURM with '
'node name "{}"'.format(name))
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV rep profiles job "{}" '
'(SLURM jobid #{}).'.format(name, out))
Status.add_job(out_dir,
'rep-profiles',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': '{}.h5'.format(name),
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def slurm(ctx, alloc, walltime, feature, memory, module, conda_env,
stdout_path):
"""slurm (Eagle) submission tool for reV supply curve aggregation."""
name = ctx.obj['NAME']
excl_fpath = ctx.obj['EXCL_FPATH']
gen_fpath = ctx.obj['GEN_FPATH']
econ_fpath = ctx.obj['ECON_FPATH']
res_fpath = ctx.obj['RES_FPATH']
tm_dset = ctx.obj['TM_DSET']
excl_dict = ctx.obj['EXCL_DICT']
check_excl_layers = ctx.obj['CHECK_LAYERS']
res_class_dset = ctx.obj['RES_CLASS_DSET']
res_class_bins = ctx.obj['RES_CLASS_BINS']
cf_dset = ctx.obj['CF_DSET']
lcoe_dset = ctx.obj['LCOE_DSET']
h5_dsets = ctx.obj['H5_DSETS']
data_layers = ctx.obj['DATA_LAYERS']
resolution = ctx.obj['RESOLUTION']
excl_area = ctx.obj['EXCL_AREA']
power_density = ctx.obj['POWER_DENSITY']
area_filter_kernel = ctx.obj['AREA_FILTER_KERNEL']
min_area = ctx.obj['MIN_AREA']
friction_fpath = ctx.obj['FRICTION_FPATH']
friction_dset = ctx.obj['FRICTION_DSET']
out_dir = ctx.obj['OUT_DIR']
log_dir = ctx.obj['LOG_DIR']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath,
tm_dset, excl_dict, check_excl_layers, res_class_dset,
res_class_bins, cf_dset, lcoe_dset, h5_dsets,
data_layers, resolution, excl_area, power_density,
area_filter_kernel, min_area, friction_fpath,
friction_dset, out_dir, log_dir, verbose)
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(out_dir,
'supply-curve-aggregation',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV SC aggregation on SLURM with '
'node name "{}"'.format(name))
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV SC aggregation job "{}" '
'(SLURM jobid #{}).'.format(name, out))
Status.add_job(out_dir,
'supply-curve-aggregation',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': '{}.csv'.format(name),
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def slurm(ctx, alloc, nodes, memory, walltime, feature, conda_env, module,
stdout_path, verbose):
"""Run generation on HPC via SLURM job submission."""
name = ctx.obj['NAME']
tech = ctx.obj['TECH']
points = ctx.obj['POINTS']
sam_files = ctx.obj['SAM_FILES']
res_file = ctx.obj['RES_FILE']
sites_per_worker = ctx.obj['SITES_PER_WORKER']
fout = ctx.obj['FOUT']
dirout = ctx.obj['DIROUT']
logdir = ctx.obj['LOGDIR']
output_request = ctx.obj['OUTPUT_REQUEST']
site_data = ctx.obj['SITE_DATA']
max_workers = ctx.obj['MAX_WORKERS']
mem_util_lim = ctx.obj['MEM_UTIL_LIM']
timeout = ctx.obj['TIMEOUT']
curtailment = ctx.obj['CURTAILMENT']
verbose = any([verbose, ctx.obj['VERBOSE']])
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
pc = get_node_pc(points, sam_files, tech, res_file, nodes)
for i, split in enumerate(pc):
node_name, fout_node = get_node_name_fout(name,
fout,
i,
pc,
hpc='slurm')
cmd = get_node_cmd(node_name,
tech,
sam_files,
res_file,
points=points,
points_range=split.split_range,
sites_per_worker=sites_per_worker,
max_workers=max_workers,
fout=fout_node,
dirout=dirout,
logdir=logdir,
output_request=output_request,
site_data=site_data,
mem_util_lim=mem_util_lim,
timeout=timeout,
curtailment=curtailment,
verbose=verbose)
status = Status.retrieve_job_status(dirout,
'generation',
node_name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(node_name, dirout))
elif 'fail' not in str(status).lower() and status is not None:
msg = (
'Job "{}" was found with status "{}", not resubmitting'.format(
node_name, status))
else:
logger.info('Running reV generation on SLURM with node name "{}" '
'for {} (points range: {}).'.format(
node_name, pc, split.split_range))
# create and submit the SLURM job
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=node_name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV generation job "{}" (SLURM jobid #{}).'.
format(node_name, out))
# add job to reV status file.
Status.add_job(dirout,
'generation',
node_name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': fout_node,
'dirout': dirout
})
click.echo(msg)
logger.info(msg)
def slurm(ctx, alloc, feature, memory, walltime, module, conda_env,
stdout_path):
"""slurm (Eagle) submission tool for reV supply curve aggregation."""
name = ctx.obj['NAME']
gen_fpath = ctx.obj['GEN_FPATH']
offshore_fpath = ctx.obj['OFFSHORE_FPATH']
project_points = ctx.obj['PROJECT_POINTS']
sam_files = ctx.obj['SAM_FILES']
log_dir = ctx.obj['LOG_DIR']
out_dir = ctx.obj['OUT_DIR']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, gen_fpath, offshore_fpath, project_points,
sam_files, log_dir, verbose)
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(out_dir,
'offshore',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV offshore aggregation on SLURM with '
'node name "{}"'.format(name))
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV offshore job "{}" '
'(SLURM jobid #{}).'.format(name, out))
Status.add_job(out_dir,
'offshore',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': '{}.csv'.format(name),
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def collect_slurm(ctx, alloc, memory, walltime, feature, conda_env, module,
stdout_path, verbose):
"""Run collection on HPC via SLURM job submission."""
name = ctx.obj['NAME']
h5_file = ctx.obj['H5_FILE']
h5_dir = ctx.obj['H5_DIR']
log_dir = ctx.obj['LOG_DIR']
project_points = ctx.obj['PROJECT_POINTS']
dsets = ctx.obj['DSETS']
file_prefix = ctx.obj['FILE_PREFIX']
purge_chunks = ctx.obj['PURGE_CHUNKS']
verbose = any([verbose, ctx.obj['VERBOSE']])
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
cmd = get_node_cmd(name,
h5_file,
h5_dir,
project_points,
dsets,
file_prefix=file_prefix,
log_dir=log_dir,
purge_chunks=purge_chunks,
verbose=verbose)
status = Status.retrieve_job_status(os.path.dirname(h5_file),
'collect',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, os.path.dirname(h5_file)))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info(
'Running reV collection on SLURM with node name "{}", '
'collecting data to "{}" from "{}" with file prefix "{}".'.format(
name, h5_file, h5_dir, file_prefix))
out = slurm_manager.sbatch(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = (
'Kicked off reV collection job "{}" (SLURM jobid #{}).'.format(
name, out))
# add job to reV status file.
Status.add_job(os.path.dirname(h5_file),
'collect',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': os.path.basename(h5_file),
'dirout': os.path.dirname(h5_file)
})
click.echo(msg)
logger.info(msg)
