def reV_h5(ctx, h5_file, out_dir, sub_dir, dsets, group, process_size,
max_workers, plot_type, cmap, log_file, verbose, terminal):
"""
Summarize and plot data for reV h5_file
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
QaQc.h5(h5_file,
qa_dir,
dsets=dsets,
group=group,
process_size=process_size,
max_workers=max_workers,
plot_type=plot_type,
cmap=cmap)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': h5_file
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def supply_curve(ctx, sc_table, out_dir, sub_dir, columns, plot_type, cmap,
lcoe, log_file, verbose, terminal):
"""
Summarize and plot reV Supply Curve data
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
QaQc.supply_curve(sc_table,
qa_dir,
columns=columns,
lcoe=lcoe,
plot_type=plot_type,
cmap=cmap)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': sc_table
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def slurm(ctx, alloc, memory, walltime, feature, conda_env, module,
stdout_path):
"""slurm (Eagle) submission tool for reV representative profiles."""
name = ctx.obj['NAME']
gen_fpath = ctx.obj['GEN_FPATH']
rev_summary = ctx.obj['REV_SUMMARY']
reg_cols = ctx.obj['REG_COLS']
cf_dset = ctx.obj['CF_DSET']
rep_method = ctx.obj['REP_METHOD']
err_method = ctx.obj['ERR_METHOD']
weight = ctx.obj['WEIGHT']
n_profiles = ctx.obj['N_PROFILES']
out_dir = ctx.obj['OUT_DIR']
log_dir = ctx.obj['LOG_DIR']
max_workers = ctx.obj['MAX_WORKERS']
aggregate_profiles = ctx.obj['AGGREGATE_PROFILES']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset,
rep_method, err_method, weight, n_profiles, out_dir,
log_dir, max_workers, aggregate_profiles, verbose)
status = Status.retrieve_job_status(out_dir, 'rep-profiles', name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
else:
logger.info('Running reV SC rep profiles on SLURM with '
'node name "{}"'.format(name))
slurm = SLURM(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)
if slurm.id:
msg = ('Kicked off reV rep profiles job "{}" '
'(SLURM jobid #{}).'.format(name, slurm.id))
Status.add_job(out_dir,
'rep-profiles',
name,
replace=True,
job_attrs={
'job_id': slurm.id,
'hardware': 'eagle',
'fout': '{}.h5'.format(name),
'dirout': out_dir
})
else:
msg = ('Was unable to kick off reV rep profiles job "{}". '
'Please see the stdout error messages'.format(name))
click.echo(msg)
logger.info(msg)
def from_config(ctx, config_file, verbose):
"""Run reV gen from a config file."""
name = ctx.obj['NAME']
# Instantiate the config object
config = MultiYearConfig(config_file)
# take name from config if not default
if config.name.lower() != 'rev':
name = config.name
ctx.obj['NAME'] = name
# Enforce verbosity if logging level is specified in the config
if config.log_level == logging.DEBUG:
verbose = True
# make output directory if does not exist
if not os.path.exists(config.dirout):
os.makedirs(config.dirout)
# initialize loggers.
init_mult(name, config.logdir, modules=[__name__, 'reV'], verbose=verbose)
# Initial log statements
logger.info(
'Running reV multi-year from config file: "{}"'.format(config_file))
logger.info('Target output directory: "{}"'.format(config.dirout))
logger.info('Target logging directory: "{}"'.format(config.logdir))
ctx.obj['MY_FILE'] = config.my_file
if config.execution_control.option == 'local':
ctx.obj['NAME'] = name
status = Status.retrieve_job_status(config.dirout, 'multi-year', name)
if status != 'successful':
Status.add_job(config.dirout,
'multi-year',
name,
replace=True,
job_attrs={
'hardware': 'local',
'fout': ctx.obj['MY_FILE'],
'dirout': config.dirout
})
group_params = json.dumps(config.group_params)
ctx.invoke(multi_year_groups, group_params=group_params)
elif config.execution_control.option in ('eagle', 'slurm'):
ctx.obj['NAME'] = name
ctx.invoke(multi_year_slurm,
alloc=config.execution_control.allocation,
walltime=config.execution_control.walltime,
feature=config.execution_control.feature,
memory=config.execution_control.memory,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module,
stdout_path=os.path.join(config.logdir, 'stdout'),
group_params=json.dumps(config.group_params),
verbose=verbose)
def test_job_exists():
"""Test job addition and exist check"""
purge()
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'job_status': 'submitted'})
exists = Status.job_exists(STATUS_DIR, 'test1')
assert exists
purge()
def test_make_file():
"""Test file creation and reading"""
purge()
Status.make_job_file(STATUS_DIR, 'generation', 'test1', TEST_1_ATTRS_1)
status = Status.retrieve_job_status(STATUS_DIR, 'generation', 'test1')
msg = 'Failed, status is "{}"'.format(status)
assert status == 'R', msg
purge()
def multi_year_slurm(ctx, group_params, alloc, walltime, feature, memory,
conda_env, module, stdout_path, verbose):
"""
Run multi year collection and means on HPC via SLURM job submission.
"""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
slurm_manager = ctx.obj.get('SLURM_MANAGER', None)
if slurm_manager is None:
slurm_manager = SLURM()
ctx.obj['SLURM_MANAGER'] = slurm_manager
status = Status.retrieve_job_status(os.path.dirname(my_file),
'multi-year',
name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, os.path.dirname(my_file)))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
name, status))
else:
logger.info('Running reV multi-year collection on SLURM with node '
' name "{}", collecting into "{}".'.format(name, my_file))
# create and submit the SLURM job
slurm_cmd = get_slurm_cmd(name, my_file, group_params, verbose=verbose)
out = slurm_manager.sbatch(slurm_cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)[0]
if out:
msg = ('Kicked off reV multi-year collection job "{}" '
'(SLURM jobid #{}).'.format(name, out))
# add job to reV status file.
Status.add_job(os.path.dirname(my_file),
'multi-year',
name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'fout': os.path.basename(my_file),
'dirout': os.path.dirname(my_file)
})
click.echo(msg)
logger.info(msg)
def direct(ctx, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method,
err_method, weight, n_profiles, out_dir, log_dir, max_workers,
aggregate_profiles, verbose):
"""reV representative profiles CLI."""
name = ctx.obj['NAME']
ctx.obj['GEN_FPATH'] = gen_fpath
ctx.obj['REV_SUMMARY'] = rev_summary
ctx.obj['REG_COLS'] = reg_cols
ctx.obj['CF_DSET'] = cf_dset
ctx.obj['REP_METHOD'] = rep_method
ctx.obj['ERR_METHOD'] = err_method
ctx.obj['WEIGHT'] = weight
ctx.obj['N_PROFILES'] = n_profiles
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['LOG_DIR'] = log_dir
ctx.obj['MAX_WORKERS'] = max_workers
ctx.obj['AGGREGATE_PROFILES'] = aggregate_profiles
ctx.obj['VERBOSE'] = verbose
if ctx.invoked_subcommand is None:
t0 = time.time()
init_mult(name, log_dir, modules=['reV', 'rex'], verbose=verbose)
fn_out = '{}.h5'.format(name)
fout = os.path.join(out_dir, fn_out)
if aggregate_profiles:
AggregatedRepProfiles.run(gen_fpath,
rev_summary,
cf_dset=cf_dset,
weight=weight,
fout=fout,
max_workers=max_workers)
else:
RepProfiles.run(gen_fpath,
rev_summary,
reg_cols,
cf_dset=cf_dset,
rep_method=rep_method,
err_method=err_method,
weight=weight,
fout=fout,
n_profiles=n_profiles,
max_workers=max_workers)
runtime = (time.time() - t0) / 60
logger.info('reV representative profiles complete. '
'Time elapsed: {:.2f} min. Target output dir: {}'.format(
runtime, out_dir))
status = {
'dirout': out_dir,
'fout': fn_out,
'job_status': 'successful',
'runtime': runtime,
'finput': [gen_fpath, rev_summary]
}
Status.make_job_file(out_dir, 'rep-profiles', name, status)
def slurm(ctx, alloc, memory, walltime, feature, module, conda_env,
stdout_path):
"""slurm (eagle) submission tool for reV supply curve."""
name = ctx.obj['NAME']
sc_points = ctx.obj['SC_POINTS']
trans_table = ctx.obj['TRANS_TABLE']
fixed_charge_rate = ctx.obj['FIXED_CHARGE_RATE']
sc_features = ctx.obj['SC_FEATURES']
transmission_costs = ctx.obj['TRANSMISSION_COSTS']
simple = ctx.obj['SIMPLE']
line_limited = ctx.obj['LINE_LIMITED']
sort_on = ctx.obj['SORT_ON']
offshore_trans_table = ctx.obj['OFFSHORE_TRANS_TABLE']
wind_dirs = ctx.obj['WIND_DIRS']
n_dirs = ctx.obj['N_DIRS']
downwind = ctx.obj['DOWNWIND']
offshore_compete = ctx.obj['OFFSHORE_COMPETE']
max_workers = ctx.obj['MAX_WORKERS']
out_dir = ctx.obj['OUT_DIR']
log_dir = ctx.obj['LOG_DIR']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, sc_points, trans_table, fixed_charge_rate,
sc_features, transmission_costs, sort_on,
offshore_trans_table, wind_dirs, n_dirs, downwind,
offshore_compete, max_workers, out_dir, log_dir,
simple, line_limited, verbose)
status = Status.retrieve_job_status(out_dir, 'supply-curve', name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'
.format(name, out_dir))
else:
logger.info('Running reV Supply Curve on SLURM with '
'node name "{}"'.format(name))
logger.debug('\t{}'.format(cmd))
slurm = SLURM(cmd, alloc=alloc, memory=memory,
walltime=walltime, feature=feature,
name=name, stdout_path=stdout_path,
conda_env=conda_env, module=module)
if slurm.id:
msg = ('Kicked off reV SC job "{}" (SLURM jobid #{}).'
.format(name, slurm.id))
Status.add_job(
out_dir, 'supply-curve', name, replace=True,
job_attrs={'job_id': slurm.id, 'hardware': 'eagle',
'fout': '{}.csv'.format(name), 'dirout': out_dir})
else:
msg = ('Was unable to kick off reV SC job "{}". Please see the '
'stdout error messages'.format(name))
click.echo(msg)
logger.info(msg)
def launch_slurm(config, verbose):
"""
Launch slurm QA/QC job
Parameters
----------
config : dict
'reV QA/QC configuration dictionary'
"""
out_dir = config.dirout
log_file = os.path.join(config.logdir, config.name + '.log')
stdout_path = os.path.join(config.logdir, 'stdout/')
node_cmd = get_multiple_cmds(config, out_dir, log_file, verbose)
slurm_manager = SLURM()
status = Status.retrieve_job_status(out_dir,
'qa-qc',
config.name,
hardware='eagle',
subprocess_manager=slurm_manager)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(config.name, out_dir))
elif 'fail' not in str(status).lower() and status is not None:
msg = ('Job "{}" was found with status "{}", not resubmitting'.format(
config.name, status))
else:
logger.info('Running reV QA-QC on SLURM with '
'node name "{}"'.format(config.name))
out = slurm_manager.sbatch(
node_cmd,
name=config.name,
alloc=config.execution_control.allocation,
memory=config.execution_control.memory,
feature=config.execution_control.feature,
walltime=config.execution_control.walltime,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module,
stdout_path=stdout_path)[0]
if out:
msg = ('Kicked off reV QA-QC job "{}" '
'(SLURM jobid #{}).'.format(config.name, out))
Status.add_job(out_dir,
'qa-qc',
config.name,
replace=True,
job_attrs={
'job_id': out,
'hardware': 'eagle',
'dirout': out_dir
})
click.echo(msg)
logger.info(msg)
def slurm(ctx, alloc, feature, memory, walltime, module, conda_env,
stdout_path):
"""slurm (Eagle) submission tool for reV supply curve aggregation."""
name = ctx.obj['NAME']
gen_fpath = ctx.obj['GEN_FPATH']
offshore_fpath = ctx.obj['OFFSHORE_FPATH']
project_points = ctx.obj['PROJECT_POINTS']
sam_files = ctx.obj['SAM_FILES']
log_dir = ctx.obj['LOG_DIR']
out_dir = ctx.obj['OUT_DIR']
verbose = ctx.obj['VERBOSE']
if stdout_path is None:
stdout_path = os.path.join(log_dir, 'stdout/')
cmd = get_node_cmd(name, gen_fpath, offshore_fpath, project_points,
sam_files, log_dir, verbose)
status = Status.retrieve_job_status(out_dir, 'offshore', name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, out_dir))
else:
logger.info('Running reV offshore aggregation on SLURM with '
'node name "{}"'.format(name))
slurm = SLURM(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)
if slurm.id:
msg = ('Kicked off reV offshore job "{}" '
'(SLURM jobid #{}).'.format(name, slurm.id))
Status.add_job(out_dir,
'offshore',
name,
replace=True,
job_attrs={
'job_id': slurm.id,
'hardware': 'eagle',
'fout': '{}.csv'.format(name),
'dirout': out_dir
})
else:
msg = ('Was unable to kick off reV offshore job "{}". Please see '
'the stdout error messages'.format(name))
click.echo(msg)
logger.info(msg)
def multi_year_groups(ctx, group_params, verbose):
"""Run multi year collection and means for multiple groups."""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize loggers for multiple modules
log_dir = os.path.dirname(my_file)
init_mult(name,
log_dir,
modules=[__name__, 'reV.handlers.multi_year'],
verbose=verbose,
node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to collection method: "{}" : "{}" '
'with type "{}"'.format(key, val, type(val)))
logger.info('Multi-year collection is being run with job name "{}". '
'Target output path is: {}'.format(name, my_file))
ts = time.time()
for group_name, group in json.loads(group_params).items():
logger.info('- Collecting datasets "{}" from "{}" into "{}/"'.format(
group['dsets'], group['source_files'], group_name))
t0 = time.time()
for dset in group['dsets']:
if MultiYear.is_profile(group['source_files'], dset):
MultiYear.collect_profiles(my_file,
group['source_files'],
dset,
group=group['group'])
else:
MultiYear.collect_means(my_file,
group['source_files'],
dset,
group=group['group'])
runtime = (time.time() - t0) / 60
logger.info('- {} collection completed in: {:.2f} min.'.format(
group_name, runtime))
runtime = (time.time() - ts) / 60
logger.info(
'Multi-year collection completed in : {:.2f} min.'.format(runtime))
# add job to reV status file.
status = {
'dirout': os.path.dirname(my_file),
'fout': os.path.basename(my_file),
'job_status': 'successful',
'runtime': runtime
}
Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def multi_year(ctx, source_files, group, dsets, pass_through_dsets, verbose):
"""Run multi year collection and means on local worker."""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize loggers for multiple modules
log_dir = os.path.dirname(my_file)
init_mult(name,
log_dir,
modules=[__name__, 'reV.handlers.multi_year'],
verbose=verbose,
node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to collection method: "{}" : "{}" '
'with type "{}"'.format(key, val, type(val)))
logger.info('Multi-year collection is being run for "{}" '
'with job name "{}" on {}. Target output path is: {}'.format(
dsets, name, source_files, my_file))
t0 = time.time()
for dset in dsets:
if MultiYear.is_profile(source_files, dset):
MultiYear.collect_profiles(my_file,
source_files,
dset,
group=group)
else:
MultiYear.collect_means(my_file, source_files, dset, group=group)
if pass_through_dsets is not None:
for dset in pass_through_dsets:
MultiYear.pass_through(my_file, source_files, dset, group=group)
runtime = (time.time() - t0) / 60
logger.info(
'Multi-year collection completed in: {:.2f} min.'.format(runtime))
# add job to reV status file.
status = {
'dirout': os.path.dirname(my_file),
'fout': os.path.basename(my_file),
'job_status': 'successful',
'runtime': runtime,
'finput': source_files
}
Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def multi_year_slurm(ctx, alloc, walltime, feature, memory, conda_env, module,
stdout_path, group_params, verbose):
"""
Run multi year collection and means on HPC via SLURM job submission.
"""
name = ctx.obj['NAME']
my_file = ctx.obj['MY_FILE']
verbose = any([verbose, ctx.obj['VERBOSE']])
status = Status.retrieve_job_status(os.path.dirname(my_file), 'multi-year',
name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, os.path.dirname(my_file)))
else:
logger.info('Running reV multi-year collection on SLURM with node '
' name "{}", collecting into "{}".'.format(name, my_file))
# create and submit the SLURM job
slurm_cmd = get_slurm_cmd(name, my_file, group_params, verbose=verbose)
slurm = SLURM(slurm_cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)
if slurm.id:
msg = ('Kicked off reV multi-year collection job "{}" '
'(SLURM jobid #{}).'.format(name, slurm.id))
# add job to reV status file.
Status.add_job(os.path.dirname(my_file),
'multi-year',
name,
replace=True,
job_attrs={
'job_id': slurm.id,
'hardware': 'eagle',
'fout': os.path.basename(my_file),
'dirout': os.path.dirname(my_file)
})
else:
msg = ('Was unable to kick off reV collection job "{}". '
'Please see the stdout error messages'.format(name))
click.echo(msg)
logger.info(msg)
def parse_previous(status_dir, module, target='fpath', target_module=None):
"""Parse output file paths from the previous pipeline step.
Parameters
----------
status_dir : str
Directory containing the status file to parse.
module : str
Current module (i.e. current pipeline step).
target : str
Parsing target of previous module.
target_module : str | None
Optional name of module to pull target data from.
Returns
-------
out : list
Arguments parsed from the status file in status_dir from
the module preceding the input module arg.
"""
status = Status(status_dir)
msg = ('Could not parse data regarding "{}" from reV status file in '
'"{}".'.format(module, status_dir))
if module in status.data:
if 'pipeline_index' in status.data[module]:
msg = None
if msg:
raise KeyError(msg)
i1 = int(status.data[module]['pipeline_index'])
i0 = i1 - 1
if i0 < 0:
i0 = 0
warn('Module "{0}" is attempting to parse a previous pipeline '
'step, but it appears to be the first step. Attempting to '
'parse data from {0}.'.format(module))
if target_module is None:
module_status = Pipeline._get_module_status(status, i0)
job_statuses = Pipeline._get_job_status(module_status)
else:
if target_module not in status.data:
raise KeyError('Target module "{}" not found in pipeline '
'status dictionary.'.format(target_module))
else:
module_status = status.data[target_module]
job_statuses = Pipeline._get_job_status(module_status)
out = []
if target == 'fpath':
for status in job_statuses:
out.append(os.path.join(status['dirout'], status['fout']))
else:
for status in job_statuses:
out.append(status[target])
return out
def test_recursive_update():
"""Test a recursive merge of two status dictionaries"""
test = Status.update_dict({'generation': TEST_1_ATTRS_1},
{'generation': TEST_1_ATTRS_2})
assert test['generation']['run_id'] == TEST_1_ATTRS_1['run_id']
assert test['generation']['job_status'] == TEST_1_ATTRS_2['job_status']
def direct(ctx, gen_fpath, offshore_fpath, points, sam_files, log_dir,
verbose):
"""Main entry point to run offshore wind aggregation"""
name = ctx.obj['NAME']
ctx.obj['GEN_FPATH'] = gen_fpath
ctx.obj['OFFSHORE_FPATH'] = offshore_fpath
ctx.obj['POINTS'] = points
ctx.obj['SAM_FILES'] = sam_files
ctx.obj['OUT_DIR'] = os.path.dirname(gen_fpath)
ctx.obj['LOG_DIR'] = log_dir
ctx.obj['VERBOSE'] = verbose
if ctx.invoked_subcommand is None:
t0 = time.time()
init_mult(name,
log_dir,
modules=[__name__, 'reV.offshore', 'reV.handlers', 'rex'],
verbose=verbose,
node=True)
fpath_out = gen_fpath.replace('.h5', '_offshore.h5')
try:
Offshore.run(gen_fpath,
offshore_fpath,
points,
sam_files,
fpath_out=fpath_out)
except Exception as e:
logger.exception('Offshore module failed, received the '
'following exception:\n{}'.format(e))
raise e
runtime = (time.time() - t0) / 60
status = {
'dirout': os.path.dirname(fpath_out),
'fout': os.path.basename(fpath_out),
'job_status': 'successful',
'runtime': runtime,
'finput': gen_fpath
}
Status.make_job_file(os.path.dirname(fpath_out), 'offshore', name,
status)
def _get_status_obj(self):
"""Get a reV pipeline status object.
Returns
-------
status : reV.pipeline.status.Status
reV job status object.
"""
status = Status(self._config.dirout,
name=self._config.name,
hardware=self._config.hardware)
return status
def exclusions(ctx, excl_fpath, out_dir, sub_dir, excl_dict,
area_filter_kernel, min_area, plot_type, cmap, plot_step,
log_file, verbose, terminal):
"""
Extract and plot reV exclusions mask
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
if isinstance(excl_dict, str):
excl_dict = dict_str_load(excl_dict)
QaQc.exclusions_mask(excl_fpath,
qa_dir,
layers_dict=excl_dict,
min_area=min_area,
kernel=area_filter_kernel,
plot_type=plot_type,
cmap=cmap,
plot_step=plot_step)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': excl_fpath
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_job_addition():
"""Test job addition and exist check"""
purge()
Status.add_job(STATUS_DIR, 'generation', 'test1')
status1 = Status(STATUS_DIR).data['generation']['test1']['job_status']
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'job_status': 'finished', 'additional': 'test'})
status2 = Status(STATUS_DIR).data['generation']['test1']['job_status']
assert status2 == status1
purge()
def test_file_collection():
"""Test file creation and collection"""
purge()
Status.make_job_file(STATUS_DIR, 'generation', 'test1', TEST_1_ATTRS_1)
Status.make_job_file(STATUS_DIR, 'generation', 'test2', TEST_2_ATTRS_1)
Status.update(STATUS_DIR)
with open(os.path.join(STATUS_DIR, 'rev_status.json'), 'r') as f:
data = json.load(f)
assert str(TEST_1_ATTRS_1) in str(data)
assert str(TEST_2_ATTRS_1) in str(data)
purge()
def test_job_replacement():
"""Test job addition and replacement"""
purge()
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'job_status': 'submitted'})
Status.add_job(STATUS_DIR, 'generation', 'test1',
job_attrs={'addition': 'test', 'job_status': 'finished'},
replace=True)
status = Status(STATUS_DIR).data['generation']['test1']['job_status']
addition = Status(STATUS_DIR).data['generation']['test1']['addition']
assert status == 'finished'
assert addition == 'test'
purge()
def from_config(ctx, config_file, verbose):
"""Run reV QA/QC from a config file."""
name = ctx.obj['NAME']
# Instantiate the config object
config = QaQcConfig(config_file)
# take name from config if not default
if config.name.lower() != 'rev':
name = config.name
ctx.obj['NAME'] = name
# Enforce verbosity if logging level is specified in the config
verbose = config.log_level == logging.DEBUG
# initialize loggers
init_mult(name, config.logdir, modules=['reV', 'rex'], verbose=verbose)
# Initial log statements
logger.info('Running reV supply curve from config '
'file: "{}"'.format(config_file))
logger.info('Target output directory: "{}"'.format(config.dirout))
logger.info('Target logging directory: "{}"'.format(config.logdir))
logger.debug('The full configuration input is as follows:\n{}'.format(
pprint.pformat(config, indent=4)))
if config.execution_control.option == 'local':
status = Status.retrieve_job_status(config.dirout, 'qa-qc', name)
if status != 'successful':
Status.add_job(config.dirout,
'qa-qc',
name,
replace=True,
job_attrs={
'hardware': 'local',
'dirout': config.dirout
})
terminal = False
for i, module in enumerate(config.module_names):
if i == len(config.module_names) - 1:
terminal = True
module_config = config.get_module_inputs(module)
fpath = module_config.fpath
if module.lower() == 'exclusions':
log_file = os.path.join(
config.logdir,
os.path.basename(fpath).replace('.h5', '.log'))
afk = module_config.area_filter_kernel
ctx.invoke(exclusions,
excl_fpath=fpath,
out_dir=config.dirout,
sub_dir=module_config.sub_dir,
excl_dict=module_config.excl_dict,
area_filter_kernel=afk,
min_area=module_config.min_area,
plot_type=module_config.plot_type,
cmap=module_config.cmap,
plot_step=module_config.plot_step,
log_file=log_file,
verbose=verbose,
terminal=terminal)
elif fpath.endswith('.h5'):
log_file = os.path.join(
config.logdir,
os.path.basename(fpath).replace('.h5', '.log'))
ctx.invoke(reV_h5,
h5_file=fpath,
out_dir=config.dirout,
sub_dir=module_config.sub_dir,
dsets=module_config.dsets,
group=module_config.group,
process_size=module_config.process_size,
max_workers=module_config.max_workers,
plot_type=module_config.plot_type,
cmap=module_config.cmap,
log_file=log_file,
verbose=verbose,
terminal=terminal)
elif fpath.endswith('.csv'):
log_file = os.path.join(
config.logdir,
os.path.basename(fpath).replace('.csv', '.log'))
ctx.invoke(supply_curve,
sc_table=fpath,
out_dir=config.dirout,
sub_dir=module_config.sub_dir,
columns=module_config.columns,
plot_type=module_config.plot_type,
cmap=module_config.cmap,
lcoe=module_config.lcoe,
log_file=log_file,
verbose=verbose,
terminal=terminal)
else:
msg = (
"Cannot run QA/QC for {}: 'fpath' must be a '*.h5' "
"or '*.csv' reV output file, but {} was given!".format(
module, fpath))
logger.error(msg)
raise ValueError(msg)
elif config.execution_control.option in ('eagle', 'slurm'):
launch_slurm(config, verbose)
def direct(ctx, sc_points, trans_table, fixed_charge_rate, sc_features,
transmission_costs, sort_on, offshore_trans_table, wind_dirs,
n_dirs, downwind, offshore_compete, max_workers, out_dir, log_dir,
simple, line_limited, verbose):
"""reV Supply Curve CLI."""
name = ctx.obj['NAME']
ctx.obj['SC_POINTS'] = sc_points
ctx.obj['TRANS_TABLE'] = trans_table
ctx.obj['FIXED_CHARGE_RATE'] = fixed_charge_rate
ctx.obj['SC_FEATURES'] = sc_features
ctx.obj['TRANSMISSION_COSTS'] = transmission_costs
ctx.obj['SORT_ON'] = sort_on
ctx.obj['OFFSHORE_TRANS_TABLE'] = offshore_trans_table
ctx.obj['WIND_DIRS'] = wind_dirs
ctx.obj['N_DIRS'] = n_dirs
ctx.obj['DOWNWIND'] = downwind
ctx.obj['MAX_WORKERS'] = max_workers
ctx.obj['OFFSHORE_COMPETE'] = offshore_compete
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['LOG_DIR'] = log_dir
ctx.obj['SIMPLE'] = simple
ctx.obj['LINE_LIMITED'] = line_limited
ctx.obj['VERBOSE'] = verbose
if ctx.invoked_subcommand is None:
t0 = time.time()
init_mult(
name,
log_dir,
modules=[__name__, 'reV.supply_curve', 'reV.handlers', 'rex'],
verbose=verbose)
if isinstance(transmission_costs, str):
transmission_costs = dict_str_load(transmission_costs)
offshore_table = offshore_trans_table
try:
if simple:
out = SupplyCurve.simple(sc_points,
trans_table,
fixed_charge_rate,
sc_features=sc_features,
transmission_costs=transmission_costs,
sort_on=sort_on,
wind_dirs=wind_dirs,
n_dirs=n_dirs,
downwind=downwind,
max_workers=max_workers,
offshore_trans_table=offshore_table,
offshore_compete=offshore_compete)
else:
out = SupplyCurve.full(sc_points,
trans_table,
fixed_charge_rate,
sc_features=sc_features,
transmission_costs=transmission_costs,
line_limited=line_limited,
sort_on=sort_on,
wind_dirs=wind_dirs,
n_dirs=n_dirs,
downwind=downwind,
max_workers=max_workers,
offshore_trans_table=offshore_table,
offshore_compete=offshore_compete)
except Exception as e:
logger.exception('Supply curve compute failed. Received the '
'following error:\n{}'.format(e))
raise e
fn_out = '{}.csv'.format(name)
fpath_out = os.path.join(out_dir, fn_out)
out.to_csv(fpath_out, index=False)
runtime = (time.time() - t0) / 60
logger.info('Supply curve complete. Time elapsed: {:.2f} min. '
'Target output dir: {}'.format(runtime, out_dir))
finput = [sc_points, trans_table]
if sc_features is not None:
finput.append(sc_features)
if transmission_costs is not None:
finput.append(transmission_costs)
# add job to reV status file.
status = {
'dirout': out_dir,
'fout': fn_out,
'job_status': 'successful',
'runtime': runtime,
'finput': finput
}
Status.make_job_file(out_dir, 'supply-curve', name, status)
def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env,
stdout_path, verbose):
"""Run econ on HPC via SLURM job submission."""
name = ctx.obj['NAME']
points = ctx.obj['POINTS']
sam_files = ctx.obj['SAM_FILES']
cf_file = ctx.obj['CF_FILE']
cf_year = ctx.obj['CF_YEAR']
site_data = ctx.obj['SITE_DATA']
sites_per_worker = ctx.obj['SITES_PER_WORKER']
max_workers = ctx.obj['MAX_WORKERS']
timeout = ctx.obj['TIMEOUT']
fout = ctx.obj['FOUT']
dirout = ctx.obj['DIROUT']
logdir = ctx.obj['LOGDIR']
output_request = ctx.obj['OUTPUT_REQUEST']
append = ctx.obj['APPEND']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize an info logger on the year level
init_mult(name,
logdir,
modules=[
__name__, 'reV.econ.econ', 'reV.config', 'reV.utilities',
'reV.SAM', 'rex.utilities'
],
verbose=False)
if append:
pc = [None]
else:
pc = get_node_pc(points, sam_files, nodes)
for i, split in enumerate(pc):
node_name, fout_node = get_node_name_fout(name,
fout,
i,
pc,
hpc='slurm')
node_name = node_name.replace('gen', 'econ')
points_range = split.split_range if split is not None else None
cmd = get_node_cmd(node_name,
sam_files,
cf_file,
cf_year=cf_year,
site_data=site_data,
points=points,
points_range=points_range,
sites_per_worker=sites_per_worker,
max_workers=max_workers,
timeout=timeout,
fout=fout_node,
dirout=dirout,
logdir=logdir,
output_request=output_request,
append=append,
verbose=verbose)
status = Status.retrieve_job_status(dirout, 'econ', node_name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(node_name, dirout))
else:
logger.info('Running reV econ on SLURM with node name "{}" for '
'{} (points range: {}).'.format(
node_name, pc, points_range))
# create and submit the SLURM job
slurm = SLURM(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=node_name,
stdout_path=stdout_path,
conda_env=conda_env,
module=module)
if slurm.id:
msg = (
'Kicked off reV econ job "{}" (SLURM jobid #{}).'.format(
node_name, slurm.id))
# add job to reV status file.
Status.add_job(dirout,
'econ',
node_name,
replace=True,
job_attrs={
'job_id': slurm.id,
'hardware': 'eagle',
'fout': fout_node,
'dirout': dirout
})
else:
msg = (
'Was unable to kick off reV econ job "{}". '
'Please see the stdout error messages'.format(node_name))
click.echo(msg)
logger.info(msg)
def collect(ctx, verbose):
"""Run collection on local worker."""
name = ctx.obj['NAME']
h5_file = ctx.obj['H5_FILE']
h5_dir = ctx.obj['H5_DIR']
project_points = ctx.obj['PROJECT_POINTS']
dsets = ctx.obj['DSETS']
file_prefix = ctx.obj['FILE_PREFIX']
log_dir = ctx.obj['LOG_DIR']
purge_chunks = ctx.obj['PURGE_CHUNKS']
verbose = any([verbose, ctx.obj['VERBOSE']])
# initialize loggers for multiple modules
init_mult(name,
log_dir,
modules=[__name__, 'reV.handlers.collection'],
verbose=verbose,
node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to collection method: "{}" : "{}" '
'with type "{}"'.format(key, val, type(val)))
logger.info('Collection is being run for "{}" with job name "{}" '
'and collection dir: {}. Target output path is: {}'.format(
dsets, name, h5_dir, h5_file))
t0 = time.time()
Collector.collect(h5_file,
h5_dir,
project_points,
dsets[0],
file_prefix=file_prefix)
if len(dsets) > 1:
for dset_name in dsets[1:]:
Collector.add_dataset(h5_file,
h5_dir,
dset_name,
file_prefix=file_prefix)
if purge_chunks:
Collector.purge_chunks(h5_file,
h5_dir,
project_points,
file_prefix=file_prefix)
else:
Collector.move_chunks(h5_file,
h5_dir,
project_points,
file_prefix=file_prefix)
runtime = (time.time() - t0) / 60
logger.info('Collection completed in: {:.2f} min.'.format(runtime))
# add job to reV status file.
status = {
'dirout': os.path.dirname(h5_file),
'fout': os.path.basename(h5_file),
'job_status': 'successful',
'runtime': runtime,
'finput': os.path.join(h5_dir, '{}*.h5'.format(file_prefix))
}
Status.make_job_file(os.path.dirname(h5_file), 'collect', name, status)
def from_config(ctx, config_file, verbose):
"""Run reV gen from a config file."""
name = ctx.obj['NAME']
# Instantiate the config object
config = CollectionConfig(config_file)
# take name from config if not default
if config.name.lower() != 'rev':
name = config.name
ctx.obj['NAME'] = name
# Enforce verbosity if logging level is specified in the config
if config.log_level == logging.DEBUG:
verbose = True
# make output directory if does not exist
if not os.path.exists(config.dirout):
os.makedirs(config.dirout)
# initialize loggers.
init_mult(name,
config.logdir,
modules=[__name__, 'reV.handlers.collection'],
verbose=verbose)
# Initial log statements
logger.info(
'Running reV collection from config file: "{}"'.format(config_file))
logger.info('Target output directory: "{}"'.format(config.dirout))
logger.info('Target logging directory: "{}"'.format(config.logdir))
logger.info('Target collection directory: "{}"'.format(config.coldir))
logger.info('The following project points were specified: "{}"'.format(
config.get('project_points', None)))
logger.debug('The full configuration input is as follows:\n{}'.format(
pprint.pformat(config, indent=4)))
# set config objects to be passed through invoke to direct methods
ctx.obj['H5_DIR'] = config.coldir
ctx.obj['LOG_DIR'] = config.logdir
ctx.obj['DSETS'] = config.dsets
ctx.obj['PROJECT_POINTS'] = config.project_points
ctx.obj['PURGE_CHUNKS'] = config.purge_chunks
ctx.obj['VERBOSE'] = verbose
for file_prefix in config.file_prefixes:
ctx.obj['NAME'] = name + '_{}'.format(file_prefix)
ctx.obj['H5_FILE'] = os.path.join(config.dirout, file_prefix + '.h5')
ctx.obj['FILE_PREFIX'] = file_prefix
if config.execution_control.option == 'local':
status = Status.retrieve_job_status(config.dirout, 'collect',
ctx.obj['NAME'])
if status != 'successful':
Status.add_job(config.dirout,
'collect',
ctx.obj['NAME'],
replace=True,
job_attrs={
'hardware': 'local',
'fout': file_prefix + '.h5',
'dirout': config.dirout
})
ctx.invoke(collect)
elif config.execution_control.option in ('eagle', 'slurm'):
ctx.invoke(collect_slurm,
alloc=config.execution_control.allocation,
memory=config.execution_control.memory,
walltime=config.execution_control.walltime,
feature=config.execution_control.feature,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module,
stdout_path=os.path.join(config.logdir, 'stdout'),
verbose=verbose)
def collect_slurm(ctx, alloc, memory, walltime, feature, conda_env, module,
stdout_path, verbose):
"""Run collection on HPC via SLURM job submission."""
name = ctx.obj['NAME']
h5_file = ctx.obj['H5_FILE']
h5_dir = ctx.obj['H5_DIR']
log_dir = ctx.obj['LOG_DIR']
project_points = ctx.obj['PROJECT_POINTS']
dsets = ctx.obj['DSETS']
file_prefix = ctx.obj['FILE_PREFIX']
purge_chunks = ctx.obj['PURGE_CHUNKS']
verbose = any([verbose, ctx.obj['VERBOSE']])
cmd = get_node_cmd(name,
h5_file,
h5_dir,
project_points,
dsets,
file_prefix=file_prefix,
log_dir=log_dir,
purge_chunks=purge_chunks,
verbose=verbose)
status = Status.retrieve_job_status(os.path.dirname(h5_file), 'collect',
name)
if status == 'successful':
msg = ('Job "{}" is successful in status json found in "{}", '
'not re-running.'.format(name, os.path.dirname(h5_file)))
else:
logger.info(
'Running reV collection on SLURM with node name "{}", '
'collecting data to "{}" from "{}" with file prefix "{}".'.format(
name, h5_file, h5_dir, file_prefix))
# create and submit the SLURM job
slurm = SLURM(cmd,
alloc=alloc,
memory=memory,
walltime=walltime,
feature=feature,
name=name,
conda_env=conda_env,
module=module,
stdout_path=stdout_path)
if slurm.id:
msg = (
'Kicked off reV collection job "{}" (SLURM jobid #{}).'.format(
name, slurm.id))
# add job to reV status file.
Status.add_job(os.path.dirname(h5_file),
'collect',
name,
replace=True,
job_attrs={
'job_id': slurm.id,
'hardware': 'eagle',
'fout': os.path.basename(h5_file),
'dirout': os.path.dirname(h5_file)
})
else:
msg = ('Was unable to kick off reV collection job "{}". '
'Please see the stdout error messages'.format(name))
click.echo(msg)
logger.info(msg)
def from_config(ctx, config_file, verbose):
"""Run reV supply curve compute from a config file."""
name = ctx.obj['NAME']
# Instantiate the config object
config = SupplyCurveConfig(config_file)
# take name from config if not default
if config.name.lower() != 'rev':
name = config.name
ctx.obj['NAME'] = name
# Enforce verbosity if logging level is specified in the config
if config.log_level == logging.DEBUG:
verbose = True
# initialize loggers
init_mult(
name,
config.logdir,
modules=[__name__, 'reV.config', 'reV.utilities', 'rex.utilities'],
verbose=verbose)
# Initial log statements
logger.info('Running reV supply curve from config '
'file: "{}"'.format(config_file))
logger.info('Target output directory: "{}"'.format(config.dirout))
logger.info('Target logging directory: "{}"'.format(config.logdir))
logger.debug('The full configuration input is as follows:\n{}'.format(
pprint.pformat(config, indent=4)))
if config.execution_control.option == 'local':
status = Status.retrieve_job_status(config.dirout, 'supply-curve',
name)
if status != 'successful':
Status.add_job(config.dirout,
'supply-curve',
name,
replace=True,
job_attrs={
'hardware': 'local',
'fout': '{}.csv'.format(name),
'dirout': config.dirout
})
ctx.invoke(direct,
sc_points=config.sc_points,
trans_table=config.trans_table,
fixed_charge_rate=config.fixed_charge_rate,
sc_features=config.sc_features,
transmission_costs=config.transmission_costs,
sort_on=config.sort_on,
wind_dirs=config.wind_dirs,
n_dirs=config.n_dirs,
downwind=config.downwind,
max_workers=config.max_workers,
out_dir=config.dirout,
log_dir=config.logdir,
simple=config.simple,
line_limited=config.line_limited,
verbose=verbose)
elif config.execution_control.option in ('eagle', 'slurm'):
ctx.obj['NAME'] = name
ctx.obj['SC_POINTS'] = config.sc_points
ctx.obj['TRANS_TABLE'] = config.trans_table
ctx.obj['FIXED_CHARGE_RATE'] = config.fixed_charge_rate
ctx.obj['SC_FEATURES'] = config.sc_features
ctx.obj['TRANSMISSION_COSTS'] = config.transmission_costs
ctx.obj['SORT_ON'] = config.sort_on
ctx.obj['OFFSHORE_TRANS_TABLE'] = config.offshore_trans_table
ctx.obj['WIND_DIRS'] = config.wind_dirs
ctx.obj['N_DIRS'] = config.n_dirs
ctx.obj['DOWNWIND'] = config.downwind
ctx.obj['OFFSHORE_COMPETE'] = config.offshore_compete
ctx.obj['MAX_WORKERS'] = config.max_workers
ctx.obj['OUT_DIR'] = config.dirout
ctx.obj['LOG_DIR'] = config.logdir
ctx.obj['SIMPLE'] = config.simple
ctx.obj['LINE_LIMITED'] = config.line_limited
ctx.obj['VERBOSE'] = verbose
ctx.invoke(slurm,
alloc=config.execution_control.allocation,
memory=config.execution_control.memory,
walltime=config.execution_control.walltime,
feature=config.execution_control.feature,
conda_env=config.execution_control.conda_env,
module=config.execution_control.module)
def local(ctx, max_workers, timeout, points_range, verbose):
"""Run econ on local worker(s)."""
name = ctx.obj['NAME']
points = ctx.obj['POINTS']
sam_files = ctx.obj['SAM_FILES']
cf_file = ctx.obj['CF_FILE']
cf_year = ctx.obj['CF_YEAR']
site_data = ctx.obj['SITE_DATA']
sites_per_worker = ctx.obj['SITES_PER_WORKER']
fout = ctx.obj['FOUT']
dirout = ctx.obj['DIROUT']
logdir = ctx.obj['LOGDIR']
output_request = ctx.obj['OUTPUT_REQUEST']
append = ctx.obj['APPEND']
verbose = any([verbose, ctx.obj['VERBOSE']])
if append:
fout = os.path.basename(cf_file)
dirout = os.path.dirname(cf_file)
# initialize loggers for multiple modules
log_modules = [
__name__, 'reV.econ.econ', 'reV.generation', 'reV.config',
'reV.utilities', 'reV.SAM', 'reV.handlers', 'rex.utilities'
]
init_mult(name, logdir, modules=log_modules, verbose=verbose, node=True)
for key, val in ctx.obj.items():
logger.debug('ctx var passed to local method: "{}" : "{}" with type '
'"{}"'.format(key, val, type(val)))
logger.info(
'Econ local is being run with with job name "{}" and '
'generation results file: {}. Target output path is: {}'.format(
name, cf_file, os.path.join(dirout, fout)))
t0 = time.time()
# Execute the Generation module with smart data flushing.
Econ.reV_run(points=points,
sam_files=sam_files,
cf_file=cf_file,
cf_year=cf_year,
site_data=site_data,
output_request=output_request,
max_workers=max_workers,
timeout=timeout,
sites_per_worker=sites_per_worker,
points_range=points_range,
fout=fout,
dirout=dirout,
append=append)
tmp_str = ' with points range {}'.format(points_range)
runtime = (time.time() - t0) / 60
logger.info('Econ compute complete for project points "{0}"{1}. '
'Time elapsed: {2:.2f} min. Target output dir: {3}'.format(
points, tmp_str if points_range else '', runtime, dirout))
# add job to reV status file.
status = {
'dirout': dirout,
'fout': fout,
'job_status': 'successful',
'runtime': runtime,
'finput': cf_file
}
Status.make_job_file(dirout, 'econ', name, status)