def main(src_h5, dst_h5, var_attrs_path, hub_height, chunk_size,
weeks_per_chunk, overwrite, meta, process_size, check_dset_attrs,
resolution, log_file, verbose):
"""
RechunkH5 CLI entry point
"""
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
dst_dir = os.path.dirname(dst_h5)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
RechunkH5.run(src_h5,
dst_h5,
var_attrs=var_attrs_path,
hub_height=hub_height,
chunk_size=chunk_size,
weeks_per_chunk=weeks_per_chunk,
overwrite=overwrite,
meta=meta,
process_size=process_size,
check_dset_attrs=check_dset_attrs,
resolution=resolution)
def test_profiles_means():
"""
Test adding means to pre-collected profiles
"""
init_logger('reV.handlers.collection')
h5_file = os.path.join(TEMP_DIR, 'cf.h5')
Collector.collect(h5_file,
H5_DIR,
POINTS_PATH,
'cf_profile',
dset_out=None,
file_prefix='peregrine_2012')
Collector.add_dataset(h5_file,
H5_DIR,
'cf_mean',
dset_out=None,
file_prefix='peregrine_2012')
with h5py.File(h5_file, 'r') as f:
assert 'cf_profile' in f
assert 'cf_mean' in f
data = f['cf_profile'][...]
node_file = os.path.join(H5_DIR, 'peregrine_2012_node01_x001.h5')
with h5py.File(node_file, 'r') as f:
source_data = f['cf_profile'][...]
assert np.allclose(source_data, data[:, -1 * source_data.shape[1]:])
if PURGE_OUT:
os.remove(h5_file)
def test_collection():
"""
Test collection on 'cf_profile' ensuring output array is correct
"""
init_logger('reV.handlers.collection')
profiles = manual_collect(H5_DIR, 'peregrine_2012', 'cf_profile')
h5_file = os.path.join(TEMP_DIR, 'collection.h5')
Collector.collect(h5_file,
H5_DIR,
POINTS_PATH,
'cf_profile',
dset_out=None,
file_prefix='peregrine_2012')
with h5py.File(h5_file, 'r') as f:
cf_profiles = f['cf_profile'][...]
diff = np.mean(np.abs(profiles - cf_profiles))
msg = "Arrays differ by {:.4f}".format(diff)
assert np.allclose(profiles, cf_profiles), msg
source_file = os.path.join(H5_DIR, "peregrine_2012_node00_x000.h5")
with h5py.File(source_file, 'r') as f_s:
def check_attrs(name, object):
object_s = f_s[name]
for k, v in object.attrs.items():
v_s = object_s.attrs[k]
assert v == v_s
with h5py.File(h5_file, 'r') as f:
f.visititems(check_attrs)
if PURGE_OUT:
os.remove(h5_file)
def reV_h5(ctx, h5_file, out_dir, sub_dir, dsets, group, process_size,
max_workers, plot_type, cmap, log_file, verbose, terminal):
"""
Summarize and plot data for reV h5_file
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
QaQc.h5(h5_file,
qa_dir,
dsets=dsets,
group=group,
process_size=process_size,
max_workers=max_workers,
plot_type=plot_type,
cmap=cmap)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': h5_file
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_low_mem_collect():
"""Test memory limited multi chunk collection"""
init_logger('reV.handlers.collection', log_level='DEBUG')
h5_file = os.path.join(TEMP_DIR, 'cf.h5')
Collector.collect(h5_file,
H5_DIR,
POINTS_PATH,
'cf_profile',
dset_out=None,
file_prefix='peregrine_2012',
mem_util_lim=0.00002)
with h5py.File(h5_file, 'r') as f:
assert 'cf_profile' in f
data = f['cf_profile'][...]
node_file = os.path.join(H5_DIR, 'peregrine_2012_node01_x001.h5')
with h5py.File(node_file, 'r') as f:
source_data = f['cf_profile'][...]
assert np.allclose(source_data, data[:, -1 * source_data.shape[1]:])
if PURGE_OUT:
os.remove(h5_file)
def supply_curve(ctx, sc_table, out_dir, sub_dir, columns, plot_type, cmap,
lcoe, log_file, verbose, terminal):
"""
Summarize and plot reV Supply Curve data
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
QaQc.supply_curve(sc_table,
qa_dir,
columns=columns,
lcoe=lcoe,
plot_type=plot_type,
cmap=cmap)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': sc_table
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def main(ctx, wave_h5, out_dir, log_file, verbose):
"""
WaveX Command Line Interface
"""
ctx.ensure_object(dict)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
ctx.obj['H5'] = wave_h5
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['CLS_KWARGS'] = {}
ctx.obj['CLS'] = WaveX
_, hsds = check_res_file(wave_h5)
if hsds:
ctx.obj['CLS_KWARGS']['hsds'] = hsds
else:
assert os.path.exists(wave_h5)
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
logger.info('Extracting wave data from {}'.format(wave_h5))
logger.info('Outputs to be stored in: {}'.format(out_dir))
def main(combined_h5, source_h5, axis, overwrite, process_size, log_file,
verbose):
"""
CombineH5 CLI entry point
"""
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
dst_dir = os.path.dirname(combined_h5)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
CombineH5.run(combined_h5,
*source_h5,
axis=axis,
overwrite=overwrite,
process_size=process_size)
def main(ctx, resource_path, out_dir, years, hsds, res_cls, log_file, verbose):
"""
Multi-year ResourceX Command Line Interface
"""
ctx.ensure_object(dict)
ctx.obj['H5'] = resource_path
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['CLS_KWARGS'] = {'years': years, 'hsds': hsds}
if res_cls == 'Resource':
ctx.obj['CLS'] = MultiYearResourceX
elif res_cls == 'NSRDB':
ctx.obj['CLS'] = MultiYearNSRDBX
elif res_cls == 'Wind':
ctx.obj['CLS'] = MultiYearWindX
elif res_cls == 'Wave':
ctx.obj['CLS'] = MultiYearWaveX
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
logger.info('Extracting Resource data from {}'.format(resource_path))
logger.info('Outputs to be stored in: {}'.format(out_dir))
def __init__(self, config, verbose=False):
"""
Parameters
----------
config : str
File path to config json or csv (str).
verbose : bool
Flag to turn on debug logging.
"""
self._job_tags = None
self._config = BatchConfig(config)
self._base_dir = self._config.config_dir
os.chdir(self._base_dir)
if 'logging' in self._config:
logging_kwargs = self._config.get('logging', {})
if verbose:
logging_kwargs['log_level'] = 'DEBUG'
init_logger('reV.batch', **logging_kwargs)
x = self._parse_config(self._config)
self._arg_combs, self._file_sets, self._set_tags = x
logger.info('Batch job initialized with {} sub jobs.'.format(
len(self._arg_combs)))
def summarize(ctx, out_dir, log_file, verbose):
"""
Summarize reV data
"""
ctx.obj['OUT_DIR'] = out_dir
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
def main(ctx, fpath, res_file, sam_file, verbose):
"""reV ProjectPoints generator"""
ctx.ensure_object(dict)
ctx.obj['FPATH'] = fpath
ctx.obj['RES_FILE'] = res_file
ctx.obj['SAM_FILE'] = sam_file
ctx.obj['VERBOSE'] = verbose
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV.config.project_points', log_level=log_level)
def test_collect_means():
"""
Test means collection:
"""
init_logger('reV.handlers.collection')
h5_file = os.path.join(TEMP_DIR, 'cf_means.h5')
Collector.collect(h5_file,
H5_DIR,
POINTS_PATH,
'cf_mean',
dset_out=None,
file_prefix='peregrine_2012')
if PURGE_OUT:
os.remove(h5_file)
def main(ctx, src_h5, dst_h5, var_attrs_path, version, meta, process_size,
check_dset_attrs, log_file, verbose):
"""
RechunkH5 CLI entry point
"""
ctx.ensure_object(dict)
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
logger = init_logger('rex.rechunk_h5.rechunk_h5',
log_file=log_file,
log_level=log_level)
ctx.obj['LOGGER'] = logger
dst_dir = os.path.dirname(dst_h5)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
RechunkH5.run(src_h5,
dst_h5,
var_attrs_path,
version=version,
meta=meta,
process_size=process_size,
check_dset_attrs=check_dset_attrs)
def __init__(self, pipeline, monitor=True, verbose=False):
"""
Parameters
----------
pipeline : str | dict
Pipeline config file path or dictionary.
monitor : bool
Flag to perform continuous monitoring of the pipeline.
verbose : bool
Flag to submit pipeline steps with -v flag for debug logging
"""
self.monitor = monitor
self.verbose = verbose
self._config = PipelineConfig(pipeline)
self._run_list = self._config.pipeline_steps
self._init_status()
# init logger for pipeline module if requested in input config
if 'logging' in self._config:
init_logger('reV.pipeline', **self._config['logging'])
def main(ctx, solar_h5, out_dir, log_file, verbose):
"""
NSRDBX Command Line Interface
"""
ctx.ensure_object(dict)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
ctx.obj['H5'] = solar_h5
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['CLS_KWARGS'] = {}
multi_h5_res, hsds = check_res_file(solar_h5)
name = os.path.splitext(os.path.basename(solar_h5))[0]
if 'nsrdb' in name:
ctx.obj['CLS'] = NSRDBX
else:
ctx.obj['CLS'] = SolarX
if multi_h5_res and not hsds:
assert os.path.exists(os.path.dirname(solar_h5))
ctx.obj['CLS'] = MultiFileNSRDBX
elif hsds:
ctx.obj['CLS_KWARGS']['hsds'] = hsds
else:
assert os.path.exists(solar_h5)
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
logger.info('Extracting solar data from {}'.format(solar_h5))
logger.info('Outputs to be stored in: {}'.format(out_dir))
def main(ctx, domain, out_dir, years, buoy, eagle, log_file, verbose):
"""
US Wave Command Line Interface
"""
ctx.ensure_object(dict)
if eagle:
path = '/datasets/US_wave/v1.0.0'
hsds = False
else:
path = '/nrel/US_wave'
hsds = True
if buoy:
path = os.path.join(path, 'virtual_buoy', domain)
else:
path = os.path.join(path, domain)
ctx.obj['H5'] = path
if not os.path.exists(out_dir):
os.makedirs(out_dir)
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['CLS_KWARGS'] = {'years': years, 'hsds': hsds}
ctx.obj['CLS'] = MultiYearWaveX
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
if log_file is not None:
log_dir = os.path.dirname(log_file)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
init_logger('rex', log_file=log_file, log_level=log_level)
logger.info('Extracting US wave data for {} from {}'.format(domain, path))
logger.info('Outputs to be stored in: {}'.format(out_dir))
def exclusions(ctx, excl_fpath, out_dir, sub_dir, excl_dict,
area_filter_kernel, min_area, plot_type, cmap, plot_step,
log_file, verbose, terminal):
"""
Extract and plot reV exclusions mask
"""
name = ctx.obj['NAME']
if any([verbose, ctx.obj['VERBOSE']]):
log_level = 'DEBUG'
else:
log_level = 'INFO'
init_logger('reV', log_file=log_file, log_level=log_level)
qa_dir = out_dir
if sub_dir is not None:
qa_dir = os.path.join(out_dir, sub_dir)
if isinstance(excl_dict, str):
excl_dict = dict_str_load(excl_dict)
QaQc.exclusions_mask(excl_fpath,
qa_dir,
layers_dict=excl_dict,
min_area=min_area,
kernel=area_filter_kernel,
plot_type=plot_type,
cmap=cmap,
plot_step=plot_step)
if terminal:
status = {
'dirout': out_dir,
'job_status': 'successful',
'finput': excl_fpath
}
Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_means_lcoe():
"""
Test adding means to pre-collected profiles
"""
init_logger('reV.handlers.collection')
h5_file = os.path.join(TEMP_DIR, 'cf_lcoe.h5')
Collector.collect(h5_file,
H5_DIR,
POINTS_PATH,
'cf_mean',
dset_out=None,
file_prefix='peregrine_2012')
Collector.add_dataset(h5_file,
H5_DIR,
'lcoe_fcr',
dset_out=None,
file_prefix='peregrine_2012')
with h5py.File(h5_file, 'r') as f:
assert 'cf_mean' in f
assert 'lcoe_fcr' in f
if PURGE_OUT:
os.remove(h5_file)
def test_eagle(year):
"""Gen PV CF profiles with write to disk and compare against rev1."""
res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year)
sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json'
rev2_out_dir = os.path.join(TESTDATADIR, 'ri_pv_reV2')
rev2_out = 'gen_ri_pv_smart_{}.h5'.format(year)
if not os.path.exists(rev2_out_dir):
os.mkdir(rev2_out_dir)
name = 'etest'
points = slice(0, 100)
verbose = True
log_level = 'DEBUG'
log_file = os.path.join(rev2_out_dir, '{}.log'.format(name))
modules = [__name__, 'reV.utilities', 'reV.generation']
for mod in modules:
init_logger(mod, log_level=log_level, log_file=log_file)
cmd = get_node_cmd(name=name, tech='pvwattsv5',
points=points, points_range=None,
sam_files=sam_files, res_file=res_file,
sites_per_worker=None, max_workers=None,
fout=rev2_out, dirout=rev2_out_dir, logdir=rev2_out_dir,
output_request=('cf_profile', 'cf_mean'),
verbose=verbose)
# create and submit the SLURM job
slurm = SLURM(cmd, alloc='rev', memory=96, walltime=0.1,
name=name, stdout_path=rev2_out_dir)
while True:
status = slurm.check_status(name, var='name')
if status == 'CG':
break
else:
time.sleep(5)
# get reV 2.0 generation profiles from disk
flist = os.listdir(rev2_out_dir)
for fname in flist:
if '.h5' in fname:
if rev2_out.strip('.h5') in fname:
full_f = os.path.join(rev2_out_dir, fname)
with Outputs(full_f, 'r') as cf:
rev2_profiles = cf['cf_profile']
break
# get reV 1.0 generation profiles
rev1_profiles = get_r1_profiles(year=year)
rev1_profiles = rev1_profiles[:, points]
result = np.allclose(rev1_profiles, rev2_profiles, rtol=RTOL, atol=ATOL)
if result:
# remove output files if test passes.
flist = os.listdir(rev2_out_dir)
for fname in flist:
os.remove(os.path.join(rev2_out_dir, fname))
assert result is True
from reV import TESTDATADIR
from rex.utilities.loggers import init_logger
TEMP_DIR = os.path.join(TESTDATADIR, 'ri_gen_collect')
H5_DIR = os.path.join(TESTDATADIR, 'gen_out')
YEARS = [2012, 2013]
H5_FILES = [
os.path.join(H5_DIR, 'gen_ri_pv_{}_x000.h5'.format(year)) for year in YEARS
]
PURGE_OUT = True
if not os.path.exists(TEMP_DIR):
os.makedirs(TEMP_DIR)
logger = init_logger('reV.handlers.multi_year', log_level='DEBUG')
def manual_means(h5_files, dset):
"""
Manually calculate means for dset in .h5 files
Parameters
----------
h5_files : list
List of .h5 files to compute means from
dset : str
Dataset to collect
Results
-------
