def test_vpd_fractional_excl():
"""Test variable power density with fractional exclusions"""
gids_subset = list(range(0, 20))
excl_dict_1 = {'ri_padus': {'exclude_values': [1]}}
s1 = SupplyCurveAggregation.summary(EXCL, GEN, TM_DSET,
excl_dict=excl_dict_1,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
power_density=FVPD,
max_workers=1, gids=gids_subset)
excl_dict_2 = {'ri_padus': {'exclude_values': [1],
'weight': 0.5}}
s2 = SupplyCurveAggregation.summary(EXCL, GEN, TM_DSET,
excl_dict=excl_dict_2,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
power_density=FVPD,
max_workers=1, gids=gids_subset)
for i in s1.index:
cap_full = s1.loc[i, 'capacity']
cap_half = s2.loc[i, 'capacity']
msg = ('Variable power density for fractional exclusions failed! '
'Index {} has cap full {} and cap half {}'
.format(i, cap_full, cap_half))
assert (cap_full / cap_half) == 2, msg
def test_parallel_agg(resolution=64):
"""Test that parallel aggregation yields the same results as serial
aggregation."""
gids = list(range(50, 70))
summary_serial = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=None,
res_class_bins=None,
resolution=resolution,
gids=gids,
max_workers=1)
summary_parallel = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=None,
res_class_bins=None,
resolution=resolution,
gids=gids,
max_workers=3)
assert all(summary_serial == summary_parallel)
def test_vpd_incomplete():
"""Test an incomplete VPD input and make sure an exception is raised"""
try:
SupplyCurveAggregation.summary(EXCL, GEN, TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1, power_density=FVPDI)
except FileInputError as e:
if '1314958' in str(e):
pass
else:
raise Exception('Test with incomplete VPD input did not throw error!')
def plot_single_gen_sc_point(gid=2, resolution=64):
"""Test the calculation of the SC points setup from exclusions tiff."""
import matplotlib.pyplot as plt
colors = ['b', 'g', 'c', 'y', 'm']
colors *= 100
_, axs = plt.subplots(1, 1)
gen_index = SupplyCurveAggregation._parse_gen_index(F_GEN)
with GenerationSupplyCurvePoint(gid, F_EXCL, F_GEN, F_TECHMAP, DSET_TM,
gen_index,
resolution=resolution) as sc:
all_gen_gids = list(set(sc._gen_gids))
excl_meta = sc.exclusions['meta', sc.rows, sc.cols]
for i, gen_gid in enumerate(all_gen_gids):
if gen_gid != -1:
mask = (sc._gen_gids == gen_gid)
axs.scatter(excl_meta.loc[mask, 'longitude'],
excl_meta.loc[mask, 'latitude'],
marker='s', c=colors[i], s=1)
axs.scatter(sc.gen.meta.loc[gen_gid, 'longitude'],
sc.gen.meta.loc[gen_gid, 'latitude'],
c='k', s=100)
axs.scatter(sc.centroid[1], sc.centroid[0], marker='x', c='k', s=200)
axs.axis('equal')
plt.show()
def test_vpd():
"""Test variable power density"""
s = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1,
power_density=FVPD)
vpd = pd.read_csv(FVPD, index_col=0)
for i in s.index:
capacity = s.loc[i, 'capacity']
area = s.loc[i, 'area_sq_km']
res_gids = np.array(s.loc[i, 'res_gids'])
gid_counts = np.array(s.loc[i, 'gid_counts'])
vpd_per_gid = vpd.loc[res_gids, 'power_density'].values
truth = area * (vpd_per_gid * gid_counts).sum() / gid_counts.sum()
diff = 100 * (capacity - truth) / truth
msg = ('Variable power density failed! Index {} has cap {} and '
'truth {}'.format(i, capacity, truth))
assert diff < 1, msg
def test_sc_agg_econ_scale():
"""Test supply curve aggregation with LCOE scaling based on plant capacity.
"""
data = {
'capital_cost': 53455000,
'fixed_operating_cost': 360000,
'fixed_charge_rate': 0.096,
'variable_operating_cost': 0
}
with tempfile.TemporaryDirectory() as td:
gen_temp = os.path.join(td, 'ri_my_pv_gen.h5')
shutil.copy(GEN, gen_temp)
with h5py.File(gen_temp, 'a') as res:
for k, v in data.items():
arr = np.full(res['meta'].shape, v)
res.create_dataset(k, res['meta'].shape, data=arr)
res[k].attrs['scale_factor'] = 1.0
eqn = '2 * capacity ** -0.3'
s = SupplyCurveAggregation.summary(EXCL,
gen_temp,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
gids=list(np.arange(10)),
max_workers=1,
cap_cost_scale=eqn)
aep = s['capacity'] * s['mean_cf'] * 8760 * 1000
true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] +
data['fixed_operating_cost']) / aep +
data['variable_operating_cost'])
true_raw_lcoe *= 1000 # convert $/kwh -> $/MWh
# Back out the fcr * capital_cost term ($)
x = ((s['raw_lcoe'] / 1000 - data['variable_operating_cost']) * aep -
data['fixed_operating_cost'])
eval_inputs = {k: s[k].values.flatten() for k in s.columns}
# pylint: disable=eval-used
scalars = eval(str(eqn), globals(), eval_inputs)
s['scalars'] = scalars
x *= scalars
true_scaled_lcoe = ((x + data['fixed_operating_cost']) / aep +
data['variable_operating_cost'])
true_scaled_lcoe *= 1000 # convert $/kwh -> $/MWh
assert np.allclose(true_scaled_lcoe, s['mean_lcoe'])
assert np.allclose(true_raw_lcoe, s['raw_lcoe'])
s = s.sort_values('capacity')
assert all(s['mean_lcoe'].diff()[1:] < 0)
for i in s.index.values:
if s.loc[i, 'scalars'] < 1:
assert s.loc[i, 'mean_lcoe'] < s.loc[i, 'raw_lcoe']
else:
assert s.loc[i, 'mean_lcoe'] >= s.loc[i, 'raw_lcoe']
def test_aggregation_scalar_excl():
"""Test the aggregation summary with exclusions of 0.5"""
gids_subset = list(range(0, 20))
excl_dict_1 = {'ri_padus': {'exclude_values': [1]}}
s1 = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=excl_dict_1,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1,
gids=gids_subset)
excl_dict_2 = {'ri_padus': {'exclude_values': [1], 'weight': 0.5}}
s2 = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=excl_dict_2,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1,
gids=gids_subset)
dsets = ['area_sq_km', 'capacity']
for dset in dsets:
diff = (s1[dset].values / s2[dset].values)
msg = ('Fractional exclusions failed for {} which has values {} and {}'
.format(dset, s1[dset].values, s2[dset].values))
assert all(diff == 2), msg
for i in s1.index:
counts_full = s1.loc[i, 'gid_counts']
counts_half = s2.loc[i, 'gid_counts']
for j, counts in enumerate(counts_full):
msg = ('GID counts for fractional exclusions failed for index {}!'.
format(i))
assert counts == 2 * counts_half[j], msg
def test_aggregation_gen_econ():
"""Test the aggregation summary method with separate gen and econ
input files."""
s1 = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1)
s2 = SupplyCurveAggregation.summary(EXCL,
ONLY_GEN,
TM_DSET,
econ_fpath=ONLY_ECON,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1)
assert_frame_equal(s1, s2)
def test_aggregation_category_layer():
"""Test aggregation of data layers with category method"""
data_layers = {
'pct_slope': {
'dset': 'ri_srtm_slope',
'method': 'mean'
},
'reeds_region': {
'dset': 'ri_reeds_regions',
'method': 'category'
},
'padus': {
'dset': 'ri_padus',
'method': 'category'
}
}
s = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=data_layers,
max_workers=1)
for i in s.index.values:
counts = s.loc[i, 'gid_counts']
rr = s.loc[i, 'reeds_region']
assert isinstance(rr, str)
rr = json.loads(rr)
assert isinstance(rr, dict)
rr_sum = sum(list(rr.values()))
padus = s.loc[i, 'padus']
assert isinstance(padus, str)
padus = json.loads(padus)
assert isinstance(padus, dict)
padus_sum = sum(list(padus.values()))
try:
assert padus_sum == sum(counts)
assert padus_sum >= rr_sum
except AssertionError:
e = ('Categorical data layer aggregation failed:\n{}'.format(
s.loc[i]))
raise RuntimeError(e)
def plot_sc_offshore(plot_var='mean_lcoe'):
"""Plot the supply curve map colored by plot_var."""
import matplotlib.pyplot as plt
s = SupplyCurveAggregation.summary(EXCL_FPATH,
OFFSHORE_BASELINE,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
cf_dset=CF_DSET,
lcoe_dset=LCOE_DSET,
data_layers=DATA_LAYERS,
max_workers=1)
plt.scatter(s['longitude'], s['latitude'], c=s[plot_var], marker='s')
plt.axis('equal')
plt.colorbar(label=plot_var)
def test_aggregation_extent(resolution=64):
"""Get the SC points aggregation summary and test that there are expected
columns and that all resource gids were found"""
summary = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=None,
res_class_bins=None,
data_layers=DATA_LAYERS,
resolution=resolution)
all_res_gids = []
for gids in summary['res_gids']:
all_res_gids += gids
assert 'sc_col_ind' in summary
assert 'sc_row_ind' in summary
assert 'gen_gids' in summary
assert len(set(all_res_gids)) == 177
def test_sc_agg_offshore():
"""Test the SC offshore aggregation and check offshore SC points against
known offshore gen points."""
s = SupplyCurveAggregation.summary(EXCL_FPATH,
OFFSHORE_BASELINE,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
cf_dset=CF_DSET,
lcoe_dset=LCOE_DSET,
data_layers=DATA_LAYERS,
max_workers=1)
for col in Offshore.DEFAULT_META_COLS:
msg = ('Offshore data column "{}" was not passed through to agg table'.
format(col))
assert col in s, msg
with Outputs(OFFSHORE_BASELINE, mode='r') as out:
meta = out.meta
offshore_mask = (meta.offshore == 1)
offshore_gids = meta.loc[offshore_mask, 'gid'].values.tolist()
for sc_gid in s.index:
if s.at[sc_gid, 'offshore']:
assert int(s.at[sc_gid, 'farm_gid']) in offshore_gids
assert all(np.array(json.loads(s.at[sc_gid, 'res_gids'])) < 3e6)
assert s.at[sc_gid, 'elevation'] == 0.0
assert s.at[sc_gid, 'capacity'] == 600
assert np.isnan(s.at[sc_gid, 'pct_slope'])
else:
for res_gid in s.at[sc_gid, 'res_gids']:
assert res_gid not in offshore_gids
for gid in offshore_gids:
assert gid in s['farm_gid'].values
def test_aggregation_summary():
"""Test the aggregation summary method against a baseline file."""
s = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1)
if not os.path.exists(AGG_BASELINE):
s.to_csv(AGG_BASELINE)
raise Exception('Aggregation summary baseline file did not exist. '
'Created: {}'.format(AGG_BASELINE))
else:
for c in ['res_gids', 'gen_gids', 'gid_counts']:
s[c] = s[c].astype(str)
s_baseline = pd.read_csv(AGG_BASELINE, index_col=0)
assert_frame_equal(s, s_baseline, check_dtype=False)
def test_agg_friction(gid):
"""Test SC Aggregation with friction by checking friction factors and LCOE
against a hand calc."""
warnings.filterwarnings('ignore')
for gid in [100, 114, 130, 181]:
s = SupplyCurveAggregation.summary(EXCL_FPATH,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
resolution=RESOLUTION,
gids=[gid],
max_workers=1,
friction_fpath=FRICTION_FPATH,
friction_dset=FRICTION_DSET)
row_slice, col_slice = EXTENT.get_excl_slices(gid)
test_e = EXCL[row_slice, col_slice]
test_f = FRICTION[row_slice, col_slice]
x = test_e * test_f
x = x.flatten()
x = x[(x != 0)]
mean_friction = x.mean()
m = ('SC point gid {} does not match mean friction hand calc'.format(
gid))
assert s['mean_friction'].values[0] == mean_friction, m
m = ('SC point gid {} does not match mean LCOE with friction hand calc'
.format(gid))
assert np.allclose(s['mean_lcoe_friction'],
s['mean_lcoe'] * mean_friction), m
def test_aggregation_extra_dsets():
"""Test aggregation with extra datasets to aggregate."""
h5_dsets = ['lcoe_fcr-2012', 'lcoe_fcr-2013', 'lcoe_fcr-stdev']
s = SupplyCurveAggregation.summary(EXCL,
ONLY_GEN,
TM_DSET,
h5_dsets=h5_dsets,
econ_fpath=ONLY_ECON,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=DATA_LAYERS,
max_workers=1)
for dset in h5_dsets:
assert 'mean_{}'.format(dset) in s.columns
check = s['mean_lcoe_fcr-2012'] == s['mean_lcoe']
assert not any(check)
check = s['mean_lcoe_fcr-2013'] == s['mean_lcoe']
assert not any(check)
avg = (s['mean_lcoe_fcr-2012'] + s['mean_lcoe_fcr-2013']) / 2
assert np.allclose(avg.values, s['mean_lcoe'].values)
def direct(ctx, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset,
excl_dict, check_excl_layers, res_class_dset, res_class_bins,
cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area,
power_density, area_filter_kernel, min_area, friction_fpath,
friction_dset, out_dir, log_dir, verbose):
"""reV Supply Curve Aggregation Summary CLI."""
name = ctx.obj['NAME']
ctx.obj['EXCL_FPATH'] = excl_fpath
ctx.obj['GEN_FPATH'] = gen_fpath
ctx.obj['ECON_FPATH'] = econ_fpath
ctx.obj['RES_FPATH'] = res_fpath
ctx.obj['TM_DSET'] = tm_dset
ctx.obj['EXCL_DICT'] = excl_dict
ctx.obj['CHECK_LAYERS'] = check_excl_layers
ctx.obj['RES_CLASS_DSET'] = res_class_dset
ctx.obj['RES_CLASS_BINS'] = res_class_bins
ctx.obj['CF_DSET'] = cf_dset
ctx.obj['LCOE_DSET'] = lcoe_dset
ctx.obj['H5_DSETS'] = h5_dsets
ctx.obj['DATA_LAYERS'] = data_layers
ctx.obj['RESOLUTION'] = resolution
ctx.obj['EXCL_AREA'] = excl_area
ctx.obj['POWER_DENSITY'] = power_density
ctx.obj['AREA_FILTER_KERNEL'] = area_filter_kernel
ctx.obj['MIN_AREA'] = min_area
ctx.obj['FRICTION_FPATH'] = friction_fpath
ctx.obj['FRICTION_DSET'] = friction_dset
ctx.obj['OUT_DIR'] = out_dir
ctx.obj['LOG_DIR'] = log_dir
ctx.obj['VERBOSE'] = verbose
if ctx.invoked_subcommand is None:
t0 = time.time()
init_mult(name,
log_dir,
modules=[__name__, 'reV.supply_curve'],
verbose=verbose)
with h5py.File(excl_fpath, mode='r') as f:
dsets = list(f)
if tm_dset not in dsets:
try:
TechMapping.run(excl_fpath, res_fpath, tm_dset)
except Exception as e:
logger.exception('TechMapping process failed. Received the '
'following error:\n{}'.format(e))
raise e
if isinstance(excl_dict, str):
excl_dict = dict_str_load(excl_dict)
if isinstance(data_layers, str):
data_layers = dict_str_load(data_layers)
try:
summary = SupplyCurveAggregation.summary(
excl_fpath,
gen_fpath,
tm_dset,
econ_fpath=econ_fpath,
excl_dict=excl_dict,
res_class_dset=res_class_dset,
res_class_bins=res_class_bins,
cf_dset=cf_dset,
lcoe_dset=lcoe_dset,
h5_dsets=h5_dsets,
data_layers=data_layers,
resolution=resolution,
excl_area=excl_area,
power_density=power_density,
area_filter_kernel=area_filter_kernel,
min_area=min_area,
friction_fpath=friction_fpath,
friction_dset=friction_dset,
check_excl_layers=check_excl_layers)
except Exception as e:
logger.exception('Supply curve Aggregation failed. Received the '
'following error:\n{}'.format(e))
raise e
fn_out = '{}.csv'.format(name)
fpath_out = os.path.join(out_dir, fn_out)
summary.to_csv(fpath_out)
runtime = (time.time() - t0) / 60
logger.info('Supply curve aggregation complete. '
'Time elapsed: {:.2f} min. Target output dir: {}'.format(
runtime, out_dir))
finput = [excl_fpath, gen_fpath]
if res_fpath is not None:
finput.append(res_fpath)
# add job to reV status file.
status = {
'dirout': out_dir,
'fout': fn_out,
'job_status': 'successful',
'runtime': runtime,
'finput': finput,
'excl_fpath': excl_fpath,
'excl_dict': excl_dict,
'area_filter_kernel': area_filter_kernel,
'min_area': min_area
}
Status.make_job_file(out_dir, 'supply-curve-aggregation', name, status)
def test_data_layer_methods():
"""Test aggregation of data layers with different methods"""
data_layers = {
'pct_slope_mean': {
'dset': 'ri_srtm_slope',
'method': 'mean'
},
'pct_slope_max': {
'dset': 'ri_srtm_slope',
'method': 'max'
},
'pct_slope_min': {
'dset': 'ri_srtm_slope',
'method': 'min'
},
'reeds_region': {
'dset': 'ri_reeds_regions',
'method': 'category'
},
'padus': {
'dset': 'ri_padus',
'method': 'category'
}
}
s = SupplyCurveAggregation.summary(EXCL,
GEN,
TM_DSET,
excl_dict=EXCL_DICT,
res_class_dset=RES_CLASS_DSET,
res_class_bins=RES_CLASS_BINS,
data_layers=data_layers,
max_workers=1)
for i in s.index.values:
# Check categorical data layers
counts = s.loc[i, 'gid_counts']
rr = s.loc[i, 'reeds_region']
assert isinstance(rr, str)
rr = json.loads(rr)
assert isinstance(rr, dict)
rr_sum = sum(list(rr.values()))
padus = s.loc[i, 'padus']
assert isinstance(padus, str)
padus = json.loads(padus)
assert isinstance(padus, dict)
padus_sum = sum(list(padus.values()))
try:
assert padus_sum == sum(counts)
assert padus_sum >= rr_sum
except AssertionError:
e = ('Categorical data layer aggregation failed:\n{}'.format(
s.loc[i]))
raise RuntimeError(e)
# Check min/mean/max of the same data layer
n = s.loc[i, 'n_gids']
slope_mean = s.loc[i, 'pct_slope_mean']
slope_max = s.loc[i, 'pct_slope_max']
slope_min = s.loc[i, 'pct_slope_min']
if n > 3: # sc points with <= 3 90m pixels can have min == mean == max
assert slope_min < slope_mean < slope_max
else:
assert slope_min <= slope_mean <= slope_max
