def test_multiple_excl_fractions():
"""
Test that multiple fraction exclusions are handled properly
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
excl_dict = {'ri_smod': {'include_values': [1, ], 'weight': 0.5,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth = f.mask
excl_dict = {'ri_padus': {'exclude_values': [1, ], 'weight': 0.25,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth = np.minimum(truth, f.mask)
excl_dict = {'ri_smod': {'include_values': [1, ], 'weight': 0.5,
'exclude_nodata': True},
'ri_padus': {'exclude_values': [1, ], 'weight': 0.25,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
test = f.mask
assert np.allclose(test, truth)
assert np.all(test[test > 0] >= 0.25)
def test_inclusion_weights():
"""
Test inclusion weights
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
excl_dict = {'ri_smod': {'include_values': [1, ], 'weight': 1,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth = f.mask
excl_dict = {'ri_smod': {'include_values': [2, 3], 'weight': 0.5,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth += f.mask
excl_dict = {'ri_smod': {'inclusion_weights': {1: 1, 2: 0.5, 3: 0.5},
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
test = f.mask
assert np.allclose(test, truth)
assert np.all(test > 0)
excl_dict = {'ri_smod': {'inclusion_weights': {1.0: 1, 2.0: 0.5, 3.0: 0.5},
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
test = f.mask
assert np.allclose(test, truth)
assert np.all(test > 0)
def __init__(self,
excl_fpath,
excl_dict=None,
area_filter_kernel='queen',
min_area=None,
check_excl_layers=False):
"""
Parameters
----------
excl_fpath : str
Filepath to exclusions h5 with techmap dataset.
excl_dict : dict | None
Dictionary of exclusion LayerMask arugments {layer: {kwarg: value}}
area_filter_kernel : str
Contiguous area filter method to use on final exclusions mask
min_area : float | None
Minimum required contiguous area filter in sq-km
check_excl_layers : bool
Run a pre-flight check on each exclusion layer to ensure they
contain un-excluded values
"""
self._excl_fpath = excl_fpath
self._excl = ExclusionMaskFromDict(excl_fpath,
layers_dict=excl_dict,
min_area=min_area,
kernel=area_filter_kernel,
check_layers=check_excl_layers)
class AbstractAggFileHandler(ABC):
"""Simple framework to handle aggregation file context managers."""
def __init__(self,
excl_fpath,
excl_dict=None,
area_filter_kernel='queen',
min_area=None,
check_excl_layers=False):
"""
Parameters
----------
excl_fpath : str
Filepath to exclusions h5 with techmap dataset.
excl_dict : dict | None
Dictionary of exclusion LayerMask arugments {layer: {kwarg: value}}
area_filter_kernel : str
Contiguous area filter method to use on final exclusions mask
min_area : float | None
Minimum required contiguous area filter in sq-km
check_excl_layers : bool
Run a pre-flight check on each exclusion layer to ensure they
contain un-excluded values
"""
self._excl_fpath = excl_fpath
self._excl = ExclusionMaskFromDict(excl_fpath,
layers_dict=excl_dict,
min_area=min_area,
kernel=area_filter_kernel,
check_layers=check_excl_layers)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
if type is not None:
raise
@abstractmethod
def close(self):
"""Close all file handlers."""
self._excl.close()
@property
def exclusions(self):
"""Get the exclusions file handler object.
Returns
-------
_excl : ExclusionMask
Exclusions h5 handler object.
"""
return self._excl
@property
def h5(self):
"""
def test_bad_layer():
"""
Test creation of inclusion mask
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
excl_dict = CONFIGS['bad']
with pytest.raises(ExclusionLayerError):
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict,
check_layers=True) as f:
# pylint: disable=pointless-statement
f.mask
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict,
check_layers=False) as f:
assert not f.mask.any()
def test_no_excl(ds_slice):
"""
Test ExclusionMask with no exclusions provided
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
with ExclusionLayers(excl_h5) as f:
shape = f.shape
truth = np.ones(shape)
with ExclusionMask(excl_h5) as f:
if ds_slice is None:
test = f.mask
else:
test = f[ds_slice]
truth = truth[ds_slice]
assert np.allclose(truth, test)
truth = np.ones(shape)
with ExclusionMaskFromDict(excl_h5) as f:
if ds_slice is None:
test = f.mask
else:
test = f[ds_slice]
truth = truth[ds_slice]
assert np.allclose(truth, test)
def test_inclusion_mask(scenario):
"""
Test creation of inclusion mask
Parameters
----------
scenario : str
Standard reV exclusion scenario
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
truth_path = os.path.join(TESTDATADIR, 'ri_exclusions',
'{}.npy'.format(scenario))
truth = np.load(truth_path)
layers_dict = CONFIGS[scenario]
min_area = AREA.get(scenario, None)
layers = []
with ExclusionLayers(excl_h5) as f:
for layer, kwargs in layers_dict.items():
nodata_value = f.get_nodata_value(layer)
kwargs['nodata_value'] = nodata_value
layers.append(LayerMask(layer, **kwargs))
mask_test = ExclusionMask.run(excl_h5, layers=layers,
min_area=min_area)
assert np.allclose(truth, mask_test)
dict_test = ExclusionMaskFromDict.run(excl_h5, layers_dict=layers_dict,
min_area=min_area)
assert np.allclose(truth, dict_test)
def exclusions_mask(cls, excl_h5, out_dir, layers_dict=None, min_area=None,
kernel='queen', hsds=False, plot_type='plotly',
cmap='viridis', plot_step=100, **kwargs):
"""
Create inclusion mask from given layers dictionary, dump to disk and
plot
Parameters
----------
excl_h5 : str
Path to exclusions .h5 file
layers_dict : dict | NoneType
Dictionary of LayerMask arugments {layer: {kwarg: value}}
min_area : float | NoneType
Minimum required contiguous area in sq-km
kernel : str
Contiguous filter method to use on final exclusions
hsds : bool
Boolean flag to use h5pyd to handle .h5 'files' hosted on AWS
behind HSDS
plot_type : str, optional
plot_type of plot to create 'plot' or 'plotly', by default 'plotly'
cmap : str, optional
Colormap name, by default 'viridis'
plot_step : int
Step between points to plot
kwargs : dict
Additional plotting kwargs
"""
try:
cls(out_dir)
excl_mask = ExclusionMaskFromDict.run(excl_h5,
layers_dict=layers_dict,
min_area=min_area,
kernel=kernel,
hsds=hsds)
excl_mask = np.round(excl_mask * 100).astype('uint8')
out_file = os.path.basename(excl_h5).replace('.h5', '_mask.npy')
out_file = os.path.join(out_dir, out_file)
np.save(out_file, excl_mask)
ExclusionsMask.plot(excl_mask, out_dir, plot_type=plot_type,
cmap=cmap, plot_step=plot_step, **kwargs)
except Exception as e:
logger.exception('QAQC failed on file: {}. Received exception:\n{}'
.format(os.path.basename(excl_h5), e))
raise e
else:
logger.info('Finished QAQC on file: {} output directory: {}'
.format(os.path.basename(excl_h5), out_dir))
def test_force_inclusion():
"""
Test force inclusion
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
excl_dict = {'ri_padus': {'exclude_values': [1, ], 'weight': 0.25,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth = f.mask
excl_dict = {'ri_smod': {'force_include_values': [1, ], 'weight': 0.5,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
truth = np.maximum(truth, f.mask)
excl_dict = {'ri_padus': {'exclude_values': [1, ], 'weight': 0.25,
'exclude_nodata': True},
'ri_smod': {'force_include_values': [1, ], 'weight': 0.5,
'exclude_nodata': True}}
with ExclusionMaskFromDict(excl_h5, layers_dict=excl_dict) as f:
test = f.mask
assert np.allclose(test, truth)
def mask(ag_dir, dst):
"""Use a supply-curve config to aggregate exclusion layers into one."""
if os.path.exists(dst):
return
# Get the aggreation configuration file
path = os.path.join(ag_dir, "config_aggregation.json")
with open(path, "r") as file:
config = json.load(file)
# Extract the needed elements from the confgi
excl_h5 = config["excl_fpath"]
layers_dict = config["excl_dict"]
if "min_area" in config:
min_area = config["min_area"]
else:
min_area = None
if "area_filter_kernel" in config:
kernel = config["area_filter_kernel"]
else:
kernel = "queen"
# Create a mask converter
masker = ExclusionMaskFromDict(excl_h5,
layers_dict=layers_dict,
min_area=min_area,
kernel=kernel)
# Get the mask and the georeferencing
mask = masker.mask
mask = mask.astype("float32")
try:
profile = masker.excl_h5.profile
except KeyError:
with h5py.File(excl_h5, "r") as ds:
for key in ds.keys():
if "profile" in ds[key].attrs.keys():
profile = ds[key].attrs["profile"]
profile = json.loads(profile)
profile["dtype"] = str(mask.dtype)
# Save
with rio.Env():
with rio.open(dst, "w", **profile) as file:
file.write(mask, 1)
def test_layer_mask(layer_name, inclusion_range, exclude_values,
include_values, weight, exclude_nodata):
"""
Test creation of layer masks
Parameters
----------
layer_name : str
Layer name
inclusion_range : tuple
(min threshold, max threshold) for values to include
exclude_values : list
list of values to exclude
Note: Only supply exclusions OR inclusions
include_values : list
List of values to include
Note: Only supply inclusions OR exclusions
"""
excl_h5 = os.path.join(TESTDATADIR, 'ri_exclusions', 'ri_exclusions.h5')
with ExclusionLayers(excl_h5) as f:
data = f[layer_name]
nodata_value = f.get_nodata_value(layer_name)
truth = mask_data(data, inclusion_range, exclude_values,
include_values, weight, exclude_nodata, nodata_value)
layer = LayerMask(layer_name, inclusion_range=inclusion_range,
exclude_values=exclude_values,
include_values=include_values, weight=weight,
exclude_nodata=exclude_nodata,
nodata_value=nodata_value)
layer_test = layer._apply_mask(data)
assert np.allclose(truth, layer_test)
mask_test = ExclusionMask.run(excl_h5, layers=layer)
assert np.allclose(truth, mask_test)
layer_dict = {layer_name: {"inclusion_range": inclusion_range,
"exclude_values": exclude_values,
"include_values": include_values,
"weight": weight,
"exclude_nodata": exclude_nodata}}
dict_test = ExclusionMaskFromDict.run(excl_h5, layers_dict=layer_dict)
assert np.allclose(truth, dict_test)
}
EXCL_DICT = {
'ri_srtm_slope': {
'inclusion_range': (None, 5),
'exclude_nodata': True
},
'ri_padus': {
'exclude_values': [1],
'exclude_nodata': True
}
}
RESOLUTION = 64
EXTENT = SupplyCurveExtent(EXCL_FPATH, resolution=RESOLUTION)
EXCL = ExclusionMaskFromDict(EXCL_FPATH, EXCL_DICT)
FRICTION = FrictionMask(FRICTION_FPATH, FRICTION_DSET)
def test_friction_mask():
"""Test the friction mask on known quantities."""
x = FRICTION[slice(700, 800), slice(300, 400)].mean()
assert x == 20.0, 'Friction for region should be 20.0, but is {}'.format(x)
x = FRICTION[slice(300, 400), slice(100, 300)].mean()
assert x == 1.0, 'Friction for nodata should be 1.0, but is {}'.format(x)
x = FRICTION[slice(300, 400), slice(800, 900)].mean()
assert x == 1.0, 'Friction for nodata should be 1.0, but is {}'.format(x)