def smooth_mask(base_mask_path, smooth_radius, target_smooth_mask_path):
"""Fill in gaps in base mask if there are neighbors.
Args:
base_mask_path (str): path to base raster, should be 0, 1 and nodata.
smooth_radius (int): how far to smooth out at a max radius?
target_smooth_mask_path (str): target smoothed file.
Returns:
None.
"""
kernel_size = smooth_radius*2+1
working_dir = tempfile.mkdtemp(
dir=os.path.dirname(target_smooth_mask_path))
kernel_path = os.path.join(working_dir, f'kernel_{kernel_size}.tif')
make_neighborhood_hat_kernel(kernel_size, kernel_path)
convolved_raster_path = os.path.join(working_dir, 'convolved_mask.tif')
byte_nodata = 255
pygeoprocessing.convolve_2d(
(base_mask_path, 1), (kernel_path, 1), convolved_raster_path,
ignore_nodata=False, working_dir=working_dir, mask_nodata=False,
target_nodata=TARGET_NODATA)
# set required proportion of coverage to turn on a pixel, lets make it a
# quarter wedge.
proportion_covered = 0.01
threshold_val = proportion_covered * 3.14159 * (smooth_radius+1)**2
pygeoprocessing.raster_calculator(
[(convolved_raster_path, 1), (threshold_val, 'raw'),
(TARGET_NODATA, 'raw'), (byte_nodata, 'raw'), ], threshold_op,
target_smooth_mask_path, gdal.GDT_Byte, byte_nodata)
def erode_one_pixel(base_raster_path, target_raster_path):
"""Take base and erode out one pixel."""
kernel_raster_path = os.path.join(os.path.dirname(target_raster_path),
'kernel.tif')
gtiff_driver = gdal.GetDriverByName('GTiff')
kernel_raster = gtiff_driver.Create(kernel_raster_path, 3, 3, 1,
gdal.GDT_Float32)
# Make some kind of geotransform, it doesn't matter what but
# will make GIS libraries behave better if it's all defined
kernel_raster.SetGeoTransform([0, 1, 0, 0, 0, -1])
srs = osr.SpatialReference()
srs.SetWellKnownGeogCS('WGS84')
kernel_raster.SetProjection(srs.ExportToWkt())
kernel_band = kernel_raster.GetRasterBand(1)
kernel_band.SetNoDataValue(-1)
kernel_band.Fill(1)
kernel_band = None
kernel_raster = None
pygeoprocessing.convolve_2d((base_raster_path, 1), (kernel_raster_path, 1),
target_raster_path,
target_datatype=gdal.GDT_Byte,
target_nodata=255)
def raster_average(raster_path, radius, kernel_path, out_path):
"""Average pixel values within a radius.
Make a search kernel where a pixel has '1' if its centerpoint is within
the radius of the center pixel's centerpoint.
For each pixel in a raster, center the search kernel on top of it. Then
its "neighborhood" includes all the pixels that are below a '1' in the
search kernel. Add up the neighborhood pixel values and divide by how
many there are.
This accounts for edge pixels and nodata pixels. For instance, if the
kernel covers a 3x3 pixel area centered on each pixel, most pixels will
have 9 valid pixels in their neighborhood, most edge pixels will have 6,
and most corner pixels will have 4. Edge and nodata pixels in the
neighborhood don't count towards the total (denominator in the average).
Args:
raster_path (str): path to the raster file to average
radius (float): distance to average around each pixel's centerpoint in
raster coordinate system units
kernel_path (str): path to write out the search kernel raster, an
intermediate output required by pygeoprocessing.convolve_2d
out_path (str): path to write out the averaged raster output
Returns:
None
"""
search_kernel = make_search_kernel(raster_path, radius)
srs = osr.SpatialReference()
srs.ImportFromEPSG(3857)
projection_wkt = srs.ExportToWkt()
pygeoprocessing.numpy_array_to_raster(
# float32 here to avoid pygeoprocessing bug issue #180
search_kernel.astype(numpy.float32),
FLOAT_NODATA,
(20, -20),
(0, 0),
projection_wkt,
kernel_path)
# convolve the signal (input raster) with the kernel and normalize
# this is equivalent to taking an average of each pixel's neighborhood
pygeoprocessing.convolve_2d(
(raster_path, 1),
(kernel_path, 1),
out_path,
# pixels with nodata or off the edge of the raster won't count towards
# the sum or the number of values to normalize by
ignore_nodata_and_edges=True,
# divide by number of valid pixels in the kernel (averaging)
normalize_kernel=True,
# output will have nodata where ratio_path has nodata
mask_nodata=True,
target_datatype=gdal.GDT_Float32,
target_nodata=FLOAT_NODATA)
def clean_convolve_2d(signal_raster_band_path,
kernel_raster_band_path,
target_raster_path,
working_dir=None):
"""Do 2D convolution but mask out any close to 0 values to 0."""
temp_workspace_dir = tempfile.mkdtemp(dir=os.path.dirname(
signal_raster_band_path[0]),
prefix='clean_convolve_2d')
temp_target_raster = os.path.join(temp_workspace_dir, 'result.tif')
pygeoprocessing.convolve_2d(signal_raster_band_path,
kernel_raster_band_path,
temp_target_raster,
working_dir=working_dir)
nodata = pygeoprocessing.get_raster_info(temp_target_raster)['nodata'][0]
pygeoprocessing.raster_calculator([(temp_target_raster, 1),
(1e-6, 'raw')], set_almost_zero_to_zero,
target_raster_path, gdal.GDT_Float32,
nodata)
try:
shutil.rmtree(temp_workspace_dir)
except OSError:
LOGGER.exception('unable to remove %s' % temp_workspace_dir)
def execute(args):
"""Habitat Quality.
Open files necessary for the portion of the habitat_quality
model.
Args:
workspace_dir (string): a path to the directory that will write output
and other temporary files (required)
lulc_cur_path (string): a path to an input land use/land cover raster
(required)
lulc_fut_path (string): a path to an input land use/land cover raster
(optional)
lulc_bas_path (string): a path to an input land use/land cover raster
(optional, but required for rarity calculations)
threat_folder (string): a path to the directory that will contain all
threat rasters (required)
threats_table_path (string): a path to an input CSV containing data
of all the considered threats. Each row is a degradation source
and each column a different attribute of the source with the
following names: 'THREAT','MAX_DIST','WEIGHT' (required).
access_vector_path (string): a path to an input polygon shapefile
containing data on the relative protection against threats (optional)
sensitivity_table_path (string): a path to an input CSV file of LULC
types, whether they are considered habitat, and their sensitivity
to each threat (required)
half_saturation_constant (float): a python float that determines
the spread and central tendency of habitat quality scores
(required)
suffix (string): a python string that will be inserted into all
raster path paths just before the file extension.
Example Args Dictionary::
{
'workspace_dir': 'path/to/workspace_dir',
'lulc_cur_path': 'path/to/lulc_cur_raster',
'lulc_fut_path': 'path/to/lulc_fut_raster',
'lulc_bas_path': 'path/to/lulc_bas_raster',
'threat_raster_folder': 'path/to/threat_rasters/',
'threats_table_path': 'path/to/threats_csv',
'access_vector_path': 'path/to/access_shapefile',
'sensitivity_table_path': 'path/to/sensitivity_csv',
'half_saturation_constant': 0.5,
'suffix': '_results',
}
Returns:
None
"""
workspace = args['workspace_dir']
# Append a _ to the suffix if it's not empty and doesn't already have one
suffix = utils.make_suffix_string(args, 'suffix')
# Check to see if each of the workspace folders exists. If not, create the
# folder in the filesystem.
inter_dir = os.path.join(workspace, 'intermediate')
out_dir = os.path.join(workspace, 'output')
kernel_dir = os.path.join(inter_dir, 'kernels')
utils.make_directories([inter_dir, out_dir, kernel_dir])
# get a handle on the folder with the threat rasters
threat_raster_dir = args['threat_raster_folder']
threat_dict = utils.build_lookup_from_csv(args['threats_table_path'],
'THREAT',
to_lower=False)
sensitivity_dict = utils.build_lookup_from_csv(
args['sensitivity_table_path'], 'LULC', to_lower=False)
# check that the required headers exist in the sensitivity table.
# Raise exception if they don't.
sens_header_list = sensitivity_dict.items()[0][1].keys()
required_sens_header_list = ['LULC', 'NAME', 'HABITAT']
missing_sens_header_list = [
h for h in required_sens_header_list if h not in sens_header_list
]
if missing_sens_header_list:
raise ValueError('Column(s) %s are missing in the sensitivity table' %
(', '.join(missing_sens_header_list)))
# check that the threat names in the threats table match with the threats
# columns in the sensitivity table. Raise exception if they don't.
for threat in threat_dict:
if 'L_' + threat not in sens_header_list:
missing_threat_header_list = (set(sens_header_list) -
set(required_sens_header_list))
raise ValueError(
'Threat "%s" does not match any column in the sensitivity '
'table. Possible columns: %s' %
(threat, missing_threat_header_list))
# get the half saturation constant
try:
half_saturation = float(args['half_saturation_constant'])
except ValueError:
raise ValueError('Half-saturation constant is not a numeric number.'
'It is: %s' % args['half_saturation_constant'])
# declare dictionaries to store the land cover and the threat rasters
# pertaining to the different threats
lulc_path_dict = {}
threat_path_dict = {}
# also store land cover and threat rasters in a list
lulc_and_threat_raster_list = []
aligned_raster_list = []
# declare a set to store unique codes from lulc rasters
raster_unique_lucodes = set()
# compile all the threat rasters associated with the land cover
for lulc_key, lulc_args in (('_c', 'lulc_cur_path'),
('_f', 'lulc_fut_path'), ('_b',
'lulc_bas_path')):
if lulc_args in args:
lulc_path = args[lulc_args]
lulc_path_dict[lulc_key] = lulc_path
# save land cover paths in a list for alignment and resize
lulc_and_threat_raster_list.append(lulc_path)
aligned_raster_list.append(
os.path.join(
inter_dir,
os.path.basename(lulc_path).replace(
'.tif', '_aligned.tif')))
# save unique codes to check if it's missing in sensitivity table
for _, lulc_block in pygeoprocessing.iterblocks((lulc_path, 1)):
raster_unique_lucodes.update(numpy.unique(lulc_block))
# Remove the nodata value from the set of landuser codes.
nodata = pygeoprocessing.get_raster_info(lulc_path)['nodata'][0]
try:
raster_unique_lucodes.remove(nodata)
except KeyError:
# KeyError when the nodata value was not encountered in the
# raster's pixel values. Same result when nodata value is
# None.
pass
# add a key to the threat dictionary that associates all threat
# rasters with this land cover
threat_path_dict['threat' + lulc_key] = {}
# for each threat given in the CSV file try opening the associated
# raster which should be found in threat_raster_folder
for threat in threat_dict:
# it's okay to have no threat raster for baseline scenario
threat_path_dict['threat' + lulc_key][threat] = (
resolve_ambiguous_raster_path(
os.path.join(threat_raster_dir, threat + lulc_key),
raise_error=(lulc_key != '_b')))
# save threat paths in a list for alignment and resize
threat_path = threat_path_dict['threat' + lulc_key][threat]
if threat_path:
lulc_and_threat_raster_list.append(threat_path)
aligned_raster_list.append(
os.path.join(
inter_dir,
os.path.basename(lulc_path).replace(
'.tif', '_aligned.tif')))
# check if there's any lucode from the LULC rasters missing in the
# sensitivity table
table_unique_lucodes = set(sensitivity_dict.keys())
missing_lucodes = raster_unique_lucodes.difference(table_unique_lucodes)
if missing_lucodes:
raise ValueError(
'The following land cover codes were found in your landcover rasters '
'but not in your sensitivity table. Check your sensitivity table '
'to see if they are missing: %s. \n\n' %
', '.join([str(x) for x in sorted(missing_lucodes)]))
# Align and resize all the land cover and threat rasters,
# and tore them in the intermediate folder
LOGGER.info('Starting aligning and resizing land cover and threat rasters')
lulc_pixel_size = (pygeoprocessing.get_raster_info(
args['lulc_cur_path']))['pixel_size']
aligned_raster_list = [
os.path.join(inter_dir,
os.path.basename(path).replace('.tif', '_aligned.tif'))
for path in lulc_and_threat_raster_list
]
pygeoprocessing.align_and_resize_raster_stack(
lulc_and_threat_raster_list, aligned_raster_list,
['near'] * len(lulc_and_threat_raster_list), lulc_pixel_size,
'intersection')
LOGGER.info('Finished aligning and resizing land cover and threat rasters')
# Modify paths in lulc_path_dict and threat_path_dict to be aligned rasters
for lulc_key, lulc_path in lulc_path_dict.iteritems():
lulc_path_dict[lulc_key] = os.path.join(
inter_dir,
os.path.basename(lulc_path).replace('.tif', '_aligned.tif'))
for threat in threat_dict:
threat_path = threat_path_dict['threat' + lulc_key][threat]
if threat_path in lulc_and_threat_raster_list:
threat_path_dict['threat' + lulc_key][threat] = os.path.join(
inter_dir,
os.path.basename(threat_path).replace(
'.tif', '_aligned.tif'))
LOGGER.info('Starting habitat_quality biophysical calculations')
# Rasterize access vector, if value is null set to 1 (fully accessible),
# else set to the value according to the ACCESS attribute
cur_lulc_path = lulc_path_dict['_c']
fill_value = 1.0
try:
LOGGER.info('Handling Access Shape')
access_raster_path = os.path.join(inter_dir,
'access_layer%s.tif' % suffix)
# create a new raster based on the raster info of current land cover
pygeoprocessing.new_raster_from_base(cur_lulc_path,
access_raster_path,
gdal.GDT_Float32, [_OUT_NODATA],
fill_value_list=[fill_value])
pygeoprocessing.rasterize(args['access_vector_path'],
access_raster_path,
burn_values=None,
option_list=['ATTRIBUTE=ACCESS'])
except KeyError:
LOGGER.info('No Access Shape Provided, access raster filled with 1s.')
# calculate the weight sum which is the sum of all the threats' weights
weight_sum = 0.0
for threat_data in threat_dict.itervalues():
# Sum weight of threats
weight_sum = weight_sum + threat_data['WEIGHT']
LOGGER.debug('lulc_path_dict : %s', lulc_path_dict)
# for each land cover raster provided compute habitat quality
for lulc_key, lulc_path in lulc_path_dict.iteritems():
LOGGER.info('Calculating habitat quality for landuse: %s', lulc_path)
# Create raster of habitat based on habitat field
habitat_raster_path = os.path.join(
inter_dir, 'habitat%s%s.tif' % (lulc_key, suffix))
map_raster_to_dict_values(lulc_path,
habitat_raster_path,
sensitivity_dict,
'HABITAT',
_OUT_NODATA,
values_required=False)
# initialize a list that will store all the threat/threat rasters
# after they have been adjusted for distance, weight, and access
deg_raster_list = []
# a list to keep track of the normalized weight for each threat
weight_list = numpy.array([])
# variable to indicate whether we should break out of calculations
# for a land cover because a threat raster was not found
exit_landcover = False
# adjust each threat/threat raster for distance, weight, and access
for threat, threat_data in threat_dict.iteritems():
LOGGER.info('Calculating threat: %s.\nThreat data: %s' %
(threat, threat_data))
# get the threat raster for the specific threat
threat_raster_path = threat_path_dict['threat' + lulc_key][threat]
LOGGER.info('threat_raster_path %s', threat_raster_path)
if threat_raster_path is None:
LOGGER.info(
'The threat raster for %s could not be found for the land '
'cover %s. Skipping Habitat Quality calculation for this '
'land cover.' % (threat, lulc_key))
exit_landcover = True
break
# need the pixel size for the threat raster so we can create
# an appropriate kernel for convolution
threat_pixel_size = pygeoprocessing.get_raster_info(
threat_raster_path)['pixel_size']
# pixel size tuple could have negative value
mean_threat_pixel_size = (abs(threat_pixel_size[0]) +
abs(threat_pixel_size[1])) / 2.0
# convert max distance (given in KM) to meters
max_dist_m = threat_data['MAX_DIST'] * 1000.0
# convert max distance from meters to the number of pixels that
# represents on the raster
max_dist_pixel = max_dist_m / mean_threat_pixel_size
LOGGER.debug('Max distance in pixels: %f', max_dist_pixel)
# blur the threat raster based on the effect of the threat over
# distance
decay_type = threat_data['DECAY']
kernel_path = os.path.join(
kernel_dir, 'kernel_%s%s%s.tif' % (threat, lulc_key, suffix))
if decay_type == 'linear':
make_linear_decay_kernel_path(max_dist_pixel, kernel_path)
elif decay_type == 'exponential':
utils.exponential_decay_kernel_raster(max_dist_pixel,
kernel_path)
else:
raise ValueError(
"Unknown type of decay in biophysical table, should be "
"either 'linear' or 'exponential'. Input was %s for threat"
" %s." % (decay_type, threat))
filtered_threat_raster_path = os.path.join(
inter_dir, 'filtered_%s%s%s.tif' % (threat, lulc_key, suffix))
pygeoprocessing.convolve_2d((threat_raster_path, 1),
(kernel_path, 1),
filtered_threat_raster_path)
# create sensitivity raster based on threat
sens_raster_path = os.path.join(
inter_dir, 'sens_%s%s%s.tif' % (threat, lulc_key, suffix))
map_raster_to_dict_values(lulc_path,
sens_raster_path,
sensitivity_dict,
'L_' + threat,
_OUT_NODATA,
values_required=True)
# get the normalized weight for each threat
weight_avg = threat_data['WEIGHT'] / weight_sum
# add the threat raster adjusted by distance and the raster
# representing sensitivity to the list to be past to
# vectorized_rasters below
deg_raster_list.append(filtered_threat_raster_path)
deg_raster_list.append(sens_raster_path)
# store the normalized weight for each threat in a list that
# will be used below in total_degradation
weight_list = numpy.append(weight_list, weight_avg)
# check to see if we got here because a threat raster was missing
# and if so then we want to skip to the next landcover
if exit_landcover:
continue
def total_degradation(*raster):
"""A vectorized function that computes the degradation value for
each pixel based on each threat and then sums them together
*rasters - a list of floats depicting the adjusted threat
value per pixel based on distance and sensitivity.
The values are in pairs so that the values for each threat
can be tracked:
[filtered_val_threat1, sens_val_threat1,
filtered_val_threat2, sens_val_threat2, ...]
There is an optional last value in the list which is the
access_raster value, but it is only present if
access_raster is not None.
returns - the total degradation score for the pixel"""
# we can not be certain how many threats the user will enter,
# so we handle each filtered threat and sensitivity raster
# in pairs
sum_degradation = numpy.zeros(raster[0].shape)
for index in range(len(raster) / 2):
step = index * 2
sum_degradation += (raster[step] * raster[step + 1] *
weight_list[index])
nodata_mask = numpy.empty(raster[0].shape, dtype=numpy.int8)
nodata_mask[:] = 0
for array in raster:
nodata_mask = nodata_mask | (array == _OUT_NODATA)
# the last element in raster is access
return numpy.where(nodata_mask, _OUT_NODATA,
sum_degradation * raster[-1])
# add the access_raster onto the end of the collected raster list. The
# access_raster will be values from the shapefile if provided or a
# raster filled with all 1's if not
deg_raster_list.append(access_raster_path)
deg_sum_raster_path = os.path.join(
out_dir, 'deg_sum' + lulc_key + suffix + '.tif')
LOGGER.info('Starting raster calculation on total_degradation')
deg_raster_band_list = [(path, 1) for path in deg_raster_list]
pygeoprocessing.raster_calculator(deg_raster_band_list,
total_degradation,
deg_sum_raster_path,
gdal.GDT_Float32, _OUT_NODATA)
LOGGER.info('Finished raster calculation on total_degradation')
# Compute habitat quality
# ksq: a term used below to compute habitat quality
ksq = half_saturation**_SCALING_PARAM
def quality_op(degradation, habitat):
"""Vectorized function that computes habitat quality given
a degradation and habitat value.
degradation - a float from the created degradation
raster above.
habitat - a float indicating habitat suitability from
from the habitat raster created above.
returns - a float representing the habitat quality
score for a pixel
"""
degredataion_clamped = numpy.where(degradation < 0, 0, degradation)
return numpy.where(
(degradation == _OUT_NODATA) | (habitat == _OUT_NODATA),
_OUT_NODATA,
(habitat * (1.0 -
((degredataion_clamped**_SCALING_PARAM) /
(degredataion_clamped**_SCALING_PARAM + ksq)))))
quality_path = os.path.join(out_dir,
'quality' + lulc_key + suffix + '.tif')
LOGGER.info('Starting raster calculation on quality_op')
deg_hab_raster_list = [deg_sum_raster_path, habitat_raster_path]
deg_hab_raster_band_list = [(path, 1) for path in deg_hab_raster_list]
pygeoprocessing.raster_calculator(deg_hab_raster_band_list, quality_op,
quality_path, gdal.GDT_Float32,
_OUT_NODATA)
LOGGER.info('Finished raster calculation on quality_op')
# Compute Rarity if user supplied baseline raster
if '_b' not in lulc_path_dict:
LOGGER.info('Baseline not provided to compute Rarity')
else:
lulc_base_path = lulc_path_dict['_b']
# get the area of a base pixel to use for computing rarity where the
# pixel sizes are different between base and cur/fut rasters
base_pixel_size = pygeoprocessing.get_raster_info(
lulc_base_path)['pixel_size']
base_area = float(abs(base_pixel_size[0]) * abs(base_pixel_size[1]))
base_nodata = pygeoprocessing.get_raster_info(
lulc_base_path)['nodata'][0]
lulc_code_count_b = raster_pixel_count(lulc_base_path)
# compute rarity for current landscape and future (if provided)
for lulc_key in ['_c', '_f']:
if lulc_key not in lulc_path_dict:
continue
lulc_path = lulc_path_dict[lulc_key]
lulc_time = 'current' if lulc_key == '_c' else 'future'
# get the area of a cur/fut pixel
lulc_pixel_size = pygeoprocessing.get_raster_info(
lulc_path)['pixel_size']
lulc_area = float(
abs(lulc_pixel_size[0]) * abs(lulc_pixel_size[1]))
lulc_nodata = pygeoprocessing.get_raster_info(
lulc_path)['nodata'][0]
def trim_op(base, cover_x):
"""Trim cover_x to the mask of base.
Parameters:
base (numpy.ndarray): base raster from 'lulc_base'
cover_x (numpy.ndarray): either future or current land
cover raster from 'lulc_path' above
Returns:
_OUT_NODATA where either array has nodata, otherwise
cover_x.
"""
return numpy.where(
(base == base_nodata) | (cover_x == lulc_nodata),
base_nodata, cover_x)
LOGGER.info('Create new cover for %s', lulc_path)
new_cover_path = os.path.join(
inter_dir, 'new_cover' + lulc_key + suffix + '.tif')
LOGGER.info('Starting masking %s land cover to base land cover.' %
lulc_time)
pygeoprocessing.raster_calculator([(lulc_base_path, 1),
(lulc_path, 1)], trim_op,
new_cover_path, gdal.GDT_Float32,
_OUT_NODATA)
LOGGER.info('Finished masking %s land cover to base land cover.' %
lulc_time)
LOGGER.info('Starting rarity computation on %s land cover.' %
lulc_time)
lulc_code_count_x = raster_pixel_count(new_cover_path)
# a dictionary to map LULC types to a number that depicts how
# rare they are considered
code_index = {}
# compute rarity index for each lulc code
# define 0.0 if an lulc code is found in the cur/fut landcover
# but not the baseline
for code in lulc_code_count_x.iterkeys():
if code in lulc_code_count_b:
numerator = lulc_code_count_x[code] * lulc_area
denominator = lulc_code_count_b[code] * base_area
ratio = 1.0 - (numerator / denominator)
code_index[code] = ratio
else:
code_index[code] = 0.0
rarity_path = os.path.join(out_dir,
'rarity' + lulc_key + suffix + '.tif')
pygeoprocessing.reclassify_raster((new_cover_path, 1), code_index,
rarity_path, gdal.GDT_Float32,
_RARITY_NODATA)
LOGGER.info('Finished rarity computation on %s land cover.' %
lulc_time)
LOGGER.info('Finished habitat_quality biophysical calculations')
def calculate_reef_population_value(shore_sample_point_vector_path,
dem_raster_path, reef_habitat_raster_path,
population_raster_path_id_target_list,
temp_workspace_dir):
"""Calculate population within protective range of reefs.
Parameters:
shore_sample_point_vector_path (str): path to a point shapefile that is
used for referencing the points of interest on the coastline where.
dem_raster_path (str): path to a dem used to mask population by height
in wgs84 lat/lng projection.
reef_habitat_raster_path (str): path to a mask raster where reef
habitat exists.
population_raster_path_id_list (list): list of
(raster_path, field_id, target_path)
tuples. The values in the raster paths will be masked where it
overlaps with < 10m dem height and convolved within 2km. That
result is in turn spread onto the habitat coverage at a distance
of the protective distance of reefs. These rasters are in
wgs84 lat/lng projection.
Returns:
None
"""
wgs84_srs = osr.SpatialReference()
wgs84_srs.ImportFromEPSG(4326)
aligned_pop_raster_list = align_raster_list(
[x[0] for x in population_raster_path_id_target_list] +
[reef_habitat_raster_path, dem_raster_path],
temp_workspace_dir,
wgs84_srs.ExportToWkt(),
align_index=len(population_raster_path_id_target_list))
raster_info = pygeoprocessing.get_raster_info(aligned_pop_raster_list[0])
target_pixel_size = raster_info['pixel_size']
n_pixels_in_prot_dist = max(
1, int(REEF_PROT_DIST / (M_PER_DEGREE * abs(target_pixel_size[0]))))
kernel_radius = [n_pixels_in_prot_dist, n_pixels_in_prot_dist]
kernel_filepath = os.path.join(temp_workspace_dir, 'reef_kernel.tif')
create_averaging_kernel_raster(kernel_radius,
kernel_filepath,
normalize=False)
buffered_point_raster_mask_path = os.path.join(temp_workspace_dir,
'reef_buffer_mask.tif')
make_buffered_point_raster_mask(shore_sample_point_vector_path,
aligned_pop_raster_list[0],
temp_workspace_dir, 'reefs_all',
REEF_PROT_DIST,
buffered_point_raster_mask_path)
for pop_index, (
_, pop_id,
target_path) in enumerate(population_raster_path_id_target_list):
# mask to < 10m
pop_height_masked_path = os.path.join(temp_workspace_dir,
'%s_masked_by_10m.tif' % pop_id)
pygeoprocessing.raster_calculator(
[
(aligned_pop_raster_list[pop_index], 1),
(aligned_pop_raster_list[-1], 1),
(10.0, 'raw'), # mask to 10 meters
(raster_info['nodata'][0], 'raw')
], # the -1 index is the dem
mask_by_height_op,
pop_height_masked_path,
gdal.GDT_Float32,
raster_info['nodata'][0])
# spread the < 10m population out 2km
n_pixels_in_2km = int(2000.0 /
(M_PER_DEGREE * abs(target_pixel_size[0])))
kernel_radius_2km = [n_pixels_in_2km, n_pixels_in_2km]
kernel_2km_filepath = os.path.join(temp_workspace_dir,
'2km_kernel_%s.tif' % pop_id)
create_averaging_kernel_raster(kernel_radius_2km,
kernel_2km_filepath,
normalize=False)
pop_sum_within_2km_path = os.path.join(
temp_workspace_dir, 'pop_sum_within_2km_%s.tif' % pop_id)
pygeoprocessing.convolve_2d((pop_height_masked_path, 1),
(kernel_2km_filepath, 1),
pop_sum_within_2km_path)
align_reef_habitat_raster_path = aligned_pop_raster_list[-2]
population_hab_spread_raster_path = os.path.join(
temp_workspace_dir, 'reef_%s_spread.tif' % (pop_id))
clean_convolve_2d((pop_sum_within_2km_path, 1), (kernel_filepath, 1),
population_hab_spread_raster_path)
hab_raster_info = pygeoprocessing.get_raster_info(
align_reef_habitat_raster_path)
# warp pop result to overlay
clipped_pop_hab_spread_raster_path = os.path.join(
temp_workspace_dir, 'reef_%s_spread_clipped.tif' % pop_id)
pygeoprocessing.warp_raster(population_hab_spread_raster_path,
hab_raster_info['pixel_size'],
clipped_pop_hab_spread_raster_path, 'near')
hab_spread_nodata = pygeoprocessing.get_raster_info(
clipped_pop_hab_spread_raster_path)['nodata'][0]
hab_nodata = hab_raster_info['nodata'][0]
pygeoprocessing.raster_calculator(
[(clipped_pop_hab_spread_raster_path, 1),
(align_reef_habitat_raster_path, 1),
(buffered_point_raster_mask_path, 1), (hab_spread_nodata, 'raw'),
(hab_nodata, 'raw')], intersect_and_mask_raster_op, target_path,
gdal.GDT_Float32, hab_spread_nodata),
ecoshard.build_overviews(target_path)
def _convert_landscape(
base_lulc_path, replacement_lucode, area_to_convert,
focal_landcover_codes, convertible_type_list, score_weight, n_steps,
smooth_distance_from_edge_path, output_landscape_raster_path,
stats_path, workspace_dir):
"""Expand replacement lucodes in relation to the focal lucodes.
If the sign on `score_weight` is positive, expansion occurs marches
away from the focal types, while if `score_weight` is negative conversion
marches toward the focal types.
Parameters:
base_lulc_path (string): path to landcover raster that will be used as
the base landcover map to agriculture pixels
replacement_lucode (int): agriculture landcover code type found in the
raster at `base_lulc_path`
area_to_convert (float): area (Ha) to convert to agriculture
focal_landcover_codes (list of int): landcover codes that are used to
calculate proximity
convertible_type_list (list of int): landcover codes that are allowable
to be converted to agriculture
score_weight (float): this value is used to multiply the distance from
the focal landcover types when prioritizing which pixels in
`convertable_type_list` are to be converted. If negative,
conversion occurs toward the focal types, if positive occurs away
from the focal types.
n_steps (int): number of steps to convert the landscape. On each step
the distance transform will be applied on the
current value of the `focal_landcover_codes` pixels in
`output_landscape_raster_path`. On the first step the distance
is calculated from `base_lulc_path`.
smooth_distance_from_edge_path (string): an intermediate output showing
the pixel distance from the edge of the base landcover types
output_landscape_raster_path (string): an output raster that will
contain the final fragmented forest layer.
stats_path (string): a path to an output csv that records the number
type, and area of pixels converted in `output_landscape_raster_path`
workspace_dir (string): workspace directory that will be used to
hold temporary files. On a successful run of this function,
the temporary directory will be removed.
Returns:
None.
"""
temp_dir = tempfile.mkdtemp(prefix='temp_dir', dir=workspace_dir)
tmp_file_registry = {
'non_base_mask': os.path.join(temp_dir, 'non_base_mask.tif'),
'base_mask': os.path.join(temp_dir, 'base_mask.tif'),
'gaussian_kernel': os.path.join(temp_dir, 'gaussian_kernel.tif'),
'distance_from_base_mask_edge': os.path.join(
temp_dir, 'distance_from_base_mask_edge.tif'),
'distance_from_non_base_mask_edge': os.path.join(
temp_dir, 'distance_from_non_base_mask_edge.tif'),
'convertible_distances': os.path.join(
temp_dir, 'convertible_distances.tif'),
'distance_from_edge': os.path.join(
temp_dir, 'distance_from_edge.tif'),
}
# a sigma of 1.0 gives nice visual results to smooth pixel level artifacts
# since a pixel is the 1.0 unit
_make_gaussian_kernel_path(1.0, tmp_file_registry['gaussian_kernel'])
# create the output raster first as a copy of the base landcover so it can
# be looped on for each step
lulc_raster_info = pygeoprocessing.get_raster_info(base_lulc_path)
lulc_nodata = lulc_raster_info['nodata'][0]
mask_nodata = 2
pygeoprocessing.raster_calculator(
[(base_lulc_path, 1)], lambda x: x, output_landscape_raster_path,
gdal.GDT_Int32, lulc_nodata)
# convert everything furthest from edge for each of n_steps
pixel_area_ha = (
abs(lulc_raster_info['pixel_size'][0]) *
abs(lulc_raster_info['pixel_size'][1])) / 10000.0
max_pixels_to_convert = int(math.ceil(area_to_convert / pixel_area_ha))
convertible_type_nodata = -1
pixels_left_to_convert = max_pixels_to_convert
pixels_to_convert = max_pixels_to_convert / n_steps
stats_cache = collections.defaultdict(int)
# pylint complains when these are defined inside the loop
invert_mask = None
distance_nodata = None
for step_index in range(n_steps):
LOGGER.info('step %d of %d', step_index+1, n_steps)
pixels_left_to_convert -= pixels_to_convert
# Often the last segement of the steps will overstep the number of
# pixels to convert, this check converts the exact amount
if pixels_left_to_convert < 0:
pixels_to_convert += pixels_left_to_convert
# create distance transforms for inside and outside the base lulc codes
LOGGER.info('create distance transform for current landcover')
for invert_mask, mask_id, distance_id in [
(False, 'non_base_mask', 'distance_from_non_base_mask_edge'),
(True, 'base_mask', 'distance_from_base_mask_edge')]:
def _mask_base_op(lulc_array):
"""Create a mask of valid non-base pixels only."""
base_mask = numpy.in1d(
lulc_array.flatten(), focal_landcover_codes).reshape(
lulc_array.shape)
if invert_mask:
base_mask = ~base_mask
return numpy.where(
lulc_array == lulc_nodata,
mask_nodata, base_mask)
pygeoprocessing.raster_calculator(
[(output_landscape_raster_path, 1)], _mask_base_op,
tmp_file_registry[mask_id], gdal.GDT_Byte,
mask_nodata)
# create distance transform for the current mask
pygeoprocessing.distance_transform_edt(
(tmp_file_registry[mask_id], 1),
tmp_file_registry[distance_id],
working_dir=temp_dir)
# combine inner and outer distance transforms into one
distance_nodata = pygeoprocessing.get_raster_info(
tmp_file_registry['distance_from_base_mask_edge'])['nodata'][0]
def _combine_masks(base_distance_array, non_base_distance_array):
"""Create a mask of valid non-base pixels only."""
result = non_base_distance_array
valid_base_mask = base_distance_array > 0.0
result[valid_base_mask] = base_distance_array[valid_base_mask]
return result
pygeoprocessing.raster_calculator(
[(tmp_file_registry['distance_from_base_mask_edge'], 1),
(tmp_file_registry['distance_from_non_base_mask_edge'], 1)],
_combine_masks, tmp_file_registry['distance_from_edge'],
gdal.GDT_Float32, distance_nodata)
# smooth the distance transform to avoid scanline artifacts
pygeoprocessing.convolve_2d(
(tmp_file_registry['distance_from_edge'], 1),
(tmp_file_registry['gaussian_kernel'], 1),
smooth_distance_from_edge_path)
# turn inside and outside masks into a single mask
def _mask_to_convertible_codes(distance_from_base_edge, lulc):
"""Mask out the distance transform to a set of lucodes."""
convertible_mask = numpy.in1d(
lulc.flatten(), convertible_type_list).reshape(lulc.shape)
return numpy.where(
convertible_mask, distance_from_base_edge,
convertible_type_nodata)
pygeoprocessing.raster_calculator(
[(smooth_distance_from_edge_path, 1),
(output_landscape_raster_path, 1)],
_mask_to_convertible_codes,
tmp_file_registry['convertible_distances'], gdal.GDT_Float32,
convertible_type_nodata)
LOGGER.info(
'convert %d pixels to lucode %d', pixels_to_convert,
replacement_lucode)
_convert_by_score(
tmp_file_registry['convertible_distances'], pixels_to_convert,
output_landscape_raster_path, replacement_lucode, stats_cache,
score_weight)
_log_stats(stats_cache, pixel_area_ha, stats_path)
try:
shutil.rmtree(temp_dir)
except OSError:
LOGGER.warn(
"Could not delete temporary working directory '%s'", temp_dir)
# Make convolution kernel
kernel_path = os.path.join(churn_dir, 'kernel.tif')
# assume square pixels
kernel_radius = int(args.protective_distance // target_pixel_size[0])
LOGGER.info(f"kernel radius: {kernel_radius}")
kernel_x, kernel_y = numpy.meshgrid(range((kernel_radius - 1) * 2 + 1),
range((kernel_radius - 1) * 2 + 1))
kernel_distance = numpy.sqrt((kernel_x - (kernel_radius - 1))**2 +
(kernel_y - (kernel_radius - 1))**2)
kernel_array = (kernel_distance <= kernel_radius).astype(numpy.int8)
pygeoprocessing.numpy_array_to_raster(kernel_array, 0, (1, -1), (0, 0),
None, kernel_path)
# Convolve CV points for coverage
convolve_target_raster_path = os.path.join(churn_dir, 'convolve_2d.tif')
pygeoprocessing.convolve_2d((shore_point_raster_path, 1), (kernel_path, 1),
convolve_target_raster_path,
ignore_nodata_and_edges=False,
mask_nodata=False,
normalize_kernel=False,
target_datatype=gdal.GDT_Float64)
target_habitat_value_raster_path = os.path.join(
args.workspace_dir, args.target_habitat_value_raster_filename)
# TODO: mask result to habitat
mask_by_nodata(convolve_target_raster_path, habitat_mask_raster_path,
target_habitat_value_raster_path)
def _create_marginal_value_layer(
future_raster_path, base_raster_path,
gaussian_blur_pixel_radius, mask_raster_path, target_raster_path):
"""Calculate marginal value layer.
Calculated by taking the difference of future from base, Gaussian blurring
that result by the given radius, and masking by the given raster mask.
Args:
future_raster_path (str): raster A, same nodata and size as B
base_raster_path (str): raster B
gaussian_blur_pixel_radius (int): number of pixels to blur out when
determining marginal value of that pixel.
mask_raster_path (str): path to raster where anything not 1 is masked
to 0/nodata.
target_diff_raster_path (str): result of A-B accounting for nodata.
Returns:
None
"""
raster_info = pygeoprocessing.get_raster_info(future_raster_path)
nodata = raster_info['nodata'][0]
def _diff_op(a_array, b_array):
"""Return a-b and consider nodata."""
result = numpy.copy(a_array)
valid_mask = ~numpy.isclose(a_array, nodata)
result[valid_mask] -= b_array[valid_mask]
return result
churn_dir = tempfile.mkdtemp(dir=os.path.dirname(target_raster_path))
diff_raster_path = os.path.join(churn_dir, 'diff.tif')
pygeoprocessing.raster_calculator(
[(future_raster_path, 1), (base_raster_path, 1)], _diff_op,
diff_raster_path, raster_info['datatype'], nodata)
# Gaussian filter
if gaussian_blur_pixel_radius is not None:
kernel_raster_path = os.path.join(churn_dir, 'kernel.tif')
mask_gf_path = os.path.join(churn_dir, 'gf.tif')
if os.path.exists(mask_gf_path):
os.remove(mask_gf_path)
make_kernel_raster(
gaussian_blur_pixel_radius, kernel_raster_path)
pygeoprocessing.convolve_2d(
(diff_raster_path, 1), (kernel_raster_path, 1), mask_gf_path,
ignore_nodata_and_edges=False, mask_nodata=True,
target_nodata=0.0)
else:
mask_gf_path = diff_raster_path
def _mask_op(base_array, mask_array):
"""Return base where mask is 1, otherwise 0 or nodata."""
result = numpy.copy(base_array)
zero_mask = (~numpy.isclose(base_array, nodata)) & (mask_array != 1)
result[zero_mask] = 0
return result
pygeoprocessing.raster_calculator(
[(mask_gf_path, 1), (mask_raster_path, 1)], _mask_op,
target_raster_path, raster_info['datatype'], nodata)
shutil.rmtree(churn_dir)
def fill_by_convolution(base_raster_path, convolve_radius,
target_filled_raster_path):
"""Clip and fill.
Clip the base raster data to the bounding box then fill any noodata
holes with a weighted distance convolution.
Args:
base_raster_path (str): path to base raster
convolve_radius (float): maximum convolution distance kernel in
projected units of base.
target_filled_raster_path (str): raster created by convolution fill,
if holes are too far from valid pixels resulting fill will be
nonsensical, perhaps NaN.
Return:
None
"""
try:
LOGGER.info(f'filling {base_raster_path}')
# create working directory in the same directory as the target with
# the same name as the target file so it can't be duplicated
# easier to spot for debugging too
working_dir = os.path.join(
os.path.dirname(target_filled_raster_path),
os.path.basename(os.path.splitext(target_filled_raster_path)[0]))
try:
os.makedirs(working_dir)
except OSError:
pass
basename = os.path.basename(target_filled_raster_path)
base_raster_info = pygeoprocessing.get_raster_info(base_raster_path)
# this ensures a minimum of 3 pixels in case the pixel size is too
# chunky
n = max(3, int(convolve_radius / base_raster_info['pixel_size'][0]))
base = numpy.zeros((n, n))
base[n // 2, n // 2] = 1
kernel_array = scipy.ndimage.filters.gaussian_filter(base, n / 3)
kernel_raster_path = os.path.join(working_dir, f'kernel_{basename}')
geotransform = base_raster_info['geotransform']
pygeoprocessing.numpy_array_to_raster(
kernel_array, None, base_raster_info['pixel_size'],
(geotransform[0], geotransform[3]),
base_raster_info['projection_wkt'], kernel_raster_path)
# scrub input raster
sanitized_base_raster_path = os.path.join(working_dir,
f'sanitized_{basename}')
sanitize_raster(base_raster_path, sanitized_base_raster_path)
# mask valid
valid_raster_path = os.path.join(working_dir, f'sanitized_{basename}')
pygeoprocessing.raster_calculator(
[(base_raster_path, 1), (base_raster_info['nodata'][0], 'raw')],
_mask_valid_op, valid_raster_path, gdal.GDT_Byte, None)
mask_kernel_raster_path = os.path.join(working_dir,
f'mask_kernel_{basename}')
geotransform = base_raster_info['geotransform']
mask_kernel_array = numpy.copy(kernel_array)
mask_kernel_array[:] = 1
pygeoprocessing.numpy_array_to_raster(
mask_kernel_array, None, base_raster_info['pixel_size'],
(geotransform[0], geotransform[3]),
base_raster_info['projection_wkt'], mask_kernel_raster_path)
coverage_raster_path = os.path.join(working_dir,
f'coverage_{basename}')
pygeoprocessing.convolve_2d((valid_raster_path, 1),
(mask_kernel_raster_path, 1),
coverage_raster_path,
mask_nodata=False,
target_nodata=-1,
target_datatype=gdal.GDT_Byte,
working_dir=working_dir)
# this raster will be filled with the entire convolution
backfill_raster_path = os.path.join(working_dir,
f'backfill_{basename}')
base_nodata = base_raster_info['nodata'][0]
if base_nodata is None:
target_datatype = gdal.GDT_Float64
else:
target_datatype = base_raster_info['datatype']
LOGGER.info(f'create backfill from {sanitized_base_raster_path} to '
f'{backfill_raster_path}')
pygeoprocessing.convolve_2d((sanitized_base_raster_path, 1),
(kernel_raster_path, 1),
backfill_raster_path,
ignore_nodata_and_edges=True,
mask_nodata=False,
normalize_kernel=True,
target_nodata=base_nodata,
target_datatype=target_datatype,
working_dir=working_dir)
LOGGER.info(
f'fill nodata of {base_raster_path} to {backfill_raster_path}')
pygeoprocessing.raster_calculator([(base_raster_path, 1),
(backfill_raster_path, 1),
(coverage_raster_path, 1),
(base_nodata, 'raw')],
_fill_nodata_op,
target_filled_raster_path,
base_raster_info['datatype'],
base_nodata)
shutil.rmtree(working_dir)
except Exception:
LOGGER.exception(
f'error on fill by convolution {target_filled_raster_path}')
raise
