def write_attributes(in_layer_path: str, output_values: dict, id_field: str, fields, field_type=ogr.OFTReal, null_values=None):
"""
Write field values to a feature class
:param feature_class: Path to feature class
:param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names
:param id_field: Unique key identifying each feature in both feature class and output_values dictionary
:param fields: List of fields in output_values to write to
:return: None
"""
log = Logger('write_attributes')
with get_shp_or_gpkg(in_layer_path, write=True) as in_layer:
# Create each field and store the name and index in a list of tuples
field_indices = [(field, in_layer.create_field(field, field_type)) for field in fields] # TODO different field types
for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
reach = feature.GetField(id_field) # TODO Error when id_field is same as FID field .GetFID() seems to work instead
if reach not in output_values:
continue
# Set all the field values and then store the feature
for field, _idx in field_indices:
if field in output_values[reach]:
if not output_values[reach][field]:
if null_values:
feature.SetField(field, null_values)
else:
log.warning('Unhandled feature class value for None type')
feature.SetField(field, None)
else:
feature.SetField(field, output_values[reach][field])
in_layer.ogr_layer.SetFeature(feature)
def get_nhd_states(inpath):
"""
Gets the list of US States that an NHD HUC encompasses
This relies on the watershed boundary ShapeFile having a column called
'States' that stores a comma separated list of state abbreviations
such as 'OR,WA'. A dcitionary is used to retrieve the full names.
:param inpath: Path to the watershed boundary ShapeFile
:return: List of full US state names that the watershed touches (.e.g. Oregon)
"""
log = Logger('RS Context')
driver = ogr.GetDriverByName("ESRI Shapefile")
data_source = driver.Open(inpath, 0)
layer = data_source.GetLayer()
states = []
for feature in layer:
value = feature.GetField('States')
[states.append(us_states[acronym]) for acronym in value.split(',')]
data_source = None
if 'Canada' in states:
if len(states) == 1:
log.error(
'HUC is entirely within Canada. No DEMs will be available.')
else:
log.warning(
'HUC is partially in Canada. Certain data will only be available for US portion.'
)
log.info('HUC intersects {} state(s): {}'.format(len(states),
', '.join(states)))
return list(dict.fromkeys(states))
def get_watershed_info(gpkg_path):
"""Query a BRAT database and get information about
the watershed being run. Assumes that all watersheds
except the one being run have been deleted.
Arguments:
database {str} -- Path to the BRAT SQLite database
Returns:
[tuple] -- WatershedID, max drainage area, EcoregionID with which
the watershed is associated.
"""
with SQLiteCon(gpkg_path) as database:
database.curs.execute(
'SELECT WatershedID, MaxDrainage, EcoregionID FROM Watersheds')
row = database.curs.fetchone()
watershed = row['WatershedID']
max_drainage = row['MaxDrainage']
ecoregion = row['EcoregionID']
log = Logger('BRAT Run')
if not watershed:
raise Exception(
'Missing watershed in BRAT datatabase {}'.format(database))
if not max_drainage:
log.warning('Missing max drainage for watershed {}'.format(watershed))
if not ecoregion:
raise Exception('Missing ecoregion for watershed {}'.format(watershed))
return watershed, max_drainage, ecoregion
def centerline_points(
in_lines: Path,
distance: float = 0.0,
transform: Transform = None) -> Dict[int, List[RiverPoint]]:
"""Generates points along each line feature at specified distances from the end as well as quarter and halfway
Args:
in_lines (Path): path of shapefile with features
distance (float, optional): distance from ends to generate points. Defaults to 0.0.
transform (Transform, optional): coordinate transformation. Defaults to None.
Returns:
[type]: [description]
"""
log = Logger('centerline_points')
with get_shp_or_gpkg(in_lines) as in_lyr:
out_group = {}
ogr_extent = in_lyr.ogr_layer.GetExtent()
extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2],
ogr_extent[1], ogr_extent[3])
for feat, _counter, progbar in in_lyr.iterate_features(
"Centerline points"):
line = VectorBase.ogr2shapely(feat, transform)
fid = feat.GetFID()
out_points = []
# Attach the FID in case we need it later
props = {'fid': fid}
pts = [
line.interpolate(distance),
line.interpolate(0.5, True),
line.interpolate(-distance)
]
if line.project(line.interpolate(0.25, True)) > distance:
pts.append(line.interpolate(0.25, True))
pts.append(line.interpolate(-0.25, True))
for pt in pts:
# Recall that interpolation can have multiple solutions due to pythagorean theorem
# Throw away anything that's not inside our bounds
if not extent.contains(pt):
progbar.erase()
log.warning('Point {} is outside of extent: {}'.format(
pt.coords[0], ogr_extent))
out_points.append(RiverPoint(pt, properties=props))
out_group[int(fid)] = out_points
feat = None
return out_group
def merge_feature_classes(feature_class_paths: List[str], boundary: BaseGeometry, out_layer_path: str):
"""[summary]
Args:
feature_class_paths (List[str]): [description]
boundary (BaseGeometry): [description]
out_layer_path (str): [description]
"""
log = Logger('merge_feature_classes')
log.info('Merging {} feature classes.'.format(len(feature_class_paths)))
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer:
fccount = 0
for in_layer_path in feature_class_paths:
fccount += 1
log.info("Merging feature class {}/{}".format(fccount, len(feature_class_paths)))
with get_shp_or_gpkg(in_layer_path) as in_layer:
in_layer.SetSpatialFilter(VectorBase.shapely2ogr(boundary))
# First input spatial ref sets the SRS for the output file
transform = in_layer.get_transform(out_layer)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
in_field_def = in_layer.ogr_layer_def.GetFieldDefn(i)
# Only create fields if we really don't have them
# NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE
if out_layer.ogr_layer_def.GetFieldIndex(in_field_def.GetName()) == -1:
out_layer.ogr_layer.CreateField(in_field_def)
log.info('Processing feature: {}/{}'.format(fccount, len(feature_class_paths)))
for feature, _counter, progbar in in_layer.iterate_features('Processing feature'):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
geom.Transform(transform)
out_feature = ogr.Feature(out_layer.ogr_layer_def)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_feature.SetGeometry(geom)
out_layer.ogr_layer.CreateFeature(out_feature)
log.info('Merge complete.')
return fccount
def get_riverpoints(inpath, epsg, attribute_filter=None):
"""[summary]
Args:
inpath ([type]): Path to a ShapeFile
epsg ([type]): Desired output spatial reference
attribute_filter ([type], optional): [description]. Defaults to None.
Returns:
[type]: List of RiverPoint objects
"""
log = Logger('get_riverpoints')
points = []
with get_shp_or_gpkg(inpath) as in_lyr:
_out_spatial_ref, transform = get_transform_from_epsg(
in_lyr.spatial_ref, epsg)
for feat, _counter, progbar in in_lyr.iterate_features(
'Getting points for use in Thiessen',
attribute_filter=attribute_filter):
new_geom = feat.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feat.GetFID()))
continue
new_geom.Transform(transform)
new_shape = VectorBase.ogr2shapely(new_geom)
if new_shape.type == 'Polygon':
new_shape = MultiPolygon([new_shape])
for poly in new_shape:
# Exterior is the shell and there is only ever 1
for pt in list(poly.exterior.coords):
points.append(RiverPoint(pt, interior=False))
# Now we consider interiors. NB: Interiors are only qualifying islands in this case
for idx, island in enumerate(poly.interiors):
for pt in list(island.coords):
points.append(RiverPoint(pt, interior=True,
island=idx))
return points
def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict:
"""[summary]
Args:
in_layer_path (str): [description]
id_field (str, optional): [description]. Defaults to None.
epsg (int, optional): [description]. Defaults to None.
spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
dict: [description]
"""
log = Logger('load_geometries')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
# Determine the transformation if user provides an EPSG
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
features = {}
for feature, _counter, progbar in in_layer.iterate_features("Loading features"):
if id_field is None:
reach = feature.GetFID()
else:
reach = feature.GetField(id_field)
geom = feature.GetGeometryRef()
geo_type = geom.GetGeometryType()
new_geom = VectorBase.ogr2shapely(geom, transform=transform)
if new_geom.is_empty:
progbar.erase() # get around the progressbar
log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
elif not new_geom.is_valid:
progbar.erase() # get around the progressbar
log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for feature with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for feature with FID={}'.format(feature.GetFID()))
else:
features[reach] = new_geom
return features
def get_geometry_union(inpath, epsg, attribute_filter=None):
"""
TODO: Remove this method and replace all references to the get_geometry_unary_union method below
Load all features from a ShapeFile and union them together into a single geometry
:param inpath: Path to a ShapeFile
:param epsg: Desired output spatial reference
:return: Single Shapely geometry of all unioned features
"""
log = Logger('Shapefile')
driver = ogr.GetDriverByName("ESRI Shapefile")
data_source = driver.Open(inpath, 0)
layer = data_source.GetLayer()
in_spatial_ref = layer.GetSpatialRef()
if attribute_filter:
layer.SetAttributeFilter(attribute_filter)
_out_spatial_ref, transform = get_transform_from_epsg(in_spatial_ref, epsg)
geom = None
progbar = ProgressBar(layer.GetFeatureCount(), 50, "Unioning features")
counter = 0
for feature in layer:
counter += 1
progbar.update(counter)
new_geom = feature.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(
feature.GetFID()))
continue
new_geom.Transform(transform)
new_shape = wkbload(new_geom.ExportToWkb())
try:
geom = geom.union(new_shape) if geom else new_shape
except Exception as e:
progbar.erase() # get around the progressbar
log.warning(
'Union failed for shape with FID={} and will be ignored'.
format(feature.GetFID()))
progbar.finish()
data_source = None
return geom
def safe_remove_file(file_path):
"""Remove a file without throwing an error
Args:
file_path ([type]): [description]
"""
log = Logger("safe_remove_file")
try:
if not os.path.isfile(file_path):
log.warning('File not found: {}'.format(file_path))
os.remove(file_path)
log.debug('File removed: {}'.format(file_path))
except Exception as e:
log.error(str(e))
def load_geometries(feature_class, id_field, epsg=None):
log = Logger('Shapefile')
# Get the input network
driver = ogr.GetDriverByName('ESRI Shapefile')
dataset = driver.Open(feature_class, 0)
layer = dataset.GetLayer()
in_spatial_ref = layer.GetSpatialRef()
# Determine the transformation if user provides an EPSG
transform = None
if epsg:
out_spatial_ref, transform = get_transform_from_epsg(
in_spatial_ref, epsg)
features = {}
progbar = ProgressBar(layer.GetFeatureCount(), 50, "Loading features")
counter = 0
for inFeature in layer:
counter += 1
progbar.update(counter)
reach = inFeature.GetField(id_field)
geom = inFeature.GetGeometryRef()
# Optional coordinate transformation
if transform:
geom.Transform(transform)
new_geom = wkbload(geom.ExportToWkb())
geo_type = new_geom.GetGeometryType()
if new_geom.is_empty:
progbar.erase() # get around the progressbar
log.warning(
'Empty feature with FID={} cannot be unioned and will be ignored'
.format(inFeature.GetFID()))
elif not new_geom.is_valid:
progbar.erase() # get around the progressbar
log.warning(
'Invalid feature with FID={} cannot be unioned and will be ignored'
.format(inFeature.GetFID()))
# Filter out zero-length lines
elif geo_type in LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for feature with FID={}'.format(
inFeature.GetFID()))
# Filter out zero-area polys
elif geo_type in POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for feature with FID={}'.format(
inFeature.GetFID()))
else:
features[reach] = new_geom
progbar.finish()
dataset = None
return features
def merge_geometries(feature_classes, epsg):
"""
Load all features from multiple feature classes into a single list of geometries
:param feature_classes:
:param epsg:
:return:
"""
log = Logger('Shapefile')
driver = ogr.GetDriverByName("ESRI Shapefile")
union = ogr.Geometry(ogr.wkbMultiLineString)
fccount = 0
for fc in feature_classes:
fccount += 1
log.info("Merging Geometries for feature class {}/{}".format(
fccount, len(feature_classes)))
data_source = driver.Open(fc, 0)
layer = data_source.GetLayer()
in_spatial_ref = layer.GetSpatialRef()
out_spatial_ref, transform = get_transform_from_epsg(
in_spatial_ref, epsg)
progbar = ProgressBar(layer.GetFeatureCount(), 50,
"Merging Geometries")
counter = 0
for feature in layer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geoemtry. Skipping'.format(
feature.GetFID()))
continue
geom.Transform(transform)
union.AddGeometry(geom)
progbar.finish()
data_source = None
return union
def buffer_by_field(in_layer_path: str, out_layer_path, field: str, epsg: int = None, min_buffer=None, centered=False) -> None:
"""generate buffered polygons by value in field
Args:
flowlines (str): feature class of line features to buffer
field (str): field with buffer value
epsg (int): output srs
min_buffer: use this buffer value for field values that are less than this
Returns:
geometry: unioned polygon geometry of buffered lines
"""
log = Logger('buffer_by_field')
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, get_shp_or_gpkg(in_layer_path) as in_layer:
conversion = in_layer.rough_convert_metres_to_vector_units(1)
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer(ogr.wkbPolygon, epsg=epsg, fields=in_layer.get_fields())
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
factor = 0.5 if centered else 1.0
for feature, _counter, progbar in in_layer.iterate_features('Buffering features', write_layers=[out_layer]):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
buffer_dist = feature.GetField(field) * conversion * factor
geom.Transform(transform)
geom_buffer = geom.Buffer(buffer_dist if buffer_dist > min_buffer else min_buffer)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom_buffer)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
def write_attributes(feature_class,
output_values,
id_field,
fields,
field_type=ogr.OFTReal,
null_values=None):
"""
Write field values to a ShapeFile feature class
:param feature_class: Path to feature class
:param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names
:param id_field: Unique key identifying each feature in both feature class and output_values dictionary
:param fields: List of fields in output_values to write to ShapeFile
:return: None
"""
log = Logger('ShapeFile')
driver = ogr.GetDriverByName('ESRI Shapefile')
dataset = driver.Open(feature_class, 1)
layer = dataset.GetLayer()
# Create each field and store the name and index in a list of tuples
field_indices = [(field, create_field(layer, field, field_type))
for field in fields]
for feature in layer:
reach = feature.GetField(id_field)
if reach not in output_values:
continue
# Set all the field values and then store the feature
for field, idx in field_indices:
if field in output_values[reach]:
if not output_values[reach][field]:
if null_values:
feature.SetField(field, null_values)
else:
log.warning('Unhandled ShapeFile value for None type')
feature.SetField(field, None)
else:
feature.SetField(field, output_values[reach][field])
layer.SetFeature(feature)
dataset = None
def calculate_hydrology(reaches: dict, equation: str, params: dict,
drainage_conversion_factor: float, field: str) -> dict:
""" Perform the actual hydrology calculation
Args:
reaches ([type]): [description]
equation ([type]): [description]
params ([type]): [description]
drainage_conversion_factor ([type]): [description]
field ([type]): [description]
Raises:
ex: [description]
Returns:
[type]: [description]
"""
results = {}
log = Logger('Hydrology')
try:
# Loop over each reach
for reachid, values in reaches.items():
# Use the drainage area for the current reach and convert to the units used in the equation
params[
DRNAREA_PARAM] = values['iGeo_DA'] * drainage_conversion_factor
# Execute the equation but restrict the use of all built-in functions
eval_result = eval(equation, {'__builtins__': None}, params)
results[reachid] = {field: eval_result}
except Exception as ex:
[
log.warning('{}: {}'.format(param, value))
for param, value in params.items()
]
log.warning('Hydrology formula failed: {}'.format(equation))
log.error('Error calculating {} hydrology')
raise ex
return results
def get_geometry_union(in_layer_path: str, epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""[summary]
Args:
in_layer_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_union')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
geom = None
for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
if feature.GetGeometryRef() is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
new_shape = VectorBase.ogr2shapely(feature, transform=transform)
try:
geom = geom.union(new_shape) if geom is not None else new_shape
except Exception:
progbar.erase() # get around the progressbar
log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID()))
return geom
def extract_mean_values_by_polygon(polys, rasters, reference_raster):
log = Logger('extract_mean_values_by_polygon')
progbar = ProgressBar(len(polys), 50, "Extracting Mean values...")
counter = 0
with rasterio.open(reference_raster) as dataset:
output_mean = {}
output_unique = {}
for reachid, poly in polys.items():
counter += 1
progbar.update(counter)
if poly.geom_type in ["Polygon", "MultiPolygon"] and poly.area > 0:
values_mean = {}
values_unique = {}
reach_raster = np.ma.masked_invalid(
features.rasterize(
[poly],
out_shape=dataset.shape,
transform=dataset.transform,
all_touched=True,
fill=np.nan))
for key, raster in rasters.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
values_mean[key] = np.ma.mean(current_raster)
values_unique[key] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True)
else:
values_mean[key] = 0.0
values_unique[key] = []
output_mean[reachid] = values_mean
output_unique[reachid] = values_unique
# log.debug(f"Reach: {reachid} | {sum([v for v in values.values() if v is not None]):.2f}")
else:
progbar.erase()
log.warning(f"Reach: {reachid} | WARNING no geom")
progbar.finish()
return output_mean, output_unique
def download_unzip(url, download_folder, unzip_folder=None, force_download=False, retries=3):
"""
A wrapper for Download() and Unzip(). WE do these things together
enough that it makes sense. Also there's the concept of retrying that
needs to be handled in a centralized way
Arguments:
url {[type]} -- [description]
download_folder {[type]} -- [description]
Keyword Arguments:
unzip_folder {[string]} -- (optional) specify the specific directory to extract files into (we still create a subfolder with the zip-file's name though)
force_download {bool} -- [description] (default: {False})
Returns:
[type] -- [description]
"""
log = Logger('Download')
# If we specified an unzip path then use it, otherwise just unzip into the folder
# with the same name as the file (minus the '.zip' extension)
dl_retry = 0
dl_success = False
while not dl_success and dl_retry < 3:
try:
zipfilepath = download_file(url, download_folder, force_download)
dl_success = True
except Exception as e:
log.debug(e)
log.warning('download failed. retrying...')
dl_retry += 1
if (not dl_success):
raise Exception('Downloading of file failed after {} attempts'.format(retries))
final_unzip_folder = unzip_folder if unzip_folder is not None else os.path.splitext(zipfilepath)[0]
unzip(zipfilepath, final_unzip_folder, force_download, retries)
return final_unzip_folder
def verify_areas(raster_path, boundary_shp):
"""[summary]
Arguments:
raster_path {[type]} -- path
boundary_shp {[type]} -- path
Raises:
Exception: [description] if raster area is zero
Exception: [description] if shapefile area is zero
Returns:
[type] -- rastio of raster area over shape file area
"""
log = Logger('Verify Areas')
log.info('Verifying raster and shape areas')
# This comes back in the raster's unit
raster_area = 0
with rasterio.open(raster_path) as ds:
cell_count = 0
gt = ds.get_transform()
cell_area = math.fabs(gt[1]) * math.fabs(gt[5])
# Incrememntally add the area of a block to the count
progbar = ProgressBar(len(list(ds.block_windows(1))), 50,
"Calculating Area")
progcount = 0
for _ji, window in ds.block_windows(1):
r = ds.read(1, window=window, masked=True)
progbar.update(progcount)
cell_count += r.count()
progcount += 1
progbar.finish()
# Multiply the count by the area of a given cell
raster_area = cell_area * cell_count
log.debug('raster area {}'.format(raster_area))
if (raster_area == 0):
raise Exception('Raster has zero area: {}'.format(raster_path))
# We could just use Rasterio's CRS object but it doesn't seem to play nice with GDAL so....
raster_ds = gdal.Open(raster_path)
raster_srs = osr.SpatialReference(wkt=raster_ds.GetProjection())
# Load and transform ownership polygons by adminstration agency
driver = ogr.GetDriverByName("ESRI Shapefile")
data_source = driver.Open(boundary_shp, 0)
layer = data_source.GetLayer()
in_spatial_ref = layer.GetSpatialRef()
# https://github.com/OSGeo/gdal/issues/1546
raster_srs.SetAxisMappingStrategy(in_spatial_ref.GetAxisMappingStrategy())
transform = osr.CoordinateTransformation(in_spatial_ref, raster_srs)
shape_area = 0
for polygon in layer:
geom = polygon.GetGeometryRef()
geom.Transform(transform)
shape_area = shape_area + geom.GetArea()
log.debug('shape file area {}'.format(shape_area))
if (shape_area == 0):
raise Exception('Shapefile has zero area: {}'.format(boundary_shp))
area_ratio = raster_area / shape_area
if (area_ratio < 0.99 and area_ratio > 0.9):
log.warning('Raster Area covers only {0:.2f}% of the shapefile'.format(
area_ratio * 100))
if (area_ratio <= 0.9):
log.error('Raster Area covers only {0:.2f}% of the shapefile'.format(
area_ratio * 100))
else:
log.info('Raster Area covers {0:.2f}% of the shapefile'.format(
area_ratio * 100))
return area_ratio
def calculate_combined_fis(feature_values: dict, veg_fis_field: str,
capacity_field: str, dam_count_field: str,
max_drainage_area: float):
"""
Calculate dam capacity and density using combined FIS
:param feature_values: Dictionary of features keyed by ReachID and values are dictionaries of attributes
:param veg_fis_field: Attribute containing the output of the vegetation FIS
:param com_capacity_field: Attribute used to store the capacity result in feature_values
:param com_density_field: Attribute used to store the capacity results in feature_values
:param max_drainage_area: Reaches with drainage area greater than this threshold will have zero capacity
:return: Insert the dam capacity and density values to the feature_values dictionary
"""
log = Logger('Combined FIS')
log.info('Initializing Combined FIS')
if not max_drainage_area:
log.warning(
'Missing max drainage area. Calculating combined FIS without max drainage threshold.'
)
# get arrays for fields of interest
feature_count = len(feature_values)
reachid_array = np.zeros(feature_count, np.int64)
veg_array = np.zeros(feature_count, np.float64)
hydq2_array = np.zeros(feature_count, np.float64)
hydlow_array = np.zeros(feature_count, np.float64)
slope_array = np.zeros(feature_count, np.float64)
drain_array = np.zeros(feature_count, np.float64)
counter = 0
for reach_id, values in feature_values.items():
reachid_array[counter] = reach_id
veg_array[counter] = values[veg_fis_field]
hydlow_array[counter] = values['iHyd_SPLow']
hydq2_array[counter] = values['iHyd_SP2']
slope_array[counter] = values['iGeo_Slope']
drain_array[counter] = values['iGeo_DA']
counter += 1
# Adjust inputs to be within FIS membership range
veg_array[veg_array < 0] = 0
veg_array[veg_array > 45] = 45
hydq2_array[hydq2_array < 0] = 0.0001
hydq2_array[hydq2_array > 10000] = 10000
hydlow_array[hydlow_array < 0] = 0.0001
hydlow_array[hydlow_array > 10000] = 10000
slope_array[slope_array > 1] = 1
# create antecedent (input) and consequent (output) objects to hold universe variables and membership functions
ovc = ctrl.Antecedent(np.arange(0, 45, 0.01), 'input1')
sp2 = ctrl.Antecedent(np.arange(0, 10000, 1), 'input2')
splow = ctrl.Antecedent(np.arange(0, 10000, 1), 'input3')
slope = ctrl.Antecedent(np.arange(0, 1, 0.0001), 'input4')
density = ctrl.Consequent(np.arange(0, 45, 0.01), 'result')
# build membership functions for each antecedent and consequent object
ovc['none'] = fuzz.trimf(ovc.universe, [0, 0, 0.1])
ovc['rare'] = fuzz.trapmf(ovc.universe, [0, 0.1, 0.5, 1.5])
ovc['occasional'] = fuzz.trapmf(ovc.universe, [0.5, 1.5, 4, 8])
ovc['frequent'] = fuzz.trapmf(ovc.universe, [4, 8, 12, 25])
ovc['pervasive'] = fuzz.trapmf(ovc.universe, [12, 25, 45, 45])
sp2['persists'] = fuzz.trapmf(sp2.universe, [0, 0, 1000, 1200])
sp2['breach'] = fuzz.trimf(sp2.universe, [1000, 1200, 1600])
sp2['oblowout'] = fuzz.trimf(sp2.universe, [1200, 1600, 2400])
sp2['blowout'] = fuzz.trapmf(sp2.universe, [1600, 2400, 10000, 10000])
splow['can'] = fuzz.trapmf(splow.universe, [0, 0, 150, 175])
splow['probably'] = fuzz.trapmf(splow.universe, [150, 175, 180, 190])
splow['cannot'] = fuzz.trapmf(splow.universe, [180, 190, 10000, 10000])
slope['flat'] = fuzz.trapmf(slope.universe, [0, 0, 0.0002, 0.005])
slope['can'] = fuzz.trapmf(slope.universe, [0.0002, 0.005, 0.12, 0.15])
slope['probably'] = fuzz.trapmf(slope.universe, [0.12, 0.15, 0.17, 0.23])
slope['cannot'] = fuzz.trapmf(slope.universe, [0.17, 0.23, 1, 1])
density['none'] = fuzz.trimf(density.universe, [0, 0, 0.1])
density['rare'] = fuzz.trapmf(density.universe, [0, 0.1, 0.5, 1.5])
density['occasional'] = fuzz.trapmf(density.universe, [0.5, 1.5, 4, 8])
density['frequent'] = fuzz.trapmf(density.universe, [4, 8, 12, 25])
density['pervasive'] = fuzz.trapmf(density.universe, [12, 25, 45, 45])
# build fis rule table
log.info('Building FIS rule table')
comb_ctrl = ctrl.ControlSystem([
ctrl.Rule(ovc['none'], density['none']),
ctrl.Rule(splow['cannot'], density['none']),
ctrl.Rule(slope['cannot'], density['none']),
ctrl.Rule(
ovc['rare'] & sp2['persists'] & splow['can'] & ~slope['cannot'],
density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['persists'] & splow['probably']
& ~slope['cannot'], density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['breach'] & splow['can'] & ~slope['cannot'],
density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['breach'] & splow['probably'] & ~slope['cannot'],
density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['oblowout'] & splow['can'] & ~slope['cannot'],
density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['oblowout'] & splow['probably']
& ~slope['cannot'], density['rare']),
ctrl.Rule(
ovc['rare'] & sp2['blowout'] & splow['can'] & ~slope['cannot'],
density['none']),
ctrl.Rule(
ovc['rare'] & sp2['blowout'] & splow['probably']
& ~slope['cannot'], density['none']),
ctrl.Rule(
ovc['occasional'] & sp2['persists'] & splow['can']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['persists'] & splow['probably']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['breach'] & splow['can']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['breach'] & splow['probably']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['oblowout'] & splow['can']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['oblowout'] & splow['probably']
& ~slope['cannot'], density['occasional']),
ctrl.Rule(
ovc['occasional'] & sp2['blowout'] & splow['can']
& ~slope['cannot'], density['rare']),
ctrl.Rule(
ovc['occasional'] & sp2['blowout'] & splow['probably']
& ~slope['cannot'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['can'] & slope['flat'],
density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['can'] & slope['can'],
density['frequent']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['can']
& slope['probably'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['probably']
& slope['flat'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['probably']
& slope['can'], density['frequent']),
ctrl.Rule(
ovc['frequent'] & sp2['persists'] & splow['probably']
& slope['probably'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['can'] & slope['flat'],
density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['can'] & slope['can'],
density['frequent']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['can'] & slope['probably'],
density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['probably']
& slope['flat'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['probably'] & slope['can'],
density['frequent']),
ctrl.Rule(
ovc['frequent'] & sp2['breach'] & splow['probably']
& slope['probably'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['can'] & slope['flat'],
density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['can'] & slope['can'],
density['frequent']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['can']
& slope['probably'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['probably']
& slope['flat'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['probably']
& slope['can'], density['occasional']),
ctrl.Rule(
ovc['frequent'] & sp2['oblowout'] & splow['probably']
& slope['probably'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['can'] & slope['flat'],
density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['can'] & slope['can'],
density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['can']
& slope['probably'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['probably']
& slope['flat'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['probably']
& slope['can'], density['rare']),
ctrl.Rule(
ovc['frequent'] & sp2['blowout'] & splow['probably']
& slope['probably'], density['rare']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['can'] & slope['flat'],
density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['can'] & slope['can'],
density['pervasive']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['can']
& slope['probably'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['probably']
& slope['flat'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['probably']
& slope['can'], density['pervasive']),
ctrl.Rule(
ovc['pervasive'] & sp2['persists'] & splow['probably']
& slope['probably'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['can'] & slope['flat'],
density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['can'] & slope['can'],
density['pervasive']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['can']
& slope['probably'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['probably']
& slope['flat'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['probably']
& slope['can'], density['pervasive']),
ctrl.Rule(
ovc['pervasive'] & sp2['breach'] & splow['probably']
& slope['probably'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['can'] & slope['flat'],
density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['can'] & slope['can'],
density['pervasive']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['can']
& slope['probably'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['probably']
& slope['flat'], density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['probably']
& slope['can'], density['frequent']),
ctrl.Rule(
ovc['pervasive'] & sp2['oblowout'] & splow['probably']
& slope['probably'], density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['can'] & slope['flat'],
density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['can'] & slope['can'],
density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['can']
& slope['probably'], density['rare']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['probably']
& slope['flat'], density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['probably']
& slope['can'], density['occasional']),
ctrl.Rule(
ovc['pervasive'] & sp2['blowout'] & splow['probably']
& slope['probably'], density['rare'])
])
comb_fis = ctrl.ControlSystemSimulation(comb_ctrl)
# calculate defuzzified centroid value for density 'none' MF group
# this will be used to re-classify output values that fall in this group
# important: will need to update the array (x) and MF values (mfx) if the
# density 'none' values are changed in the model
x_vals = np.arange(0, 45, 0.01)
mfx = fuzz.trimf(x_vals, [0, 0, 0.1])
defuzz_centroid = round(fuzz.defuzz(x_vals, mfx, 'centroid'), 6)
progbar = ProgressBar(len(reachid_array), 50, "Combined FIS")
counter = 0
for i, reach_id in enumerate(reachid_array):
capacity = 0.0
# Only compute FIS if the reach has less than user-defined max drainage area.
# this enforces a stream size threshold above which beaver dams won't persist and/or won't be built
if not max_drainage_area or drain_array[i] < max_drainage_area:
comb_fis.input['input1'] = veg_array[i]
comb_fis.input['input2'] = hydq2_array[i]
comb_fis.input['input3'] = hydlow_array[i]
comb_fis.input['input4'] = slope_array[i]
comb_fis.compute()
capacity = comb_fis.output['result']
# Combined FIS result cannot be higher than limiting vegetation FIS result
if capacity > veg_array[i]:
capacity = veg_array[i]
if round(capacity, 6) == defuzz_centroid:
capacity = 0.0
count = capacity * (feature_values[reach_id]['iGeo_Len'] / 1000.0)
count = 1.0 if 0 < count < 1 else count
feature_values[reach_id][capacity_field] = round(capacity, 2)
feature_values[reach_id][dam_count_field] = round(count, 2)
counter += 1
progbar.update(counter)
progbar.finish()
log.info('Done')
def vegetation_summary(outputs_gpkg_path: str, label: str, veg_raster: str,
buffer: float):
""" Loop through every reach in a BRAT database and
retrieve the values from a vegetation raster within
the specified buffer. Then store the tally of
vegetation values in the BRAT database.
Arguments:
database {str} -- Path to BRAT database
veg_raster {str} -- Path to vegetation raster
buffer {float} -- Distance to buffer the reach polylines
"""
log = Logger('Vegetation')
log.info('Summarizing {}m vegetation buffer from {}'.format(
int(buffer), veg_raster))
# Retrieve the raster spatial reference and geotransformation
dataset = gdal.Open(veg_raster)
geo_transform = dataset.GetGeoTransform()
raster_buffer = VectorBase.rough_convert_metres_to_raster_units(
veg_raster, buffer)
# Calculate the area of each raster cell in square metres
conversion_factor = VectorBase.rough_convert_metres_to_raster_units(
veg_raster, 1.0)
cell_area = abs(geo_transform[1] * geo_transform[5]) / conversion_factor**2
# Open the raster and then loop over all polyline features
veg_counts = []
with rasterio.open(veg_raster) as src, GeopackageLayer(
os.path.join(outputs_gpkg_path, 'ReachGeometry')) as lyr:
_srs, transform = VectorBase.get_transform_from_raster(
lyr.spatial_ref, veg_raster)
for feature, _counter, _progbar in lyr.iterate_features(label):
reach_id = feature.GetFID()
geom = feature.GetGeometryRef()
if transform:
geom.Transform(transform)
polygon = VectorBase.ogr2shapely(geom).buffer(raster_buffer)
try:
# retrieve an array for the cells under the polygon
raw_raster = mask(src, [polygon], crop=True)[0]
mask_raster = np.ma.masked_values(raw_raster, src.nodata)
# print(mask_raster)
# Reclass the raster to dam suitability. Loop over present values for performance
for oldvalue in np.unique(mask_raster):
if oldvalue is not np.ma.masked:
cell_count = np.count_nonzero(mask_raster == oldvalue)
veg_counts.append([
reach_id,
int(oldvalue), buffer, cell_count * cell_area,
cell_count
])
except Exception as ex:
log.warning(
'Error obtaining vegetation raster values for ReachID {}'.
format(reach_id))
log.warning(ex)
# Write the reach vegetation values to the database
# Because sqlite3 doesn't give us any feedback we do this in batches so that we can figure out what values
# Are causing constraint errors
with SQLiteCon(outputs_gpkg_path) as database:
errs = 0
batch_count = 0
for veg_record in veg_counts:
batch_count += 1
try:
database.conn.execute(
'INSERT INTO ReachVegetation (ReachID, VegetationID, Buffer, Area, CellCount) VALUES (?, ?, ?, ?, ?)',
veg_record)
# Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
except sqlite3.IntegrityError as err:
# THis is likely a constraint error.
errstr = "Integrity Error when inserting records: ReachID: {} VegetationID: {}".format(
veg_record[0], veg_record[1])
log.error(errstr)
errs += 1
except sqlite3.Error as err:
# This is any other kind of error
errstr = "SQL Error when inserting records: ReachID: {} VegetationID: {} ERROR: {}".format(
veg_record[0], veg_record[1], str(err))
log.error(errstr)
errs += 1
if errs > 0:
raise Exception(
'Errors were found inserting records into the database. Cannot continue.'
)
database.conn.commit()
log.info('Vegetation summary complete')
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""Load all features from a ShapeFile and union them together into a single geometry
Args:
in_layer_path (str): path to layer
epsg (int, optional): EPSG to project to. Defaults to None.
spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None.
attribute_filter (str, optional): Filter to a set of attributes. Defaults to None.
clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_unary_union')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
geom_list = []
for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
new_geom = feature.GetGeometryRef()
geo_type = new_geom.GetGeometryType()
# We can't union non-valid shapes but sometimes a buffer by 0 can help
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID()))
try:
new_geom = new_geom.Buffer(0)
if not new_geom.IsValid():
log.warning(' Still invalid. Skipping this geometry')
continue
except Exception:
log.warning('Exception raised during buffer 0 technique. skipping this file')
continue
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for shape with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for shape with FID={}'.format(feature.GetFID()))
else:
geom_list.append(VectorBase.ogr2shapely(new_geom, transform))
# IF we get past a certain size then run the union
if len(geom_list) >= 500:
geom_list = [unary_union(geom_list)]
new_geom = None
log.debug('finished iterating with list of size: {}'.format(len(geom_list)))
if len(geom_list) > 1:
log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list)))
# Do a final union to clean up anything that might still be in the list
geom_union = unary_union(geom_list)
elif len(geom_list) == 0:
log.warning('No geometry found to union')
return None
else:
log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list)))
geom_union = geom_list[0]
log.debug(' done')
print_geom_size(log, geom_union)
log.debug('Complete')
# Return a shapely object
return geom_union
def process_modis(out_sqlite, modis_folder, nhd_folder, verbose, debug_flag):
"""Generate land surface temperature sqlite db from NHD+ and MODIS data
"""
log = Logger("Process LST")
if os.path.isfile(out_sqlite):
os.remove(out_sqlite)
# Create sqlite database
conn = sqlite3.connect(out_sqlite)
cursor = conn.cursor()
# test if table exists?
cursor.execute(
"""SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='MODIS_LST' """
)
log.info('Creating DB')
if cursor.fetchone()[0] == 0:
cursor.execute("""
CREATE TABLE MODIS_LST (
NHDPlusID INTEGER NOT NULL,
MODIS_Scene DATETIME NOT NULL,
LST REAL,
PRIMARY KEY (
NHDPlusID,
MODIS_Scene
)
)
WITHOUT ROWID;
""")
conn.commit()
# populate list of modis files
modis_files = glob.glob(os.path.join(modis_folder, "*.tif"))
# Load NHD Layers
log.info(f"Processing NHD Data: {nhd_folder}")
in_driver = ogr.GetDriverByName("OpenFileGDB")
in_datasource = in_driver.Open(nhd_folder, 0)
layer_hucs = in_datasource.GetLayer(r"WBDHU8_reproject")
# Process HUC
huc_counter = 0
total_hucs = layer_hucs.GetFeatureCount()
for huc in layer_hucs:
huc_counter += 1
huc_id = huc.GetField(r"HUC8")
log.info('Processing huc:{} ({}/{})'.format(huc_id, huc_counter,
total_hucs))
log.info(f"HUC: {huc_id}")
huc_geom = huc.GetGeometryRef()
layer_catchments = None
layer_catchments = in_datasource.GetLayer(
r"NHDPlusCatchment_reproject")
# layer_catchments.SetSpatialFilter(huc_geom) catchments not perfectly aligned with hucs
layer_catchments.SetAttributeFilter(f"""HUC8 = {huc_id}""")
huc_bounds = huc_geom.GetEnvelope()
bbox = box(huc_bounds[0], huc_bounds[2], huc_bounds[1], huc_bounds[3])
# open a single MODIS raster and load its projection and transform based on current huc
with rasterio.open(f"{modis_files[0]}") as dataset:
data, modis_transform = mask(dataset, [bbox],
all_touched=True,
crop=True)
# Assuming there is only one band we can drop the first dimenson and get (36,78) instead of (1,36,78)
modis_shape = data.shape[1:]
# Read all MODIS Scences into array
modis_array_raw = np.ma.array(
[load_cropped_raster(image, bbox) for image in modis_files])
modis_array_sds = np.ma.masked_where(modis_array_raw == 0,
modis_array_raw)
# Make sure we mask out the invalid data
modis_array_K = modis_array_sds * 0.02
modis_array_C = modis_array_K - 273.15 # K to C
# Generate list of MODIS scene dates
modis_dates = np.array([
os.path.basename(image).lstrip("A").rstrip(".tif")
for image in modis_files
])
# Calcuate average LST per Catchemnt Layer
progbar = ProgressBar(layer_catchments.GetFeatureCount(), 50,
'Processing HUC: {}'.format(huc_id))
reach_counter = 0
progbar.update(reach_counter)
# loop_timer = LoopTimer("LoopTime", useMs=True)
for reach in layer_catchments:
reach_counter += 1
progbar.update(reach_counter)
# If debug flag is set then drop a CSV for every 5000 reaches
debug_drop = debug_flag is True and reach_counter % 5000 == 1
# For Debugging performance
# loop_timer.tick()
# loop_timer.progprint()
nhd_id = int(reach.GetField("NHDPlusID"))
# load_catchment_polygon and transform to raster SRS
reach_geom = reach.GetGeometryRef()
catch_poly = loads(reach_geom.ExportToWkb())
# Catchment polygons are vectorized rasters and they can have invalid geometries
if not catch_poly.is_valid:
log.warning(
'Invalid catchment polygon detected. Trying the buffer technique: {}'
.format(nhd_id))
catch_poly = catch_poly.buffer(0)
# Generate mask raster of catchment pixels
reach_raster = np.ma.masked_invalid(
rasterio.features.rasterize([catch_poly],
out_shape=modis_shape,
transform=modis_transform,
all_touched=True,
fill=np.nan))
# Now assign ascending integers to each cell. THis is so the rasterio.features.shapes gives us a unique shape for every cell
reach_raster_idx = np.ma.masked_array(
np.arange(modis_shape[0] * modis_shape[1],
dtype=np.int32).reshape(modis_shape),
# pylint: disable=E1101
reach_raster.mask)
# Generate a unique shape for each valid pixel
geoms = [{
'properties': {
'name': 'modis_pixel',
'raster_val': int(v),
'valid': v > 0
},
'geometry': geom
} for i, (geom, v) in enumerate(
rasterio.features.shapes(reach_raster_idx,
transform=modis_transform))
if test_pixel_geom(geom)]
# Now create our weights array. Start with weights of 0 so we can rule out any weird points
weights_raster_arr = np.ma.masked_array(
np.full(modis_shape, 0, dtype=np.float32),
# pylint: disable=E1101
reach_raster.mask,
)
for geom in geoms:
pxl = shape(geom['geometry'])
poly_intersect = pxl.intersection(catch_poly)
idx, idy = find_indeces(geom['properties']['raster_val'],
modis_shape)
weight = poly_intersect.area / catch_poly.area
# For debugging
if debug_drop:
geom['type'] = "Feature"
geom['properties']['weight'] = weight
geom['properties']['raster_coords'] = [idx, idy]
geom['properties']['world_coords'] = [
pxl.centroid.coords[0][0], pxl.centroid.coords[0][1]
]
weights_raster_arr[idx][idy] = weight
# Calculate average weighted modis
ave = np.ma.average(modis_array_C,
axis=(1, 2),
weights=np.broadcast_to(
weights_raster_arr, modis_array_C.shape))
# Just some useful debugging stuff
if debug_drop:
progbar.erase()
file_prefix = '{}-{}-debug'.format(huc_id, nhd_id)
log.debug('Dropping files: {}'.format(file_prefix))
# PrintArr(reach_raster_idx)
# Dump some useful shapes to a geojson Object
_debug_shape = DebugGeoJSON(
os.path.join(os.path.dirname(out_sqlite),
'{}.geojson'.format(file_prefix)))
_debug_shape.add_shapely(bbox, {"name": "bbox"})
_debug_shape.add_shapely(catch_poly, {"name": "catch_poly"})
[_debug_shape.add_geojson(gj) for gj in geoms]
_debug_shape.write()
# Now dump an CSV array report for fun
csv_file = os.path.join(os.path.dirname(out_sqlite),
'{}.csv'.format(file_prefix))
with open(csv_file, 'w') as csv_file:
csvw = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csvw.writerow(['HUC', 'NHDPlusID', 'Area'])
csvw.writerow([huc_id, nhd_id, catch_poly.area])
csvw.writerow([])
debug_weights = []
# Summary of intersected pixels
for geom in geoms:
debug_weights.append(
(geom['properties']['weight'],
geom['properties']['raster_coords']))
# Dump the weights Cell values so we can use excel to calculate them manually
# Write the average and the
csvw.writerow(['Intersecting Cells:'] +
[' ' for g in geoms])
for key, name in {
'raster_val': 'cell_id',
'raster_coords': '[row,col]',
'world_coords': '[x,y]',
'weight': 'weight'
}.items():
csvw.writerow([name] +
[g['properties'][key] for g in geoms])
csvw.writerow([])
csvw.writerow(['Date'] + [' ' for g in geoms] +
['np.ma.average'])
for didx, ave_val in enumerate(ave):
csvw.writerow([modis_dates[didx]] + [
modis_array_sds[didx][w[1][0]][w[1][1]]
for w in debug_weights
] + [ave_val])
# insert_lst_into_sqlite
cursor.executemany("""INSERT INTO MODIS_LST VALUES(?,?,?)""", [
(nhd_id, datetime.datetime.strptime(modis_date, "%Y%j").date(),
float(v) if float(v) != 0 else None)
for (modis_date, v) in zip(modis_dates, ave.data)
])
# Write data to sqlite after each reach
conn.commit()
# Close database connection
conn.close()
return
def get_geometry_unary_union_from_wkt(inpath, to_sr_wkt):
"""
Load all features from a ShapeFile and union them together into a single geometry
:param inpath: Path to a ShapeFile
:param epsg: Desired output spatial reference
:return: Single Shapely geometry of all unioned features
"""
log = Logger('Unary Union')
driver = ogr.GetDriverByName("ESRI Shapefile")
data_source = driver.Open(inpath, 0)
layer = data_source.GetLayer()
in_spatial_ref = layer.GetSpatialRef()
out_spatial_ref, transform = get_transform_from_wkt(
in_spatial_ref, to_sr_wkt)
fcount = layer.GetFeatureCount()
progbar = ProgressBar(fcount, 50, "Unary Unioning features")
counter = 0
def unionize(wkb_lst):
return unary_union([wkbload(g) for g in wkb_lst]).wkb
geom_list = []
for feature in layer:
counter += 1
progbar.update(counter)
new_geom = feature.GetGeometryRef()
geo_type = new_geom.GetGeometryType()
# We can't union non-valid shapes but sometimes a buffer by 0 can help
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning(
'Invalid shape with FID={} trying the Buffer0 technique...'.
format(feature.GetFID()))
try:
new_geom = new_geom.Buffer(0)
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning(' Still invalid. Skipping this geometry')
continue
except Exception as e:
progbar.erase() # get around the progressbar
log.warning(
'Exception raised during buffer 0 technique. skipping this file'
)
continue
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geoemtry. Skipping'.format(
feature.GetFID()))
# Filter out zero-length lines
elif geo_type in LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for shape with FID={}'.format(
feature.GetFID()))
# Filter out zero-area polys
elif geo_type in POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for shape with FID={}'.format(
feature.GetFID()))
else:
new_geom.Transform(transform)
geom_list.append(new_geom.ExportToWkb())
# IF we get past a certain size then run the union
if len(geom_list) >= 500:
geom_list = [unionize(geom_list)]
new_geom = None
log.debug('finished iterating with list of size: {}'.format(
len(geom_list)))
progbar.finish()
if len(geom_list) > 1:
log.debug('Starting final union of geom_list of size: {}'.format(
len(geom_list)))
# Do a final union to clean up anything that might still be in the list
geom_union = wkbload(unionize(geom_list))
elif len(geom_list) == 0:
log.warning('No geometry found to union')
return None
else:
log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list)))
geom_union = wkbload(geom_list[0])
log.debug(' done')
print_geom_size(log, geom_union)
log.debug('Complete')
data_source = None
return geom_union
def copy_feature_class(inpath,
epsg,
outpath,
clip_shape=None,
attribute_filter=None):
"""Copy a Shapefile from one location to another
This method is capable of reprojecting the geometries as they are copied.
It is also possible to filter features by both attributes and also clip the
features to another geometryNone
Arguments:
inpath {str} -- File path to input Shapefile that will be copied.
epsg {int} -- Output coordinate system
outpath {str} -- File path where the output Shapefile will be generated.
Keyword Arguments:
clip_shape {shape} -- Shapely polygon geometry in the output EPSG used to clip the input geometries (default: {None})
attribute_filter {str} -- Attribute filter used to limit the input features that will be copied. (default: {None})
"""
log = Logger('Shapefile')
# if os.path.isfile(outpath):
# log.info('Skipping copy of feature classes because output file exists.')
# return
driver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = driver.Open(inpath, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
geom_type = inLayer.GetGeomType()
# Optionally limit which features are copied by using an attribute filter
if attribute_filter:
inLayer.SetAttributeFilter(attribute_filter)
# If there's a clip geometry provided then limit the features copied to
# those that intersect (partially or entirely) by this clip feature.
# Note that this makes the subsequent intersection process a lot more
# performant because the SetSaptialFilter() uses the ShapeFile's spatial
# index which is much faster than manually checking if all pairs of features intersect.
clip_geom = None
if clip_shape:
clip_geom = ogr.CreateGeometryFromWkb(clip_shape.wkb)
inLayer.SetSpatialFilter(clip_geom)
outpath_dir = os.path.dirname(outpath)
safe_makedirs(outpath_dir)
# Create the output shapefile
outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg)
outDataSource = driver.CreateDataSource(outpath)
outLayer = outDataSource.CreateLayer('network',
outSpatialRef,
geom_type=geom_type)
outLayerDefn = outLayer.GetLayerDefn()
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Copying features")
counter = 0
for feature in inLayer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(
feature.GetFID()))
continue
geom.Transform(transform)
# if clip_shape:
# raw = shape(json.loads(geom.ExportToJson()))
# try:
# clip = raw.intersection(clip_shape)
# geom = ogr.CreateGeometryFromJson(json.dumps(mapping(clip)))
# except Exception as e:
# progbar.erase() # get around the progressbar
# log.warning('Invalid shape with FID={} cannot be intersected'.format(feature.GetFID()))
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
outFeature.SetGeometry(geom)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
outLayer.CreateFeature(outFeature)
outFeature = None
progbar.finish()
inDataSource = None
outDataSource = None
def copy_feature_class(in_layer_path: str, out_layer_path: str,
epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None,
buffer: float = 0,
) -> None:
"""Copy a Shapefile from one location to another
This method is capable of reprojecting the geometries as they are copied.
It is also possible to filter features by both attributes and also clip the
features to another geometryNone
Args:
in_layer (str): Input layer path
epsg ([type]): EPSG Code to use for the transformation
out_layer (str): Output layer path
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
buffer (float): Buffer the output features (in meters).
"""
log = Logger('copy_feature_class')
# NOTE: open the outlayer first so that write gets the dataset open priority
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, \
get_shp_or_gpkg(in_layer_path) as in_layer:
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer_from_ref(in_layer, epsg=epsg)
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
buffer_convert = 0
if buffer != 0:
buffer_convert = in_layer.rough_convert_metres_to_vector_units(buffer)
# This is the callback method that will be run on each feature
for feature, _counter, progbar in in_layer.iterate_features("Copying features", write_layers=[out_layer], clip_shape=clip_shape, clip_rect=clip_rect, attribute_filter=attribute_filter):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
if geom.GetGeometryType() in VectorBase.LINE_TYPES:
if geom.Length() == 0.0:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no Length. Skipping'.format(feature.GetFID()))
continue
# Buffer the shape if we need to
if buffer_convert != 0:
geom = geom.Buffer(buffer_convert)
geom.Transform(transform)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
class BratReport(RSReport):
"""In order to write a report we will extend the RSReport class from the rscommons
module which has useful styles and building blocks like Tables from lists etc.
"""
def __init__(self, database, report_path, rs_project):
# Need to call the constructor of the inherited class:
super().__init__(rs_project, report_path)
self.log = Logger('BratReport')
self.database = database
# The report has a core CSS file but we can extend it with our own if we want:
css_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'brat_report.css')
self.add_css(css_path)
self.images_dir = os.path.join(os.path.dirname(report_path), 'images')
safe_makedirs(self.images_dir)
# Now we just need to write the sections we want in the report in order
self.report_intro()
self.reach_attribute_summary()
self.dam_capacity()
self.hydrology_plots()
self.ownership()
self.vegetation()
self.conservation()
def report_intro(self):
# Create a section node to start adding things to. Section nodes are added to the table of contents if
# they have a title. If you don't specify a el_parent argument these sections will simply be added
# to the report body in the order you call them.
section = self.section('ReportIntro', 'Introduction')
# This project has a db so we'll need a connection
conn = sqlite3.connect(self.database)
conn.row_factory = _dict_factory
curs = conn.cursor()
row = curs.execute(
'SELECT Sum(iGeo_Len) AS TotalLength, Count(ReachID) AS TotalReaches FROM vwReaches'
).fetchone()
values = {
'Number of reaches':
'{0:,d}'.format(row['TotalReaches']),
'Total reach length (km)':
'{0:,.0f}'.format(row['TotalLength'] / 1000),
'Total reach length (miles)':
'{0:,.0f}'.format(row['TotalLength'] * 0.000621371)
}
row = curs.execute('''
SELECT WatershedID "Watershed ID", W.Name "Watershed Name", E.Name Ecoregion, CAST(AreaSqKm AS TEXT) "Area (Sqkm)", States
FROM Watersheds W
INNER JOIN Ecoregions E ON W.EcoregionID = E.EcoregionID
''').fetchone()
values.update(row)
curs.execute('SELECT KeyInfo, ValueInfo FROM Metadata')
values.update({
row['KeyInfo'].replace('_', ' '): row['ValueInfo']
for row in curs.fetchall()
})
# Here we're creating a new <div> to wrap around the table for stylign purposes
table_wrapper = ET.Element('div', attrib={'class': 'tableWrapper'})
RSReport.create_table_from_dict(values,
table_wrapper,
attrib={'id': 'SummTable'})
RSReport.create_table_from_sql(
['Reach Type', 'Total Length (km)', '% of Total'],
'SELECT ReachType, Sum(iGeo_Len) / 1000 As Length, 100 * Sum(iGeo_Len) / TotalLength AS TotalLength '
'FROM vwReaches INNER JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches) GROUP BY ReachType',
self.database,
table_wrapper,
attrib={'id': 'SummTable_sql'})
# Append my table_wrapper div (which now contains both tables above) to the section
section.append(table_wrapper)
def reach_attribute(self, attribute, units, parent_el):
# Use a class here because it repeats
section = self.section(None, attribute, parent_el, level=2)
conn = sqlite3.connect(self.database)
conn.row_factory = _dict_factory
curs = conn.cursor()
# Summary statistics (min, max etc) for the current attribute
curs.execute(
'SELECT Count({0}) "Values", Max({0}) Maximum, Min({0}) Minimum, Avg({0}) Average FROM vwReaches WHERE {0} IS NOT NULL'
.format(attribute))
values = curs.fetchone()
reach_wrapper_inner = ET.Element(
'div', attrib={'class': 'reachAtributeInner'})
section.append(reach_wrapper_inner)
# Add the number of NULL values
curs.execute(
'SELECT Count({0}) "NULL Values" FROM vwReaches WHERE {0} IS NULL'.
format(attribute))
values.update(curs.fetchone())
RSReport.create_table_from_dict(values, reach_wrapper_inner)
# Box plot
image_path = os.path.join(self.images_dir,
'attribute_{}.png'.format(attribute))
curs.execute('SELECT {0} FROM vwReaches WHERE {0} IS NOT NULL'.format(
attribute))
values = [row[attribute] for row in curs.fetchall()]
box_plot(values, attribute, attribute, image_path)
img_wrap = ET.Element('div', attrib={'class': 'imgWrap'})
img = ET.Element('img',
attrib={
'class':
'boxplot',
'alt':
'boxplot',
'src':
'{}/{}'.format(os.path.basename(self.images_dir),
os.path.basename(image_path))
})
img_wrap.append(img)
reach_wrapper_inner.append(img_wrap)
def dam_capacity(self):
section = self.section('DamCapacity', 'BRAT Dam Capacity Results')
conn = sqlite3.connect(self.database)
conn.row_factory = _dict_factory
curs = conn.cursor()
fields = [('Existing complex size', 'Sum(mCC_EX_CT)'),
('Historic complex size', 'Sum(mCC_HPE_CT)'),
('Existing vegetation capacity',
'Sum((iGeo_len / 1000) * oVC_EX)'),
('Historic vegetation capacity',
'Sum((iGeo_len / 1000) * oVC_HPE)'),
('Existing capacity', 'Sum((iGeo_len / 1000) * oCC_EX)'),
('Historic capacity', 'Sum((iGeo_len / 1000) * oCC_HPE)')]
curs.execute('SELECT {} FROM vwReaches'.format(', '.join(
[field for label, field in fields])))
row = curs.fetchone()
table_dict = {
fields[i][0]: row[fields[i][1]]
for i in range(len(fields))
}
RSReport.create_table_from_dict(table_dict, section)
self.dam_capacity_lengths('oCC_EX', section)
self.dam_capacity_lengths('oCC_HPE', section)
def dam_capacity_lengths(self, capacity_field, elParent):
conn = sqlite3.connect(self.database)
curs = conn.cursor()
curs.execute(
'SELECT Name, MaxCapacity FROM DamCapacities ORDER BY MaxCapacity')
bins = [(row[0], row[1]) for row in curs.fetchall()]
curs.execute('SELECT Sum(iGeo_Len) / 1000 FROM vwReaches')
total_length_km = curs.fetchone()[0]
data = []
last_bin = 0
cumulative_length_km = 0
for name, max_capacity in bins:
curs.execute(
'SELECT Sum(iGeo_len) / 1000 FROM vwReaches WHERE {} <= {}'.
format(capacity_field, max_capacity))
rowi = curs.fetchone()
if not rowi or rowi[0] is None:
bin_km = 0
else:
bin_km = rowi[0] - cumulative_length_km
cumulative_length_km = rowi[0]
data.append(('{}: {} - {}'.format(name, last_bin,
max_capacity), bin_km,
bin_km * 0.621371, 100 * bin_km / total_length_km))
last_bin = max_capacity
data.append(
('Total', cumulative_length_km, cumulative_length_km * 0.621371,
100 * cumulative_length_km / total_length_km))
RSReport.create_table_from_tuple_list(
(capacity_field, 'Stream Length (km)', 'Stream Length (mi)',
'Percent'), data, elParent)
def hydrology_plots(self):
section = self.section('HydrologyPlots', 'Hydrology')
conn = sqlite3.connect(self.database)
curs = conn.cursor()
curs.execute('SELECT MaxDrainage, QLow, Q2 FROM Watersheds')
row = curs.fetchone()
RSReport.create_table_from_dict(
{
'Max Draiange (sqkm)': row[0],
'Baseflow': row[1],
'Peak Flow': row[2]
},
section,
attrib={'class': 'fullwidth'})
RSReport.header(3, 'Hydrological Parameters', section)
RSReport.create_table_from_sql(
[
'Parameter', 'Data Value', 'Data Units', 'Conversion Factor',
'Equation Value', 'Equation Units'
],
'SELECT Parameter, Value, DataUnits, Conversion, ConvertedValue, EquationUnits FROM vwHydroParams',
self.database,
section,
attrib={'class': 'fullwidth'})
variables = [('iHyd_QLow', 'Baseflow (CFS)'),
('iHyd_Q2', 'Peak Flow (CFS)'),
('iHyd_SPLow', 'Baseflow Stream Power (Watts)'),
('iHyd_SP2', 'Peak Flow Stream Power (Watts)'),
('iGeo_Slope', 'Slope (degrees)')]
plot_wrapper = ET.Element('div', attrib={'class': 'hydroPlotWrapper'})
section.append(plot_wrapper)
for variable, ylabel in variables:
self.log.info(
'Generating XY scatter for {} against drainage area.'.format(
variable))
image_path = os.path.join(
self.images_dir,
'drainage_area_{}.png'.format(variable.lower()))
curs.execute('SELECT iGeo_DA, {} FROM vwReaches'.format(variable))
values = [(row[0], row[1]) for row in curs.fetchall()]
xyscatter(values, 'Drainage Area (sqkm)', ylabel, variable,
image_path)
img_wrap = ET.Element('div', attrib={'class': 'imgWrap'})
img = ET.Element('img',
attrib={
'src':
'{}/{}'.format(
os.path.basename(self.images_dir),
os.path.basename(image_path)),
'alt':
'boxplot'
})
img_wrap.append(img)
plot_wrapper.append(img_wrap)
def reach_attribute_summary(self):
section = self.section('ReachAttributeSummary',
'Geophysical Attributes')
attribs = [('iGeo_Slope', 'Slope', 'ratio'),
('iGeo_ElMax', 'Max Elevation', 'metres'),
('iGeo_ElMin', 'Min Elevation', 'metres'),
('iGeo_Len', 'Length', 'metres'),
('iGeo_DA', 'Drainage Area', 'Sqkm')]
plot_wrapper = ET.Element('div', attrib={'class': 'plots'})
[
self.reach_attribute(attribute, units, plot_wrapper)
for attribute, name, units in attribs
]
section.append(plot_wrapper)\
def ownership(self):
section = self.section('Ownership', 'Ownership')
RSReport.create_table_from_sql(
[
'Ownership Agency', 'Number of Reach Segments', 'Length (km)',
'% of Total Length'
],
'SELECT IFNULL(Agency, "None"), Count(ReachID), Sum(iGeo_Len) / 1000, 100* Sum(iGeo_Len) / TotalLength FROM vwReaches'
' INNER JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches) GROUP BY Agency',
self.database,
section,
attrib={'class': 'fullwidth'})
def vegetation(self):
section = self.section('Vegetation', 'Vegetation')
conn = sqlite3.connect(self.database)
# conn.row_factory = _dict_factory
curs = conn.cursor()
for epochid, veg_type in [(2, 'Historic Vegetation'),
(1, 'Existing Vegetation')]:
RSReport.header(3, veg_type, section)
pEl = ET.Element('p')
pEl.text = 'The 30 most common {} types within the 100m reach buffer.'.format(
veg_type.lower())
section.append(pEl)
RSReport.create_table_from_sql([
'Vegetation ID', 'Vegetation Type', 'Total Area (sqkm)',
'Default Suitability', 'Override Suitability',
'Effective Suitability'
], """
SELECT VegetationID,
Name, (CAST(TotalArea AS REAL) / 1000000) AS TotalAreaSqKm,
DefaultSuitability,
OverrideSuitability,
EffectiveSuitability
FROM vwReachVegetationTypes WHERE (EpochID = {}) AND (Buffer = 100) ORDER BY TotalArea DESC LIMIT 30"""
.format(epochid), self.database,
section)
try:
# Calculate the area weighted suitability
curs.execute("""
SELECT WeightedSum / SumTotalArea FROM
(SELECT Sum(CAST(TotalArea AS REAL) * CAST(EffectiveSuitability AS REAL) / 1000000) WeightedSum FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100)
JOIN
(SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SumTotalArea FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100)"""
.format(epochid))
area_weighted_avg_suitability = curs.fetchone()[0]
RSReport.header(3, 'Suitability Breakdown', section)
pEl = ET.Element('p')
pEl.text = """The area weighted average {} suitability is {}.
The breakdown of the percentage of the 100m buffer within each suitability class
across all reaches in the watershed.""".format(
veg_type.lower(),
RSReport.format_value(area_weighted_avg_suitability)[0])
section.append(pEl)
RSReport.create_table_from_sql(
['Suitability Class', '% with 100m Buffer'],
"""
SELECT EffectiveSuitability, 100.0 * SArea / SumTotalArea FROM
(
SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SArea, EffectiveSuitability
FROM vwReachVegetationTypes
WHERE EpochID = {0} AND Buffer = 100 GROUP BY EffectiveSuitability
)
JOIN
(
SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SumTotalArea
FROM vwReachVegetationTypes
WHERE EpochID = {0} AND Buffer = 100
)
ORDER BY EffectiveSuitability
""".format(epochid),
self.database,
section,
id_cols=id_cols)
except Exception as ex:
self.log.warning('Error calculating vegetation report')
def conservation(self):
section = self.section('Conservation', 'Conservation')
fields = [('Risk', 'DamRisks', 'RiskID'),
('Opportunity', 'DamOpportunities', 'OpportunityID'),
('Limitation', 'DamLimitations', 'LimitationID')]
for label, table, idfield in fields:
RSReport.header(3, label, section)
RSReport.create_table_from_sql(
[label, 'Total Length (km)', 'Reach Count', '%'],
'SELECT DR.Name, Sum(iGeo_Len) / 1000, Count(R.{1}), 100 * Sum(iGeo_Len) / TotalLength'
' FROM {0} DR LEFT JOIN vwReaches R ON DR.{1} = R.{1}'
' JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches)'
' GROUP BY DR.{1}'.format(table,
idfield), self.database, section)
RSReport.header(3, 'Conflict Attributes', section)
for attribute in ['iPC_Canal', 'iPC_DivPts', 'iPC_Privat']:
self.reach_attribute(attribute, 'meters', section)
def merge_feature_classes(feature_classes, epsg, boundary, outpath):
log = Logger('Shapefile')
if os.path.isfile(outpath):
log.info('Skipping merging feature classes because file exists.')
return
safe_makedirs(os.path.dirname(outpath))
log.info('Merging {} feature classes.'.format(len(feature_classes)))
driver = ogr.GetDriverByName("ESRI Shapefile")
# Create the output shapefile
outDataSource = driver.CreateDataSource(outpath)
outLayer = None
outSpatialRef = None
transform = None
fccount = 0
for inpath in feature_classes:
fccount += 1
log.info("Merging feature class {}/{}".format(fccount,
len(feature_classes)))
inDataSource = driver.Open(inpath, 0)
inLayer = inDataSource.GetLayer()
inSpatialRef = inLayer.GetSpatialRef()
inLayer.SetSpatialFilter(ogr.CreateGeometryFromWkb(boundary.wkb))
# First input spatial ref sets the SRS for the output file
outSpatialRefTmp, transform = get_transform_from_epsg(
inSpatialRef, epsg)
if outLayer is None:
outSpatialRef = outSpatialRefTmp
outLayer = outDataSource.CreateLayer(
'network', outSpatialRef, geom_type=ogr.wkbMultiLineString)
outLayerDefn = outLayer.GetLayerDefn()
# Transfer fields over
inLayerDef = inLayer.GetLayerDefn()
for i in range(inLayerDef.GetFieldCount()):
inFieldDef = inLayerDef.GetFieldDefn(i)
# Only create fields if we really don't have them
# NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE
if outLayerDefn.GetFieldIndex(inFieldDef.GetName()) == -1:
outLayer.CreateField(inFieldDef)
progbar = ProgressBar(inLayer.GetFeatureCount(), 50,
"Processing features")
outLayerDefn = outLayer.GetLayerDefn()
counter = 0
for feature in inLayer:
counter += 1
progbar.update(counter)
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feature.GetFID()))
continue
geom.Transform(transform)
# get a Shapely representation of the line
# featobj = json.loads(geom.ExportToJson())
# polyline = shape(featobj)
# if boundary.intersects(polyline):
# clipped = boundary.intersection(polyline)
outFeature = ogr.Feature(outLayerDefn)
for i in range(inLayerDef.GetFieldCount()):
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
feature.GetField(i))
outFeature.SetGeometry(geom)
outLayer.CreateFeature(outFeature)
feature = None
outFeature = None
progbar.finish()
inDataSource = None
outDataSource = None
log.info('Merge complete.')
def download_dem(vector_path,
_epsg,
buffer_dist,
download_folder,
unzip_folder,
force_download=False):
"""
Identify rasters within HUC8, download them and mosaic into single GeoTIF
:param vector_path: Path to bounding polygon ShapeFile
:param epsg: Output spatial reference
:param buffer_dist: Distance in DEGREES to buffer the bounding polygon
:param unzip_folder: Temporary folder where downloaded rasters will be saved
:param force_download: The download will always be performed if this is true.
:return:
"""
log = Logger('DEM')
# Query Science Base API for NED 10m DEM rasters within HUC 8 boundary
urls = get_dem_urls(vector_path, buffer_dist)
log.info('{} DEM raster(s) identified on Science Base.'.format(len(urls)))
rasters = {}
for url in urls:
base_path = os.path.basename(os.path.splitext(url)[0])
final_unzip_path = os.path.join(unzip_folder, base_path)
if url.lower().endswith('.zip'):
file_path = download_unzip(url, download_folder, final_unzip_path,
force_download)
raster_path = find_rasters(file_path)
else:
raster_path = download_file(url, download_folder, force_download)
# Sanity check that all rasters going into the VRT share the same cell resolution.
dataset = gdal.Open(raster_path)
gt = dataset.GetGeoTransform()
# Store the geotransform for later
gtHelper = Geotransform(gt)
# Create a tuple of useful numbers to use when trying to figure out if we have a problem with cellwidths
rasters[raster_path] = gtHelper
dataset = None
if (len(rasters.keys()) == 0):
raise Exception('No DEM urls were found')
# Pick one result to compare with and loop over to see if all the rasters have the same, exact dimensions
elif len(rasters.keys()) > 1:
widthStdDev = statistics.stdev(
[gt.CellWidth() for gt in rasters.values()])
heightStdDev = statistics.stdev(
[gt.CellHeight() for gt in rasters.values()])
# This is the broad-strokes check. If the cell widths are vastly different then this won't work and you'll get stitching artifacts when you resample
# so we bail out.
if widthStdDev > CELL_SIZE_MAX_STDDEV or heightStdDev > CELL_SIZE_MAX_STDDEV:
log.warning('Multiple DEM raster cells widths encountered.')
for rp, gt in rasters.items():
log.warning('cell width {} :: ({}, {})'.format(
rp, gt.CellWidth(), gt.CellHeight()))
# raise Exception('Cannot continue. Raster cell sizes are too different and resampling will cause edge effects in the stitched raster')
# Now that we know we have a problem we need to figure out where the truth is:
# if widthStdDev > CELL_SIZE_THRESH_STDDEV or heightStdDev > CELL_SIZE_THRESH_STDDEV:
# for rpath, gt in rasters.items():
# log.warning('Correcting Raster: {} from:({},{}) to:({},{})'.format(rpath, gt.CellWidth(), gt.CellHeight(), widthAvg, heightAvg))
# gt.SetCellWidth(widthAvg)
# gt.SetCellHeight(heightAvg)
# dataset = gdal.Open(raster_path)
# dataset.SetGeoTransform(gt.gt)
# dataset = None
return list(rasters.keys()), urls
def confinement(huc: int,
flowlines_orig: Path,
confining_polygon_orig: Path,
output_folder: Path,
buffer_field: str,
confinement_type: str,
reach_codes: List[str],
min_buffer: float = 0.0,
bankfull_expansion_factor: float = 1.0,
debug: bool = False,
meta=None):
"""Generate confinement attribute for a stream network
Args:
huc (integer): Huc identifier
flowlines (path): input flowlines layer
confining_polygon (path): valley bottom or other boundary defining confining margins
output_folder (path): location to store confinement project and output geopackage
buffer_field (string): name of float field with buffer values in meters (i.e. 'BFWidth')
confinement_type (string): name of type of confinement generated
reach_codes (List[int]): NHD reach codes for features to include in outputs
min_buffer (float): minimum bankfull value to use in buffers e.g. raster cell resolution
bankfull_expansion_factor (float): factor to expand bankfull on each side of bank
debug (bool): run tool in debug mode (save intermediate outputs). Default = False
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("Confinement")
log.info(f'Confinement v.{cfg.version}') # .format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
# Make the projectXML
project, _realization, proj_nodes, report_path = create_project(
huc, output_folder, {'ConfinementType': confinement_type})
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
# Copy input shapes to a geopackage
flowlines_path = os.path.join(
output_folder, LayerTypes['INPUTS'].rel_path,
LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
confining_path = os.path.join(
output_folder, LayerTypes['INPUTS'].rel_path,
LayerTypes['INPUTS'].sub_layers['CONFINING_POLYGON'].rel_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(confining_polygon_orig,
confining_path,
epsg=cfg.OUTPUT_EPSG)
_nd, _inputs_gpkg_path, inputs_gpkg_lyrs = project.add_project_geopackage(
proj_nodes['Inputs'], LayerTypes['INPUTS'])
output_gpkg = os.path.join(output_folder,
LayerTypes['CONFINEMENT'].rel_path)
intermediates_gpkg = os.path.join(output_folder,
LayerTypes['INTERMEDIATES'].rel_path)
# Creates an empty geopackage and replaces the old one
GeopackageLayer(output_gpkg, delete_dataset=True)
GeopackageLayer(intermediates_gpkg, delete_dataset=True)
# Add the flowlines file with some metadata
project.add_metadata({'BufferField': buffer_field},
inputs_gpkg_lyrs['FLOWLINES'][0])
# Add the confinement polygon
project.add_project_geopackage(proj_nodes['Intermediates'],
LayerTypes['INTERMEDIATES'])
_nd, _inputs_gpkg_path, out_gpkg_lyrs = project.add_project_geopackage(
proj_nodes['Outputs'], LayerTypes['CONFINEMENT'])
# Additional Metadata
project.add_metadata(
{
'Min Buffer': str(min_buffer),
"Expansion Factor": str(bankfull_expansion_factor)
}, out_gpkg_lyrs['CONFINEMENT_BUFFERS'][0])
# Generate confining margins
log.info(f"Preparing output geopackage: {output_gpkg}")
log.info(f"Generating Confinement from buffer field: {buffer_field}")
# Load input datasets and set the global srs and a meter conversion factor
with GeopackageLayer(flowlines_path) as flw_lyr:
srs = flw_lyr.spatial_ref
meter_conversion = flw_lyr.rough_convert_metres_to_vector_units(1)
geom_confining_polygon = get_geometry_unary_union(confining_path,
cfg.OUTPUT_EPSG)
# Calculate Spatial Constants
# Get a very rough conversion factor for 1m to whatever units the shapefile uses
offset = 0.1 * meter_conversion
selection_buffer = 0.1 * meter_conversion
# Standard Outputs
field_lookup = {
'side': ogr.FieldDefn("Side", ogr.OFTString),
'flowlineID': ogr.FieldDefn(
"NHDPlusID", ogr.OFTString
), # ArcGIS cannot read Int64 and will show up as 0, however data is stored correctly in GPKG
'confinement_type': ogr.FieldDefn("Confinement_Type", ogr.OFTString),
'confinement_ratio': ogr.FieldDefn("Confinement_Ratio", ogr.OFTReal),
'constriction_ratio': ogr.FieldDefn("Constriction_Ratio", ogr.OFTReal),
'length': ogr.FieldDefn("ApproxLeng", ogr.OFTReal),
'confined_length': ogr.FieldDefn("ConfinLeng", ogr.OFTReal),
'constricted_length': ogr.FieldDefn("ConstrLeng", ogr.OFTReal),
'bankfull_width': ogr.FieldDefn("Bankfull_Width", ogr.OFTReal),
'buffer_width': ogr.FieldDefn("Buffer_Width", ogr.OFTReal),
# Couple of Debug fields too
'process': ogr.FieldDefn("ErrorProcess", ogr.OFTString),
'message': ogr.FieldDefn("ErrorMessage", ogr.OFTString)
}
field_lookup['side'].SetWidth(5)
field_lookup['confinement_type'].SetWidth(5)
# Here we open all the necessary output layers and write the fields to them. There's no harm in quickly
# Opening these layers to instantiate them
# Standard Outputs
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_MARGINS"].rel_path,
write=True) as margins_lyr:
margins_lyr.create(ogr.wkbLineString, spatial_ref=srs)
margins_lyr.ogr_layer.CreateField(field_lookup['side'])
margins_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
margins_lyr.ogr_layer.CreateField(field_lookup['length'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_RAW"].rel_path,
write=True) as raw_lyr:
raw_lyr.create(ogr.wkbLineString, spatial_ref=srs)
raw_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
raw_lyr.ogr_layer.CreateField(field_lookup['confinement_type'])
raw_lyr.ogr_layer.CreateField(field_lookup['length'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_RATIO"].rel_path,
write=True) as ratio_lyr:
ratio_lyr.create(ogr.wkbLineString, spatial_ref=srs)
ratio_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
ratio_lyr.ogr_layer.CreateField(field_lookup['confinement_ratio'])
ratio_lyr.ogr_layer.CreateField(field_lookup['constriction_ratio'])
ratio_lyr.ogr_layer.CreateField(field_lookup['length'])
ratio_lyr.ogr_layer.CreateField(field_lookup['confined_length'])
ratio_lyr.ogr_layer.CreateField(field_lookup['constricted_length'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
lyr.ogr_layer.CreateField(field_lookup['bankfull_width'])
lyr.ogr_layer.CreateField(field_lookup['buffer_width'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_BUFFERS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
lyr.ogr_layer.CreateField(field_lookup['bankfull_width'])
lyr.ogr_layer.CreateField(field_lookup['buffer_width'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["SPLIT_POINTS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPoint, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["FLOWLINE_SEGMENTS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbLineString, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["ERROR_POLYLINES"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbLineString, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['process'])
lyr.ogr_layer.CreateField(field_lookup['message'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["ERROR_POLYGONS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['process'])
lyr.ogr_layer.CreateField(field_lookup['message'])
# Generate confinement per Flowline
with GeopackageLayer(flowlines_path) as flw_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_MARGINS"].rel_path, write=True) as margins_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RAW"].rel_path, write=True) as raw_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RATIO"].rel_path, write=True) as ratio_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["SPLIT_POINTS"].rel_path, write=True) as dbg_splitpts_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["FLOWLINE_SEGMENTS"].rel_path, write=True) as dbg_flwseg_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path, write=True) as conf_buff_split_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_BUFFERS"].rel_path, write=True) as buff_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYLINES"].rel_path, write=True) as dbg_err_lines_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYGONS"].rel_path, write=True) as dbg_err_polygons_lyr:
err_count = 0
for flowline, _counter, progbar in flw_lyr.iterate_features(
"Generating confinement for flowlines",
attribute_filter="FCode IN ({0})".format(','.join(
[key for key in reach_codes])),
write_layers=[
margins_lyr, raw_lyr, ratio_lyr, dbg_splitpts_lyr,
dbg_flwseg_lyr, buff_lyr, conf_buff_split_lyr,
dbg_err_lines_lyr, dbg_err_polygons_lyr
]):
# Load Flowline
flowlineID = int(flowline.GetFieldAsInteger64("NHDPlusID"))
bankfull_width = flowline.GetField(buffer_field)
buffer_value = max(bankfull_width, min_buffer)
geom_flowline = GeopackageLayer.ogr2shapely(flowline)
if not geom_flowline.is_valid or geom_flowline.is_empty or geom_flowline.length == 0:
progbar.erase()
log.warning("Invalid flowline with id: {}".format(flowlineID))
continue
# Generate buffer on each side of the flowline
geom_buffer = geom_flowline.buffer(
((buffer_value * meter_conversion) / 2) *
bankfull_expansion_factor,
cap_style=2)
# inital cleanup if geom is multipolygon
if geom_buffer.geom_type == "MultiPolygon":
log.warning(f"Cleaning multipolygon for id{flowlineID}")
polys = [g for g in geom_buffer if g.intersects(geom_flowline)]
if len(polys) == 1:
geom_buffer = polys[0]
if not geom_buffer.is_valid or geom_buffer.is_empty or geom_buffer.area == 0 or geom_buffer.geom_type not in [
"Polygon"
]:
progbar.erase()
log.warning("Invalid flowline (after buffering) id: {}".format(
flowlineID))
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess": "Generate Buffer",
"ErrorMessage": "Invalid Buffer"
})
err_count += 1
continue
buff_lyr.create_feature(
geom_buffer, {
"NHDPlusID": flowlineID,
"Buffer_Width": buffer_value,
"Bankfull_Width": bankfull_width
})
# Split the Buffer by the flowline
geom_buffer_splits = split(
geom_buffer, geom_flowline
) # snap(geom, geom_buffer)) <--shapely does not snap vertex to edge. need to make new function for this to ensure more buffers have 2 split polygons
# Process only if 2 buffers exist
if len(geom_buffer_splits) != 2:
# Lets try to split this again by slightly extending the line
geom_newline = scale(geom_flowline, 1.1, 1.1, origin='center')
geom_buffer_splits = split(geom_buffer, geom_newline)
if len(geom_buffer_splits) != 2:
# triage the polygon if still cannot split it
error_message = f"WARNING: Flowline FID {flowline.GetFID()} | Incorrect number of split buffer polygons: {len(geom_buffer_splits)}"
progbar.erase()
log.warning(error_message)
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
err_count += 1
if len(geom_buffer_splits) > 1:
for geom in geom_buffer_splits:
dbg_err_polygons_lyr.create_feature(
geom, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
else:
dbg_err_polygons_lyr.create_feature(
geom_buffer_splits, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
continue
# Generate point to test side of flowline
geom_offset = geom_flowline.parallel_offset(offset, "left")
if not geom_offset.is_valid or geom_offset.is_empty or geom_offset.length == 0:
progbar.erase()
log.warning("Invalid flowline (after offset) id: {}".format(
flowlineID))
err_count += 1
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess":
"Offset Error",
"ErrorMessage":
"Invalid flowline (after offset) id: {}".format(
flowlineID)
})
continue
geom_side_point = geom_offset.interpolate(0.5, True)
# Store output segements
lgeoms_right_confined_flowline_segments = []
lgeoms_left_confined_flowline_segments = []
for geom_side in geom_buffer_splits:
# Identify side of flowline
side = "LEFT" if geom_side.contains(
geom_side_point) else "RIGHT"
# Save the polygon
conf_buff_split_lyr.create_feature(
geom_side, {
"Side": side,
"NHDPlusID": flowlineID,
"Buffer_Width": buffer_value,
"Bankfull_Width": bankfull_width
})
# Generate Confining margins
geom_confined_margins = geom_confining_polygon.boundary.intersection(
geom_side) # make sure intersection splits lines
if geom_confined_margins.is_empty:
continue
# Multilinestring to individual linestrings
lines = [
line for line in geom_confined_margins
] if geom_confined_margins.geom_type == 'MultiLineString' else [
geom_confined_margins
]
for line in lines:
margins_lyr.create_feature(
line, {
"Side": side,
"NHDPlusID": flowlineID,
"ApproxLeng": line.length / meter_conversion
})
# Split flowline by Near Geometry
pt_start = nearest_points(Point(line.coords[0]),
geom_flowline)[1]
pt_end = nearest_points(Point(line.coords[-1]),
geom_flowline)[1]
for point in [pt_start, pt_end]:
dbg_splitpts_lyr.create_feature(
point, {
"Side": side,
"NHDPlusID": flowlineID
})
distance_sorted = sorted([
geom_flowline.project(pt_start),
geom_flowline.project(pt_end)
])
segment = substring(geom_flowline, distance_sorted[0],
distance_sorted[1])
# Store the segment by flowline side
if segment.is_valid and segment.geom_type in [
"LineString", "MultiLineString"
]:
if side == "LEFT":
lgeoms_left_confined_flowline_segments.append(
segment)
else:
lgeoms_right_confined_flowline_segments.append(
segment)
dbg_flwseg_lyr.create_feature(segment, {
"Side": side,
"NHDPlusID": flowlineID
})
# Raw Confinement Output
# Prepare flowline splits
splitpoints = [
Point(x, y)
for line in lgeoms_left_confined_flowline_segments +
lgeoms_right_confined_flowline_segments for x, y in line.coords
]
cut_distances = sorted(
list(
set([
geom_flowline.project(point) for point in splitpoints
])))
lgeoms_flowlines_split = []
current_line = geom_flowline
cumulative_distance = 0.0
while len(cut_distances) > 0:
distance = cut_distances.pop(0) - cumulative_distance
if not distance == 0.0:
outline = cut(current_line, distance)
if len(outline) == 1:
current_line = outline[0]
else:
current_line = outline[1]
lgeoms_flowlines_split.append(outline[0])
cumulative_distance = cumulative_distance + distance
lgeoms_flowlines_split.append(current_line)
# Confined Segments
lgeoms_confined_left_split = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_left_confined_flowline_segments,
buffer=selection_buffer)
lgeoms_confined_right_split = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_right_confined_flowline_segments,
buffer=selection_buffer)
lgeoms_confined_left = select_geoms_by_intersection(
lgeoms_confined_left_split,
lgeoms_confined_right_split,
buffer=selection_buffer,
inverse=True)
lgeoms_confined_right = select_geoms_by_intersection(
lgeoms_confined_right_split,
lgeoms_confined_left_split,
buffer=selection_buffer,
inverse=True)
geom_confined = unary_union(lgeoms_confined_left_split +
lgeoms_confined_right_split)
# Constricted Segments
lgeoms_constricted_l = select_geoms_by_intersection(
lgeoms_confined_left_split,
lgeoms_confined_right_split,
buffer=selection_buffer)
lgeoms_constrcited_r = select_geoms_by_intersection(
lgeoms_confined_right_split,
lgeoms_confined_left_split,
buffer=selection_buffer)
lgeoms_constricted = []
for geom in lgeoms_constricted_l + lgeoms_constrcited_r:
if not any(g.equals(geom) for g in lgeoms_constricted):
lgeoms_constricted.append(geom)
geom_constricted = MultiLineString(lgeoms_constricted)
# Unconfined Segments
lgeoms_unconfined = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_confined_left_split + lgeoms_confined_right_split,
buffer=selection_buffer,
inverse=True)
# Save Raw Confinement
for con_type, geoms in zip(["Left", "Right", "Both", "None"], [
lgeoms_confined_left, lgeoms_confined_right,
lgeoms_constricted, lgeoms_unconfined
]):
for g in geoms:
if g.geom_type == "LineString":
raw_lyr.create_feature(
g, {
"NHDPlusID": flowlineID,
"Confinement_Type": con_type,
"ApproxLeng": g.length / meter_conversion
})
elif geoms.geom_type in ["Point", "MultiPoint"]:
progbar.erase()
log.warning(
f"Flowline FID: {flowline.GetFID()} | Point geometry identified generating outputs for Raw Confinement."
)
else:
progbar.erase()
log.warning(
f"Flowline FID: {flowline.GetFID()} | Unknown geometry identified generating outputs for Raw Confinement."
)
# Calculated Confinement per Flowline
confinement_ratio = geom_confined.length / geom_flowline.length if geom_confined else 0.0
constricted_ratio = geom_constricted.length / geom_flowline.length if geom_constricted else 0.0
# Save Confinement Ratio
attributes = {
"NHDPlusID":
flowlineID,
"Confinement_Ratio":
confinement_ratio,
"Constriction_Ratio":
constricted_ratio,
"ApproxLeng":
geom_flowline.length / meter_conversion,
"ConfinLeng":
geom_confined.length /
meter_conversion if geom_confined else 0.0,
"ConstrLeng":
geom_constricted.length /
meter_conversion if geom_constricted else 0.0
}
ratio_lyr.create_feature(geom_flowline, attributes)
# Write a report
report = ConfinementReport(output_gpkg, report_path, project)
report.write()
progbar.finish()
log.info(f"Count of Flowline segments with errors: {err_count}")
log.info('Confinement Finished')
return
def sanitize(name: str,
in_path: str,
out_path: str,
buff_dist: float,
select_features=None):
"""
It's important to make sure we have the right kinds of geometries.
Args:
name (str): Mainly just for good logging
in_path (str): [description]
out_path (str): [description]
buff_dist (float): [description]
"""
log = Logger('VBET Simplify')
with GeopackageLayer(out_path, write=True) as out_lyr, \
TempGeopackage('sanitize_temp') as tempgpkg, \
GeopackageLayer(in_path) as in_lyr:
out_lyr.create_layer(ogr.wkbPolygon, spatial_ref=in_lyr.spatial_ref)
pts = 0
square_buff = buff_dist * buff_dist
# NOTE: Order of operations really matters here.
in_pts = 0
out_pts = 0
with GeopackageLayer(tempgpkg.filepath, "sanitize_{}".format(str(uuid4())), write=True, delete_dataset=True) as tmp_lyr, \
GeopackageLayer(select_features) as lyr_select_features:
tmp_lyr.create_layer_from_ref(in_lyr)
def geom_validity_fix(geom_in):
f_geom = geom_in
# Only clean if there's a problem:
if not f_geom.IsValid():
f_geom = f_geom.Buffer(0)
if not f_geom.IsValid():
f_geom = f_geom.Buffer(buff_dist)
f_geom = f_geom.Buffer(-buff_dist)
return f_geom
# Only keep features intersected with network
tmp_lyr.create_layer_from_ref(in_lyr)
for candidate_feat, _c2, _p1 in in_lyr.iterate_features(
"Finding interesected features"):
candidate_geom = candidate_feat.GetGeometryRef()
for select_feat, _counter, _progbar in lyr_select_features.iterate_features(
):
select_geom = select_feat.GetGeometryRef()
if select_geom.Intersects(candidate_geom):
feat = ogr.Feature(tmp_lyr.ogr_layer_def)
feat.SetGeometry(candidate_geom)
tmp_lyr.ogr_layer.CreateFeature(feat)
feat = None
break
# Second loop is about filtering bad areas and simplifying
for in_feat, _counter, _progbar in tmp_lyr.iterate_features(
"Filtering out non-relevant shapes for {}".format(name)):
fid = in_feat.GetFID()
geom = in_feat.GetGeometryRef()
area = geom.Area()
pts += geom.GetBoundary().GetPointCount()
# First check. Just make sure this is a valid shape we can work with
# Make sure the area is greater than the square of the cell width
# Make sure we're not significantly disconnected from the main shape
# Make sure we intersect the main shape
if geom.IsEmpty() \
or area < square_buff:
# or biggest_area[3].Distance(geom) > 2 * buff_dist:
continue
f_geom = geom.SimplifyPreserveTopology(buff_dist)
# # Only fix things that need fixing
f_geom = geom_validity_fix(f_geom)
# Second check here for validity after simplification
# Then write to a temporary geopackage layer
if not f_geom.IsEmpty() and f_geom.Area() > 0:
out_feature = ogr.Feature(out_lyr.ogr_layer_def)
out_feature.SetGeometry(f_geom)
out_feature.SetFID(fid)
out_lyr.ogr_layer.CreateFeature(out_feature)
in_pts += pts
out_pts += f_geom.GetBoundary().GetPointCount()
else:
log.warning(
'Invalid GEOM with fid: {} for layer {}'.format(
fid, name))
log.info('Writing to disk for layer {}'.format(name))