def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict:
"""[summary]
Args:
in_layer_path (str): [description]
id_field (str, optional): [description]. Defaults to None.
epsg (int, optional): [description]. Defaults to None.
spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
dict: [description]
"""
log = Logger('load_geometries')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
# Determine the transformation if user provides an EPSG
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
features = {}
for feature, _counter, progbar in in_layer.iterate_features("Loading features"):
if id_field is None:
reach = feature.GetFID()
else:
reach = feature.GetField(id_field)
geom = feature.GetGeometryRef()
geo_type = geom.GetGeometryType()
new_geom = VectorBase.ogr2shapely(geom, transform=transform)
if new_geom.is_empty:
progbar.erase() # get around the progressbar
log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
elif not new_geom.is_valid:
progbar.erase() # get around the progressbar
log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for feature with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for feature with FID={}'.format(feature.GetFID()))
else:
features[reach] = new_geom
return features
def include_features(source_layer: VectorBase,
out_layer: VectorBase,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
excluded_fids: list = None):
included_fids = []
excluded_fids = [] if excluded_fids is None else excluded_fids
for feature, _counter, _progbar in source_layer.iterate_features(
'Including Features',
write_layers=[out_layer],
attribute_filter=attribute_filter,
clip_shape=clip_shape):
out_feature = ogr.Feature(out_layer.ogr_layer_def)
if feature.GetFID() not in excluded_fids:
included_fids.append(feature.GetFID())
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(
out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(),
feature.GetField(i))
geom = feature.GetGeometryRef()
out_feature.SetGeometry(geom.Clone())
out_layer.ogr_layer.CreateFeature(out_feature)
return included_fids
def load_geoms(in_lines):
out = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Loading geometry"):
shapely_geom = VectorBase.ogr2shapely(feat)
out.append(shapely_geom)
return out
def network_statistics(label: str, vector_layer_path: str):
log = Logger('network_statistics')
log.info('Network ShapeFile Summary: {}'.format(vector_layer_path))
results = {}
total_length = 0.0
min_length = None
max_length = None
invalid_features = 0
no_geometry = 0
with get_shp_or_gpkg(vector_layer_path) as vector_layer:
# Delete output column from network ShapeFile if it exists and then recreate it
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
results[vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()] = 0
for feature, _counter, _progbar in vector_layer.iterate_features("Calculating Stats"):
geom = feature.GetGeometryRef()
if geom is None:
no_geometry += 1
return
shapely_obj = VectorBase.ogr2shapely(geom)
length = shapely_obj.length
if shapely_obj.is_empty or shapely_obj.is_valid is False:
invalid_features += 1
total_length += length
min_length = length if not min_length or min_length > length else min_length
max_length = length if not max_length or max_length < length else max_length
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
field = vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()
if field not in results:
results[field] = 0
results[field] += 0 if feature.GetField(field) else 1
features = vector_layer.ogr_layer.GetFeatureCount()
results['Feature Count'] = features
results['Invalid Features'] = invalid_features
results['Features without geometry'] = no_geometry
results['Min Length'] = min_length
results['Max Length'] = max_length
results['Avg Length'] = (total_length / features) if features > 0 and total_length != 0 else 0.0
results['Total Length'] = total_length
for key, value in results.items():
if value > 0:
log.info('{}, {} with {:,} NULL values'.format(label, key, value))
return results
def get_geometry_union(in_layer_path: str, epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""[summary]
Args:
in_layer_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_union')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
geom = None
for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
if feature.GetGeometryRef() is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
new_shape = VectorBase.ogr2shapely(feature, transform=transform)
try:
geom = geom.union(new_shape) if geom is not None else new_shape
except Exception:
progbar.erase() # get around the progressbar
log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID()))
return geom
def midpoints(in_lines):
out_points = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat in in_lyr.iterate_features('Getting Midpoints'):
geom = feat.GetGeometryRef()
line = VectorBase.ogr2shapely(geom)
out_points.append(RiverPoint(line.interpolate(0.5, True)))
feat = None
return out_points
def centerline_points(
in_lines: Path,
distance: float = 0.0,
transform: Transform = None) -> Dict[int, List[RiverPoint]]:
"""Generates points along each line feature at specified distances from the end as well as quarter and halfway
Args:
in_lines (Path): path of shapefile with features
distance (float, optional): distance from ends to generate points. Defaults to 0.0.
transform (Transform, optional): coordinate transformation. Defaults to None.
Returns:
[type]: [description]
"""
log = Logger('centerline_points')
with get_shp_or_gpkg(in_lines) as in_lyr:
out_group = {}
ogr_extent = in_lyr.ogr_layer.GetExtent()
extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2],
ogr_extent[1], ogr_extent[3])
for feat, _counter, progbar in in_lyr.iterate_features(
"Centerline points"):
line = VectorBase.ogr2shapely(feat, transform)
fid = feat.GetFID()
out_points = []
# Attach the FID in case we need it later
props = {'fid': fid}
pts = [
line.interpolate(distance),
line.interpolate(0.5, True),
line.interpolate(-distance)
]
if line.project(line.interpolate(0.25, True)) > distance:
pts.append(line.interpolate(0.25, True))
pts.append(line.interpolate(-0.25, True))
for pt in pts:
# Recall that interpolation can have multiple solutions due to pythagorean theorem
# Throw away anything that's not inside our bounds
if not extent.contains(pt):
progbar.erase()
log.warning('Point {} is outside of extent: {}'.format(
pt.coords[0], ogr_extent))
out_points.append(RiverPoint(pt, properties=props))
out_group[int(fid)] = out_points
feat = None
return out_group
def get_riverpoints(inpath, epsg, attribute_filter=None):
"""[summary]
Args:
inpath ([type]): Path to a ShapeFile
epsg ([type]): Desired output spatial reference
attribute_filter ([type], optional): [description]. Defaults to None.
Returns:
[type]: List of RiverPoint objects
"""
log = Logger('get_riverpoints')
points = []
with get_shp_or_gpkg(inpath) as in_lyr:
_out_spatial_ref, transform = get_transform_from_epsg(
in_lyr.spatial_ref, epsg)
for feat, _counter, progbar in in_lyr.iterate_features(
'Getting points for use in Thiessen',
attribute_filter=attribute_filter):
new_geom = feat.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feat.GetFID()))
continue
new_geom.Transform(transform)
new_shape = VectorBase.ogr2shapely(new_geom)
if new_shape.type == 'Polygon':
new_shape = MultiPolygon([new_shape])
for poly in new_shape:
# Exterior is the shell and there is only ever 1
for pt in list(poly.exterior.coords):
points.append(RiverPoint(pt, interior=False))
# Now we consider interiors. NB: Interiors are only qualifying islands in this case
for idx, island in enumerate(poly.interiors):
for pt in list(island.coords):
points.append(RiverPoint(pt, interior=True,
island=idx))
return points
def merge_feature_classes(feature_class_paths: List[str], boundary: BaseGeometry, out_layer_path: str):
"""[summary]
Args:
feature_class_paths (List[str]): [description]
boundary (BaseGeometry): [description]
out_layer_path (str): [description]
"""
log = Logger('merge_feature_classes')
log.info('Merging {} feature classes.'.format(len(feature_class_paths)))
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer:
fccount = 0
for in_layer_path in feature_class_paths:
fccount += 1
log.info("Merging feature class {}/{}".format(fccount, len(feature_class_paths)))
with get_shp_or_gpkg(in_layer_path) as in_layer:
in_layer.SetSpatialFilter(VectorBase.shapely2ogr(boundary))
# First input spatial ref sets the SRS for the output file
transform = in_layer.get_transform(out_layer)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
in_field_def = in_layer.ogr_layer_def.GetFieldDefn(i)
# Only create fields if we really don't have them
# NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE
if out_layer.ogr_layer_def.GetFieldIndex(in_field_def.GetName()) == -1:
out_layer.ogr_layer.CreateField(in_field_def)
log.info('Processing feature: {}/{}'.format(fccount, len(feature_class_paths)))
for feature, _counter, progbar in in_layer.iterate_features('Processing feature'):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
geom.Transform(transform)
out_feature = ogr.Feature(out_layer.ogr_layer_def)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_feature.SetGeometry(geom)
out_layer.ogr_layer.CreateFeature(out_feature)
log.info('Merge complete.')
return fccount
def buffer_by_field(in_layer_path: str, out_layer_path, field: str, epsg: int = None, min_buffer=None, centered=False) -> None:
"""generate buffered polygons by value in field
Args:
flowlines (str): feature class of line features to buffer
field (str): field with buffer value
epsg (int): output srs
min_buffer: use this buffer value for field values that are less than this
Returns:
geometry: unioned polygon geometry of buffered lines
"""
log = Logger('buffer_by_field')
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, get_shp_or_gpkg(in_layer_path) as in_layer:
conversion = in_layer.rough_convert_metres_to_vector_units(1)
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer(ogr.wkbPolygon, epsg=epsg, fields=in_layer.get_fields())
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
factor = 0.5 if centered else 1.0
for feature, _counter, progbar in in_layer.iterate_features('Buffering features', write_layers=[out_layer]):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
buffer_dist = feature.GetField(field) * conversion * factor
geom.Transform(transform)
geom_buffer = geom.Buffer(buffer_dist if buffer_dist > min_buffer else min_buffer)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom_buffer)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
def rasterize(in_lyr_path, out_raster_path, template_path):
"""Rasterizing an input
Args:
in_lyr_path ([type]): [description]
out_raster_ ([type]): [description]
template_path ([type]): [description]
"""
log = Logger('VBETRasterize')
ds_path, lyr_path = VectorBase.path_sorter(in_lyr_path)
progbar = ProgressBar(100, 50, "Rasterizing ")
with rasterio.open(template_path) as raster:
t = raster.transform
raster_bounds = raster.bounds
def poly_progress(progress, _msg, _data):
progbar.update(int(progress * 100))
# Rasterize the features (roads, rail etc) and calculate a raster of Euclidean distance from these features
progbar.update(0)
# Rasterize the polygon to a temporary file
with TempRaster('vbet_rasterize') as tempfile:
log.debug('Temporary file: {}'.format(tempfile.filepath))
gdal.Rasterize(
tempfile.filepath,
ds_path,
layers=[lyr_path],
xRes=t[0],
yRes=t[4],
burnValues=1,
outputType=gdal.GDT_Int16,
creationOptions=['COMPRESS=LZW'],
# outputBounds --- assigned output bounds: [minx, miny, maxx, maxy]
outputBounds=[
raster_bounds.left, raster_bounds.bottom, raster_bounds.right,
raster_bounds.top
],
callback=poly_progress)
progbar.finish()
# Now mask the output correctly
mask_rasters_nodata(tempfile.filepath, template_path, out_raster_path)
def load_geometries(database, target_srs=None, where_clause=None):
transform = None
if target_srs:
db_srs = get_db_srs(database)
# https://github.com/OSGeo/gdal/issues/1546
target_srs.SetAxisMappingStrategy(db_srs.GetAxisMappingStrategy())
transform = osr.CoordinateTransformation(db_srs, target_srs)
conn = sqlite3.connect(database)
curs = conn.execute('SELECT ReachID, Geometry FROM Reaches {}'.format('WHERE {}'.format(where_clause) if where_clause else ''))
reaches = {}
for row in curs.fetchall():
geom = ogr.CreateGeometryFromJson(row[1])
if transform:
geom.Transform(transform)
reaches[row[0]] = VectorBase.ogr2shapely(geom)
return reaches
def calc_max_drainage(huc_search, precip_raster, wbd, bankfull):
""" Temporary script to calculate the BRAT maximum drainage area threshold
Takes HUC8 watershed boundary polygons and finds the mean annual
precipitation then uses the inverted Beechi and Imaki formula
to derive drainage area at the specified constant bankfull width.
Args:
huc_search (str): feature layer attribute filter string (e.g. '17%' for hydro region 17)
precip_raster (str): path to the national PRISM annual precipitation raster
wbd (str): file path to the national watershed boundary dataset (WBD) file geodatabase
bankfull (float): bankfull width at which drainage area threshold is calculated
"""
# open watershed boundary file geodatabase
driver = ogr.GetDriverByName('OpenFileGDB')
data_source = driver.Open(wbd, 0)
wbd_layer = data_source.GetLayer('WBDHU8')
wbd_layer.SetAttributeFilter('HUC8 LIKE \'{}\''.format(huc_search))
# Need to convert watersheds to the PESG:4269 used by the PRISM raster
_srs, transform = get_transform_from_epsg(wbd_layer.GetSpatialRef(), 4269)
watersheds = {}
for feature in wbd_layer:
huc = feature.GetField('HUC8')
states = feature.GetField('states')
if 'cn' not in states.lower():
watersheds[huc] = VectorBase.ogr2shapely(feature, transform)
stats = raster_buffer_stats2(watersheds, precip_raster)
for huc, stat in stats.items():
# PRISM precipitation is in mm but Beechie and Imaki require it in cm
mean_precip_cm = stat['Mean'] / 10.0
max_drain = pow(bankfull / (0.177) / (pow(mean_precip_cm, 0.453)),
1 / 0.397)
print(
"UPDATE watersheds SET max_drainage = {} WHERE watershed_id = '{}' AND ((max_drainage IS NULL) OR (max_drainage = 0)); -- {}"
.format(int(max_drain), huc, mean_precip_cm))
def centerline_vertex_between_distance(in_lines, distance=0.0):
out_group = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Centerline points between distance"):
line = VectorBase.ogr2shapely(feat)
out_points = []
out_points.append(RiverPoint(line.interpolate(distance)))
out_points.append(RiverPoint(line.interpolate(-distance)))
max_distance = line.length - distance
for vertex in list(line.coords):
test_dist = line.project(Point(vertex))
if test_dist > distance and test_dist < max_distance:
out_points.append(RiverPoint(Point(vertex)))
out_group.append(out_points)
feat = None
return out_group
def hand_rasterize(in_lyr_path: str, template_dem_path: str,
out_raster_path: str):
log = Logger('hand_rasterize')
ds_path, lyr_path = VectorBase.path_sorter(in_lyr_path)
g = gdal.Open(template_dem_path)
geo_t = g.GetGeoTransform()
width, height = g.RasterXSize, g.RasterYSize
xmin = min(geo_t[0], geo_t[0] + width * geo_t[1])
xmax = max(geo_t[0], geo_t[0] + width * geo_t[1])
ymin = min(geo_t[3], geo_t[3] + geo_t[-1] * height)
ymax = max(geo_t[3], geo_t[3] + geo_t[-1] * height)
# Close our dataset
g = None
progbar = ProgressBar(100, 50, "Rasterizing for HAND")
def poly_progress(progress, _msg, _data):
progbar.update(int(progress * 100))
# https://gdal.org/programs/gdal_rasterize.html
# https://gdal.org/python/osgeo.gdal-module.html#RasterizeOptions
gdal.Rasterize(
out_raster_path,
ds_path,
layers=[lyr_path],
height=height,
width=width,
burnValues=1,
outputType=gdal.GDT_CFloat32,
creationOptions=['COMPRESS=LZW'],
# outputBounds --- assigned output bounds: [minx, miny, maxx, maxy]
outputBounds=[xmin, ymin, xmax, ymax],
callback=poly_progress)
progbar.finish()
# Rasterize the features (roads, rail etc) and calculate a raster of Euclidean distance from these features
progbar.update(0)
def simple_save(list_geoms, ogr_type, srs, layer_name, gpkg_path):
with GeopackageLayer(gpkg_path, layer_name, write=True) as lyr:
lyr.create_layer(ogr_type, spatial_ref=srs)
progbar = ProgressBar(len(list_geoms), 50, f"Saving {gpkg_path}/{layer_name}")
counter = 0
progbar.update(counter)
lyr.ogr_layer.StartTransaction()
for geom in list_geoms:
counter += 1
progbar.update(counter)
feature = ogr.Feature(lyr.ogr_layer_def)
geom_ogr = VectorBase.shapely2ogr(geom)
feature.SetGeometry(geom_ogr)
# if attributes:
# for field, value in attributes.items():
# feature.SetField(field, value)
lyr.ogr_layer.CreateFeature(feature)
feature = None
progbar.finish()
lyr.ogr_layer.CommitTransaction()
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""Load all features from a ShapeFile and union them together into a single geometry
Args:
in_layer_path (str): path to layer
epsg (int, optional): EPSG to project to. Defaults to None.
spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None.
attribute_filter (str, optional): Filter to a set of attributes. Defaults to None.
clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_unary_union')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
geom_list = []
for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
new_geom = feature.GetGeometryRef()
geo_type = new_geom.GetGeometryType()
# We can't union non-valid shapes but sometimes a buffer by 0 can help
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID()))
try:
new_geom = new_geom.Buffer(0)
if not new_geom.IsValid():
log.warning(' Still invalid. Skipping this geometry')
continue
except Exception:
log.warning('Exception raised during buffer 0 technique. skipping this file')
continue
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for shape with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for shape with FID={}'.format(feature.GetFID()))
else:
geom_list.append(VectorBase.ogr2shapely(new_geom, transform))
# IF we get past a certain size then run the union
if len(geom_list) >= 500:
geom_list = [unary_union(geom_list)]
new_geom = None
log.debug('finished iterating with list of size: {}'.format(len(geom_list)))
if len(geom_list) > 1:
log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list)))
# Do a final union to clean up anything that might still be in the list
geom_union = unary_union(geom_list)
elif len(geom_list) == 0:
log.warning('No geometry found to union')
return None
else:
log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list)))
geom_union = geom_list[0]
log.debug(' done')
print_geom_size(log, geom_union)
log.debug('Complete')
# Return a shapely object
return geom_union
def segment_network(inpath: str,
outpath: str,
interval: float,
minimum: float,
watershed_id: str,
create_layer=False):
"""
Chop the lines in a polyline feature class at the specified interval unless
this would create a line less than the minimum in which case the line is not segmented.
:param inpath: Original network feature class
:param outpath: Output segmented network feature class
:param interval: Distance at which to segment each line feature (map units)
:param minimum: Minimum length below which lines are not segmented (map units)
:param watershed_id: Give this watershed an id (str)
:param create_layer: This layer may be created earlier. We can choose to create it. Defaults to false (bool)
:return: None
"""
log = Logger('Segment Network')
if interval <= 0:
log.info('Skipping segmentation.')
else:
log.info(
'Segmenting network to {}m, with minimum feature length of {}m'.
format(interval, minimum))
log.info('Segmenting network from {0}'.format(inpath))
# NOTE: Remember to always open the 'write' layer first in case it's the same geopackage
with get_shp_or_gpkg(
outpath, write=True) as out_lyr, get_shp_or_gpkg(inpath) as in_lyr:
# Get the input NHD flow lines layer
srs = in_lyr.spatial_ref
feature_count = in_lyr.ogr_layer.GetFeatureCount()
log.info('Input feature count {:,}'.format(feature_count))
# Get the closest EPSG possible to calculate length
extent_poly = ogr.Geometry(ogr.wkbPolygon)
extent_centroid = extent_poly.Centroid()
utm_epsg = get_utm_zone_epsg(extent_centroid.GetX())
transform_ref, transform = VectorBase.get_transform_from_epsg(
in_lyr.spatial_ref, utm_epsg)
# IN order to get accurate lengths we are going to need to project into some coordinate system
transform_back = osr.CoordinateTransformation(transform_ref, srs)
# Create the output shapefile
if create_layer is True:
out_lyr.create_layer_from_ref(in_lyr)
# We add two features to this
out_lyr.create_fields({
'ReachID': ogr.OFTInteger,
'WatershedID': ogr.OFTString
})
# Retrieve all input features keeping track of which ones have GNIS names or not
named_features = {}
all_features = []
junctions = []
# Omit pipelines with FCode 428**
attribute_filter = 'FCode < 42800 OR FCode > 42899'
log.info('Filtering out pipelines ({})'.format(attribute_filter))
for in_feature, _counter, _progbar in in_lyr.iterate_features(
"Loading Network", attribute_filter=attribute_filter):
# Store relevant items as a tuple:
# (name, FID, StartPt, EndPt, Length, FCode)
s_feat = SegmentFeature(in_feature, transform)
# Add the end points of all lines to a single list
junctions.extend([s_feat.start, s_feat.end])
if not s_feat.name or len(s_feat.name) < 1 or interval <= 0:
# Add features without a GNIS name to list. Also add to list if not segmenting
all_features.append(s_feat)
else:
# Build separate lists for each unique GNIS name
if s_feat.name not in named_features:
named_features[s_feat.name] = [s_feat]
else:
named_features[s_feat.name].append(s_feat)
# Loop over all features with the same GNIS name.
# Only merge them if they meet at a junction where no other lines meet.
log.info('Merging simple features with the same GNIS name...')
for name, features in named_features.items():
log.debug(' {} x{}'.format(name.encode('utf-8'), len(features)))
all_features.extend(features)
log.info(
'{:,} features after merging. Starting segmentation...'.format(
len(all_features)))
# Segment the features at the desired interval
# rid = 0
log.info('Segmenting Network...')
progbar = ProgressBar(len(all_features), 50, "Segmenting")
counter = 0
for orig_feat in all_features:
counter += 1
progbar.update(counter)
old_feat = in_lyr.ogr_layer.GetFeature(orig_feat.fid)
old_geom = old_feat.GetGeometryRef()
# Anything that produces reach shorter than the minimum just gets added. Also just add features if not segmenting
if orig_feat.length_m < (interval + minimum) or interval <= 0:
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
new_ogr_feat.SetGeometry(old_geom)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
else:
# From here on out we use shapely and project to UTM. We'll transform back before writing to disk.
new_geom = old_geom.Clone()
new_geom.Transform(transform)
remaining = LineString(new_geom.GetPoints())
while remaining and remaining.length >= (interval + minimum):
part1shply, part2shply = cut(remaining, interval)
remaining = part2shply
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
geo = ogr.CreateGeometryFromWkt(part1shply.wkt)
geo.Transform(transform_back)
new_ogr_feat.SetGeometry(geo)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
# Add any remaining line to outGeometries
if remaining:
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
geo = ogr.CreateGeometryFromWkt(remaining.wkt)
geo.Transform(transform_back)
new_ogr_feat.SetGeometry(geo)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
progbar.finish()
log.info(('{:,} features written to {:}'.format(
out_lyr.ogr_layer.GetFeatureCount(), outpath)))
log.info('Process completed successfully.')
def rvd(huc: int, flowlines_orig: Path, existing_veg_orig: Path, historic_veg_orig: Path,
valley_bottom_orig: Path, output_folder: Path, reach_codes: List[str], flow_areas_orig: Path, waterbodies_orig: Path, meta=None):
"""[Generate segmented reaches on flowline network and calculate RVD from historic and existing vegetation rasters
Args:
huc (integer): Watershed ID
flowlines_orig (Path): Segmented flowlines feature layer
existing_veg_orig (Path): LANDFIRE version 2.00 evt raster, with adjacent xml metadata file
historic_veg_orig (Path): LANDFIRE version 2.00 bps raster, with adjacent xml metadata file
valley_bottom_orig (Path): Vbet polygon feature layer
output_folder (Path): destination folder for project output
reach_codes (List[int]): NHD reach codes for features to include in outputs
flow_areas_orig (Path): NHD flow area polygon feature layer
waterbodies (Path): NHD waterbodies polygon feature layer
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("RVD")
log.info('RVD v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception('Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
safe_makedirs(output_folder)
project, _realization, proj_nodes = create_project(huc, output_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
log.info('Adding inputs to project')
_prj_existing_path_node, prj_existing_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['EXVEG'], existing_veg_orig)
_prj_historic_path_node, prj_historic_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HISTVEG'], historic_veg_orig)
# TODO: Don't forget the att_filter
# _prj_flowlines_node, prj_flowlines = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'], flowlines, att_filter="\"ReachCode\" Like '{}%'".format(huc))
# Copy in the vectors we need
inputs_gpkg_path = os.path.join(output_folder, LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path)
outputs_gpkg_path = os.path.join(output_folder, LayerTypes['OUTPUTS'].rel_path)
# Make sure we're starting with empty/fresh geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
GeopackageLayer.delete(outputs_gpkg_path)
# Copy our input layers and also find the difference in the geometry for the valley bottom
flowlines_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
vbottom_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['VALLEY_BOTTOM'].rel_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(valley_bottom_orig, vbottom_path, epsg=cfg.OUTPUT_EPSG)
with GeopackageLayer(flowlines_path) as flow_lyr:
# Set the output spatial ref as this for the whole project
out_srs = flow_lyr.spatial_ref
meter_conversion = flow_lyr.rough_convert_metres_to_vector_units(1)
distance_buffer = flow_lyr.rough_convert_metres_to_vector_units(1)
# Transform issues reading 102003 as espg id. Using sr wkt seems to work, however arcgis has problems loading feature classes with this method...
raster_srs = ogr.osr.SpatialReference()
ds = gdal.Open(prj_existing_path, 0)
raster_srs.ImportFromWkt(ds.GetProjectionRef())
raster_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
transform_shp_to_raster = VectorBase.get_transform(out_srs, raster_srs)
gt = ds.GetGeoTransform()
cell_area = ((gt[1] / meter_conversion) * (-gt[5] / meter_conversion))
# Create the output feature class fields
with GeopackageLayer(outputs_gpkg_path, layer_name='ReachGeometry', delete_dataset=True) as out_lyr:
out_lyr.create_layer(ogr.wkbMultiLineString, spatial_ref=out_srs, options=['FID=ReachID'], fields={
'GNIS_NAME': ogr.OFTString,
'ReachCode': ogr.OFTString,
'TotDASqKm': ogr.OFTReal,
'NHDPlusID': ogr.OFTReal,
'WatershedID': ogr.OFTInteger
})
metadata = {
'RVD_DateTime': datetime.datetime.now().isoformat(),
'Reach_Codes': reach_codes
}
# Execute the SQL to create the lookup tables in the RVD geopackage SQLite database
watershed_name = create_database(huc, outputs_gpkg_path, metadata, cfg.OUTPUT_EPSG, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'rvd_schema.sql'))
project.add_metadata({'Watershed': watershed_name})
geom_vbottom = get_geometry_unary_union(vbottom_path, spatial_ref=raster_srs)
flowareas_path = None
if flow_areas_orig:
flowareas_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
copy_feature_class(flow_areas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
geom_flow_areas = get_geometry_unary_union(flowareas_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_flow_areas)
else:
del LayerTypes['INPUTS'].sub_layers['FLOW_AREA']
waterbodies_path = None
if waterbodies_orig:
waterbodies_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['WATERBODIES'].rel_path)
copy_feature_class(waterbodies_orig, waterbodies_path, epsg=cfg.OUTPUT_EPSG)
geom_waterbodies = get_geometry_unary_union(waterbodies_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_waterbodies)
else:
del LayerTypes['INPUTS'].sub_layers['WATERBODIES']
# Add the inputs to the XML
_nd, _in_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Filter the flow lines to just the required features and then segment to desired length
# TODO: These are brat methods that need to be refactored to use VectorBase layers
cleaned_path = os.path.join(outputs_gpkg_path, 'ReachGeometry')
build_network(flowlines_path, flowareas_path, cleaned_path, waterbodies_path=waterbodies_path, epsg=cfg.OUTPUT_EPSG, reach_codes=reach_codes, create_layer=False)
# Generate Voroni polygons
log.info("Calculating Voronoi Polygons...")
# Add all the points (including islands) to the list
flowline_thiessen_points_groups = centerline_points(cleaned_path, distance_buffer, transform_shp_to_raster)
flowline_thiessen_points = [pt for group in flowline_thiessen_points_groups.values() for pt in group]
simple_save([pt.point for pt in flowline_thiessen_points], ogr.wkbPoint, raster_srs, "Thiessen_Points", intermediates_gpkg_path)
# Exterior is the shell and there is only ever 1
myVorL = NARVoronoi(flowline_thiessen_points)
# Generate Thiessen Polys
myVorL.createshapes()
# Dissolve by flowlines
log.info("Dissolving Thiessen Polygons")
dissolved_polys = myVorL.dissolve_by_property('fid')
# Clip Thiessen Polys
log.info("Clipping Thiessen Polygons to Valley Bottom")
clipped_thiessen = clip_polygons(geom_vbottom, dissolved_polys)
# Save Intermediates
simple_save(clipped_thiessen.values(), ogr.wkbPolygon, raster_srs, "Thiessen", intermediates_gpkg_path)
simple_save(dissolved_polys.values(), ogr.wkbPolygon, raster_srs, "ThiessenPolygonsDissolved", intermediates_gpkg_path)
simple_save(myVorL.polys, ogr.wkbPolygon, raster_srs, "ThiessenPolygonsRaw", intermediates_gpkg_path)
_nd, _inter_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES'])
# OLD METHOD FOR AUDIT
# dissolved_polys2 = dissolve_by_points(flowline_thiessen_points_groups, myVorL.polys)
# simple_save(dissolved_polys2.values(), ogr.wkbPolygon, out_srs, "ThiessenPolygonsDissolved_OLD", intermediates_gpkg_path)
# Load Vegetation Rasters
log.info(f"Loading Existing and Historic Vegetation Rasters")
vegetation = {}
vegetation["EXISTING"] = load_vegetation_raster(prj_existing_path, outputs_gpkg_path, True, output_folder=os.path.join(output_folder, 'Intermediates'))
vegetation["HISTORIC"] = load_vegetation_raster(prj_historic_path, outputs_gpkg_path, False, output_folder=os.path.join(output_folder, 'Intermediates'))
for epoch in vegetation.keys():
for name in vegetation[epoch].keys():
if not f"{epoch}_{name}" == "HISTORIC_LUI":
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[f"{epoch}_{name}"])
if vegetation["EXISTING"]["RAW"].shape != vegetation["HISTORIC"]["RAW"].shape:
raise Exception('Vegetation raster shapes are not equal Existing={} Historic={}. Cannot continue'.format(vegetation["EXISTING"]["RAW"].shape, vegetation["HISTORIC"]["RAW"].shape))
# Vegetation zone calculations
riparian_zone_arrays = {}
riparian_zone_arrays["RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["RIPARIAN"] + vegetation["HISTORIC"]["RIPARIAN"]) > 0) * 1
riparian_zone_arrays["NATIVE_RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["NATIVE_RIPARIAN"] + vegetation["HISTORIC"]["NATIVE_RIPARIAN"]) > 0) * 1
riparian_zone_arrays["VEGETATION_ZONES"] = ((vegetation["EXISTING"]["VEGETATED"] + vegetation["HISTORIC"]["VEGETATED"]) > 0) * 1
# Save Intermediate Rasters
for name, raster in riparian_zone_arrays.items():
save_intarr_to_geotiff(raster, os.path.join(output_folder, "Intermediates", f"{name}.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[name])
# Calculate Riparian Departure per Reach
riparian_arrays = {f"{epoch.capitalize()}{(name.capitalize()).replace('Native_riparian', 'NativeRiparian')}Mean": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name in ["RIPARIAN", "NATIVE_RIPARIAN"]}
# Vegetation Cell Counts
raw_arrays = {f"{epoch}": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name == "RAW"}
# Generate Vegetation Conversions
vegetation_change = (vegetation["HISTORIC"]["CONVERSION"] - vegetation["EXISTING"]["CONVERSION"])
save_intarr_to_geotiff(vegetation_change, os.path.join(output_folder, "Intermediates", "Conversion_Raster.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['VEGETATION_CONVERSION'])
# load conversion types dictionary from database
conn = sqlite3.connect(outputs_gpkg_path)
conn.row_factory = dict_factory
curs = conn.cursor()
curs.execute('SELECT * FROM ConversionTypes')
conversion_classifications = curs.fetchall()
curs.execute('SELECT * FROM vwConversions')
conversion_ids = curs.fetchall()
# Split vegetation change classes into binary arrays
vegetation_change_arrays = {
c['FieldName']: (vegetation_change == int(c["TypeValue"])) * 1 if int(c["TypeValue"]) in np.unique(vegetation_change) else None
for c in conversion_classifications
}
# Calcuate average and unique cell counts per reach
progbar = ProgressBar(len(clipped_thiessen.keys()), 50, "Extracting array values by reach...")
counter = 0
discarded = 0
with rasterio.open(prj_existing_path) as dataset:
unique_vegetation_counts = {}
reach_average_riparian = {}
reach_average_change = {}
for reachid, poly in clipped_thiessen.items():
counter += 1
progbar.update(counter)
# we can discount a lot of shapes here.
if not poly.is_valid or poly.is_empty or poly.area == 0 or poly.geom_type not in ["Polygon", "MultiPolygon"]:
discarded += 1
continue
raw_values_unique = {}
change_values_mean = {}
riparian_values_mean = {}
reach_raster = np.ma.masked_invalid(
features.rasterize(
[poly],
out_shape=dataset.shape,
transform=dataset.transform,
all_touched=True,
fill=np.nan))
for raster_name, raster in raw_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
raw_values_unique[raster_name] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True)
else:
raw_values_unique[raster_name] = []
for raster_name, raster in riparian_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
riparian_values_mean[raster_name] = np.ma.mean(current_raster)
else:
riparian_values_mean[raster_name] = 0.0
for raster_name, raster in vegetation_change_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
change_values_mean[raster_name] = np.ma.mean(current_raster)
else:
change_values_mean[raster_name] = 0.0
unique_vegetation_counts[reachid] = raw_values_unique
reach_average_riparian[reachid] = riparian_values_mean
reach_average_change[reachid] = change_values_mean
progbar.finish()
with SQLiteCon(outputs_gpkg_path) as gpkg:
# Ensure all reaches are present in the ReachAttributes table before storing RVD output values
gpkg.curs.execute('INSERT INTO ReachAttributes (ReachID) SELECT ReachID FROM ReachGeometry;')
errs = 0
for reachid, epochs in unique_vegetation_counts.items():
for epoch in epochs.values():
insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
try:
gpkg.curs.executemany('''INSERT INTO ReachVegetation (
ReachID,
VegetationID,
Area,
CellCount)
VALUES (?,?,?,?)''', insert_values)
# Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
except sqlite3.IntegrityError as err:
# THis is likely a constraint error.
errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
log.error(errstr)
errs += 1
except sqlite3.Error as err:
# This is any other kind of error
errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
log.error(errstr)
errs += 1
if errs > 0:
raise Exception('Errors were found inserting records into the database. Cannot continue.')
gpkg.conn.commit()
# load RVD departure levels from DepartureLevels database table
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxRVD FROM DepartureLevels ORDER BY MaxRVD ASC')
departure_levels = gpkg.curs.fetchall()
# Calcuate Average Departure for Riparian and Native Riparian
riparian_departure_values = riparian_departure(reach_average_riparian, departure_levels)
write_db_attributes(outputs_gpkg_path, riparian_departure_values, departure_type_columns)
# Add Conversion Code, Type to Vegetation Conversion
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxValue, NAME FROM ConversionLevels ORDER BY MaxValue ASC')
conversion_levels = gpkg.curs.fetchall()
reach_values_with_conversion_codes = classify_conversions(reach_average_change, conversion_ids, conversion_levels)
write_db_attributes(outputs_gpkg_path, reach_values_with_conversion_codes, rvd_columns)
# # Write Output to GPKG table
# log.info('Insert values to GPKG tables')
# # TODO move this to write_attirubtes method
# with get_shp_or_gpkg(outputs_gpkg_path, layer_name='ReachAttributes', write=True, ) as in_layer:
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "FromConifer": ogr.OFTReal,
# "FromDevegetated": ogr.OFTReal,
# "FromGrassShrubland": ogr.OFTReal,
# "FromDeciduous": ogr.OFTReal,
# "NoChange": ogr.OFTReal,
# "Deciduous": ogr.OFTReal,
# "GrassShrubland": ogr.OFTReal,
# "Devegetation": ogr.OFTReal,
# "Conifer": ogr.OFTReal,
# "Invasive": ogr.OFTReal,
# "Development": ogr.OFTReal,
# "Agriculture": ogr.OFTReal,
# "ConversionCode": ogr.OFTInteger,
# "ConversionType": ogr.OFTString}.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in reach_values_with_conversion_codes:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in reach_values_with_conversion_codes[reach]:
# if not reach_values_with_conversion_codes[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, reach_values_with_conversion_codes[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "EXISTING_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_RIPARIAN_MEAN": ogr.OFTReal,
# "RIPARIAN_DEPARTURE": ogr.OFTReal,
# "EXISTING_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "NATIVE_RIPARIAN_DEPARTURE": ogr.OFTReal, }.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in riparian_departure_values:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in riparian_departure_values[reach]:
# if not riparian_departure_values[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, riparian_departure_values[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# with sqlite3.connect(outputs_gpkg_path) as conn:
# cursor = conn.cursor()
# errs = 0
# for reachid, epochs in unique_vegetation_counts.items():
# for epoch in epochs.values():
# insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
# try:
# cursor.executemany('''INSERT INTO ReachVegetation (
# ReachID,
# VegetationID,
# Area,
# CellCount)
# VALUES (?,?,?,?)''', insert_values)
# # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
# except sqlite3.IntegrityError as err:
# # THis is likely a constraint error.
# errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
# log.error(errstr)
# errs += 1
# except sqlite3.Error as err:
# # This is any other kind of error
# errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
# log.error(errstr)
# errs += 1
# if errs > 0:
# raise Exception('Errors were found inserting records into the database. Cannot continue.')
# conn.commit()
# Add intermediates and the report to the XML
# project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) already
# added above
project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['OUTPUTS'])
# Add the report to the XML
report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True)
report = RVDReport(report_path, project)
report.write()
log.info('RVD complete')
def build_network(flowlines_path: str,
flowareas_path: str,
out_path: str,
epsg: int = None,
reach_codes: List[str] = None,
waterbodies_path: str = None,
waterbody_max_size=None,
create_layer: bool = True):
"""[summary]
Args:
flowlines_path (str): [description]
flowareas_path (str): [description]
out_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
reach_codes (List[str], optional): [description]. Defaults to None.
waterbodies_path (str, optional): [description]. Defaults to None.
waterbody_max_size ([type], optional): [description]. Defaults to None.
create_layer (bool, optional): [description]. Defaults to True.
Returns:
[type]: [description]
"""
log = Logger('Build Network')
log.info("Building network from flow lines {0}".format(flowlines_path))
if reach_codes:
for r in reach_codes:
log.info('Retaining {} reaches with code {}'.format(
FCodeValues[int(r)], r))
else:
log.info('Retaining all reaches. No reach filtering.')
# Get the transformation required to convert to the target spatial reference
if epsg is not None:
with get_shp_or_gpkg(flowareas_path) as flowareas_lyr:
out_spatial_ref, transform = VectorBase.get_transform_from_epsg(
flowareas_lyr.spatial_ref, epsg)
# Process all perennial/intermittment/ephemeral reaches first
attribute_filter = None
if reach_codes and len(reach_codes) > 0:
_result = [
log.info("{0} {1} network features (FCode {2})".format(
'Retaining', FCodeValues[int(key)], key))
for key in reach_codes
]
attribute_filter = "FCode IN ({0})".format(','.join(
[key for key in reach_codes]))
if create_layer is True:
with get_shp_or_gpkg(flowlines_path) as flowlines_lyr, get_shp_or_gpkg(
out_path, write=True) as out_lyr:
out_lyr.create_layer_from_ref(flowlines_lyr)
log.info('Processing all reaches')
process_reaches(flowlines_path,
out_path,
attribute_filter=attribute_filter)
# Process artifical paths through small waterbodies
if waterbodies_path is not None and waterbody_max_size is not None:
small_waterbodies = get_geometry_unary_union(
waterbodies_path, epsg,
'AreaSqKm <= ({0})'.format(waterbody_max_size))
log.info(
'Retaining artificial features within waterbody features smaller than {0}km2'
.format(waterbody_max_size))
process_reaches(
flowlines_path,
out_path,
transform=transform,
attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES),
clip_shape=small_waterbodies)
# Retain artifical paths through flow areas
if flowareas_path:
flow_polygons = get_geometry_unary_union(flowareas_path, epsg)
if flow_polygons:
log.info('Retaining artificial features within flow area features')
process_reaches(
flowlines_path,
out_path,
transform=transform,
attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES),
clip_shape=flow_polygons)
else:
log.info('Zero artifical paths to be retained.')
with get_shp_or_gpkg(out_path) as out_lyr:
log.info(('{:,} features written to {:}'.format(
out_lyr.ogr_layer.GetFeatureCount(), out_path)))
log.info('Process completed successfully.')
return out_spatial_ref
def copy_feature_class(in_layer_path: str, out_layer_path: str,
epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None,
buffer: float = 0,
) -> None:
"""Copy a Shapefile from one location to another
This method is capable of reprojecting the geometries as they are copied.
It is also possible to filter features by both attributes and also clip the
features to another geometryNone
Args:
in_layer (str): Input layer path
epsg ([type]): EPSG Code to use for the transformation
out_layer (str): Output layer path
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
buffer (float): Buffer the output features (in meters).
"""
log = Logger('copy_feature_class')
# NOTE: open the outlayer first so that write gets the dataset open priority
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, \
get_shp_or_gpkg(in_layer_path) as in_layer:
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer_from_ref(in_layer, epsg=epsg)
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
buffer_convert = 0
if buffer != 0:
buffer_convert = in_layer.rough_convert_metres_to_vector_units(buffer)
# This is the callback method that will be run on each feature
for feature, _counter, progbar in in_layer.iterate_features("Copying features", write_layers=[out_layer], clip_shape=clip_shape, clip_rect=clip_rect, attribute_filter=attribute_filter):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
if geom.GetGeometryType() in VectorBase.LINE_TYPES:
if geom.Length() == 0.0:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no Length. Skipping'.format(feature.GetFID()))
continue
# Buffer the shape if we need to
if buffer_convert != 0:
geom = geom.Buffer(buffer_convert)
geom.Transform(transform)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
def raster_warp(inraster: str,
outraster: str,
epsg,
clip=None,
warp_options: dict = {}):
"""
Reproject a raster to a different coordinate system.
:param inraster: Input dataset
:param outraster: Output dataset
:param epsg: Output spatial reference EPSG identifier
:param log: Log file object
:param clip: Optional Polygon dataset to clip the output.
:param warp_options: Extra GDALWarpOptions.
:return: None
https://gdal.org/python/osgeo.gdal-module.html#WarpOptions
"""
log = Logger('Raster Warp')
if os.path.isfile(outraster):
log.info(
'Skipping raster warp because output exists {}'.format(outraster))
return None
log.info('Raster Warp input raster {}'.format(inraster))
log.info('Raster Warp output raster {}'.format(outraster))
log.info('Output spatial reference EPSG: {}'.format(epsg))
output_folder = os.path.dirname(outraster)
if not os.path.isdir(output_folder):
os.mkdir(output_folder)
warpvrt = os.path.join(os.path.dirname(outraster),
'temp_gdal_warp_output.vrt')
log.info('Performing GDAL warp to temporary VRT file.')
if clip:
log.info('Clipping to polygons using {}'.format(clip))
clip_ds, clip_layer = VectorBase.path_sorter(clip)
warp_options_obj = gdal.WarpOptions(dstSRS='EPSG:{}'.format(epsg),
format='vrt',
cutlineDSName=clip_ds,
cutlineLayer=clip_layer,
cropToCutline=True,
**warp_options)
else:
warp_options_obj = gdal.WarpOptions(dstSRS='EPSG:{}'.format(epsg),
format='vrt',
**warp_options)
ds = gdal.Warp(warpvrt, inraster, options=warp_options_obj)
log.info(
'Using GDAL translate to convert VRT to compressed raster format.')
translateoptions = gdal.TranslateOptions(
gdal.ParseCommandLine("-of Gtiff -co COMPRESS=DEFLATE"))
gdal.Translate(outraster, ds, options=translateoptions)
# Cleanup the temporary VRT file
os.remove(warpvrt)
if ds:
log.info('Process completed successfully.')
else:
log.error('Error running GDAL Warp')
def reach_geometry(flow_lines: Path, dem_path: Path, buffer_distance: float):
""" Calculate reach geometry BRAT attributes
Args:
flow_lines (Path): [description]
dem_path (Path): [description]
buffer_distance (float): [description]
"""
log = Logger('Reach Geometry')
# Determine the best projected coordinate system based on the raster
dataset = gdal.Open(dem_path)
geo_transform = dataset.GetGeoTransform()
xcentre = geo_transform[0] + (dataset.RasterXSize * geo_transform[1]) / 2.0
epsg = get_utm_zone_epsg(xcentre)
with rasterio.open(dem_path) as raster:
bounds = raster.bounds
extent = box(*bounds)
# Buffer the start and end point of each reach
line_start_polygons = {}
line_end_polygons = {}
reaches = {}
with get_shp_or_gpkg(flow_lines) as lyr:
# Transformations from original flow line features to metric EPSG, and to raster spatial reference
_srs, transform_to_metres = VectorBase.get_transform_from_epsg(lyr.spatial_ref, epsg)
_srs, transform_to_raster = VectorBase.get_transform_from_raster(lyr.spatial_ref, dem_path)
# Buffer distance converted to the units of the raster spatial reference
vector_buffer = VectorBase.rough_convert_metres_to_raster_units(dem_path, buffer_distance)
for feature, _counter, _progbar in lyr.iterate_features("Processing reaches"):
reach_id = feature.GetFID()
geom = feature.GetGeometryRef()
geom_clone = geom.Clone()
# Calculate the reach length in the output spatial reference
if transform_to_metres is not None:
geom.Transform(transform_to_metres)
reaches[reach_id] = {'iGeo_Len': geom.Length(), 'iGeo_Slope': 0.0, 'iGeo_ElMin': None, 'IGeo_ElMax': None}
if transform_to_raster is not None:
geom_clone.Transform(transform_to_raster)
# Buffer the ends of the reach polyline in the raster spatial reference
pt_start = Point(VectorBase.ogr2shapely(geom_clone, transform_to_raster).coords[0])
pt_end = Point(VectorBase.ogr2shapely(geom_clone, transform_to_raster).coords[-1])
if extent.contains(pt_start) and extent.contains(pt_end):
line_start_polygons[reach_id] = pt_start.buffer(vector_buffer)
line_end_polygons[reach_id] = pt_end.buffer(vector_buffer)
# Retrieve the mean elevation of start and end of point
line_start_elevations = raster_buffer_stats2(line_start_polygons, dem_path)
line_end_elevations = raster_buffer_stats2(line_end_polygons, dem_path)
for reach_id, data in reaches.items():
if reach_id in line_start_elevations and reach_id in line_end_elevations:
sta_data = line_start_elevations[reach_id]
end_data = line_end_elevations[reach_id]
data['iGeo_ElMax'] = _max_ignore_none(sta_data['Maximum'], end_data['Maximum'])
data['iGeo_ElMin'] = _min_ignore_none(sta_data['Minimum'], end_data['Minimum'])
if sta_data['Mean'] is not None and end_data['Mean'] is not None and sta_data['Mean'] != end_data['Mean']:
data['iGeo_Slope'] = abs(sta_data['Mean'] - end_data['Mean']) / data['iGeo_Len']
else:
log.warning('{:,} features skipped because one or both ends of polyline not on DEM raster'.format(reach_id))
write_db_attributes(os.path.dirname(flow_lines), reaches, ['iGeo_Len', 'iGeo_ElMax', 'iGeo_ElMin', 'iGeo_Slope'])