def merge_feature_classes(feature_class_paths: List[str], boundary: BaseGeometry, out_layer_path: str):
"""[summary]
Args:
feature_class_paths (List[str]): [description]
boundary (BaseGeometry): [description]
out_layer_path (str): [description]
"""
log = Logger('merge_feature_classes')
log.info('Merging {} feature classes.'.format(len(feature_class_paths)))
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer:
fccount = 0
for in_layer_path in feature_class_paths:
fccount += 1
log.info("Merging feature class {}/{}".format(fccount, len(feature_class_paths)))
with get_shp_or_gpkg(in_layer_path) as in_layer:
in_layer.SetSpatialFilter(VectorBase.shapely2ogr(boundary))
# First input spatial ref sets the SRS for the output file
transform = in_layer.get_transform(out_layer)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
in_field_def = in_layer.ogr_layer_def.GetFieldDefn(i)
# Only create fields if we really don't have them
# NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE
if out_layer.ogr_layer_def.GetFieldIndex(in_field_def.GetName()) == -1:
out_layer.ogr_layer.CreateField(in_field_def)
log.info('Processing feature: {}/{}'.format(fccount, len(feature_class_paths)))
for feature, _counter, progbar in in_layer.iterate_features('Processing feature'):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
geom.Transform(transform)
out_feature = ogr.Feature(out_layer.ogr_layer_def)
for i in range(in_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_feature.SetGeometry(geom)
out_layer.ogr_layer.CreateFeature(out_feature)
log.info('Merge complete.')
return fccount
def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict:
"""[summary]
Args:
in_layer_path (str): [description]
id_field (str, optional): [description]. Defaults to None.
epsg (int, optional): [description]. Defaults to None.
spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
dict: [description]
"""
log = Logger('load_geometries')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
# Determine the transformation if user provides an EPSG
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
features = {}
for feature, _counter, progbar in in_layer.iterate_features("Loading features"):
if id_field is None:
reach = feature.GetFID()
else:
reach = feature.GetField(id_field)
geom = feature.GetGeometryRef()
geo_type = geom.GetGeometryType()
new_geom = VectorBase.ogr2shapely(geom, transform=transform)
if new_geom.is_empty:
progbar.erase() # get around the progressbar
log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
elif not new_geom.is_valid:
progbar.erase() # get around the progressbar
log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for feature with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for feature with FID={}'.format(feature.GetFID()))
else:
features[reach] = new_geom
return features
def load_attributes(in_layer_path: str, id_field: str, fields: list) -> dict:
"""
Load ShapeFile attributes fields into a dictionary keyed by the id_field
:param network: Full, absolute path to a ShapeFile
:param id_field: Field that uniquely identifies each feature
:param fields: List of fields to load into the dictionary
:return: Dictionary with id_field as key and each feature as dictionary of values keyed by the field name
"""
# Verify that all the fields are present or throw an exception
with get_shp_or_gpkg(in_layer_path) as in_layer:
[in_layer.verify_field(field) for field in fields]
# Only calculate the combined FIS where all the inputs exist
# [networkLr.SetAttributeFilter('{} is not null'.format(field)) for field in [veg_field, drain_field, hydq2_field, hydlow_field, length_field, slope_field]]
# layer.SetAttributeFilter("iGeo_Slope > 0 and iGeo_DA > 0")
print('{:,} features in polygon ShapeFile {}'.format(in_layer.ogr_layer.GetFeatureCount(), in_layer.filepath))
feature_values = {}
for feature, _counter, _progbar in in_layer.iterate_features("loading attributes"):
reach = feature.GetField(id_field)
feature_values[reach] = {}
for field in fields:
feature_values[reach][field] = feature.GetField(field)
return feature_values
def write_attributes(in_layer_path: str, output_values: dict, id_field: str, fields, field_type=ogr.OFTReal, null_values=None):
"""
Write field values to a feature class
:param feature_class: Path to feature class
:param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names
:param id_field: Unique key identifying each feature in both feature class and output_values dictionary
:param fields: List of fields in output_values to write to
:return: None
"""
log = Logger('write_attributes')
with get_shp_or_gpkg(in_layer_path, write=True) as in_layer:
# Create each field and store the name and index in a list of tuples
field_indices = [(field, in_layer.create_field(field, field_type)) for field in fields] # TODO different field types
for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
reach = feature.GetField(id_field) # TODO Error when id_field is same as FID field .GetFID() seems to work instead
if reach not in output_values:
continue
# Set all the field values and then store the feature
for field, _idx in field_indices:
if field in output_values[reach]:
if not output_values[reach][field]:
if null_values:
feature.SetField(field, null_values)
else:
log.warning('Unhandled feature class value for None type')
feature.SetField(field, None)
else:
feature.SetField(field, output_values[reach][field])
in_layer.ogr_layer.SetFeature(feature)
def polygonize(raster_path: str, band: int, out_layer_path: str, epsg: int = None):
# mapping between gdal type and ogr field type
type_mapping = {
gdal.GDT_Byte: ogr.OFTInteger,
gdal.GDT_UInt16: ogr.OFTInteger,
gdal.GDT_Int16: ogr.OFTInteger,
gdal.GDT_UInt32: ogr.OFTInteger,
gdal.GDT_Int32: ogr.OFTInteger,
gdal.GDT_Float32: ogr.OFTReal,
gdal.GDT_Float64: ogr.OFTReal,
gdal.GDT_CInt16: ogr.OFTInteger,
gdal.GDT_CInt32: ogr.OFTInteger,
gdal.GDT_CFloat32: ogr.OFTReal,
gdal.GDT_CFloat64: ogr.OFTReal
}
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer:
out_layer.create_layer(ogr.wkbPolygon, epsg=epsg)
src_ds = gdal.Open(raster_path)
src_band = src_ds.GetRasterBand(band)
out_layer.create_field('id', field_type=type_mapping[src_band.DataType])
progbar = ProgressBar(100, 50, "Polygonizing raster")
def poly_progress(progress, _msg, _data):
# double dfProgress, char const * pszMessage=None, void * pData=None
progbar.update(int(progress * 100))
gdal.Polygonize(src_band, src_ds.GetRasterBand(band), out_layer.ogr_layer, 0, [], callback=poly_progress)
progbar.finish()
src_ds = None
def buffer_by_field(in_layer_path: str, out_layer_path, field: str, epsg: int = None, min_buffer=None, centered=False) -> None:
"""generate buffered polygons by value in field
Args:
flowlines (str): feature class of line features to buffer
field (str): field with buffer value
epsg (int): output srs
min_buffer: use this buffer value for field values that are less than this
Returns:
geometry: unioned polygon geometry of buffered lines
"""
log = Logger('buffer_by_field')
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, get_shp_or_gpkg(in_layer_path) as in_layer:
conversion = in_layer.rough_convert_metres_to_vector_units(1)
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer(ogr.wkbPolygon, epsg=epsg, fields=in_layer.get_fields())
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
factor = 0.5 if centered else 1.0
for feature, _counter, progbar in in_layer.iterate_features('Buffering features', write_layers=[out_layer]):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
buffer_dist = feature.GetField(field) * conversion * factor
geom.Transform(transform)
geom_buffer = geom.Buffer(buffer_dist if buffer_dist > min_buffer else min_buffer)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom_buffer)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
def load_geoms(in_lines):
out = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Loading geometry"):
shapely_geom = VectorBase.ogr2shapely(feat)
out.append(shapely_geom)
return out
def network_statistics(label: str, vector_layer_path: str):
log = Logger('network_statistics')
log.info('Network ShapeFile Summary: {}'.format(vector_layer_path))
results = {}
total_length = 0.0
min_length = None
max_length = None
invalid_features = 0
no_geometry = 0
with get_shp_or_gpkg(vector_layer_path) as vector_layer:
# Delete output column from network ShapeFile if it exists and then recreate it
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
results[vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()] = 0
for feature, _counter, _progbar in vector_layer.iterate_features("Calculating Stats"):
geom = feature.GetGeometryRef()
if geom is None:
no_geometry += 1
return
shapely_obj = VectorBase.ogr2shapely(geom)
length = shapely_obj.length
if shapely_obj.is_empty or shapely_obj.is_valid is False:
invalid_features += 1
total_length += length
min_length = length if not min_length or min_length > length else min_length
max_length = length if not max_length or max_length < length else max_length
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
field = vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()
if field not in results:
results[field] = 0
results[field] += 0 if feature.GetField(field) else 1
features = vector_layer.ogr_layer.GetFeatureCount()
results['Feature Count'] = features
results['Invalid Features'] = invalid_features
results['Features without geometry'] = no_geometry
results['Min Length'] = min_length
results['Max Length'] = max_length
results['Avg Length'] = (total_length / features) if features > 0 and total_length != 0 else 0.0
results['Total Length'] = total_length
for key, value in results.items():
if value > 0:
log.info('{}, {} with {:,} NULL values'.format(label, key, value))
return results
def midpoints(in_lines):
out_points = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat in in_lyr.iterate_features('Getting Midpoints'):
geom = feat.GetGeometryRef()
line = VectorBase.ogr2shapely(geom)
out_points.append(RiverPoint(line.interpolate(0.5, True)))
feat = None
return out_points
def centerline_points(
in_lines: Path,
distance: float = 0.0,
transform: Transform = None) -> Dict[int, List[RiverPoint]]:
"""Generates points along each line feature at specified distances from the end as well as quarter and halfway
Args:
in_lines (Path): path of shapefile with features
distance (float, optional): distance from ends to generate points. Defaults to 0.0.
transform (Transform, optional): coordinate transformation. Defaults to None.
Returns:
[type]: [description]
"""
log = Logger('centerline_points')
with get_shp_or_gpkg(in_lines) as in_lyr:
out_group = {}
ogr_extent = in_lyr.ogr_layer.GetExtent()
extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2],
ogr_extent[1], ogr_extent[3])
for feat, _counter, progbar in in_lyr.iterate_features(
"Centerline points"):
line = VectorBase.ogr2shapely(feat, transform)
fid = feat.GetFID()
out_points = []
# Attach the FID in case we need it later
props = {'fid': fid}
pts = [
line.interpolate(distance),
line.interpolate(0.5, True),
line.interpolate(-distance)
]
if line.project(line.interpolate(0.25, True)) > distance:
pts.append(line.interpolate(0.25, True))
pts.append(line.interpolate(-0.25, True))
for pt in pts:
# Recall that interpolation can have multiple solutions due to pythagorean theorem
# Throw away anything that's not inside our bounds
if not extent.contains(pt):
progbar.erase()
log.warning('Point {} is outside of extent: {}'.format(
pt.coords[0], ogr_extent))
out_points.append(RiverPoint(pt, properties=props))
out_group[int(fid)] = out_points
feat = None
return out_group
def process_reaches(in_path: str,
out_path: str,
attribute_filter=None,
transform=None,
clip_shape=None):
"""[summary]
Args:
in_path (str): [description]
out_path (str): [description]
attribute_filter ([type], optional): [description]. Defaults to None.
transform ([type], optional): [description]. Defaults to None.
clip_shape ([type], optional): [description]. Defaults to None.
"""
with get_shp_or_gpkg(in_path) as in_lyr, get_shp_or_gpkg(
out_path, write=True) as out_lyr:
for feature, _counter, _progbar in in_lyr.iterate_features(
"Processing reaches",
attribute_filter=attribute_filter,
clip_shape=clip_shape):
# get the input geometry and reproject the coordinates
geom = feature.GetGeometryRef()
if transform is not None:
geom.Transform(transform)
# Create output Feature
out_feature = ogr.Feature(out_lyr.ogr_layer_def)
# Add field values from input Layer
for i in range(0, out_lyr.ogr_layer_def.GetFieldCount()):
field_name = out_lyr.ogr_layer_def.GetFieldDefn(i).GetNameRef()
output_field_index = feature.GetFieldIndex(field_name)
if output_field_index >= 0:
out_feature.SetField(field_name,
feature.GetField(output_field_index))
# Add new feature to output Layer
out_feature.SetGeometry(geom)
out_lyr.ogr_layer.CreateFeature(out_feature)
def get_riverpoints(inpath, epsg, attribute_filter=None):
"""[summary]
Args:
inpath ([type]): Path to a ShapeFile
epsg ([type]): Desired output spatial reference
attribute_filter ([type], optional): [description]. Defaults to None.
Returns:
[type]: List of RiverPoint objects
"""
log = Logger('get_riverpoints')
points = []
with get_shp_or_gpkg(inpath) as in_lyr:
_out_spatial_ref, transform = get_transform_from_epsg(
in_lyr.spatial_ref, epsg)
for feat, _counter, progbar in in_lyr.iterate_features(
'Getting points for use in Thiessen',
attribute_filter=attribute_filter):
new_geom = feat.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feat.GetFID()))
continue
new_geom.Transform(transform)
new_shape = VectorBase.ogr2shapely(new_geom)
if new_shape.type == 'Polygon':
new_shape = MultiPolygon([new_shape])
for poly in new_shape:
# Exterior is the shell and there is only ever 1
for pt in list(poly.exterior.coords):
points.append(RiverPoint(pt, interior=False))
# Now we consider interiors. NB: Interiors are only qualifying islands in this case
for idx, island in enumerate(poly.interiors):
for pt in list(island.coords):
points.append(RiverPoint(pt, interior=True,
island=idx))
return points
def collect_feature_class(feature_class_path: str,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> ogr.Geometry:
"""Collect simple types into Multi types. Does not use Shapely
Args:
feature_class_path (str): [description]
attribute_filter (str, optional): Attribute Query like "HUC = 17060104". Defaults to None.
clip_shape (BaseGeometry, optional): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Raises:
Exception: [description]
Returns:
ogr.Geometry: [description]
"""
log = Logger('collect_feature_class')
log.info('Collecting {} feature class.'.format(len(feature_class_path)))
with get_shp_or_gpkg(feature_class_path) as in_lyr:
in_geom_type = in_lyr.ogr_layer.GetGeomType()
output_geom_type = None
for tp, varr in VectorBase.MULTI_TYPES.items():
if in_geom_type in varr:
output_geom_type = tp
break
if output_geom_type is None:
raise Exception('collect_feature_class: Type "{}" not supported'.format(ogr.GeometryTypeToName(in_geom_type)))
new_geom = ogr.Geometry(output_geom_type)
for feat, _counter, _progbar in in_lyr.iterate_features('Collecting Geometry', attribute_filter=attribute_filter, clip_rect=clip_rect, clip_shape=clip_shape):
geom = feat.GetGeometryRef()
if geom.IsValid() and not geom.IsEmpty():
if geom.IsMeasured() > 0 or geom.Is3D() > 0:
geom.FlattenTo2D()
# Do the flatten first to speed up the potential transform
if geom.GetGeometryType() in VectorBase.MULTI_TYPES.keys():
sub_geoms = list(geom)
else:
sub_geoms = [geom]
for subg in sub_geoms:
new_geom.AddGeometry(subg)
log.info('Collect complete.')
return new_geom
def vbet_network(flow_lines_path: str,
flow_areas_path: str,
out_path: str,
epsg: int = None,
fcodes: List[str] = None):
log = Logger('VBET Network')
log.info('Generating perennial network')
fcodes = ["46006"] if fcodes is None else fcodes
with get_shp_or_gpkg(out_path, write=True) as vbet_net, \
get_shp_or_gpkg(flow_lines_path) as flow_lines_lyr:
# Add input Layer Fields to the output Layer if it is the one we want
vbet_net.create_layer_from_ref(flow_lines_lyr, epsg=epsg)
# Perennial features
log.info('Incorporating perennial features')
fcode_filter = "FCode = " + " or FCode = ".join([
f"'{fcode}'" for fcode in fcodes
]) if len(
fcodes) > 0 else "" # e.g. "FCode = '46006' or FCode = '55800'"
fids = include_features(flow_lines_lyr, vbet_net, fcode_filter)
# Flow area features
polygon = get_geometry_unary_union(flow_areas_path, epsg=epsg)
if polygon is not None:
log.info('Incorporating flow areas.')
include_features(flow_lines_lyr,
vbet_net,
"FCode <> '46006'",
polygon,
excluded_fids=fids)
fcount = flow_lines_lyr.ogr_layer.GetFeatureCount()
log.info('VBET network generated with {} features'.format(fcount))
def get_geometry_union(in_layer_path: str, epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""[summary]
Args:
in_layer_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_union')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
geom = None
for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
if feature.GetGeometryRef() is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
new_shape = VectorBase.ogr2shapely(feature, transform=transform)
try:
geom = geom.union(new_shape) if geom is not None else new_shape
except Exception:
progbar.erase() # get around the progressbar
log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID()))
return geom
def centerline_vertex_between_distance(in_lines, distance=0.0):
out_group = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Centerline points between distance"):
line = VectorBase.ogr2shapely(feat)
out_points = []
out_points.append(RiverPoint(line.interpolate(distance)))
out_points.append(RiverPoint(line.interpolate(-distance)))
max_distance = line.length - distance
for vertex in list(line.coords):
test_dist = line.project(Point(vertex))
if test_dist > distance and test_dist < max_distance:
out_points.append(RiverPoint(Point(vertex)))
out_group.append(out_points)
feat = None
return out_group
def build_network(flowlines_path: str,
flowareas_path: str,
out_path: str,
epsg: int = None,
reach_codes: List[str] = None,
waterbodies_path: str = None,
waterbody_max_size=None,
create_layer: bool = True):
"""[summary]
Args:
flowlines_path (str): [description]
flowareas_path (str): [description]
out_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
reach_codes (List[str], optional): [description]. Defaults to None.
waterbodies_path (str, optional): [description]. Defaults to None.
waterbody_max_size ([type], optional): [description]. Defaults to None.
create_layer (bool, optional): [description]. Defaults to True.
Returns:
[type]: [description]
"""
log = Logger('Build Network')
log.info("Building network from flow lines {0}".format(flowlines_path))
if reach_codes:
for r in reach_codes:
log.info('Retaining {} reaches with code {}'.format(
FCodeValues[int(r)], r))
else:
log.info('Retaining all reaches. No reach filtering.')
# Get the transformation required to convert to the target spatial reference
if epsg is not None:
with get_shp_or_gpkg(flowareas_path) as flowareas_lyr:
out_spatial_ref, transform = VectorBase.get_transform_from_epsg(
flowareas_lyr.spatial_ref, epsg)
# Process all perennial/intermittment/ephemeral reaches first
attribute_filter = None
if reach_codes and len(reach_codes) > 0:
_result = [
log.info("{0} {1} network features (FCode {2})".format(
'Retaining', FCodeValues[int(key)], key))
for key in reach_codes
]
attribute_filter = "FCode IN ({0})".format(','.join(
[key for key in reach_codes]))
if create_layer is True:
with get_shp_or_gpkg(flowlines_path) as flowlines_lyr, get_shp_or_gpkg(
out_path, write=True) as out_lyr:
out_lyr.create_layer_from_ref(flowlines_lyr)
log.info('Processing all reaches')
process_reaches(flowlines_path,
out_path,
attribute_filter=attribute_filter)
# Process artifical paths through small waterbodies
if waterbodies_path is not None and waterbody_max_size is not None:
small_waterbodies = get_geometry_unary_union(
waterbodies_path, epsg,
'AreaSqKm <= ({0})'.format(waterbody_max_size))
log.info(
'Retaining artificial features within waterbody features smaller than {0}km2'
.format(waterbody_max_size))
process_reaches(
flowlines_path,
out_path,
transform=transform,
attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES),
clip_shape=small_waterbodies)
# Retain artifical paths through flow areas
if flowareas_path:
flow_polygons = get_geometry_unary_union(flowareas_path, epsg)
if flow_polygons:
log.info('Retaining artificial features within flow area features')
process_reaches(
flowlines_path,
out_path,
transform=transform,
attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES),
clip_shape=flow_polygons)
else:
log.info('Zero artifical paths to be retained.')
with get_shp_or_gpkg(out_path) as out_lyr:
log.info(('{:,} features written to {:}'.format(
out_lyr.ogr_layer.GetFeatureCount(), out_path)))
log.info('Process completed successfully.')
return out_spatial_ref
def rs_segmentation(nhd_flowlines_path: str, roads_path: str,
railways_path: str, ownership_path: str, out_gpkg: str,
interval: float, minimum: float, watershed_id: str):
"""Segment the network in a few different ways
Args:
nhd_flowlines_path (str): Path to shapefile or geopackage containing the original network
roads_path (str): Roads linestring shapefile or geopackage
railways_path (str): Rails lienstring shapefile or geopackage
ownership_path (str): Ownership polygon shapefile or geopackage
out_gpkg (str): Output geopackage for all the output layers
interval (float): Preferred segmentation distance split
minimum (float): Minimum possible segment size
watershed_id (str): Watershed ID
"""
log = Logger('rs_segmentation')
# First make a copy of the network.
# TODO: When we migrate to geopackages we may need to revisit this.
log.info('Copying raw network')
network_copy_path = os.path.join(out_gpkg, 'network')
copy_feature_class(nhd_flowlines_path, network_copy_path)
# Segment the raw network without doing any intersections
log.info('Segmenting the raw network')
segment_network(network_copy_path,
os.path.join(out_gpkg, 'network_300m'),
interval,
minimum,
watershed_id,
create_layer=True)
# If a point needs to be split we store the split pieces here
split_feats = {}
# Intersection points are useful in other tools so we keep them
intersect_pts = {}
log.info('Finding road intersections')
intersect_pts['roads'] = split_geoms(network_copy_path, roads_path,
split_feats)
log.info('Finding rail intersections')
intersect_pts['rail'] = split_geoms(network_copy_path, railways_path,
split_feats)
# With ownership we need to convert polygons to polylines (linestrings) to get the crossing points
# We can't use intersect_geometry_with_feature_class for this so we need to do something a little more manual
log.info('Finding ownership intersections')
ownership_lines_path = os.path.join(out_gpkg, "ownership_lines")
with GeopackageLayer(ownership_lines_path,
write=True) as out_layer, get_shp_or_gpkg(
ownership_path) as own_lyr:
out_layer.create_layer(ogr.wkbLineString,
spatial_ref=own_lyr.spatial_ref)
network_owener_collect = collect_feature_class(network_copy_path)
for feat, _counter, _progbar in own_lyr.iterate_features(
'Converting ownership polygons to polylines',
clip_shape=network_owener_collect):
geom = feat.GetGeometryRef()
# Check that this feature has valid geometry. Really important since ownership shape layers are
# Usually pretty messy.
if geom.IsValid() and not geom.IsEmpty():
# Flatten to 2D first to speed up the potential transform
if geom.IsMeasured() > 0 or geom.Is3D() > 0:
geom.FlattenTo2D()
# Get the boundary linestring
boundary = geom.GetBoundary()
b_type = boundary.GetGeometryType()
# If the boundary is a multilinestring that's fine
if b_type == ogr.wkbMultiLineString:
pass
# if it's just one linestring we make it a multilinestring of one.
elif b_type == ogr.wkbLineString:
boundary = [boundary]
else:
raise Exception('Unsupported type: {}'.format(
ogr.GeometryTypeToName(b_type)))
# Now write each individual linestring back to our output layer
for b_line in boundary:
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(b_line)
out_layer.ogr_layer.CreateFeature(out_feature)
# Now, finally, we're ready to do the actual intersection and splitting
intersect_pts['ownership'] = split_geoms(network_copy_path,
ownership_lines_path, split_feats)
# Let's write our crossings to layers for later use. This can be used in BRAT or our other tools
with GeopackageLayer(out_gpkg, layer_name='network_crossings', write=True) as out_lyr, \
GeopackageLayer(network_copy_path) as in_lyr:
out_lyr.create_layer(ogr.wkbPoint,
spatial_ref=in_lyr.spatial_ref,
fields={'type': ogr.OFTString})
for geom_type_name, ogr_geom in intersect_pts.items():
for pt in list(ogr_geom):
out_feature = ogr.Feature(out_lyr.ogr_layer_def)
out_feature.SetGeometry(GeopackageLayer.shapely2ogr(pt))
out_feature.SetField('type', geom_type_name)
out_lyr.ogr_layer.CreateFeature(out_feature)
# We're done with the original. Let that memory go.
intersect_pts = None
# Now, finally, write all the shapes, substituting splits where necessary
network_crossings_path = os.path.join(out_gpkg, 'network_intersected')
with GeopackageLayer(network_crossings_path, write=True) as out_lyr, \
GeopackageLayer(network_copy_path) as net_lyr:
out_lyr.create_layer_from_ref(net_lyr)
fcounter = 0
for feat, _counter, _progbar in net_lyr.iterate_features(
'Writing split features'):
fid = feat.GetFID()
# If a split happened then write the split geometries to the file.
if fid in split_feats:
for split_geom in split_feats[fid]:
new_feat = feat.Clone()
new_feat.SetFID(fcounter)
new_feat.SetGeometry(
GeopackageLayer.shapely2ogr(split_geom))
out_lyr.ogr_layer.CreateFeature(new_feat)
fcounter += 1
# If no split was found, write the feature as-is
else:
new_feat = feat.Clone()
new_feat.SetFID(fcounter)
out_lyr.ogr_layer.CreateFeature(new_feat)
fcounter += 1
# Finally, segment this new layer the same way we did the raw network above.
log.info('Segmenting the intersected network')
segment_network(network_crossings_path,
os.path.join(out_gpkg, 'network_intersected_300m'),
interval,
minimum,
watershed_id,
create_layer=True)
log.info('Segmentation Complete')
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""Load all features from a ShapeFile and union them together into a single geometry
Args:
in_layer_path (str): path to layer
epsg (int, optional): EPSG to project to. Defaults to None.
spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None.
attribute_filter (str, optional): Filter to a set of attributes. Defaults to None.
clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_unary_union')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
geom_list = []
for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
new_geom = feature.GetGeometryRef()
geo_type = new_geom.GetGeometryType()
# We can't union non-valid shapes but sometimes a buffer by 0 can help
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID()))
try:
new_geom = new_geom.Buffer(0)
if not new_geom.IsValid():
log.warning(' Still invalid. Skipping this geometry')
continue
except Exception:
log.warning('Exception raised during buffer 0 technique. skipping this file')
continue
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for shape with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for shape with FID={}'.format(feature.GetFID()))
else:
geom_list.append(VectorBase.ogr2shapely(new_geom, transform))
# IF we get past a certain size then run the union
if len(geom_list) >= 500:
geom_list = [unary_union(geom_list)]
new_geom = None
log.debug('finished iterating with list of size: {}'.format(len(geom_list)))
if len(geom_list) > 1:
log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list)))
# Do a final union to clean up anything that might still be in the list
geom_union = unary_union(geom_list)
elif len(geom_list) == 0:
log.warning('No geometry found to union')
return None
else:
log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list)))
geom_union = geom_list[0]
log.debug(' done')
print_geom_size(log, geom_union)
log.debug('Complete')
# Return a shapely object
return geom_union
def split_geoms(base_feature_path: str, intersect_feature_path: str,
split_feats: Dict[int, List[LineString]]) -> List[Point]:
"""Loop over base_feature_path and split it everywhere we find it intersecting with intersect_feature_path
This creates the splits to be used later
Args:
base_feature_path (str): [description]
intersect_feature_path (str): [description]
split_feats (Dict[List[LineString]]): [description]
Returns:
(List[Point]): Returns all the intersection points.
"""
log = Logger('split_geoms')
log.info('Finding intersections')
# We collect the raw NHD to use as a filter only
base_collection = collect_feature_class(base_feature_path)
# Then we use the same collection method to get a collection of intersected features that are likely to touch
# our base_collection. This seems a bit redundantly redundant but it does speed things up.
intersect_collection = GeopackageLayer.ogr2shapely(
collect_feature_class(intersect_feature_path,
clip_shape=base_collection))
intersection_pts = []
# Now go through using a clip_shape filter and do the actual splits. These features are likely to intersect
# but not guaranteed so we still need to check.
with get_shp_or_gpkg(base_feature_path) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Finding intersections", clip_shape=intersect_collection):
fid = feat.GetFID()
shply_geom = GeopackageLayer.ogr2shapely(feat)
if fid in split_feats:
# If a previous incarnation of split_geoms already split this feature we have to work on the splits.
candidates = split_feats[fid]
else:
candidates = [shply_geom]
new_splits = []
for candidate in candidates:
# This call is not really related to the segmentation but we write it back to a point layer
# for use in other tools.
intersection = candidate.intersection(intersect_collection)
# Split this candidate geometry by the intersect collection
geom_split = split(candidate, intersect_collection)
new_splits += list(geom_split)
# Now add the intersection points to the list
# >1 length means there was an intersection
if len(geom_split) > 1:
if isinstance(intersection, Point):
intersection_pts.append(intersection)
elif isinstance(intersection, MultiPoint):
intersection_pts += list(intersection)
else:
raise Exception('Unhandled type: {}'.format(
intersection.type))
split_feats[fid] = new_splits
return intersection_pts
def segment_network(inpath: str,
outpath: str,
interval: float,
minimum: float,
watershed_id: str,
create_layer=False):
"""
Chop the lines in a polyline feature class at the specified interval unless
this would create a line less than the minimum in which case the line is not segmented.
:param inpath: Original network feature class
:param outpath: Output segmented network feature class
:param interval: Distance at which to segment each line feature (map units)
:param minimum: Minimum length below which lines are not segmented (map units)
:param watershed_id: Give this watershed an id (str)
:param create_layer: This layer may be created earlier. We can choose to create it. Defaults to false (bool)
:return: None
"""
log = Logger('Segment Network')
if interval <= 0:
log.info('Skipping segmentation.')
else:
log.info(
'Segmenting network to {}m, with minimum feature length of {}m'.
format(interval, minimum))
log.info('Segmenting network from {0}'.format(inpath))
# NOTE: Remember to always open the 'write' layer first in case it's the same geopackage
with get_shp_or_gpkg(
outpath, write=True) as out_lyr, get_shp_or_gpkg(inpath) as in_lyr:
# Get the input NHD flow lines layer
srs = in_lyr.spatial_ref
feature_count = in_lyr.ogr_layer.GetFeatureCount()
log.info('Input feature count {:,}'.format(feature_count))
# Get the closest EPSG possible to calculate length
extent_poly = ogr.Geometry(ogr.wkbPolygon)
extent_centroid = extent_poly.Centroid()
utm_epsg = get_utm_zone_epsg(extent_centroid.GetX())
transform_ref, transform = VectorBase.get_transform_from_epsg(
in_lyr.spatial_ref, utm_epsg)
# IN order to get accurate lengths we are going to need to project into some coordinate system
transform_back = osr.CoordinateTransformation(transform_ref, srs)
# Create the output shapefile
if create_layer is True:
out_lyr.create_layer_from_ref(in_lyr)
# We add two features to this
out_lyr.create_fields({
'ReachID': ogr.OFTInteger,
'WatershedID': ogr.OFTString
})
# Retrieve all input features keeping track of which ones have GNIS names or not
named_features = {}
all_features = []
junctions = []
# Omit pipelines with FCode 428**
attribute_filter = 'FCode < 42800 OR FCode > 42899'
log.info('Filtering out pipelines ({})'.format(attribute_filter))
for in_feature, _counter, _progbar in in_lyr.iterate_features(
"Loading Network", attribute_filter=attribute_filter):
# Store relevant items as a tuple:
# (name, FID, StartPt, EndPt, Length, FCode)
s_feat = SegmentFeature(in_feature, transform)
# Add the end points of all lines to a single list
junctions.extend([s_feat.start, s_feat.end])
if not s_feat.name or len(s_feat.name) < 1 or interval <= 0:
# Add features without a GNIS name to list. Also add to list if not segmenting
all_features.append(s_feat)
else:
# Build separate lists for each unique GNIS name
if s_feat.name not in named_features:
named_features[s_feat.name] = [s_feat]
else:
named_features[s_feat.name].append(s_feat)
# Loop over all features with the same GNIS name.
# Only merge them if they meet at a junction where no other lines meet.
log.info('Merging simple features with the same GNIS name...')
for name, features in named_features.items():
log.debug(' {} x{}'.format(name.encode('utf-8'), len(features)))
all_features.extend(features)
log.info(
'{:,} features after merging. Starting segmentation...'.format(
len(all_features)))
# Segment the features at the desired interval
# rid = 0
log.info('Segmenting Network...')
progbar = ProgressBar(len(all_features), 50, "Segmenting")
counter = 0
for orig_feat in all_features:
counter += 1
progbar.update(counter)
old_feat = in_lyr.ogr_layer.GetFeature(orig_feat.fid)
old_geom = old_feat.GetGeometryRef()
# Anything that produces reach shorter than the minimum just gets added. Also just add features if not segmenting
if orig_feat.length_m < (interval + minimum) or interval <= 0:
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
new_ogr_feat.SetGeometry(old_geom)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
else:
# From here on out we use shapely and project to UTM. We'll transform back before writing to disk.
new_geom = old_geom.Clone()
new_geom.Transform(transform)
remaining = LineString(new_geom.GetPoints())
while remaining and remaining.length >= (interval + minimum):
part1shply, part2shply = cut(remaining, interval)
remaining = part2shply
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
geo = ogr.CreateGeometryFromWkt(part1shply.wkt)
geo.Transform(transform_back)
new_ogr_feat.SetGeometry(geo)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
# Add any remaining line to outGeometries
if remaining:
new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def)
copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def,
out_lyr.ogr_layer_def)
# Set the attributes using the values from the delimited text file
new_ogr_feat.SetField("GNIS_NAME", orig_feat.name)
new_ogr_feat.SetField("WatershedID", watershed_id)
geo = ogr.CreateGeometryFromWkt(remaining.wkt)
geo.Transform(transform_back)
new_ogr_feat.SetGeometry(geo)
out_lyr.ogr_layer.CreateFeature(new_ogr_feat)
# rid += 1
progbar.finish()
log.info(('{:,} features written to {:}'.format(
out_lyr.ogr_layer.GetFeatureCount(), outpath)))
log.info('Process completed successfully.')
def copy_feature_class(in_layer_path: str, out_layer_path: str,
epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None,
buffer: float = 0,
) -> None:
"""Copy a Shapefile from one location to another
This method is capable of reprojecting the geometries as they are copied.
It is also possible to filter features by both attributes and also clip the
features to another geometryNone
Args:
in_layer (str): Input layer path
epsg ([type]): EPSG Code to use for the transformation
out_layer (str): Output layer path
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
buffer (float): Buffer the output features (in meters).
"""
log = Logger('copy_feature_class')
# NOTE: open the outlayer first so that write gets the dataset open priority
with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, \
get_shp_or_gpkg(in_layer_path) as in_layer:
# Add input Layer Fields to the output Layer if it is the one we want
out_layer.create_layer_from_ref(in_layer, epsg=epsg)
transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref)
buffer_convert = 0
if buffer != 0:
buffer_convert = in_layer.rough_convert_metres_to_vector_units(buffer)
# This is the callback method that will be run on each feature
for feature, _counter, progbar in in_layer.iterate_features("Copying features", write_layers=[out_layer], clip_shape=clip_shape, clip_rect=clip_rect, attribute_filter=attribute_filter):
geom = feature.GetGeometryRef()
if geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
if geom.GetGeometryType() in VectorBase.LINE_TYPES:
if geom.Length() == 0.0:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no Length. Skipping'.format(feature.GetFID()))
continue
# Buffer the shape if we need to
if buffer_convert != 0:
geom = geom.Buffer(buffer_convert)
geom.Transform(transform)
# Create output Feature
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(geom)
# Add field values from input Layer
for i in range(0, out_layer.ogr_layer_def.GetFieldCount()):
out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i))
out_layer.ogr_layer.CreateFeature(out_feature)
out_feature = None
