def merge_feature_classes(feature_class_paths: List[str], boundary: BaseGeometry, out_layer_path: str): """[summary] Args: feature_class_paths (List[str]): [description] boundary (BaseGeometry): [description] out_layer_path (str): [description] """ log = Logger('merge_feature_classes') log.info('Merging {} feature classes.'.format(len(feature_class_paths))) with get_shp_or_gpkg(out_layer_path, write=True) as out_layer: fccount = 0 for in_layer_path in feature_class_paths: fccount += 1 log.info("Merging feature class {}/{}".format(fccount, len(feature_class_paths))) with get_shp_or_gpkg(in_layer_path) as in_layer: in_layer.SetSpatialFilter(VectorBase.shapely2ogr(boundary)) # First input spatial ref sets the SRS for the output file transform = in_layer.get_transform(out_layer) for i in range(in_layer.ogr_layer_def.GetFieldCount()): in_field_def = in_layer.ogr_layer_def.GetFieldDefn(i) # Only create fields if we really don't have them # NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE if out_layer.ogr_layer_def.GetFieldIndex(in_field_def.GetName()) == -1: out_layer.ogr_layer.CreateField(in_field_def) log.info('Processing feature: {}/{}'.format(fccount, len(feature_class_paths))) for feature, _counter, progbar in in_layer.iterate_features('Processing feature'): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue geom.Transform(transform) out_feature = ogr.Feature(out_layer.ogr_layer_def) for i in range(in_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_feature.SetGeometry(geom) out_layer.ogr_layer.CreateFeature(out_feature) log.info('Merge complete.') return fccount
def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict: """[summary] Args: in_layer_path (str): [description] id_field (str, optional): [description]. Defaults to None. epsg (int, optional): [description]. Defaults to None. spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None. Raises: VectorBaseException: [description] Returns: dict: [description] """ log = Logger('load_geometries') if epsg is not None and spatial_ref is not None: raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both') with get_shp_or_gpkg(in_layer_path) as in_layer: # Determine the transformation if user provides an EPSG transform = None if epsg is not None: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) elif spatial_ref is not None: transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref) features = {} for feature, _counter, progbar in in_layer.iterate_features("Loading features"): if id_field is None: reach = feature.GetFID() else: reach = feature.GetField(id_field) geom = feature.GetGeometryRef() geo_type = geom.GetGeometryType() new_geom = VectorBase.ogr2shapely(geom, transform=transform) if new_geom.is_empty: progbar.erase() # get around the progressbar log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID())) elif not new_geom.is_valid: progbar.erase() # get around the progressbar log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID())) # Filter out zero-length lines elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for feature with FID={}'.format(feature.GetFID())) # Filter out zero-area polys elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for feature with FID={}'.format(feature.GetFID())) else: features[reach] = new_geom return features
def load_attributes(in_layer_path: str, id_field: str, fields: list) -> dict: """ Load ShapeFile attributes fields into a dictionary keyed by the id_field :param network: Full, absolute path to a ShapeFile :param id_field: Field that uniquely identifies each feature :param fields: List of fields to load into the dictionary :return: Dictionary with id_field as key and each feature as dictionary of values keyed by the field name """ # Verify that all the fields are present or throw an exception with get_shp_or_gpkg(in_layer_path) as in_layer: [in_layer.verify_field(field) for field in fields] # Only calculate the combined FIS where all the inputs exist # [networkLr.SetAttributeFilter('{} is not null'.format(field)) for field in [veg_field, drain_field, hydq2_field, hydlow_field, length_field, slope_field]] # layer.SetAttributeFilter("iGeo_Slope > 0 and iGeo_DA > 0") print('{:,} features in polygon ShapeFile {}'.format(in_layer.ogr_layer.GetFeatureCount(), in_layer.filepath)) feature_values = {} for feature, _counter, _progbar in in_layer.iterate_features("loading attributes"): reach = feature.GetField(id_field) feature_values[reach] = {} for field in fields: feature_values[reach][field] = feature.GetField(field) return feature_values
def write_attributes(in_layer_path: str, output_values: dict, id_field: str, fields, field_type=ogr.OFTReal, null_values=None): """ Write field values to a feature class :param feature_class: Path to feature class :param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names :param id_field: Unique key identifying each feature in both feature class and output_values dictionary :param fields: List of fields in output_values to write to :return: None """ log = Logger('write_attributes') with get_shp_or_gpkg(in_layer_path, write=True) as in_layer: # Create each field and store the name and index in a list of tuples field_indices = [(field, in_layer.create_field(field, field_type)) for field in fields] # TODO different field types for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]): reach = feature.GetField(id_field) # TODO Error when id_field is same as FID field .GetFID() seems to work instead if reach not in output_values: continue # Set all the field values and then store the feature for field, _idx in field_indices: if field in output_values[reach]: if not output_values[reach][field]: if null_values: feature.SetField(field, null_values) else: log.warning('Unhandled feature class value for None type') feature.SetField(field, None) else: feature.SetField(field, output_values[reach][field]) in_layer.ogr_layer.SetFeature(feature)
def polygonize(raster_path: str, band: int, out_layer_path: str, epsg: int = None): # mapping between gdal type and ogr field type type_mapping = { gdal.GDT_Byte: ogr.OFTInteger, gdal.GDT_UInt16: ogr.OFTInteger, gdal.GDT_Int16: ogr.OFTInteger, gdal.GDT_UInt32: ogr.OFTInteger, gdal.GDT_Int32: ogr.OFTInteger, gdal.GDT_Float32: ogr.OFTReal, gdal.GDT_Float64: ogr.OFTReal, gdal.GDT_CInt16: ogr.OFTInteger, gdal.GDT_CInt32: ogr.OFTInteger, gdal.GDT_CFloat32: ogr.OFTReal, gdal.GDT_CFloat64: ogr.OFTReal } with get_shp_or_gpkg(out_layer_path, write=True) as out_layer: out_layer.create_layer(ogr.wkbPolygon, epsg=epsg) src_ds = gdal.Open(raster_path) src_band = src_ds.GetRasterBand(band) out_layer.create_field('id', field_type=type_mapping[src_band.DataType]) progbar = ProgressBar(100, 50, "Polygonizing raster") def poly_progress(progress, _msg, _data): # double dfProgress, char const * pszMessage=None, void * pData=None progbar.update(int(progress * 100)) gdal.Polygonize(src_band, src_ds.GetRasterBand(band), out_layer.ogr_layer, 0, [], callback=poly_progress) progbar.finish() src_ds = None
def buffer_by_field(in_layer_path: str, out_layer_path, field: str, epsg: int = None, min_buffer=None, centered=False) -> None: """generate buffered polygons by value in field Args: flowlines (str): feature class of line features to buffer field (str): field with buffer value epsg (int): output srs min_buffer: use this buffer value for field values that are less than this Returns: geometry: unioned polygon geometry of buffered lines """ log = Logger('buffer_by_field') with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, get_shp_or_gpkg(in_layer_path) as in_layer: conversion = in_layer.rough_convert_metres_to_vector_units(1) # Add input Layer Fields to the output Layer if it is the one we want out_layer.create_layer(ogr.wkbPolygon, epsg=epsg, fields=in_layer.get_fields()) transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref) factor = 0.5 if centered else 1.0 for feature, _counter, progbar in in_layer.iterate_features('Buffering features', write_layers=[out_layer]): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue buffer_dist = feature.GetField(field) * conversion * factor geom.Transform(transform) geom_buffer = geom.Buffer(buffer_dist if buffer_dist > min_buffer else min_buffer) # Create output Feature out_feature = ogr.Feature(out_layer.ogr_layer_def) out_feature.SetGeometry(geom_buffer) # Add field values from input Layer for i in range(0, out_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_layer.ogr_layer.CreateFeature(out_feature) out_feature = None
def load_geoms(in_lines): out = [] with get_shp_or_gpkg(in_lines) as in_lyr: for feat, _counter, _progbar in in_lyr.iterate_features( "Loading geometry"): shapely_geom = VectorBase.ogr2shapely(feat) out.append(shapely_geom) return out
def network_statistics(label: str, vector_layer_path: str): log = Logger('network_statistics') log.info('Network ShapeFile Summary: {}'.format(vector_layer_path)) results = {} total_length = 0.0 min_length = None max_length = None invalid_features = 0 no_geometry = 0 with get_shp_or_gpkg(vector_layer_path) as vector_layer: # Delete output column from network ShapeFile if it exists and then recreate it for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()): results[vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()] = 0 for feature, _counter, _progbar in vector_layer.iterate_features("Calculating Stats"): geom = feature.GetGeometryRef() if geom is None: no_geometry += 1 return shapely_obj = VectorBase.ogr2shapely(geom) length = shapely_obj.length if shapely_obj.is_empty or shapely_obj.is_valid is False: invalid_features += 1 total_length += length min_length = length if not min_length or min_length > length else min_length max_length = length if not max_length or max_length < length else max_length for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()): field = vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName() if field not in results: results[field] = 0 results[field] += 0 if feature.GetField(field) else 1 features = vector_layer.ogr_layer.GetFeatureCount() results['Feature Count'] = features results['Invalid Features'] = invalid_features results['Features without geometry'] = no_geometry results['Min Length'] = min_length results['Max Length'] = max_length results['Avg Length'] = (total_length / features) if features > 0 and total_length != 0 else 0.0 results['Total Length'] = total_length for key, value in results.items(): if value > 0: log.info('{}, {} with {:,} NULL values'.format(label, key, value)) return results
def midpoints(in_lines): out_points = [] with get_shp_or_gpkg(in_lines) as in_lyr: for feat in in_lyr.iterate_features('Getting Midpoints'): geom = feat.GetGeometryRef() line = VectorBase.ogr2shapely(geom) out_points.append(RiverPoint(line.interpolate(0.5, True))) feat = None return out_points
def centerline_points( in_lines: Path, distance: float = 0.0, transform: Transform = None) -> Dict[int, List[RiverPoint]]: """Generates points along each line feature at specified distances from the end as well as quarter and halfway Args: in_lines (Path): path of shapefile with features distance (float, optional): distance from ends to generate points. Defaults to 0.0. transform (Transform, optional): coordinate transformation. Defaults to None. Returns: [type]: [description] """ log = Logger('centerline_points') with get_shp_or_gpkg(in_lines) as in_lyr: out_group = {} ogr_extent = in_lyr.ogr_layer.GetExtent() extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2], ogr_extent[1], ogr_extent[3]) for feat, _counter, progbar in in_lyr.iterate_features( "Centerline points"): line = VectorBase.ogr2shapely(feat, transform) fid = feat.GetFID() out_points = [] # Attach the FID in case we need it later props = {'fid': fid} pts = [ line.interpolate(distance), line.interpolate(0.5, True), line.interpolate(-distance) ] if line.project(line.interpolate(0.25, True)) > distance: pts.append(line.interpolate(0.25, True)) pts.append(line.interpolate(-0.25, True)) for pt in pts: # Recall that interpolation can have multiple solutions due to pythagorean theorem # Throw away anything that's not inside our bounds if not extent.contains(pt): progbar.erase() log.warning('Point {} is outside of extent: {}'.format( pt.coords[0], ogr_extent)) out_points.append(RiverPoint(pt, properties=props)) out_group[int(fid)] = out_points feat = None return out_group
def process_reaches(in_path: str, out_path: str, attribute_filter=None, transform=None, clip_shape=None): """[summary] Args: in_path (str): [description] out_path (str): [description] attribute_filter ([type], optional): [description]. Defaults to None. transform ([type], optional): [description]. Defaults to None. clip_shape ([type], optional): [description]. Defaults to None. """ with get_shp_or_gpkg(in_path) as in_lyr, get_shp_or_gpkg( out_path, write=True) as out_lyr: for feature, _counter, _progbar in in_lyr.iterate_features( "Processing reaches", attribute_filter=attribute_filter, clip_shape=clip_shape): # get the input geometry and reproject the coordinates geom = feature.GetGeometryRef() if transform is not None: geom.Transform(transform) # Create output Feature out_feature = ogr.Feature(out_lyr.ogr_layer_def) # Add field values from input Layer for i in range(0, out_lyr.ogr_layer_def.GetFieldCount()): field_name = out_lyr.ogr_layer_def.GetFieldDefn(i).GetNameRef() output_field_index = feature.GetFieldIndex(field_name) if output_field_index >= 0: out_feature.SetField(field_name, feature.GetField(output_field_index)) # Add new feature to output Layer out_feature.SetGeometry(geom) out_lyr.ogr_layer.CreateFeature(out_feature)
def get_riverpoints(inpath, epsg, attribute_filter=None): """[summary] Args: inpath ([type]): Path to a ShapeFile epsg ([type]): Desired output spatial reference attribute_filter ([type], optional): [description]. Defaults to None. Returns: [type]: List of RiverPoint objects """ log = Logger('get_riverpoints') points = [] with get_shp_or_gpkg(inpath) as in_lyr: _out_spatial_ref, transform = get_transform_from_epsg( in_lyr.spatial_ref, epsg) for feat, _counter, progbar in in_lyr.iterate_features( 'Getting points for use in Thiessen', attribute_filter=attribute_filter): new_geom = feat.GetGeometryRef() if new_geom is None: progbar.erase() # get around the progressbar log.warning( 'Feature with FID={} has no geometry. Skipping'.format( feat.GetFID())) continue new_geom.Transform(transform) new_shape = VectorBase.ogr2shapely(new_geom) if new_shape.type == 'Polygon': new_shape = MultiPolygon([new_shape]) for poly in new_shape: # Exterior is the shell and there is only ever 1 for pt in list(poly.exterior.coords): points.append(RiverPoint(pt, interior=False)) # Now we consider interiors. NB: Interiors are only qualifying islands in this case for idx, island in enumerate(poly.interiors): for pt in list(island.coords): points.append(RiverPoint(pt, interior=True, island=idx)) return points
def collect_feature_class(feature_class_path: str, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None ) -> ogr.Geometry: """Collect simple types into Multi types. Does not use Shapely Args: feature_class_path (str): [description] attribute_filter (str, optional): Attribute Query like "HUC = 17060104". Defaults to None. clip_shape (BaseGeometry, optional): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. Raises: Exception: [description] Returns: ogr.Geometry: [description] """ log = Logger('collect_feature_class') log.info('Collecting {} feature class.'.format(len(feature_class_path))) with get_shp_or_gpkg(feature_class_path) as in_lyr: in_geom_type = in_lyr.ogr_layer.GetGeomType() output_geom_type = None for tp, varr in VectorBase.MULTI_TYPES.items(): if in_geom_type in varr: output_geom_type = tp break if output_geom_type is None: raise Exception('collect_feature_class: Type "{}" not supported'.format(ogr.GeometryTypeToName(in_geom_type))) new_geom = ogr.Geometry(output_geom_type) for feat, _counter, _progbar in in_lyr.iterate_features('Collecting Geometry', attribute_filter=attribute_filter, clip_rect=clip_rect, clip_shape=clip_shape): geom = feat.GetGeometryRef() if geom.IsValid() and not geom.IsEmpty(): if geom.IsMeasured() > 0 or geom.Is3D() > 0: geom.FlattenTo2D() # Do the flatten first to speed up the potential transform if geom.GetGeometryType() in VectorBase.MULTI_TYPES.keys(): sub_geoms = list(geom) else: sub_geoms = [geom] for subg in sub_geoms: new_geom.AddGeometry(subg) log.info('Collect complete.') return new_geom
def vbet_network(flow_lines_path: str, flow_areas_path: str, out_path: str, epsg: int = None, fcodes: List[str] = None): log = Logger('VBET Network') log.info('Generating perennial network') fcodes = ["46006"] if fcodes is None else fcodes with get_shp_or_gpkg(out_path, write=True) as vbet_net, \ get_shp_or_gpkg(flow_lines_path) as flow_lines_lyr: # Add input Layer Fields to the output Layer if it is the one we want vbet_net.create_layer_from_ref(flow_lines_lyr, epsg=epsg) # Perennial features log.info('Incorporating perennial features') fcode_filter = "FCode = " + " or FCode = ".join([ f"'{fcode}'" for fcode in fcodes ]) if len( fcodes) > 0 else "" # e.g. "FCode = '46006' or FCode = '55800'" fids = include_features(flow_lines_lyr, vbet_net, fcode_filter) # Flow area features polygon = get_geometry_unary_union(flow_areas_path, epsg=epsg) if polygon is not None: log.info('Incorporating flow areas.') include_features(flow_lines_lyr, vbet_net, "FCode <> '46006'", polygon, excluded_fids=fids) fcount = flow_lines_lyr.ogr_layer.GetFeatureCount() log.info('VBET network generated with {} features'.format(fcount))
def get_geometry_union(in_layer_path: str, epsg: int = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None ) -> BaseGeometry: """[summary] Args: in_layer_path (str): [description] epsg (int, optional): [description]. Defaults to None. attribute_filter (str, optional): [description]. Defaults to None. clip_shape (BaseGeometry, optional): [description]. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. Returns: BaseGeometry: [description] """ log = Logger('get_geometry_union') with get_shp_or_gpkg(in_layer_path) as in_layer: transform = None if epsg: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) geom = None for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect): if feature.GetGeometryRef() is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue new_shape = VectorBase.ogr2shapely(feature, transform=transform) try: geom = geom.union(new_shape) if geom is not None else new_shape except Exception: progbar.erase() # get around the progressbar log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID())) return geom
def centerline_vertex_between_distance(in_lines, distance=0.0): out_group = [] with get_shp_or_gpkg(in_lines) as in_lyr: for feat, _counter, _progbar in in_lyr.iterate_features( "Centerline points between distance"): line = VectorBase.ogr2shapely(feat) out_points = [] out_points.append(RiverPoint(line.interpolate(distance))) out_points.append(RiverPoint(line.interpolate(-distance))) max_distance = line.length - distance for vertex in list(line.coords): test_dist = line.project(Point(vertex)) if test_dist > distance and test_dist < max_distance: out_points.append(RiverPoint(Point(vertex))) out_group.append(out_points) feat = None return out_group
def build_network(flowlines_path: str, flowareas_path: str, out_path: str, epsg: int = None, reach_codes: List[str] = None, waterbodies_path: str = None, waterbody_max_size=None, create_layer: bool = True): """[summary] Args: flowlines_path (str): [description] flowareas_path (str): [description] out_path (str): [description] epsg (int, optional): [description]. Defaults to None. reach_codes (List[str], optional): [description]. Defaults to None. waterbodies_path (str, optional): [description]. Defaults to None. waterbody_max_size ([type], optional): [description]. Defaults to None. create_layer (bool, optional): [description]. Defaults to True. Returns: [type]: [description] """ log = Logger('Build Network') log.info("Building network from flow lines {0}".format(flowlines_path)) if reach_codes: for r in reach_codes: log.info('Retaining {} reaches with code {}'.format( FCodeValues[int(r)], r)) else: log.info('Retaining all reaches. No reach filtering.') # Get the transformation required to convert to the target spatial reference if epsg is not None: with get_shp_or_gpkg(flowareas_path) as flowareas_lyr: out_spatial_ref, transform = VectorBase.get_transform_from_epsg( flowareas_lyr.spatial_ref, epsg) # Process all perennial/intermittment/ephemeral reaches first attribute_filter = None if reach_codes and len(reach_codes) > 0: _result = [ log.info("{0} {1} network features (FCode {2})".format( 'Retaining', FCodeValues[int(key)], key)) for key in reach_codes ] attribute_filter = "FCode IN ({0})".format(','.join( [key for key in reach_codes])) if create_layer is True: with get_shp_or_gpkg(flowlines_path) as flowlines_lyr, get_shp_or_gpkg( out_path, write=True) as out_lyr: out_lyr.create_layer_from_ref(flowlines_lyr) log.info('Processing all reaches') process_reaches(flowlines_path, out_path, attribute_filter=attribute_filter) # Process artifical paths through small waterbodies if waterbodies_path is not None and waterbody_max_size is not None: small_waterbodies = get_geometry_unary_union( waterbodies_path, epsg, 'AreaSqKm <= ({0})'.format(waterbody_max_size)) log.info( 'Retaining artificial features within waterbody features smaller than {0}km2' .format(waterbody_max_size)) process_reaches( flowlines_path, out_path, transform=transform, attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES), clip_shape=small_waterbodies) # Retain artifical paths through flow areas if flowareas_path: flow_polygons = get_geometry_unary_union(flowareas_path, epsg) if flow_polygons: log.info('Retaining artificial features within flow area features') process_reaches( flowlines_path, out_path, transform=transform, attribute_filter='FCode = {0}'.format(ARTIFICIAL_REACHES), clip_shape=flow_polygons) else: log.info('Zero artifical paths to be retained.') with get_shp_or_gpkg(out_path) as out_lyr: log.info(('{:,} features written to {:}'.format( out_lyr.ogr_layer.GetFeatureCount(), out_path))) log.info('Process completed successfully.') return out_spatial_ref
def rs_segmentation(nhd_flowlines_path: str, roads_path: str, railways_path: str, ownership_path: str, out_gpkg: str, interval: float, minimum: float, watershed_id: str): """Segment the network in a few different ways Args: nhd_flowlines_path (str): Path to shapefile or geopackage containing the original network roads_path (str): Roads linestring shapefile or geopackage railways_path (str): Rails lienstring shapefile or geopackage ownership_path (str): Ownership polygon shapefile or geopackage out_gpkg (str): Output geopackage for all the output layers interval (float): Preferred segmentation distance split minimum (float): Minimum possible segment size watershed_id (str): Watershed ID """ log = Logger('rs_segmentation') # First make a copy of the network. # TODO: When we migrate to geopackages we may need to revisit this. log.info('Copying raw network') network_copy_path = os.path.join(out_gpkg, 'network') copy_feature_class(nhd_flowlines_path, network_copy_path) # Segment the raw network without doing any intersections log.info('Segmenting the raw network') segment_network(network_copy_path, os.path.join(out_gpkg, 'network_300m'), interval, minimum, watershed_id, create_layer=True) # If a point needs to be split we store the split pieces here split_feats = {} # Intersection points are useful in other tools so we keep them intersect_pts = {} log.info('Finding road intersections') intersect_pts['roads'] = split_geoms(network_copy_path, roads_path, split_feats) log.info('Finding rail intersections') intersect_pts['rail'] = split_geoms(network_copy_path, railways_path, split_feats) # With ownership we need to convert polygons to polylines (linestrings) to get the crossing points # We can't use intersect_geometry_with_feature_class for this so we need to do something a little more manual log.info('Finding ownership intersections') ownership_lines_path = os.path.join(out_gpkg, "ownership_lines") with GeopackageLayer(ownership_lines_path, write=True) as out_layer, get_shp_or_gpkg( ownership_path) as own_lyr: out_layer.create_layer(ogr.wkbLineString, spatial_ref=own_lyr.spatial_ref) network_owener_collect = collect_feature_class(network_copy_path) for feat, _counter, _progbar in own_lyr.iterate_features( 'Converting ownership polygons to polylines', clip_shape=network_owener_collect): geom = feat.GetGeometryRef() # Check that this feature has valid geometry. Really important since ownership shape layers are # Usually pretty messy. if geom.IsValid() and not geom.IsEmpty(): # Flatten to 2D first to speed up the potential transform if geom.IsMeasured() > 0 or geom.Is3D() > 0: geom.FlattenTo2D() # Get the boundary linestring boundary = geom.GetBoundary() b_type = boundary.GetGeometryType() # If the boundary is a multilinestring that's fine if b_type == ogr.wkbMultiLineString: pass # if it's just one linestring we make it a multilinestring of one. elif b_type == ogr.wkbLineString: boundary = [boundary] else: raise Exception('Unsupported type: {}'.format( ogr.GeometryTypeToName(b_type))) # Now write each individual linestring back to our output layer for b_line in boundary: out_feature = ogr.Feature(out_layer.ogr_layer_def) out_feature.SetGeometry(b_line) out_layer.ogr_layer.CreateFeature(out_feature) # Now, finally, we're ready to do the actual intersection and splitting intersect_pts['ownership'] = split_geoms(network_copy_path, ownership_lines_path, split_feats) # Let's write our crossings to layers for later use. This can be used in BRAT or our other tools with GeopackageLayer(out_gpkg, layer_name='network_crossings', write=True) as out_lyr, \ GeopackageLayer(network_copy_path) as in_lyr: out_lyr.create_layer(ogr.wkbPoint, spatial_ref=in_lyr.spatial_ref, fields={'type': ogr.OFTString}) for geom_type_name, ogr_geom in intersect_pts.items(): for pt in list(ogr_geom): out_feature = ogr.Feature(out_lyr.ogr_layer_def) out_feature.SetGeometry(GeopackageLayer.shapely2ogr(pt)) out_feature.SetField('type', geom_type_name) out_lyr.ogr_layer.CreateFeature(out_feature) # We're done with the original. Let that memory go. intersect_pts = None # Now, finally, write all the shapes, substituting splits where necessary network_crossings_path = os.path.join(out_gpkg, 'network_intersected') with GeopackageLayer(network_crossings_path, write=True) as out_lyr, \ GeopackageLayer(network_copy_path) as net_lyr: out_lyr.create_layer_from_ref(net_lyr) fcounter = 0 for feat, _counter, _progbar in net_lyr.iterate_features( 'Writing split features'): fid = feat.GetFID() # If a split happened then write the split geometries to the file. if fid in split_feats: for split_geom in split_feats[fid]: new_feat = feat.Clone() new_feat.SetFID(fcounter) new_feat.SetGeometry( GeopackageLayer.shapely2ogr(split_geom)) out_lyr.ogr_layer.CreateFeature(new_feat) fcounter += 1 # If no split was found, write the feature as-is else: new_feat = feat.Clone() new_feat.SetFID(fcounter) out_lyr.ogr_layer.CreateFeature(new_feat) fcounter += 1 # Finally, segment this new layer the same way we did the raw network above. log.info('Segmenting the intersected network') segment_network(network_crossings_path, os.path.join(out_gpkg, 'network_intersected_300m'), interval, minimum, watershed_id, create_layer=True) log.info('Segmentation Complete')
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None ) -> BaseGeometry: """Load all features from a ShapeFile and union them together into a single geometry Args: in_layer_path (str): path to layer epsg (int, optional): EPSG to project to. Defaults to None. spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None. attribute_filter (str, optional): Filter to a set of attributes. Defaults to None. clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. Raises: VectorBaseException: [description] Returns: BaseGeometry: [description] """ log = Logger('get_geometry_unary_union') if epsg is not None and spatial_ref is not None: raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both') with get_shp_or_gpkg(in_layer_path) as in_layer: transform = None if epsg is not None: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) elif spatial_ref is not None: transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref) geom_list = [] for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect): new_geom = feature.GetGeometryRef() geo_type = new_geom.GetGeometryType() # We can't union non-valid shapes but sometimes a buffer by 0 can help if not new_geom.IsValid(): progbar.erase() # get around the progressbar log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID())) try: new_geom = new_geom.Buffer(0) if not new_geom.IsValid(): log.warning(' Still invalid. Skipping this geometry') continue except Exception: log.warning('Exception raised during buffer 0 technique. skipping this file') continue if new_geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID())) # Filter out zero-length lines elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for shape with FID={}'.format(feature.GetFID())) # Filter out zero-area polys elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for shape with FID={}'.format(feature.GetFID())) else: geom_list.append(VectorBase.ogr2shapely(new_geom, transform)) # IF we get past a certain size then run the union if len(geom_list) >= 500: geom_list = [unary_union(geom_list)] new_geom = None log.debug('finished iterating with list of size: {}'.format(len(geom_list))) if len(geom_list) > 1: log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list))) # Do a final union to clean up anything that might still be in the list geom_union = unary_union(geom_list) elif len(geom_list) == 0: log.warning('No geometry found to union') return None else: log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list))) geom_union = geom_list[0] log.debug(' done') print_geom_size(log, geom_union) log.debug('Complete') # Return a shapely object return geom_union
def split_geoms(base_feature_path: str, intersect_feature_path: str, split_feats: Dict[int, List[LineString]]) -> List[Point]: """Loop over base_feature_path and split it everywhere we find it intersecting with intersect_feature_path This creates the splits to be used later Args: base_feature_path (str): [description] intersect_feature_path (str): [description] split_feats (Dict[List[LineString]]): [description] Returns: (List[Point]): Returns all the intersection points. """ log = Logger('split_geoms') log.info('Finding intersections') # We collect the raw NHD to use as a filter only base_collection = collect_feature_class(base_feature_path) # Then we use the same collection method to get a collection of intersected features that are likely to touch # our base_collection. This seems a bit redundantly redundant but it does speed things up. intersect_collection = GeopackageLayer.ogr2shapely( collect_feature_class(intersect_feature_path, clip_shape=base_collection)) intersection_pts = [] # Now go through using a clip_shape filter and do the actual splits. These features are likely to intersect # but not guaranteed so we still need to check. with get_shp_or_gpkg(base_feature_path) as in_lyr: for feat, _counter, _progbar in in_lyr.iterate_features( "Finding intersections", clip_shape=intersect_collection): fid = feat.GetFID() shply_geom = GeopackageLayer.ogr2shapely(feat) if fid in split_feats: # If a previous incarnation of split_geoms already split this feature we have to work on the splits. candidates = split_feats[fid] else: candidates = [shply_geom] new_splits = [] for candidate in candidates: # This call is not really related to the segmentation but we write it back to a point layer # for use in other tools. intersection = candidate.intersection(intersect_collection) # Split this candidate geometry by the intersect collection geom_split = split(candidate, intersect_collection) new_splits += list(geom_split) # Now add the intersection points to the list # >1 length means there was an intersection if len(geom_split) > 1: if isinstance(intersection, Point): intersection_pts.append(intersection) elif isinstance(intersection, MultiPoint): intersection_pts += list(intersection) else: raise Exception('Unhandled type: {}'.format( intersection.type)) split_feats[fid] = new_splits return intersection_pts
def segment_network(inpath: str, outpath: str, interval: float, minimum: float, watershed_id: str, create_layer=False): """ Chop the lines in a polyline feature class at the specified interval unless this would create a line less than the minimum in which case the line is not segmented. :param inpath: Original network feature class :param outpath: Output segmented network feature class :param interval: Distance at which to segment each line feature (map units) :param minimum: Minimum length below which lines are not segmented (map units) :param watershed_id: Give this watershed an id (str) :param create_layer: This layer may be created earlier. We can choose to create it. Defaults to false (bool) :return: None """ log = Logger('Segment Network') if interval <= 0: log.info('Skipping segmentation.') else: log.info( 'Segmenting network to {}m, with minimum feature length of {}m'. format(interval, minimum)) log.info('Segmenting network from {0}'.format(inpath)) # NOTE: Remember to always open the 'write' layer first in case it's the same geopackage with get_shp_or_gpkg( outpath, write=True) as out_lyr, get_shp_or_gpkg(inpath) as in_lyr: # Get the input NHD flow lines layer srs = in_lyr.spatial_ref feature_count = in_lyr.ogr_layer.GetFeatureCount() log.info('Input feature count {:,}'.format(feature_count)) # Get the closest EPSG possible to calculate length extent_poly = ogr.Geometry(ogr.wkbPolygon) extent_centroid = extent_poly.Centroid() utm_epsg = get_utm_zone_epsg(extent_centroid.GetX()) transform_ref, transform = VectorBase.get_transform_from_epsg( in_lyr.spatial_ref, utm_epsg) # IN order to get accurate lengths we are going to need to project into some coordinate system transform_back = osr.CoordinateTransformation(transform_ref, srs) # Create the output shapefile if create_layer is True: out_lyr.create_layer_from_ref(in_lyr) # We add two features to this out_lyr.create_fields({ 'ReachID': ogr.OFTInteger, 'WatershedID': ogr.OFTString }) # Retrieve all input features keeping track of which ones have GNIS names or not named_features = {} all_features = [] junctions = [] # Omit pipelines with FCode 428** attribute_filter = 'FCode < 42800 OR FCode > 42899' log.info('Filtering out pipelines ({})'.format(attribute_filter)) for in_feature, _counter, _progbar in in_lyr.iterate_features( "Loading Network", attribute_filter=attribute_filter): # Store relevant items as a tuple: # (name, FID, StartPt, EndPt, Length, FCode) s_feat = SegmentFeature(in_feature, transform) # Add the end points of all lines to a single list junctions.extend([s_feat.start, s_feat.end]) if not s_feat.name or len(s_feat.name) < 1 or interval <= 0: # Add features without a GNIS name to list. Also add to list if not segmenting all_features.append(s_feat) else: # Build separate lists for each unique GNIS name if s_feat.name not in named_features: named_features[s_feat.name] = [s_feat] else: named_features[s_feat.name].append(s_feat) # Loop over all features with the same GNIS name. # Only merge them if they meet at a junction where no other lines meet. log.info('Merging simple features with the same GNIS name...') for name, features in named_features.items(): log.debug(' {} x{}'.format(name.encode('utf-8'), len(features))) all_features.extend(features) log.info( '{:,} features after merging. Starting segmentation...'.format( len(all_features))) # Segment the features at the desired interval # rid = 0 log.info('Segmenting Network...') progbar = ProgressBar(len(all_features), 50, "Segmenting") counter = 0 for orig_feat in all_features: counter += 1 progbar.update(counter) old_feat = in_lyr.ogr_layer.GetFeature(orig_feat.fid) old_geom = old_feat.GetGeometryRef() # Anything that produces reach shorter than the minimum just gets added. Also just add features if not segmenting if orig_feat.length_m < (interval + minimum) or interval <= 0: new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def) copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def, out_lyr.ogr_layer_def) # Set the attributes using the values from the delimited text file new_ogr_feat.SetField("GNIS_NAME", orig_feat.name) new_ogr_feat.SetField("WatershedID", watershed_id) new_ogr_feat.SetGeometry(old_geom) out_lyr.ogr_layer.CreateFeature(new_ogr_feat) # rid += 1 else: # From here on out we use shapely and project to UTM. We'll transform back before writing to disk. new_geom = old_geom.Clone() new_geom.Transform(transform) remaining = LineString(new_geom.GetPoints()) while remaining and remaining.length >= (interval + minimum): part1shply, part2shply = cut(remaining, interval) remaining = part2shply new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def) copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def, out_lyr.ogr_layer_def) # Set the attributes using the values from the delimited text file new_ogr_feat.SetField("GNIS_NAME", orig_feat.name) new_ogr_feat.SetField("WatershedID", watershed_id) geo = ogr.CreateGeometryFromWkt(part1shply.wkt) geo.Transform(transform_back) new_ogr_feat.SetGeometry(geo) out_lyr.ogr_layer.CreateFeature(new_ogr_feat) # rid += 1 # Add any remaining line to outGeometries if remaining: new_ogr_feat = ogr.Feature(out_lyr.ogr_layer_def) copy_fields(old_feat, new_ogr_feat, in_lyr.ogr_layer_def, out_lyr.ogr_layer_def) # Set the attributes using the values from the delimited text file new_ogr_feat.SetField("GNIS_NAME", orig_feat.name) new_ogr_feat.SetField("WatershedID", watershed_id) geo = ogr.CreateGeometryFromWkt(remaining.wkt) geo.Transform(transform_back) new_ogr_feat.SetGeometry(geo) out_lyr.ogr_layer.CreateFeature(new_ogr_feat) # rid += 1 progbar.finish() log.info(('{:,} features written to {:}'.format( out_lyr.ogr_layer.GetFeatureCount(), outpath))) log.info('Process completed successfully.')
def copy_feature_class(in_layer_path: str, out_layer_path: str, epsg: int = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None, buffer: float = 0, ) -> None: """Copy a Shapefile from one location to another This method is capable of reprojecting the geometries as they are copied. It is also possible to filter features by both attributes and also clip the features to another geometryNone Args: in_layer (str): Input layer path epsg ([type]): EPSG Code to use for the transformation out_layer (str): Output layer path attribute_filter (str, optional): [description]. Defaults to None. clip_shape (BaseGeometry, optional): [description]. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. buffer (float): Buffer the output features (in meters). """ log = Logger('copy_feature_class') # NOTE: open the outlayer first so that write gets the dataset open priority with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, \ get_shp_or_gpkg(in_layer_path) as in_layer: # Add input Layer Fields to the output Layer if it is the one we want out_layer.create_layer_from_ref(in_layer, epsg=epsg) transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref) buffer_convert = 0 if buffer != 0: buffer_convert = in_layer.rough_convert_metres_to_vector_units(buffer) # This is the callback method that will be run on each feature for feature, _counter, progbar in in_layer.iterate_features("Copying features", write_layers=[out_layer], clip_shape=clip_shape, clip_rect=clip_rect, attribute_filter=attribute_filter): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue if geom.GetGeometryType() in VectorBase.LINE_TYPES: if geom.Length() == 0.0: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no Length. Skipping'.format(feature.GetFID())) continue # Buffer the shape if we need to if buffer_convert != 0: geom = geom.Buffer(buffer_convert) geom.Transform(transform) # Create output Feature out_feature = ogr.Feature(out_layer.ogr_layer_def) out_feature.SetGeometry(geom) # Add field values from input Layer for i in range(0, out_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_layer.ogr_layer.CreateFeature(out_feature) out_feature = None