def write_attributes(in_layer_path: str, output_values: dict, id_field: str, fields, field_type=ogr.OFTReal, null_values=None): """ Write field values to a feature class :param feature_class: Path to feature class :param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names :param id_field: Unique key identifying each feature in both feature class and output_values dictionary :param fields: List of fields in output_values to write to :return: None """ log = Logger('write_attributes') with get_shp_or_gpkg(in_layer_path, write=True) as in_layer: # Create each field and store the name and index in a list of tuples field_indices = [(field, in_layer.create_field(field, field_type)) for field in fields] # TODO different field types for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]): reach = feature.GetField(id_field) # TODO Error when id_field is same as FID field .GetFID() seems to work instead if reach not in output_values: continue # Set all the field values and then store the feature for field, _idx in field_indices: if field in output_values[reach]: if not output_values[reach][field]: if null_values: feature.SetField(field, null_values) else: log.warning('Unhandled feature class value for None type') feature.SetField(field, None) else: feature.SetField(field, output_values[reach][field]) in_layer.ogr_layer.SetFeature(feature)
def get_nhd_states(inpath): """ Gets the list of US States that an NHD HUC encompasses This relies on the watershed boundary ShapeFile having a column called 'States' that stores a comma separated list of state abbreviations such as 'OR,WA'. A dcitionary is used to retrieve the full names. :param inpath: Path to the watershed boundary ShapeFile :return: List of full US state names that the watershed touches (.e.g. Oregon) """ log = Logger('RS Context') driver = ogr.GetDriverByName("ESRI Shapefile") data_source = driver.Open(inpath, 0) layer = data_source.GetLayer() states = [] for feature in layer: value = feature.GetField('States') [states.append(us_states[acronym]) for acronym in value.split(',')] data_source = None if 'Canada' in states: if len(states) == 1: log.error( 'HUC is entirely within Canada. No DEMs will be available.') else: log.warning( 'HUC is partially in Canada. Certain data will only be available for US portion.' ) log.info('HUC intersects {} state(s): {}'.format(len(states), ', '.join(states))) return list(dict.fromkeys(states))
def get_watershed_info(gpkg_path): """Query a BRAT database and get information about the watershed being run. Assumes that all watersheds except the one being run have been deleted. Arguments: database {str} -- Path to the BRAT SQLite database Returns: [tuple] -- WatershedID, max drainage area, EcoregionID with which the watershed is associated. """ with SQLiteCon(gpkg_path) as database: database.curs.execute( 'SELECT WatershedID, MaxDrainage, EcoregionID FROM Watersheds') row = database.curs.fetchone() watershed = row['WatershedID'] max_drainage = row['MaxDrainage'] ecoregion = row['EcoregionID'] log = Logger('BRAT Run') if not watershed: raise Exception( 'Missing watershed in BRAT datatabase {}'.format(database)) if not max_drainage: log.warning('Missing max drainage for watershed {}'.format(watershed)) if not ecoregion: raise Exception('Missing ecoregion for watershed {}'.format(watershed)) return watershed, max_drainage, ecoregion
def centerline_points( in_lines: Path, distance: float = 0.0, transform: Transform = None) -> Dict[int, List[RiverPoint]]: """Generates points along each line feature at specified distances from the end as well as quarter and halfway Args: in_lines (Path): path of shapefile with features distance (float, optional): distance from ends to generate points. Defaults to 0.0. transform (Transform, optional): coordinate transformation. Defaults to None. Returns: [type]: [description] """ log = Logger('centerline_points') with get_shp_or_gpkg(in_lines) as in_lyr: out_group = {} ogr_extent = in_lyr.ogr_layer.GetExtent() extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2], ogr_extent[1], ogr_extent[3]) for feat, _counter, progbar in in_lyr.iterate_features( "Centerline points"): line = VectorBase.ogr2shapely(feat, transform) fid = feat.GetFID() out_points = [] # Attach the FID in case we need it later props = {'fid': fid} pts = [ line.interpolate(distance), line.interpolate(0.5, True), line.interpolate(-distance) ] if line.project(line.interpolate(0.25, True)) > distance: pts.append(line.interpolate(0.25, True)) pts.append(line.interpolate(-0.25, True)) for pt in pts: # Recall that interpolation can have multiple solutions due to pythagorean theorem # Throw away anything that's not inside our bounds if not extent.contains(pt): progbar.erase() log.warning('Point {} is outside of extent: {}'.format( pt.coords[0], ogr_extent)) out_points.append(RiverPoint(pt, properties=props)) out_group[int(fid)] = out_points feat = None return out_group
def merge_feature_classes(feature_class_paths: List[str], boundary: BaseGeometry, out_layer_path: str): """[summary] Args: feature_class_paths (List[str]): [description] boundary (BaseGeometry): [description] out_layer_path (str): [description] """ log = Logger('merge_feature_classes') log.info('Merging {} feature classes.'.format(len(feature_class_paths))) with get_shp_or_gpkg(out_layer_path, write=True) as out_layer: fccount = 0 for in_layer_path in feature_class_paths: fccount += 1 log.info("Merging feature class {}/{}".format(fccount, len(feature_class_paths))) with get_shp_or_gpkg(in_layer_path) as in_layer: in_layer.SetSpatialFilter(VectorBase.shapely2ogr(boundary)) # First input spatial ref sets the SRS for the output file transform = in_layer.get_transform(out_layer) for i in range(in_layer.ogr_layer_def.GetFieldCount()): in_field_def = in_layer.ogr_layer_def.GetFieldDefn(i) # Only create fields if we really don't have them # NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE if out_layer.ogr_layer_def.GetFieldIndex(in_field_def.GetName()) == -1: out_layer.ogr_layer.CreateField(in_field_def) log.info('Processing feature: {}/{}'.format(fccount, len(feature_class_paths))) for feature, _counter, progbar in in_layer.iterate_features('Processing feature'): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue geom.Transform(transform) out_feature = ogr.Feature(out_layer.ogr_layer_def) for i in range(in_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_feature.SetGeometry(geom) out_layer.ogr_layer.CreateFeature(out_feature) log.info('Merge complete.') return fccount
def get_riverpoints(inpath, epsg, attribute_filter=None): """[summary] Args: inpath ([type]): Path to a ShapeFile epsg ([type]): Desired output spatial reference attribute_filter ([type], optional): [description]. Defaults to None. Returns: [type]: List of RiverPoint objects """ log = Logger('get_riverpoints') points = [] with get_shp_or_gpkg(inpath) as in_lyr: _out_spatial_ref, transform = get_transform_from_epsg( in_lyr.spatial_ref, epsg) for feat, _counter, progbar in in_lyr.iterate_features( 'Getting points for use in Thiessen', attribute_filter=attribute_filter): new_geom = feat.GetGeometryRef() if new_geom is None: progbar.erase() # get around the progressbar log.warning( 'Feature with FID={} has no geometry. Skipping'.format( feat.GetFID())) continue new_geom.Transform(transform) new_shape = VectorBase.ogr2shapely(new_geom) if new_shape.type == 'Polygon': new_shape = MultiPolygon([new_shape]) for poly in new_shape: # Exterior is the shell and there is only ever 1 for pt in list(poly.exterior.coords): points.append(RiverPoint(pt, interior=False)) # Now we consider interiors. NB: Interiors are only qualifying islands in this case for idx, island in enumerate(poly.interiors): for pt in list(island.coords): points.append(RiverPoint(pt, interior=True, island=idx)) return points
def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict: """[summary] Args: in_layer_path (str): [description] id_field (str, optional): [description]. Defaults to None. epsg (int, optional): [description]. Defaults to None. spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None. Raises: VectorBaseException: [description] Returns: dict: [description] """ log = Logger('load_geometries') if epsg is not None and spatial_ref is not None: raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both') with get_shp_or_gpkg(in_layer_path) as in_layer: # Determine the transformation if user provides an EPSG transform = None if epsg is not None: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) elif spatial_ref is not None: transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref) features = {} for feature, _counter, progbar in in_layer.iterate_features("Loading features"): if id_field is None: reach = feature.GetFID() else: reach = feature.GetField(id_field) geom = feature.GetGeometryRef() geo_type = geom.GetGeometryType() new_geom = VectorBase.ogr2shapely(geom, transform=transform) if new_geom.is_empty: progbar.erase() # get around the progressbar log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID())) elif not new_geom.is_valid: progbar.erase() # get around the progressbar log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID())) # Filter out zero-length lines elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for feature with FID={}'.format(feature.GetFID())) # Filter out zero-area polys elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for feature with FID={}'.format(feature.GetFID())) else: features[reach] = new_geom return features
def get_geometry_union(inpath, epsg, attribute_filter=None): """ TODO: Remove this method and replace all references to the get_geometry_unary_union method below Load all features from a ShapeFile and union them together into a single geometry :param inpath: Path to a ShapeFile :param epsg: Desired output spatial reference :return: Single Shapely geometry of all unioned features """ log = Logger('Shapefile') driver = ogr.GetDriverByName("ESRI Shapefile") data_source = driver.Open(inpath, 0) layer = data_source.GetLayer() in_spatial_ref = layer.GetSpatialRef() if attribute_filter: layer.SetAttributeFilter(attribute_filter) _out_spatial_ref, transform = get_transform_from_epsg(in_spatial_ref, epsg) geom = None progbar = ProgressBar(layer.GetFeatureCount(), 50, "Unioning features") counter = 0 for feature in layer: counter += 1 progbar.update(counter) new_geom = feature.GetGeometryRef() if new_geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format( feature.GetFID())) continue new_geom.Transform(transform) new_shape = wkbload(new_geom.ExportToWkb()) try: geom = geom.union(new_shape) if geom else new_shape except Exception as e: progbar.erase() # get around the progressbar log.warning( 'Union failed for shape with FID={} and will be ignored'. format(feature.GetFID())) progbar.finish() data_source = None return geom
def safe_remove_file(file_path): """Remove a file without throwing an error Args: file_path ([type]): [description] """ log = Logger("safe_remove_file") try: if not os.path.isfile(file_path): log.warning('File not found: {}'.format(file_path)) os.remove(file_path) log.debug('File removed: {}'.format(file_path)) except Exception as e: log.error(str(e))
def load_geometries(feature_class, id_field, epsg=None): log = Logger('Shapefile') # Get the input network driver = ogr.GetDriverByName('ESRI Shapefile') dataset = driver.Open(feature_class, 0) layer = dataset.GetLayer() in_spatial_ref = layer.GetSpatialRef() # Determine the transformation if user provides an EPSG transform = None if epsg: out_spatial_ref, transform = get_transform_from_epsg( in_spatial_ref, epsg) features = {} progbar = ProgressBar(layer.GetFeatureCount(), 50, "Loading features") counter = 0 for inFeature in layer: counter += 1 progbar.update(counter) reach = inFeature.GetField(id_field) geom = inFeature.GetGeometryRef() # Optional coordinate transformation if transform: geom.Transform(transform) new_geom = wkbload(geom.ExportToWkb()) geo_type = new_geom.GetGeometryType() if new_geom.is_empty: progbar.erase() # get around the progressbar log.warning( 'Empty feature with FID={} cannot be unioned and will be ignored' .format(inFeature.GetFID())) elif not new_geom.is_valid: progbar.erase() # get around the progressbar log.warning( 'Invalid feature with FID={} cannot be unioned and will be ignored' .format(inFeature.GetFID())) # Filter out zero-length lines elif geo_type in LINE_TYPES and new_geom.Length() == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for feature with FID={}'.format( inFeature.GetFID())) # Filter out zero-area polys elif geo_type in POLY_TYPES and new_geom.Area() == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for feature with FID={}'.format( inFeature.GetFID())) else: features[reach] = new_geom progbar.finish() dataset = None return features
def merge_geometries(feature_classes, epsg): """ Load all features from multiple feature classes into a single list of geometries :param feature_classes: :param epsg: :return: """ log = Logger('Shapefile') driver = ogr.GetDriverByName("ESRI Shapefile") union = ogr.Geometry(ogr.wkbMultiLineString) fccount = 0 for fc in feature_classes: fccount += 1 log.info("Merging Geometries for feature class {}/{}".format( fccount, len(feature_classes))) data_source = driver.Open(fc, 0) layer = data_source.GetLayer() in_spatial_ref = layer.GetSpatialRef() out_spatial_ref, transform = get_transform_from_epsg( in_spatial_ref, epsg) progbar = ProgressBar(layer.GetFeatureCount(), 50, "Merging Geometries") counter = 0 for feature in layer: counter += 1 progbar.update(counter) geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning( 'Feature with FID={} has no geoemtry. Skipping'.format( feature.GetFID())) continue geom.Transform(transform) union.AddGeometry(geom) progbar.finish() data_source = None return union
def buffer_by_field(in_layer_path: str, out_layer_path, field: str, epsg: int = None, min_buffer=None, centered=False) -> None: """generate buffered polygons by value in field Args: flowlines (str): feature class of line features to buffer field (str): field with buffer value epsg (int): output srs min_buffer: use this buffer value for field values that are less than this Returns: geometry: unioned polygon geometry of buffered lines """ log = Logger('buffer_by_field') with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, get_shp_or_gpkg(in_layer_path) as in_layer: conversion = in_layer.rough_convert_metres_to_vector_units(1) # Add input Layer Fields to the output Layer if it is the one we want out_layer.create_layer(ogr.wkbPolygon, epsg=epsg, fields=in_layer.get_fields()) transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref) factor = 0.5 if centered else 1.0 for feature, _counter, progbar in in_layer.iterate_features('Buffering features', write_layers=[out_layer]): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue buffer_dist = feature.GetField(field) * conversion * factor geom.Transform(transform) geom_buffer = geom.Buffer(buffer_dist if buffer_dist > min_buffer else min_buffer) # Create output Feature out_feature = ogr.Feature(out_layer.ogr_layer_def) out_feature.SetGeometry(geom_buffer) # Add field values from input Layer for i in range(0, out_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_layer.ogr_layer.CreateFeature(out_feature) out_feature = None
def write_attributes(feature_class, output_values, id_field, fields, field_type=ogr.OFTReal, null_values=None): """ Write field values to a ShapeFile feature class :param feature_class: Path to feature class :param output_values: Dictionary of values keyed by id_field. Each feature is dictionary keyed by field names :param id_field: Unique key identifying each feature in both feature class and output_values dictionary :param fields: List of fields in output_values to write to ShapeFile :return: None """ log = Logger('ShapeFile') driver = ogr.GetDriverByName('ESRI Shapefile') dataset = driver.Open(feature_class, 1) layer = dataset.GetLayer() # Create each field and store the name and index in a list of tuples field_indices = [(field, create_field(layer, field, field_type)) for field in fields] for feature in layer: reach = feature.GetField(id_field) if reach not in output_values: continue # Set all the field values and then store the feature for field, idx in field_indices: if field in output_values[reach]: if not output_values[reach][field]: if null_values: feature.SetField(field, null_values) else: log.warning('Unhandled ShapeFile value for None type') feature.SetField(field, None) else: feature.SetField(field, output_values[reach][field]) layer.SetFeature(feature) dataset = None
def calculate_hydrology(reaches: dict, equation: str, params: dict, drainage_conversion_factor: float, field: str) -> dict: """ Perform the actual hydrology calculation Args: reaches ([type]): [description] equation ([type]): [description] params ([type]): [description] drainage_conversion_factor ([type]): [description] field ([type]): [description] Raises: ex: [description] Returns: [type]: [description] """ results = {} log = Logger('Hydrology') try: # Loop over each reach for reachid, values in reaches.items(): # Use the drainage area for the current reach and convert to the units used in the equation params[ DRNAREA_PARAM] = values['iGeo_DA'] * drainage_conversion_factor # Execute the equation but restrict the use of all built-in functions eval_result = eval(equation, {'__builtins__': None}, params) results[reachid] = {field: eval_result} except Exception as ex: [ log.warning('{}: {}'.format(param, value)) for param, value in params.items() ] log.warning('Hydrology formula failed: {}'.format(equation)) log.error('Error calculating {} hydrology') raise ex return results
def get_geometry_union(in_layer_path: str, epsg: int = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None ) -> BaseGeometry: """[summary] Args: in_layer_path (str): [description] epsg (int, optional): [description]. Defaults to None. attribute_filter (str, optional): [description]. Defaults to None. clip_shape (BaseGeometry, optional): [description]. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. Returns: BaseGeometry: [description] """ log = Logger('get_geometry_union') with get_shp_or_gpkg(in_layer_path) as in_layer: transform = None if epsg: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) geom = None for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect): if feature.GetGeometryRef() is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue new_shape = VectorBase.ogr2shapely(feature, transform=transform) try: geom = geom.union(new_shape) if geom is not None else new_shape except Exception: progbar.erase() # get around the progressbar log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID())) return geom
def extract_mean_values_by_polygon(polys, rasters, reference_raster): log = Logger('extract_mean_values_by_polygon') progbar = ProgressBar(len(polys), 50, "Extracting Mean values...") counter = 0 with rasterio.open(reference_raster) as dataset: output_mean = {} output_unique = {} for reachid, poly in polys.items(): counter += 1 progbar.update(counter) if poly.geom_type in ["Polygon", "MultiPolygon"] and poly.area > 0: values_mean = {} values_unique = {} reach_raster = np.ma.masked_invalid( features.rasterize( [poly], out_shape=dataset.shape, transform=dataset.transform, all_touched=True, fill=np.nan)) for key, raster in rasters.items(): if raster is not None: current_raster = np.ma.masked_array(raster, mask=reach_raster.mask) values_mean[key] = np.ma.mean(current_raster) values_unique[key] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True) else: values_mean[key] = 0.0 values_unique[key] = [] output_mean[reachid] = values_mean output_unique[reachid] = values_unique # log.debug(f"Reach: {reachid} | {sum([v for v in values.values() if v is not None]):.2f}") else: progbar.erase() log.warning(f"Reach: {reachid} | WARNING no geom") progbar.finish() return output_mean, output_unique
def download_unzip(url, download_folder, unzip_folder=None, force_download=False, retries=3): """ A wrapper for Download() and Unzip(). WE do these things together enough that it makes sense. Also there's the concept of retrying that needs to be handled in a centralized way Arguments: url {[type]} -- [description] download_folder {[type]} -- [description] Keyword Arguments: unzip_folder {[string]} -- (optional) specify the specific directory to extract files into (we still create a subfolder with the zip-file's name though) force_download {bool} -- [description] (default: {False}) Returns: [type] -- [description] """ log = Logger('Download') # If we specified an unzip path then use it, otherwise just unzip into the folder # with the same name as the file (minus the '.zip' extension) dl_retry = 0 dl_success = False while not dl_success and dl_retry < 3: try: zipfilepath = download_file(url, download_folder, force_download) dl_success = True except Exception as e: log.debug(e) log.warning('download failed. retrying...') dl_retry += 1 if (not dl_success): raise Exception('Downloading of file failed after {} attempts'.format(retries)) final_unzip_folder = unzip_folder if unzip_folder is not None else os.path.splitext(zipfilepath)[0] unzip(zipfilepath, final_unzip_folder, force_download, retries) return final_unzip_folder
def verify_areas(raster_path, boundary_shp): """[summary] Arguments: raster_path {[type]} -- path boundary_shp {[type]} -- path Raises: Exception: [description] if raster area is zero Exception: [description] if shapefile area is zero Returns: [type] -- rastio of raster area over shape file area """ log = Logger('Verify Areas') log.info('Verifying raster and shape areas') # This comes back in the raster's unit raster_area = 0 with rasterio.open(raster_path) as ds: cell_count = 0 gt = ds.get_transform() cell_area = math.fabs(gt[1]) * math.fabs(gt[5]) # Incrememntally add the area of a block to the count progbar = ProgressBar(len(list(ds.block_windows(1))), 50, "Calculating Area") progcount = 0 for _ji, window in ds.block_windows(1): r = ds.read(1, window=window, masked=True) progbar.update(progcount) cell_count += r.count() progcount += 1 progbar.finish() # Multiply the count by the area of a given cell raster_area = cell_area * cell_count log.debug('raster area {}'.format(raster_area)) if (raster_area == 0): raise Exception('Raster has zero area: {}'.format(raster_path)) # We could just use Rasterio's CRS object but it doesn't seem to play nice with GDAL so.... raster_ds = gdal.Open(raster_path) raster_srs = osr.SpatialReference(wkt=raster_ds.GetProjection()) # Load and transform ownership polygons by adminstration agency driver = ogr.GetDriverByName("ESRI Shapefile") data_source = driver.Open(boundary_shp, 0) layer = data_source.GetLayer() in_spatial_ref = layer.GetSpatialRef() # https://github.com/OSGeo/gdal/issues/1546 raster_srs.SetAxisMappingStrategy(in_spatial_ref.GetAxisMappingStrategy()) transform = osr.CoordinateTransformation(in_spatial_ref, raster_srs) shape_area = 0 for polygon in layer: geom = polygon.GetGeometryRef() geom.Transform(transform) shape_area = shape_area + geom.GetArea() log.debug('shape file area {}'.format(shape_area)) if (shape_area == 0): raise Exception('Shapefile has zero area: {}'.format(boundary_shp)) area_ratio = raster_area / shape_area if (area_ratio < 0.99 and area_ratio > 0.9): log.warning('Raster Area covers only {0:.2f}% of the shapefile'.format( area_ratio * 100)) if (area_ratio <= 0.9): log.error('Raster Area covers only {0:.2f}% of the shapefile'.format( area_ratio * 100)) else: log.info('Raster Area covers {0:.2f}% of the shapefile'.format( area_ratio * 100)) return area_ratio
def calculate_combined_fis(feature_values: dict, veg_fis_field: str, capacity_field: str, dam_count_field: str, max_drainage_area: float): """ Calculate dam capacity and density using combined FIS :param feature_values: Dictionary of features keyed by ReachID and values are dictionaries of attributes :param veg_fis_field: Attribute containing the output of the vegetation FIS :param com_capacity_field: Attribute used to store the capacity result in feature_values :param com_density_field: Attribute used to store the capacity results in feature_values :param max_drainage_area: Reaches with drainage area greater than this threshold will have zero capacity :return: Insert the dam capacity and density values to the feature_values dictionary """ log = Logger('Combined FIS') log.info('Initializing Combined FIS') if not max_drainage_area: log.warning( 'Missing max drainage area. Calculating combined FIS without max drainage threshold.' ) # get arrays for fields of interest feature_count = len(feature_values) reachid_array = np.zeros(feature_count, np.int64) veg_array = np.zeros(feature_count, np.float64) hydq2_array = np.zeros(feature_count, np.float64) hydlow_array = np.zeros(feature_count, np.float64) slope_array = np.zeros(feature_count, np.float64) drain_array = np.zeros(feature_count, np.float64) counter = 0 for reach_id, values in feature_values.items(): reachid_array[counter] = reach_id veg_array[counter] = values[veg_fis_field] hydlow_array[counter] = values['iHyd_SPLow'] hydq2_array[counter] = values['iHyd_SP2'] slope_array[counter] = values['iGeo_Slope'] drain_array[counter] = values['iGeo_DA'] counter += 1 # Adjust inputs to be within FIS membership range veg_array[veg_array < 0] = 0 veg_array[veg_array > 45] = 45 hydq2_array[hydq2_array < 0] = 0.0001 hydq2_array[hydq2_array > 10000] = 10000 hydlow_array[hydlow_array < 0] = 0.0001 hydlow_array[hydlow_array > 10000] = 10000 slope_array[slope_array > 1] = 1 # create antecedent (input) and consequent (output) objects to hold universe variables and membership functions ovc = ctrl.Antecedent(np.arange(0, 45, 0.01), 'input1') sp2 = ctrl.Antecedent(np.arange(0, 10000, 1), 'input2') splow = ctrl.Antecedent(np.arange(0, 10000, 1), 'input3') slope = ctrl.Antecedent(np.arange(0, 1, 0.0001), 'input4') density = ctrl.Consequent(np.arange(0, 45, 0.01), 'result') # build membership functions for each antecedent and consequent object ovc['none'] = fuzz.trimf(ovc.universe, [0, 0, 0.1]) ovc['rare'] = fuzz.trapmf(ovc.universe, [0, 0.1, 0.5, 1.5]) ovc['occasional'] = fuzz.trapmf(ovc.universe, [0.5, 1.5, 4, 8]) ovc['frequent'] = fuzz.trapmf(ovc.universe, [4, 8, 12, 25]) ovc['pervasive'] = fuzz.trapmf(ovc.universe, [12, 25, 45, 45]) sp2['persists'] = fuzz.trapmf(sp2.universe, [0, 0, 1000, 1200]) sp2['breach'] = fuzz.trimf(sp2.universe, [1000, 1200, 1600]) sp2['oblowout'] = fuzz.trimf(sp2.universe, [1200, 1600, 2400]) sp2['blowout'] = fuzz.trapmf(sp2.universe, [1600, 2400, 10000, 10000]) splow['can'] = fuzz.trapmf(splow.universe, [0, 0, 150, 175]) splow['probably'] = fuzz.trapmf(splow.universe, [150, 175, 180, 190]) splow['cannot'] = fuzz.trapmf(splow.universe, [180, 190, 10000, 10000]) slope['flat'] = fuzz.trapmf(slope.universe, [0, 0, 0.0002, 0.005]) slope['can'] = fuzz.trapmf(slope.universe, [0.0002, 0.005, 0.12, 0.15]) slope['probably'] = fuzz.trapmf(slope.universe, [0.12, 0.15, 0.17, 0.23]) slope['cannot'] = fuzz.trapmf(slope.universe, [0.17, 0.23, 1, 1]) density['none'] = fuzz.trimf(density.universe, [0, 0, 0.1]) density['rare'] = fuzz.trapmf(density.universe, [0, 0.1, 0.5, 1.5]) density['occasional'] = fuzz.trapmf(density.universe, [0.5, 1.5, 4, 8]) density['frequent'] = fuzz.trapmf(density.universe, [4, 8, 12, 25]) density['pervasive'] = fuzz.trapmf(density.universe, [12, 25, 45, 45]) # build fis rule table log.info('Building FIS rule table') comb_ctrl = ctrl.ControlSystem([ ctrl.Rule(ovc['none'], density['none']), ctrl.Rule(splow['cannot'], density['none']), ctrl.Rule(slope['cannot'], density['none']), ctrl.Rule( ovc['rare'] & sp2['persists'] & splow['can'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['persists'] & splow['probably'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['breach'] & splow['can'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['breach'] & splow['probably'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['oblowout'] & splow['can'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['oblowout'] & splow['probably'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['rare'] & sp2['blowout'] & splow['can'] & ~slope['cannot'], density['none']), ctrl.Rule( ovc['rare'] & sp2['blowout'] & splow['probably'] & ~slope['cannot'], density['none']), ctrl.Rule( ovc['occasional'] & sp2['persists'] & splow['can'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['persists'] & splow['probably'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['breach'] & splow['can'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['breach'] & splow['probably'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['oblowout'] & splow['can'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['oblowout'] & splow['probably'] & ~slope['cannot'], density['occasional']), ctrl.Rule( ovc['occasional'] & sp2['blowout'] & splow['can'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['occasional'] & sp2['blowout'] & splow['probably'] & ~slope['cannot'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['can'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['can'] & slope['can'], density['frequent']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['can'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['probably'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['probably'] & slope['can'], density['frequent']), ctrl.Rule( ovc['frequent'] & sp2['persists'] & splow['probably'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['can'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['can'] & slope['can'], density['frequent']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['can'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['probably'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['probably'] & slope['can'], density['frequent']), ctrl.Rule( ovc['frequent'] & sp2['breach'] & splow['probably'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['can'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['can'] & slope['can'], density['frequent']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['can'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['probably'] & slope['flat'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['probably'] & slope['can'], density['occasional']), ctrl.Rule( ovc['frequent'] & sp2['oblowout'] & splow['probably'] & slope['probably'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['can'] & slope['flat'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['can'] & slope['can'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['can'] & slope['probably'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['probably'] & slope['flat'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['probably'] & slope['can'], density['rare']), ctrl.Rule( ovc['frequent'] & sp2['blowout'] & splow['probably'] & slope['probably'], density['rare']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['can'] & slope['flat'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['can'] & slope['can'], density['pervasive']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['can'] & slope['probably'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['probably'] & slope['flat'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['probably'] & slope['can'], density['pervasive']), ctrl.Rule( ovc['pervasive'] & sp2['persists'] & splow['probably'] & slope['probably'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['can'] & slope['flat'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['can'] & slope['can'], density['pervasive']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['can'] & slope['probably'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['probably'] & slope['flat'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['probably'] & slope['can'], density['pervasive']), ctrl.Rule( ovc['pervasive'] & sp2['breach'] & splow['probably'] & slope['probably'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['can'] & slope['flat'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['can'] & slope['can'], density['pervasive']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['can'] & slope['probably'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['probably'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['probably'] & slope['can'], density['frequent']), ctrl.Rule( ovc['pervasive'] & sp2['oblowout'] & splow['probably'] & slope['probably'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['can'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['can'] & slope['can'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['can'] & slope['probably'], density['rare']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['probably'] & slope['flat'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['probably'] & slope['can'], density['occasional']), ctrl.Rule( ovc['pervasive'] & sp2['blowout'] & splow['probably'] & slope['probably'], density['rare']) ]) comb_fis = ctrl.ControlSystemSimulation(comb_ctrl) # calculate defuzzified centroid value for density 'none' MF group # this will be used to re-classify output values that fall in this group # important: will need to update the array (x) and MF values (mfx) if the # density 'none' values are changed in the model x_vals = np.arange(0, 45, 0.01) mfx = fuzz.trimf(x_vals, [0, 0, 0.1]) defuzz_centroid = round(fuzz.defuzz(x_vals, mfx, 'centroid'), 6) progbar = ProgressBar(len(reachid_array), 50, "Combined FIS") counter = 0 for i, reach_id in enumerate(reachid_array): capacity = 0.0 # Only compute FIS if the reach has less than user-defined max drainage area. # this enforces a stream size threshold above which beaver dams won't persist and/or won't be built if not max_drainage_area or drain_array[i] < max_drainage_area: comb_fis.input['input1'] = veg_array[i] comb_fis.input['input2'] = hydq2_array[i] comb_fis.input['input3'] = hydlow_array[i] comb_fis.input['input4'] = slope_array[i] comb_fis.compute() capacity = comb_fis.output['result'] # Combined FIS result cannot be higher than limiting vegetation FIS result if capacity > veg_array[i]: capacity = veg_array[i] if round(capacity, 6) == defuzz_centroid: capacity = 0.0 count = capacity * (feature_values[reach_id]['iGeo_Len'] / 1000.0) count = 1.0 if 0 < count < 1 else count feature_values[reach_id][capacity_field] = round(capacity, 2) feature_values[reach_id][dam_count_field] = round(count, 2) counter += 1 progbar.update(counter) progbar.finish() log.info('Done')
def vegetation_summary(outputs_gpkg_path: str, label: str, veg_raster: str, buffer: float): """ Loop through every reach in a BRAT database and retrieve the values from a vegetation raster within the specified buffer. Then store the tally of vegetation values in the BRAT database. Arguments: database {str} -- Path to BRAT database veg_raster {str} -- Path to vegetation raster buffer {float} -- Distance to buffer the reach polylines """ log = Logger('Vegetation') log.info('Summarizing {}m vegetation buffer from {}'.format( int(buffer), veg_raster)) # Retrieve the raster spatial reference and geotransformation dataset = gdal.Open(veg_raster) geo_transform = dataset.GetGeoTransform() raster_buffer = VectorBase.rough_convert_metres_to_raster_units( veg_raster, buffer) # Calculate the area of each raster cell in square metres conversion_factor = VectorBase.rough_convert_metres_to_raster_units( veg_raster, 1.0) cell_area = abs(geo_transform[1] * geo_transform[5]) / conversion_factor**2 # Open the raster and then loop over all polyline features veg_counts = [] with rasterio.open(veg_raster) as src, GeopackageLayer( os.path.join(outputs_gpkg_path, 'ReachGeometry')) as lyr: _srs, transform = VectorBase.get_transform_from_raster( lyr.spatial_ref, veg_raster) for feature, _counter, _progbar in lyr.iterate_features(label): reach_id = feature.GetFID() geom = feature.GetGeometryRef() if transform: geom.Transform(transform) polygon = VectorBase.ogr2shapely(geom).buffer(raster_buffer) try: # retrieve an array for the cells under the polygon raw_raster = mask(src, [polygon], crop=True)[0] mask_raster = np.ma.masked_values(raw_raster, src.nodata) # print(mask_raster) # Reclass the raster to dam suitability. Loop over present values for performance for oldvalue in np.unique(mask_raster): if oldvalue is not np.ma.masked: cell_count = np.count_nonzero(mask_raster == oldvalue) veg_counts.append([ reach_id, int(oldvalue), buffer, cell_count * cell_area, cell_count ]) except Exception as ex: log.warning( 'Error obtaining vegetation raster values for ReachID {}'. format(reach_id)) log.warning(ex) # Write the reach vegetation values to the database # Because sqlite3 doesn't give us any feedback we do this in batches so that we can figure out what values # Are causing constraint errors with SQLiteCon(outputs_gpkg_path) as database: errs = 0 batch_count = 0 for veg_record in veg_counts: batch_count += 1 try: database.conn.execute( 'INSERT INTO ReachVegetation (ReachID, VegetationID, Buffer, Area, CellCount) VALUES (?, ?, ?, ?, ?)', veg_record) # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is except sqlite3.IntegrityError as err: # THis is likely a constraint error. errstr = "Integrity Error when inserting records: ReachID: {} VegetationID: {}".format( veg_record[0], veg_record[1]) log.error(errstr) errs += 1 except sqlite3.Error as err: # This is any other kind of error errstr = "SQL Error when inserting records: ReachID: {} VegetationID: {} ERROR: {}".format( veg_record[0], veg_record[1], str(err)) log.error(errstr) errs += 1 if errs > 0: raise Exception( 'Errors were found inserting records into the database. Cannot continue.' ) database.conn.commit() log.info('Vegetation summary complete')
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None ) -> BaseGeometry: """Load all features from a ShapeFile and union them together into a single geometry Args: in_layer_path (str): path to layer epsg (int, optional): EPSG to project to. Defaults to None. spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None. attribute_filter (str, optional): Filter to a set of attributes. Defaults to None. clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. Raises: VectorBaseException: [description] Returns: BaseGeometry: [description] """ log = Logger('get_geometry_unary_union') if epsg is not None and spatial_ref is not None: raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both') with get_shp_or_gpkg(in_layer_path) as in_layer: transform = None if epsg is not None: _outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg) elif spatial_ref is not None: transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref) geom_list = [] for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect): new_geom = feature.GetGeometryRef() geo_type = new_geom.GetGeometryType() # We can't union non-valid shapes but sometimes a buffer by 0 can help if not new_geom.IsValid(): progbar.erase() # get around the progressbar log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID())) try: new_geom = new_geom.Buffer(0) if not new_geom.IsValid(): log.warning(' Still invalid. Skipping this geometry') continue except Exception: log.warning('Exception raised during buffer 0 technique. skipping this file') continue if new_geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID())) # Filter out zero-length lines elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for shape with FID={}'.format(feature.GetFID())) # Filter out zero-area polys elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for shape with FID={}'.format(feature.GetFID())) else: geom_list.append(VectorBase.ogr2shapely(new_geom, transform)) # IF we get past a certain size then run the union if len(geom_list) >= 500: geom_list = [unary_union(geom_list)] new_geom = None log.debug('finished iterating with list of size: {}'.format(len(geom_list))) if len(geom_list) > 1: log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list))) # Do a final union to clean up anything that might still be in the list geom_union = unary_union(geom_list) elif len(geom_list) == 0: log.warning('No geometry found to union') return None else: log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list))) geom_union = geom_list[0] log.debug(' done') print_geom_size(log, geom_union) log.debug('Complete') # Return a shapely object return geom_union
def process_modis(out_sqlite, modis_folder, nhd_folder, verbose, debug_flag): """Generate land surface temperature sqlite db from NHD+ and MODIS data """ log = Logger("Process LST") if os.path.isfile(out_sqlite): os.remove(out_sqlite) # Create sqlite database conn = sqlite3.connect(out_sqlite) cursor = conn.cursor() # test if table exists? cursor.execute( """SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='MODIS_LST' """ ) log.info('Creating DB') if cursor.fetchone()[0] == 0: cursor.execute(""" CREATE TABLE MODIS_LST ( NHDPlusID INTEGER NOT NULL, MODIS_Scene DATETIME NOT NULL, LST REAL, PRIMARY KEY ( NHDPlusID, MODIS_Scene ) ) WITHOUT ROWID; """) conn.commit() # populate list of modis files modis_files = glob.glob(os.path.join(modis_folder, "*.tif")) # Load NHD Layers log.info(f"Processing NHD Data: {nhd_folder}") in_driver = ogr.GetDriverByName("OpenFileGDB") in_datasource = in_driver.Open(nhd_folder, 0) layer_hucs = in_datasource.GetLayer(r"WBDHU8_reproject") # Process HUC huc_counter = 0 total_hucs = layer_hucs.GetFeatureCount() for huc in layer_hucs: huc_counter += 1 huc_id = huc.GetField(r"HUC8") log.info('Processing huc:{} ({}/{})'.format(huc_id, huc_counter, total_hucs)) log.info(f"HUC: {huc_id}") huc_geom = huc.GetGeometryRef() layer_catchments = None layer_catchments = in_datasource.GetLayer( r"NHDPlusCatchment_reproject") # layer_catchments.SetSpatialFilter(huc_geom) catchments not perfectly aligned with hucs layer_catchments.SetAttributeFilter(f"""HUC8 = {huc_id}""") huc_bounds = huc_geom.GetEnvelope() bbox = box(huc_bounds[0], huc_bounds[2], huc_bounds[1], huc_bounds[3]) # open a single MODIS raster and load its projection and transform based on current huc with rasterio.open(f"{modis_files[0]}") as dataset: data, modis_transform = mask(dataset, [bbox], all_touched=True, crop=True) # Assuming there is only one band we can drop the first dimenson and get (36,78) instead of (1,36,78) modis_shape = data.shape[1:] # Read all MODIS Scences into array modis_array_raw = np.ma.array( [load_cropped_raster(image, bbox) for image in modis_files]) modis_array_sds = np.ma.masked_where(modis_array_raw == 0, modis_array_raw) # Make sure we mask out the invalid data modis_array_K = modis_array_sds * 0.02 modis_array_C = modis_array_K - 273.15 # K to C # Generate list of MODIS scene dates modis_dates = np.array([ os.path.basename(image).lstrip("A").rstrip(".tif") for image in modis_files ]) # Calcuate average LST per Catchemnt Layer progbar = ProgressBar(layer_catchments.GetFeatureCount(), 50, 'Processing HUC: {}'.format(huc_id)) reach_counter = 0 progbar.update(reach_counter) # loop_timer = LoopTimer("LoopTime", useMs=True) for reach in layer_catchments: reach_counter += 1 progbar.update(reach_counter) # If debug flag is set then drop a CSV for every 5000 reaches debug_drop = debug_flag is True and reach_counter % 5000 == 1 # For Debugging performance # loop_timer.tick() # loop_timer.progprint() nhd_id = int(reach.GetField("NHDPlusID")) # load_catchment_polygon and transform to raster SRS reach_geom = reach.GetGeometryRef() catch_poly = loads(reach_geom.ExportToWkb()) # Catchment polygons are vectorized rasters and they can have invalid geometries if not catch_poly.is_valid: log.warning( 'Invalid catchment polygon detected. Trying the buffer technique: {}' .format(nhd_id)) catch_poly = catch_poly.buffer(0) # Generate mask raster of catchment pixels reach_raster = np.ma.masked_invalid( rasterio.features.rasterize([catch_poly], out_shape=modis_shape, transform=modis_transform, all_touched=True, fill=np.nan)) # Now assign ascending integers to each cell. THis is so the rasterio.features.shapes gives us a unique shape for every cell reach_raster_idx = np.ma.masked_array( np.arange(modis_shape[0] * modis_shape[1], dtype=np.int32).reshape(modis_shape), # pylint: disable=E1101 reach_raster.mask) # Generate a unique shape for each valid pixel geoms = [{ 'properties': { 'name': 'modis_pixel', 'raster_val': int(v), 'valid': v > 0 }, 'geometry': geom } for i, (geom, v) in enumerate( rasterio.features.shapes(reach_raster_idx, transform=modis_transform)) if test_pixel_geom(geom)] # Now create our weights array. Start with weights of 0 so we can rule out any weird points weights_raster_arr = np.ma.masked_array( np.full(modis_shape, 0, dtype=np.float32), # pylint: disable=E1101 reach_raster.mask, ) for geom in geoms: pxl = shape(geom['geometry']) poly_intersect = pxl.intersection(catch_poly) idx, idy = find_indeces(geom['properties']['raster_val'], modis_shape) weight = poly_intersect.area / catch_poly.area # For debugging if debug_drop: geom['type'] = "Feature" geom['properties']['weight'] = weight geom['properties']['raster_coords'] = [idx, idy] geom['properties']['world_coords'] = [ pxl.centroid.coords[0][0], pxl.centroid.coords[0][1] ] weights_raster_arr[idx][idy] = weight # Calculate average weighted modis ave = np.ma.average(modis_array_C, axis=(1, 2), weights=np.broadcast_to( weights_raster_arr, modis_array_C.shape)) # Just some useful debugging stuff if debug_drop: progbar.erase() file_prefix = '{}-{}-debug'.format(huc_id, nhd_id) log.debug('Dropping files: {}'.format(file_prefix)) # PrintArr(reach_raster_idx) # Dump some useful shapes to a geojson Object _debug_shape = DebugGeoJSON( os.path.join(os.path.dirname(out_sqlite), '{}.geojson'.format(file_prefix))) _debug_shape.add_shapely(bbox, {"name": "bbox"}) _debug_shape.add_shapely(catch_poly, {"name": "catch_poly"}) [_debug_shape.add_geojson(gj) for gj in geoms] _debug_shape.write() # Now dump an CSV array report for fun csv_file = os.path.join(os.path.dirname(out_sqlite), '{}.csv'.format(file_prefix)) with open(csv_file, 'w') as csv_file: csvw = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csvw.writerow(['HUC', 'NHDPlusID', 'Area']) csvw.writerow([huc_id, nhd_id, catch_poly.area]) csvw.writerow([]) debug_weights = [] # Summary of intersected pixels for geom in geoms: debug_weights.append( (geom['properties']['weight'], geom['properties']['raster_coords'])) # Dump the weights Cell values so we can use excel to calculate them manually # Write the average and the csvw.writerow(['Intersecting Cells:'] + [' ' for g in geoms]) for key, name in { 'raster_val': 'cell_id', 'raster_coords': '[row,col]', 'world_coords': '[x,y]', 'weight': 'weight' }.items(): csvw.writerow([name] + [g['properties'][key] for g in geoms]) csvw.writerow([]) csvw.writerow(['Date'] + [' ' for g in geoms] + ['np.ma.average']) for didx, ave_val in enumerate(ave): csvw.writerow([modis_dates[didx]] + [ modis_array_sds[didx][w[1][0]][w[1][1]] for w in debug_weights ] + [ave_val]) # insert_lst_into_sqlite cursor.executemany("""INSERT INTO MODIS_LST VALUES(?,?,?)""", [ (nhd_id, datetime.datetime.strptime(modis_date, "%Y%j").date(), float(v) if float(v) != 0 else None) for (modis_date, v) in zip(modis_dates, ave.data) ]) # Write data to sqlite after each reach conn.commit() # Close database connection conn.close() return
def get_geometry_unary_union_from_wkt(inpath, to_sr_wkt): """ Load all features from a ShapeFile and union them together into a single geometry :param inpath: Path to a ShapeFile :param epsg: Desired output spatial reference :return: Single Shapely geometry of all unioned features """ log = Logger('Unary Union') driver = ogr.GetDriverByName("ESRI Shapefile") data_source = driver.Open(inpath, 0) layer = data_source.GetLayer() in_spatial_ref = layer.GetSpatialRef() out_spatial_ref, transform = get_transform_from_wkt( in_spatial_ref, to_sr_wkt) fcount = layer.GetFeatureCount() progbar = ProgressBar(fcount, 50, "Unary Unioning features") counter = 0 def unionize(wkb_lst): return unary_union([wkbload(g) for g in wkb_lst]).wkb geom_list = [] for feature in layer: counter += 1 progbar.update(counter) new_geom = feature.GetGeometryRef() geo_type = new_geom.GetGeometryType() # We can't union non-valid shapes but sometimes a buffer by 0 can help if not new_geom.IsValid(): progbar.erase() # get around the progressbar log.warning( 'Invalid shape with FID={} trying the Buffer0 technique...'. format(feature.GetFID())) try: new_geom = new_geom.Buffer(0) if not new_geom.IsValid(): progbar.erase() # get around the progressbar log.warning(' Still invalid. Skipping this geometry') continue except Exception as e: progbar.erase() # get around the progressbar log.warning( 'Exception raised during buffer 0 technique. skipping this file' ) continue if new_geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geoemtry. Skipping'.format( feature.GetFID())) # Filter out zero-length lines elif geo_type in LINE_TYPES and new_geom.Length() == 0: progbar.erase() # get around the progressbar log.warning('Zero Length for shape with FID={}'.format( feature.GetFID())) # Filter out zero-area polys elif geo_type in POLY_TYPES and new_geom.Area() == 0: progbar.erase() # get around the progressbar log.warning('Zero Area for shape with FID={}'.format( feature.GetFID())) else: new_geom.Transform(transform) geom_list.append(new_geom.ExportToWkb()) # IF we get past a certain size then run the union if len(geom_list) >= 500: geom_list = [unionize(geom_list)] new_geom = None log.debug('finished iterating with list of size: {}'.format( len(geom_list))) progbar.finish() if len(geom_list) > 1: log.debug('Starting final union of geom_list of size: {}'.format( len(geom_list))) # Do a final union to clean up anything that might still be in the list geom_union = wkbload(unionize(geom_list)) elif len(geom_list) == 0: log.warning('No geometry found to union') return None else: log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list))) geom_union = wkbload(geom_list[0]) log.debug(' done') print_geom_size(log, geom_union) log.debug('Complete') data_source = None return geom_union
def copy_feature_class(inpath, epsg, outpath, clip_shape=None, attribute_filter=None): """Copy a Shapefile from one location to another This method is capable of reprojecting the geometries as they are copied. It is also possible to filter features by both attributes and also clip the features to another geometryNone Arguments: inpath {str} -- File path to input Shapefile that will be copied. epsg {int} -- Output coordinate system outpath {str} -- File path where the output Shapefile will be generated. Keyword Arguments: clip_shape {shape} -- Shapely polygon geometry in the output EPSG used to clip the input geometries (default: {None}) attribute_filter {str} -- Attribute filter used to limit the input features that will be copied. (default: {None}) """ log = Logger('Shapefile') # if os.path.isfile(outpath): # log.info('Skipping copy of feature classes because output file exists.') # return driver = ogr.GetDriverByName("ESRI Shapefile") inDataSource = driver.Open(inpath, 0) inLayer = inDataSource.GetLayer() inSpatialRef = inLayer.GetSpatialRef() geom_type = inLayer.GetGeomType() # Optionally limit which features are copied by using an attribute filter if attribute_filter: inLayer.SetAttributeFilter(attribute_filter) # If there's a clip geometry provided then limit the features copied to # those that intersect (partially or entirely) by this clip feature. # Note that this makes the subsequent intersection process a lot more # performant because the SetSaptialFilter() uses the ShapeFile's spatial # index which is much faster than manually checking if all pairs of features intersect. clip_geom = None if clip_shape: clip_geom = ogr.CreateGeometryFromWkb(clip_shape.wkb) inLayer.SetSpatialFilter(clip_geom) outpath_dir = os.path.dirname(outpath) safe_makedirs(outpath_dir) # Create the output shapefile outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg) outDataSource = driver.CreateDataSource(outpath) outLayer = outDataSource.CreateLayer('network', outSpatialRef, geom_type=geom_type) outLayerDefn = outLayer.GetLayerDefn() # Add input Layer Fields to the output Layer if it is the one we want inLayerDefn = inLayer.GetLayerDefn() for i in range(0, inLayerDefn.GetFieldCount()): fieldDefn = inLayerDefn.GetFieldDefn(i) outLayer.CreateField(fieldDefn) # Get the output Layer's Feature Definition outLayerDefn = outLayer.GetLayerDefn() progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Copying features") counter = 0 for feature in inLayer: counter += 1 progbar.update(counter) geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format( feature.GetFID())) continue geom.Transform(transform) # if clip_shape: # raw = shape(json.loads(geom.ExportToJson())) # try: # clip = raw.intersection(clip_shape) # geom = ogr.CreateGeometryFromJson(json.dumps(mapping(clip))) # except Exception as e: # progbar.erase() # get around the progressbar # log.warning('Invalid shape with FID={} cannot be intersected'.format(feature.GetFID())) # Create output Feature outFeature = ogr.Feature(outLayerDefn) outFeature.SetGeometry(geom) # Add field values from input Layer for i in range(0, outLayerDefn.GetFieldCount()): outFeature.SetField( outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) outLayer.CreateFeature(outFeature) outFeature = None progbar.finish() inDataSource = None outDataSource = None
def copy_feature_class(in_layer_path: str, out_layer_path: str, epsg: int = None, attribute_filter: str = None, clip_shape: BaseGeometry = None, clip_rect: List[float] = None, buffer: float = 0, ) -> None: """Copy a Shapefile from one location to another This method is capable of reprojecting the geometries as they are copied. It is also possible to filter features by both attributes and also clip the features to another geometryNone Args: in_layer (str): Input layer path epsg ([type]): EPSG Code to use for the transformation out_layer (str): Output layer path attribute_filter (str, optional): [description]. Defaults to None. clip_shape (BaseGeometry, optional): [description]. Defaults to None. clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None. buffer (float): Buffer the output features (in meters). """ log = Logger('copy_feature_class') # NOTE: open the outlayer first so that write gets the dataset open priority with get_shp_or_gpkg(out_layer_path, write=True) as out_layer, \ get_shp_or_gpkg(in_layer_path) as in_layer: # Add input Layer Fields to the output Layer if it is the one we want out_layer.create_layer_from_ref(in_layer, epsg=epsg) transform = VectorBase.get_transform(in_layer.spatial_ref, out_layer.spatial_ref) buffer_convert = 0 if buffer != 0: buffer_convert = in_layer.rough_convert_metres_to_vector_units(buffer) # This is the callback method that will be run on each feature for feature, _counter, progbar in in_layer.iterate_features("Copying features", write_layers=[out_layer], clip_shape=clip_shape, clip_rect=clip_rect, attribute_filter=attribute_filter): geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID())) continue if geom.GetGeometryType() in VectorBase.LINE_TYPES: if geom.Length() == 0.0: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no Length. Skipping'.format(feature.GetFID())) continue # Buffer the shape if we need to if buffer_convert != 0: geom = geom.Buffer(buffer_convert) geom.Transform(transform) # Create output Feature out_feature = ogr.Feature(out_layer.ogr_layer_def) out_feature.SetGeometry(geom) # Add field values from input Layer for i in range(0, out_layer.ogr_layer_def.GetFieldCount()): out_feature.SetField(out_layer.ogr_layer_def.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) out_layer.ogr_layer.CreateFeature(out_feature) out_feature = None
class BratReport(RSReport): """In order to write a report we will extend the RSReport class from the rscommons module which has useful styles and building blocks like Tables from lists etc. """ def __init__(self, database, report_path, rs_project): # Need to call the constructor of the inherited class: super().__init__(rs_project, report_path) self.log = Logger('BratReport') self.database = database # The report has a core CSS file but we can extend it with our own if we want: css_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'brat_report.css') self.add_css(css_path) self.images_dir = os.path.join(os.path.dirname(report_path), 'images') safe_makedirs(self.images_dir) # Now we just need to write the sections we want in the report in order self.report_intro() self.reach_attribute_summary() self.dam_capacity() self.hydrology_plots() self.ownership() self.vegetation() self.conservation() def report_intro(self): # Create a section node to start adding things to. Section nodes are added to the table of contents if # they have a title. If you don't specify a el_parent argument these sections will simply be added # to the report body in the order you call them. section = self.section('ReportIntro', 'Introduction') # This project has a db so we'll need a connection conn = sqlite3.connect(self.database) conn.row_factory = _dict_factory curs = conn.cursor() row = curs.execute( 'SELECT Sum(iGeo_Len) AS TotalLength, Count(ReachID) AS TotalReaches FROM vwReaches' ).fetchone() values = { 'Number of reaches': '{0:,d}'.format(row['TotalReaches']), 'Total reach length (km)': '{0:,.0f}'.format(row['TotalLength'] / 1000), 'Total reach length (miles)': '{0:,.0f}'.format(row['TotalLength'] * 0.000621371) } row = curs.execute(''' SELECT WatershedID "Watershed ID", W.Name "Watershed Name", E.Name Ecoregion, CAST(AreaSqKm AS TEXT) "Area (Sqkm)", States FROM Watersheds W INNER JOIN Ecoregions E ON W.EcoregionID = E.EcoregionID ''').fetchone() values.update(row) curs.execute('SELECT KeyInfo, ValueInfo FROM Metadata') values.update({ row['KeyInfo'].replace('_', ' '): row['ValueInfo'] for row in curs.fetchall() }) # Here we're creating a new <div> to wrap around the table for stylign purposes table_wrapper = ET.Element('div', attrib={'class': 'tableWrapper'}) RSReport.create_table_from_dict(values, table_wrapper, attrib={'id': 'SummTable'}) RSReport.create_table_from_sql( ['Reach Type', 'Total Length (km)', '% of Total'], 'SELECT ReachType, Sum(iGeo_Len) / 1000 As Length, 100 * Sum(iGeo_Len) / TotalLength AS TotalLength ' 'FROM vwReaches INNER JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches) GROUP BY ReachType', self.database, table_wrapper, attrib={'id': 'SummTable_sql'}) # Append my table_wrapper div (which now contains both tables above) to the section section.append(table_wrapper) def reach_attribute(self, attribute, units, parent_el): # Use a class here because it repeats section = self.section(None, attribute, parent_el, level=2) conn = sqlite3.connect(self.database) conn.row_factory = _dict_factory curs = conn.cursor() # Summary statistics (min, max etc) for the current attribute curs.execute( 'SELECT Count({0}) "Values", Max({0}) Maximum, Min({0}) Minimum, Avg({0}) Average FROM vwReaches WHERE {0} IS NOT NULL' .format(attribute)) values = curs.fetchone() reach_wrapper_inner = ET.Element( 'div', attrib={'class': 'reachAtributeInner'}) section.append(reach_wrapper_inner) # Add the number of NULL values curs.execute( 'SELECT Count({0}) "NULL Values" FROM vwReaches WHERE {0} IS NULL'. format(attribute)) values.update(curs.fetchone()) RSReport.create_table_from_dict(values, reach_wrapper_inner) # Box plot image_path = os.path.join(self.images_dir, 'attribute_{}.png'.format(attribute)) curs.execute('SELECT {0} FROM vwReaches WHERE {0} IS NOT NULL'.format( attribute)) values = [row[attribute] for row in curs.fetchall()] box_plot(values, attribute, attribute, image_path) img_wrap = ET.Element('div', attrib={'class': 'imgWrap'}) img = ET.Element('img', attrib={ 'class': 'boxplot', 'alt': 'boxplot', 'src': '{}/{}'.format(os.path.basename(self.images_dir), os.path.basename(image_path)) }) img_wrap.append(img) reach_wrapper_inner.append(img_wrap) def dam_capacity(self): section = self.section('DamCapacity', 'BRAT Dam Capacity Results') conn = sqlite3.connect(self.database) conn.row_factory = _dict_factory curs = conn.cursor() fields = [('Existing complex size', 'Sum(mCC_EX_CT)'), ('Historic complex size', 'Sum(mCC_HPE_CT)'), ('Existing vegetation capacity', 'Sum((iGeo_len / 1000) * oVC_EX)'), ('Historic vegetation capacity', 'Sum((iGeo_len / 1000) * oVC_HPE)'), ('Existing capacity', 'Sum((iGeo_len / 1000) * oCC_EX)'), ('Historic capacity', 'Sum((iGeo_len / 1000) * oCC_HPE)')] curs.execute('SELECT {} FROM vwReaches'.format(', '.join( [field for label, field in fields]))) row = curs.fetchone() table_dict = { fields[i][0]: row[fields[i][1]] for i in range(len(fields)) } RSReport.create_table_from_dict(table_dict, section) self.dam_capacity_lengths('oCC_EX', section) self.dam_capacity_lengths('oCC_HPE', section) def dam_capacity_lengths(self, capacity_field, elParent): conn = sqlite3.connect(self.database) curs = conn.cursor() curs.execute( 'SELECT Name, MaxCapacity FROM DamCapacities ORDER BY MaxCapacity') bins = [(row[0], row[1]) for row in curs.fetchall()] curs.execute('SELECT Sum(iGeo_Len) / 1000 FROM vwReaches') total_length_km = curs.fetchone()[0] data = [] last_bin = 0 cumulative_length_km = 0 for name, max_capacity in bins: curs.execute( 'SELECT Sum(iGeo_len) / 1000 FROM vwReaches WHERE {} <= {}'. format(capacity_field, max_capacity)) rowi = curs.fetchone() if not rowi or rowi[0] is None: bin_km = 0 else: bin_km = rowi[0] - cumulative_length_km cumulative_length_km = rowi[0] data.append(('{}: {} - {}'.format(name, last_bin, max_capacity), bin_km, bin_km * 0.621371, 100 * bin_km / total_length_km)) last_bin = max_capacity data.append( ('Total', cumulative_length_km, cumulative_length_km * 0.621371, 100 * cumulative_length_km / total_length_km)) RSReport.create_table_from_tuple_list( (capacity_field, 'Stream Length (km)', 'Stream Length (mi)', 'Percent'), data, elParent) def hydrology_plots(self): section = self.section('HydrologyPlots', 'Hydrology') conn = sqlite3.connect(self.database) curs = conn.cursor() curs.execute('SELECT MaxDrainage, QLow, Q2 FROM Watersheds') row = curs.fetchone() RSReport.create_table_from_dict( { 'Max Draiange (sqkm)': row[0], 'Baseflow': row[1], 'Peak Flow': row[2] }, section, attrib={'class': 'fullwidth'}) RSReport.header(3, 'Hydrological Parameters', section) RSReport.create_table_from_sql( [ 'Parameter', 'Data Value', 'Data Units', 'Conversion Factor', 'Equation Value', 'Equation Units' ], 'SELECT Parameter, Value, DataUnits, Conversion, ConvertedValue, EquationUnits FROM vwHydroParams', self.database, section, attrib={'class': 'fullwidth'}) variables = [('iHyd_QLow', 'Baseflow (CFS)'), ('iHyd_Q2', 'Peak Flow (CFS)'), ('iHyd_SPLow', 'Baseflow Stream Power (Watts)'), ('iHyd_SP2', 'Peak Flow Stream Power (Watts)'), ('iGeo_Slope', 'Slope (degrees)')] plot_wrapper = ET.Element('div', attrib={'class': 'hydroPlotWrapper'}) section.append(plot_wrapper) for variable, ylabel in variables: self.log.info( 'Generating XY scatter for {} against drainage area.'.format( variable)) image_path = os.path.join( self.images_dir, 'drainage_area_{}.png'.format(variable.lower())) curs.execute('SELECT iGeo_DA, {} FROM vwReaches'.format(variable)) values = [(row[0], row[1]) for row in curs.fetchall()] xyscatter(values, 'Drainage Area (sqkm)', ylabel, variable, image_path) img_wrap = ET.Element('div', attrib={'class': 'imgWrap'}) img = ET.Element('img', attrib={ 'src': '{}/{}'.format( os.path.basename(self.images_dir), os.path.basename(image_path)), 'alt': 'boxplot' }) img_wrap.append(img) plot_wrapper.append(img_wrap) def reach_attribute_summary(self): section = self.section('ReachAttributeSummary', 'Geophysical Attributes') attribs = [('iGeo_Slope', 'Slope', 'ratio'), ('iGeo_ElMax', 'Max Elevation', 'metres'), ('iGeo_ElMin', 'Min Elevation', 'metres'), ('iGeo_Len', 'Length', 'metres'), ('iGeo_DA', 'Drainage Area', 'Sqkm')] plot_wrapper = ET.Element('div', attrib={'class': 'plots'}) [ self.reach_attribute(attribute, units, plot_wrapper) for attribute, name, units in attribs ] section.append(plot_wrapper)\ def ownership(self): section = self.section('Ownership', 'Ownership') RSReport.create_table_from_sql( [ 'Ownership Agency', 'Number of Reach Segments', 'Length (km)', '% of Total Length' ], 'SELECT IFNULL(Agency, "None"), Count(ReachID), Sum(iGeo_Len) / 1000, 100* Sum(iGeo_Len) / TotalLength FROM vwReaches' ' INNER JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches) GROUP BY Agency', self.database, section, attrib={'class': 'fullwidth'}) def vegetation(self): section = self.section('Vegetation', 'Vegetation') conn = sqlite3.connect(self.database) # conn.row_factory = _dict_factory curs = conn.cursor() for epochid, veg_type in [(2, 'Historic Vegetation'), (1, 'Existing Vegetation')]: RSReport.header(3, veg_type, section) pEl = ET.Element('p') pEl.text = 'The 30 most common {} types within the 100m reach buffer.'.format( veg_type.lower()) section.append(pEl) RSReport.create_table_from_sql([ 'Vegetation ID', 'Vegetation Type', 'Total Area (sqkm)', 'Default Suitability', 'Override Suitability', 'Effective Suitability' ], """ SELECT VegetationID, Name, (CAST(TotalArea AS REAL) / 1000000) AS TotalAreaSqKm, DefaultSuitability, OverrideSuitability, EffectiveSuitability FROM vwReachVegetationTypes WHERE (EpochID = {}) AND (Buffer = 100) ORDER BY TotalArea DESC LIMIT 30""" .format(epochid), self.database, section) try: # Calculate the area weighted suitability curs.execute(""" SELECT WeightedSum / SumTotalArea FROM (SELECT Sum(CAST(TotalArea AS REAL) * CAST(EffectiveSuitability AS REAL) / 1000000) WeightedSum FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100) JOIN (SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SumTotalArea FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100)""" .format(epochid)) area_weighted_avg_suitability = curs.fetchone()[0] RSReport.header(3, 'Suitability Breakdown', section) pEl = ET.Element('p') pEl.text = """The area weighted average {} suitability is {}. The breakdown of the percentage of the 100m buffer within each suitability class across all reaches in the watershed.""".format( veg_type.lower(), RSReport.format_value(area_weighted_avg_suitability)[0]) section.append(pEl) RSReport.create_table_from_sql( ['Suitability Class', '% with 100m Buffer'], """ SELECT EffectiveSuitability, 100.0 * SArea / SumTotalArea FROM ( SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SArea, EffectiveSuitability FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100 GROUP BY EffectiveSuitability ) JOIN ( SELECT CAST(Sum(TotalArea) AS REAL) / 1000000 SumTotalArea FROM vwReachVegetationTypes WHERE EpochID = {0} AND Buffer = 100 ) ORDER BY EffectiveSuitability """.format(epochid), self.database, section, id_cols=id_cols) except Exception as ex: self.log.warning('Error calculating vegetation report') def conservation(self): section = self.section('Conservation', 'Conservation') fields = [('Risk', 'DamRisks', 'RiskID'), ('Opportunity', 'DamOpportunities', 'OpportunityID'), ('Limitation', 'DamLimitations', 'LimitationID')] for label, table, idfield in fields: RSReport.header(3, label, section) RSReport.create_table_from_sql( [label, 'Total Length (km)', 'Reach Count', '%'], 'SELECT DR.Name, Sum(iGeo_Len) / 1000, Count(R.{1}), 100 * Sum(iGeo_Len) / TotalLength' ' FROM {0} DR LEFT JOIN vwReaches R ON DR.{1} = R.{1}' ' JOIN (SELECT Sum(iGeo_Len) AS TotalLength FROM vwReaches)' ' GROUP BY DR.{1}'.format(table, idfield), self.database, section) RSReport.header(3, 'Conflict Attributes', section) for attribute in ['iPC_Canal', 'iPC_DivPts', 'iPC_Privat']: self.reach_attribute(attribute, 'meters', section)
def merge_feature_classes(feature_classes, epsg, boundary, outpath): log = Logger('Shapefile') if os.path.isfile(outpath): log.info('Skipping merging feature classes because file exists.') return safe_makedirs(os.path.dirname(outpath)) log.info('Merging {} feature classes.'.format(len(feature_classes))) driver = ogr.GetDriverByName("ESRI Shapefile") # Create the output shapefile outDataSource = driver.CreateDataSource(outpath) outLayer = None outSpatialRef = None transform = None fccount = 0 for inpath in feature_classes: fccount += 1 log.info("Merging feature class {}/{}".format(fccount, len(feature_classes))) inDataSource = driver.Open(inpath, 0) inLayer = inDataSource.GetLayer() inSpatialRef = inLayer.GetSpatialRef() inLayer.SetSpatialFilter(ogr.CreateGeometryFromWkb(boundary.wkb)) # First input spatial ref sets the SRS for the output file outSpatialRefTmp, transform = get_transform_from_epsg( inSpatialRef, epsg) if outLayer is None: outSpatialRef = outSpatialRefTmp outLayer = outDataSource.CreateLayer( 'network', outSpatialRef, geom_type=ogr.wkbMultiLineString) outLayerDefn = outLayer.GetLayerDefn() # Transfer fields over inLayerDef = inLayer.GetLayerDefn() for i in range(inLayerDef.GetFieldCount()): inFieldDef = inLayerDef.GetFieldDefn(i) # Only create fields if we really don't have them # NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE if outLayerDefn.GetFieldIndex(inFieldDef.GetName()) == -1: outLayer.CreateField(inFieldDef) progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Processing features") outLayerDefn = outLayer.GetLayerDefn() counter = 0 for feature in inLayer: counter += 1 progbar.update(counter) geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning( 'Feature with FID={} has no geometry. Skipping'.format( feature.GetFID())) continue geom.Transform(transform) # get a Shapely representation of the line # featobj = json.loads(geom.ExportToJson()) # polyline = shape(featobj) # if boundary.intersects(polyline): # clipped = boundary.intersection(polyline) outFeature = ogr.Feature(outLayerDefn) for i in range(inLayerDef.GetFieldCount()): outFeature.SetField( outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) outFeature.SetGeometry(geom) outLayer.CreateFeature(outFeature) feature = None outFeature = None progbar.finish() inDataSource = None outDataSource = None log.info('Merge complete.')
def download_dem(vector_path, _epsg, buffer_dist, download_folder, unzip_folder, force_download=False): """ Identify rasters within HUC8, download them and mosaic into single GeoTIF :param vector_path: Path to bounding polygon ShapeFile :param epsg: Output spatial reference :param buffer_dist: Distance in DEGREES to buffer the bounding polygon :param unzip_folder: Temporary folder where downloaded rasters will be saved :param force_download: The download will always be performed if this is true. :return: """ log = Logger('DEM') # Query Science Base API for NED 10m DEM rasters within HUC 8 boundary urls = get_dem_urls(vector_path, buffer_dist) log.info('{} DEM raster(s) identified on Science Base.'.format(len(urls))) rasters = {} for url in urls: base_path = os.path.basename(os.path.splitext(url)[0]) final_unzip_path = os.path.join(unzip_folder, base_path) if url.lower().endswith('.zip'): file_path = download_unzip(url, download_folder, final_unzip_path, force_download) raster_path = find_rasters(file_path) else: raster_path = download_file(url, download_folder, force_download) # Sanity check that all rasters going into the VRT share the same cell resolution. dataset = gdal.Open(raster_path) gt = dataset.GetGeoTransform() # Store the geotransform for later gtHelper = Geotransform(gt) # Create a tuple of useful numbers to use when trying to figure out if we have a problem with cellwidths rasters[raster_path] = gtHelper dataset = None if (len(rasters.keys()) == 0): raise Exception('No DEM urls were found') # Pick one result to compare with and loop over to see if all the rasters have the same, exact dimensions elif len(rasters.keys()) > 1: widthStdDev = statistics.stdev( [gt.CellWidth() for gt in rasters.values()]) heightStdDev = statistics.stdev( [gt.CellHeight() for gt in rasters.values()]) # This is the broad-strokes check. If the cell widths are vastly different then this won't work and you'll get stitching artifacts when you resample # so we bail out. if widthStdDev > CELL_SIZE_MAX_STDDEV or heightStdDev > CELL_SIZE_MAX_STDDEV: log.warning('Multiple DEM raster cells widths encountered.') for rp, gt in rasters.items(): log.warning('cell width {} :: ({}, {})'.format( rp, gt.CellWidth(), gt.CellHeight())) # raise Exception('Cannot continue. Raster cell sizes are too different and resampling will cause edge effects in the stitched raster') # Now that we know we have a problem we need to figure out where the truth is: # if widthStdDev > CELL_SIZE_THRESH_STDDEV or heightStdDev > CELL_SIZE_THRESH_STDDEV: # for rpath, gt in rasters.items(): # log.warning('Correcting Raster: {} from:({},{}) to:({},{})'.format(rpath, gt.CellWidth(), gt.CellHeight(), widthAvg, heightAvg)) # gt.SetCellWidth(widthAvg) # gt.SetCellHeight(heightAvg) # dataset = gdal.Open(raster_path) # dataset.SetGeoTransform(gt.gt) # dataset = None return list(rasters.keys()), urls
def confinement(huc: int, flowlines_orig: Path, confining_polygon_orig: Path, output_folder: Path, buffer_field: str, confinement_type: str, reach_codes: List[str], min_buffer: float = 0.0, bankfull_expansion_factor: float = 1.0, debug: bool = False, meta=None): """Generate confinement attribute for a stream network Args: huc (integer): Huc identifier flowlines (path): input flowlines layer confining_polygon (path): valley bottom or other boundary defining confining margins output_folder (path): location to store confinement project and output geopackage buffer_field (string): name of float field with buffer values in meters (i.e. 'BFWidth') confinement_type (string): name of type of confinement generated reach_codes (List[int]): NHD reach codes for features to include in outputs min_buffer (float): minimum bankfull value to use in buffers e.g. raster cell resolution bankfull_expansion_factor (float): factor to expand bankfull on each side of bank debug (bool): run tool in debug mode (save intermediate outputs). Default = False meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs """ log = Logger("Confinement") log.info(f'Confinement v.{cfg.version}') # .format(cfg.version)) try: int(huc) except ValueError: raise Exception( 'Invalid HUC identifier "{}". Must be an integer'.format(huc)) if not (len(huc) == 4 or len(huc) == 8): raise Exception('Invalid HUC identifier. Must be four digit integer') # Make the projectXML project, _realization, proj_nodes, report_path = create_project( huc, output_folder, {'ConfinementType': confinement_type}) # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) # Copy input shapes to a geopackage flowlines_path = os.path.join( output_folder, LayerTypes['INPUTS'].rel_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path) confining_path = os.path.join( output_folder, LayerTypes['INPUTS'].rel_path, LayerTypes['INPUTS'].sub_layers['CONFINING_POLYGON'].rel_path) copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG) copy_feature_class(confining_polygon_orig, confining_path, epsg=cfg.OUTPUT_EPSG) _nd, _inputs_gpkg_path, inputs_gpkg_lyrs = project.add_project_geopackage( proj_nodes['Inputs'], LayerTypes['INPUTS']) output_gpkg = os.path.join(output_folder, LayerTypes['CONFINEMENT'].rel_path) intermediates_gpkg = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path) # Creates an empty geopackage and replaces the old one GeopackageLayer(output_gpkg, delete_dataset=True) GeopackageLayer(intermediates_gpkg, delete_dataset=True) # Add the flowlines file with some metadata project.add_metadata({'BufferField': buffer_field}, inputs_gpkg_lyrs['FLOWLINES'][0]) # Add the confinement polygon project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) _nd, _inputs_gpkg_path, out_gpkg_lyrs = project.add_project_geopackage( proj_nodes['Outputs'], LayerTypes['CONFINEMENT']) # Additional Metadata project.add_metadata( { 'Min Buffer': str(min_buffer), "Expansion Factor": str(bankfull_expansion_factor) }, out_gpkg_lyrs['CONFINEMENT_BUFFERS'][0]) # Generate confining margins log.info(f"Preparing output geopackage: {output_gpkg}") log.info(f"Generating Confinement from buffer field: {buffer_field}") # Load input datasets and set the global srs and a meter conversion factor with GeopackageLayer(flowlines_path) as flw_lyr: srs = flw_lyr.spatial_ref meter_conversion = flw_lyr.rough_convert_metres_to_vector_units(1) geom_confining_polygon = get_geometry_unary_union(confining_path, cfg.OUTPUT_EPSG) # Calculate Spatial Constants # Get a very rough conversion factor for 1m to whatever units the shapefile uses offset = 0.1 * meter_conversion selection_buffer = 0.1 * meter_conversion # Standard Outputs field_lookup = { 'side': ogr.FieldDefn("Side", ogr.OFTString), 'flowlineID': ogr.FieldDefn( "NHDPlusID", ogr.OFTString ), # ArcGIS cannot read Int64 and will show up as 0, however data is stored correctly in GPKG 'confinement_type': ogr.FieldDefn("Confinement_Type", ogr.OFTString), 'confinement_ratio': ogr.FieldDefn("Confinement_Ratio", ogr.OFTReal), 'constriction_ratio': ogr.FieldDefn("Constriction_Ratio", ogr.OFTReal), 'length': ogr.FieldDefn("ApproxLeng", ogr.OFTReal), 'confined_length': ogr.FieldDefn("ConfinLeng", ogr.OFTReal), 'constricted_length': ogr.FieldDefn("ConstrLeng", ogr.OFTReal), 'bankfull_width': ogr.FieldDefn("Bankfull_Width", ogr.OFTReal), 'buffer_width': ogr.FieldDefn("Buffer_Width", ogr.OFTReal), # Couple of Debug fields too 'process': ogr.FieldDefn("ErrorProcess", ogr.OFTString), 'message': ogr.FieldDefn("ErrorMessage", ogr.OFTString) } field_lookup['side'].SetWidth(5) field_lookup['confinement_type'].SetWidth(5) # Here we open all the necessary output layers and write the fields to them. There's no harm in quickly # Opening these layers to instantiate them # Standard Outputs with GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT']. sub_layers["CONFINEMENT_MARGINS"].rel_path, write=True) as margins_lyr: margins_lyr.create(ogr.wkbLineString, spatial_ref=srs) margins_lyr.ogr_layer.CreateField(field_lookup['side']) margins_lyr.ogr_layer.CreateField(field_lookup['flowlineID']) margins_lyr.ogr_layer.CreateField(field_lookup['length']) with GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT']. sub_layers["CONFINEMENT_RAW"].rel_path, write=True) as raw_lyr: raw_lyr.create(ogr.wkbLineString, spatial_ref=srs) raw_lyr.ogr_layer.CreateField(field_lookup['flowlineID']) raw_lyr.ogr_layer.CreateField(field_lookup['confinement_type']) raw_lyr.ogr_layer.CreateField(field_lookup['length']) with GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT']. sub_layers["CONFINEMENT_RATIO"].rel_path, write=True) as ratio_lyr: ratio_lyr.create(ogr.wkbLineString, spatial_ref=srs) ratio_lyr.ogr_layer.CreateField(field_lookup['flowlineID']) ratio_lyr.ogr_layer.CreateField(field_lookup['confinement_ratio']) ratio_lyr.ogr_layer.CreateField(field_lookup['constriction_ratio']) ratio_lyr.ogr_layer.CreateField(field_lookup['length']) ratio_lyr.ogr_layer.CreateField(field_lookup['confined_length']) ratio_lyr.ogr_layer.CreateField(field_lookup['constricted_length']) with GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES']. sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path, write=True) as lyr: lyr.create(ogr.wkbPolygon, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['side']) lyr.ogr_layer.CreateField(field_lookup['flowlineID']) lyr.ogr_layer.CreateField(field_lookup['bankfull_width']) lyr.ogr_layer.CreateField(field_lookup['buffer_width']) with GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT']. sub_layers["CONFINEMENT_BUFFERS"].rel_path, write=True) as lyr: lyr.create(ogr.wkbPolygon, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['flowlineID']) lyr.ogr_layer.CreateField(field_lookup['bankfull_width']) lyr.ogr_layer.CreateField(field_lookup['buffer_width']) with GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES']. sub_layers["SPLIT_POINTS"].rel_path, write=True) as lyr: lyr.create(ogr.wkbPoint, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['side']) lyr.ogr_layer.CreateField(field_lookup['flowlineID']) with GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES']. sub_layers["FLOWLINE_SEGMENTS"].rel_path, write=True) as lyr: lyr.create(ogr.wkbLineString, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['side']) lyr.ogr_layer.CreateField(field_lookup['flowlineID']) with GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES']. sub_layers["ERROR_POLYLINES"].rel_path, write=True) as lyr: lyr.create(ogr.wkbLineString, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['process']) lyr.ogr_layer.CreateField(field_lookup['message']) with GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES']. sub_layers["ERROR_POLYGONS"].rel_path, write=True) as lyr: lyr.create(ogr.wkbPolygon, spatial_ref=srs) lyr.ogr_layer.CreateField(field_lookup['process']) lyr.ogr_layer.CreateField(field_lookup['message']) # Generate confinement per Flowline with GeopackageLayer(flowlines_path) as flw_lyr, \ GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_MARGINS"].rel_path, write=True) as margins_lyr, \ GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RAW"].rel_path, write=True) as raw_lyr, \ GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RATIO"].rel_path, write=True) as ratio_lyr, \ GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["SPLIT_POINTS"].rel_path, write=True) as dbg_splitpts_lyr, \ GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["FLOWLINE_SEGMENTS"].rel_path, write=True) as dbg_flwseg_lyr, \ GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path, write=True) as conf_buff_split_lyr, \ GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_BUFFERS"].rel_path, write=True) as buff_lyr, \ GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYLINES"].rel_path, write=True) as dbg_err_lines_lyr, \ GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYGONS"].rel_path, write=True) as dbg_err_polygons_lyr: err_count = 0 for flowline, _counter, progbar in flw_lyr.iterate_features( "Generating confinement for flowlines", attribute_filter="FCode IN ({0})".format(','.join( [key for key in reach_codes])), write_layers=[ margins_lyr, raw_lyr, ratio_lyr, dbg_splitpts_lyr, dbg_flwseg_lyr, buff_lyr, conf_buff_split_lyr, dbg_err_lines_lyr, dbg_err_polygons_lyr ]): # Load Flowline flowlineID = int(flowline.GetFieldAsInteger64("NHDPlusID")) bankfull_width = flowline.GetField(buffer_field) buffer_value = max(bankfull_width, min_buffer) geom_flowline = GeopackageLayer.ogr2shapely(flowline) if not geom_flowline.is_valid or geom_flowline.is_empty or geom_flowline.length == 0: progbar.erase() log.warning("Invalid flowline with id: {}".format(flowlineID)) continue # Generate buffer on each side of the flowline geom_buffer = geom_flowline.buffer( ((buffer_value * meter_conversion) / 2) * bankfull_expansion_factor, cap_style=2) # inital cleanup if geom is multipolygon if geom_buffer.geom_type == "MultiPolygon": log.warning(f"Cleaning multipolygon for id{flowlineID}") polys = [g for g in geom_buffer if g.intersects(geom_flowline)] if len(polys) == 1: geom_buffer = polys[0] if not geom_buffer.is_valid or geom_buffer.is_empty or geom_buffer.area == 0 or geom_buffer.geom_type not in [ "Polygon" ]: progbar.erase() log.warning("Invalid flowline (after buffering) id: {}".format( flowlineID)) dbg_err_lines_lyr.create_feature( geom_flowline, { "ErrorProcess": "Generate Buffer", "ErrorMessage": "Invalid Buffer" }) err_count += 1 continue buff_lyr.create_feature( geom_buffer, { "NHDPlusID": flowlineID, "Buffer_Width": buffer_value, "Bankfull_Width": bankfull_width }) # Split the Buffer by the flowline geom_buffer_splits = split( geom_buffer, geom_flowline ) # snap(geom, geom_buffer)) <--shapely does not snap vertex to edge. need to make new function for this to ensure more buffers have 2 split polygons # Process only if 2 buffers exist if len(geom_buffer_splits) != 2: # Lets try to split this again by slightly extending the line geom_newline = scale(geom_flowline, 1.1, 1.1, origin='center') geom_buffer_splits = split(geom_buffer, geom_newline) if len(geom_buffer_splits) != 2: # triage the polygon if still cannot split it error_message = f"WARNING: Flowline FID {flowline.GetFID()} | Incorrect number of split buffer polygons: {len(geom_buffer_splits)}" progbar.erase() log.warning(error_message) dbg_err_lines_lyr.create_feature( geom_flowline, { "ErrorProcess": "Buffer Split", "ErrorMessage": error_message }) err_count += 1 if len(geom_buffer_splits) > 1: for geom in geom_buffer_splits: dbg_err_polygons_lyr.create_feature( geom, { "ErrorProcess": "Buffer Split", "ErrorMessage": error_message }) else: dbg_err_polygons_lyr.create_feature( geom_buffer_splits, { "ErrorProcess": "Buffer Split", "ErrorMessage": error_message }) continue # Generate point to test side of flowline geom_offset = geom_flowline.parallel_offset(offset, "left") if not geom_offset.is_valid or geom_offset.is_empty or geom_offset.length == 0: progbar.erase() log.warning("Invalid flowline (after offset) id: {}".format( flowlineID)) err_count += 1 dbg_err_lines_lyr.create_feature( geom_flowline, { "ErrorProcess": "Offset Error", "ErrorMessage": "Invalid flowline (after offset) id: {}".format( flowlineID) }) continue geom_side_point = geom_offset.interpolate(0.5, True) # Store output segements lgeoms_right_confined_flowline_segments = [] lgeoms_left_confined_flowline_segments = [] for geom_side in geom_buffer_splits: # Identify side of flowline side = "LEFT" if geom_side.contains( geom_side_point) else "RIGHT" # Save the polygon conf_buff_split_lyr.create_feature( geom_side, { "Side": side, "NHDPlusID": flowlineID, "Buffer_Width": buffer_value, "Bankfull_Width": bankfull_width }) # Generate Confining margins geom_confined_margins = geom_confining_polygon.boundary.intersection( geom_side) # make sure intersection splits lines if geom_confined_margins.is_empty: continue # Multilinestring to individual linestrings lines = [ line for line in geom_confined_margins ] if geom_confined_margins.geom_type == 'MultiLineString' else [ geom_confined_margins ] for line in lines: margins_lyr.create_feature( line, { "Side": side, "NHDPlusID": flowlineID, "ApproxLeng": line.length / meter_conversion }) # Split flowline by Near Geometry pt_start = nearest_points(Point(line.coords[0]), geom_flowline)[1] pt_end = nearest_points(Point(line.coords[-1]), geom_flowline)[1] for point in [pt_start, pt_end]: dbg_splitpts_lyr.create_feature( point, { "Side": side, "NHDPlusID": flowlineID }) distance_sorted = sorted([ geom_flowline.project(pt_start), geom_flowline.project(pt_end) ]) segment = substring(geom_flowline, distance_sorted[0], distance_sorted[1]) # Store the segment by flowline side if segment.is_valid and segment.geom_type in [ "LineString", "MultiLineString" ]: if side == "LEFT": lgeoms_left_confined_flowline_segments.append( segment) else: lgeoms_right_confined_flowline_segments.append( segment) dbg_flwseg_lyr.create_feature(segment, { "Side": side, "NHDPlusID": flowlineID }) # Raw Confinement Output # Prepare flowline splits splitpoints = [ Point(x, y) for line in lgeoms_left_confined_flowline_segments + lgeoms_right_confined_flowline_segments for x, y in line.coords ] cut_distances = sorted( list( set([ geom_flowline.project(point) for point in splitpoints ]))) lgeoms_flowlines_split = [] current_line = geom_flowline cumulative_distance = 0.0 while len(cut_distances) > 0: distance = cut_distances.pop(0) - cumulative_distance if not distance == 0.0: outline = cut(current_line, distance) if len(outline) == 1: current_line = outline[0] else: current_line = outline[1] lgeoms_flowlines_split.append(outline[0]) cumulative_distance = cumulative_distance + distance lgeoms_flowlines_split.append(current_line) # Confined Segments lgeoms_confined_left_split = select_geoms_by_intersection( lgeoms_flowlines_split, lgeoms_left_confined_flowline_segments, buffer=selection_buffer) lgeoms_confined_right_split = select_geoms_by_intersection( lgeoms_flowlines_split, lgeoms_right_confined_flowline_segments, buffer=selection_buffer) lgeoms_confined_left = select_geoms_by_intersection( lgeoms_confined_left_split, lgeoms_confined_right_split, buffer=selection_buffer, inverse=True) lgeoms_confined_right = select_geoms_by_intersection( lgeoms_confined_right_split, lgeoms_confined_left_split, buffer=selection_buffer, inverse=True) geom_confined = unary_union(lgeoms_confined_left_split + lgeoms_confined_right_split) # Constricted Segments lgeoms_constricted_l = select_geoms_by_intersection( lgeoms_confined_left_split, lgeoms_confined_right_split, buffer=selection_buffer) lgeoms_constrcited_r = select_geoms_by_intersection( lgeoms_confined_right_split, lgeoms_confined_left_split, buffer=selection_buffer) lgeoms_constricted = [] for geom in lgeoms_constricted_l + lgeoms_constrcited_r: if not any(g.equals(geom) for g in lgeoms_constricted): lgeoms_constricted.append(geom) geom_constricted = MultiLineString(lgeoms_constricted) # Unconfined Segments lgeoms_unconfined = select_geoms_by_intersection( lgeoms_flowlines_split, lgeoms_confined_left_split + lgeoms_confined_right_split, buffer=selection_buffer, inverse=True) # Save Raw Confinement for con_type, geoms in zip(["Left", "Right", "Both", "None"], [ lgeoms_confined_left, lgeoms_confined_right, lgeoms_constricted, lgeoms_unconfined ]): for g in geoms: if g.geom_type == "LineString": raw_lyr.create_feature( g, { "NHDPlusID": flowlineID, "Confinement_Type": con_type, "ApproxLeng": g.length / meter_conversion }) elif geoms.geom_type in ["Point", "MultiPoint"]: progbar.erase() log.warning( f"Flowline FID: {flowline.GetFID()} | Point geometry identified generating outputs for Raw Confinement." ) else: progbar.erase() log.warning( f"Flowline FID: {flowline.GetFID()} | Unknown geometry identified generating outputs for Raw Confinement." ) # Calculated Confinement per Flowline confinement_ratio = geom_confined.length / geom_flowline.length if geom_confined else 0.0 constricted_ratio = geom_constricted.length / geom_flowline.length if geom_constricted else 0.0 # Save Confinement Ratio attributes = { "NHDPlusID": flowlineID, "Confinement_Ratio": confinement_ratio, "Constriction_Ratio": constricted_ratio, "ApproxLeng": geom_flowline.length / meter_conversion, "ConfinLeng": geom_confined.length / meter_conversion if geom_confined else 0.0, "ConstrLeng": geom_constricted.length / meter_conversion if geom_constricted else 0.0 } ratio_lyr.create_feature(geom_flowline, attributes) # Write a report report = ConfinementReport(output_gpkg, report_path, project) report.write() progbar.finish() log.info(f"Count of Flowline segments with errors: {err_count}") log.info('Confinement Finished') return
def sanitize(name: str, in_path: str, out_path: str, buff_dist: float, select_features=None): """ It's important to make sure we have the right kinds of geometries. Args: name (str): Mainly just for good logging in_path (str): [description] out_path (str): [description] buff_dist (float): [description] """ log = Logger('VBET Simplify') with GeopackageLayer(out_path, write=True) as out_lyr, \ TempGeopackage('sanitize_temp') as tempgpkg, \ GeopackageLayer(in_path) as in_lyr: out_lyr.create_layer(ogr.wkbPolygon, spatial_ref=in_lyr.spatial_ref) pts = 0 square_buff = buff_dist * buff_dist # NOTE: Order of operations really matters here. in_pts = 0 out_pts = 0 with GeopackageLayer(tempgpkg.filepath, "sanitize_{}".format(str(uuid4())), write=True, delete_dataset=True) as tmp_lyr, \ GeopackageLayer(select_features) as lyr_select_features: tmp_lyr.create_layer_from_ref(in_lyr) def geom_validity_fix(geom_in): f_geom = geom_in # Only clean if there's a problem: if not f_geom.IsValid(): f_geom = f_geom.Buffer(0) if not f_geom.IsValid(): f_geom = f_geom.Buffer(buff_dist) f_geom = f_geom.Buffer(-buff_dist) return f_geom # Only keep features intersected with network tmp_lyr.create_layer_from_ref(in_lyr) for candidate_feat, _c2, _p1 in in_lyr.iterate_features( "Finding interesected features"): candidate_geom = candidate_feat.GetGeometryRef() for select_feat, _counter, _progbar in lyr_select_features.iterate_features( ): select_geom = select_feat.GetGeometryRef() if select_geom.Intersects(candidate_geom): feat = ogr.Feature(tmp_lyr.ogr_layer_def) feat.SetGeometry(candidate_geom) tmp_lyr.ogr_layer.CreateFeature(feat) feat = None break # Second loop is about filtering bad areas and simplifying for in_feat, _counter, _progbar in tmp_lyr.iterate_features( "Filtering out non-relevant shapes for {}".format(name)): fid = in_feat.GetFID() geom = in_feat.GetGeometryRef() area = geom.Area() pts += geom.GetBoundary().GetPointCount() # First check. Just make sure this is a valid shape we can work with # Make sure the area is greater than the square of the cell width # Make sure we're not significantly disconnected from the main shape # Make sure we intersect the main shape if geom.IsEmpty() \ or area < square_buff: # or biggest_area[3].Distance(geom) > 2 * buff_dist: continue f_geom = geom.SimplifyPreserveTopology(buff_dist) # # Only fix things that need fixing f_geom = geom_validity_fix(f_geom) # Second check here for validity after simplification # Then write to a temporary geopackage layer if not f_geom.IsEmpty() and f_geom.Area() > 0: out_feature = ogr.Feature(out_lyr.ogr_layer_def) out_feature.SetGeometry(f_geom) out_feature.SetFID(fid) out_lyr.ogr_layer.CreateFeature(out_feature) in_pts += pts out_pts += f_geom.GetBoundary().GetPointCount() else: log.warning( 'Invalid GEOM with fid: {} for layer {}'.format( fid, name)) log.info('Writing to disk for layer {}'.format(name))