def load_geometries(in_layer_path: str, id_field: str = None, epsg: int = None, spatial_ref: osr.SpatialReference = None) -> dict:
"""[summary]
Args:
in_layer_path (str): [description]
id_field (str, optional): [description]. Defaults to None.
epsg (int, optional): [description]. Defaults to None.
spatial_ref (osr.SpatialReference, optional): [description]. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
dict: [description]
"""
log = Logger('load_geometries')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
# Determine the transformation if user provides an EPSG
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
features = {}
for feature, _counter, progbar in in_layer.iterate_features("Loading features"):
if id_field is None:
reach = feature.GetFID()
else:
reach = feature.GetField(id_field)
geom = feature.GetGeometryRef()
geo_type = geom.GetGeometryType()
new_geom = VectorBase.ogr2shapely(geom, transform=transform)
if new_geom.is_empty:
progbar.erase() # get around the progressbar
log.warning('Empty feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
elif not new_geom.is_valid:
progbar.erase() # get around the progressbar
log.warning('Invalid feature with FID={} cannot be unioned and will be ignored'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.length == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for feature with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.area == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for feature with FID={}'.format(feature.GetFID()))
else:
features[reach] = new_geom
return features
def load_geoms(in_lines):
out = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Loading geometry"):
shapely_geom = VectorBase.ogr2shapely(feat)
out.append(shapely_geom)
return out
def network_statistics(label: str, vector_layer_path: str):
log = Logger('network_statistics')
log.info('Network ShapeFile Summary: {}'.format(vector_layer_path))
results = {}
total_length = 0.0
min_length = None
max_length = None
invalid_features = 0
no_geometry = 0
with get_shp_or_gpkg(vector_layer_path) as vector_layer:
# Delete output column from network ShapeFile if it exists and then recreate it
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
results[vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()] = 0
for feature, _counter, _progbar in vector_layer.iterate_features("Calculating Stats"):
geom = feature.GetGeometryRef()
if geom is None:
no_geometry += 1
return
shapely_obj = VectorBase.ogr2shapely(geom)
length = shapely_obj.length
if shapely_obj.is_empty or shapely_obj.is_valid is False:
invalid_features += 1
total_length += length
min_length = length if not min_length or min_length > length else min_length
max_length = length if not max_length or max_length < length else max_length
for fieldidx in range(0, vector_layer.ogr_layer_def.GetFieldCount()):
field = vector_layer.ogr_layer_def.GetFieldDefn(fieldidx).GetName()
if field not in results:
results[field] = 0
results[field] += 0 if feature.GetField(field) else 1
features = vector_layer.ogr_layer.GetFeatureCount()
results['Feature Count'] = features
results['Invalid Features'] = invalid_features
results['Features without geometry'] = no_geometry
results['Min Length'] = min_length
results['Max Length'] = max_length
results['Avg Length'] = (total_length / features) if features > 0 and total_length != 0 else 0.0
results['Total Length'] = total_length
for key, value in results.items():
if value > 0:
log.info('{}, {} with {:,} NULL values'.format(label, key, value))
return results
def midpoints(in_lines):
out_points = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat in in_lyr.iterate_features('Getting Midpoints'):
geom = feat.GetGeometryRef()
line = VectorBase.ogr2shapely(geom)
out_points.append(RiverPoint(line.interpolate(0.5, True)))
feat = None
return out_points
def centerline_points(
in_lines: Path,
distance: float = 0.0,
transform: Transform = None) -> Dict[int, List[RiverPoint]]:
"""Generates points along each line feature at specified distances from the end as well as quarter and halfway
Args:
in_lines (Path): path of shapefile with features
distance (float, optional): distance from ends to generate points. Defaults to 0.0.
transform (Transform, optional): coordinate transformation. Defaults to None.
Returns:
[type]: [description]
"""
log = Logger('centerline_points')
with get_shp_or_gpkg(in_lines) as in_lyr:
out_group = {}
ogr_extent = in_lyr.ogr_layer.GetExtent()
extent = Polygon.from_bounds(ogr_extent[0], ogr_extent[2],
ogr_extent[1], ogr_extent[3])
for feat, _counter, progbar in in_lyr.iterate_features(
"Centerline points"):
line = VectorBase.ogr2shapely(feat, transform)
fid = feat.GetFID()
out_points = []
# Attach the FID in case we need it later
props = {'fid': fid}
pts = [
line.interpolate(distance),
line.interpolate(0.5, True),
line.interpolate(-distance)
]
if line.project(line.interpolate(0.25, True)) > distance:
pts.append(line.interpolate(0.25, True))
pts.append(line.interpolate(-0.25, True))
for pt in pts:
# Recall that interpolation can have multiple solutions due to pythagorean theorem
# Throw away anything that's not inside our bounds
if not extent.contains(pt):
progbar.erase()
log.warning('Point {} is outside of extent: {}'.format(
pt.coords[0], ogr_extent))
out_points.append(RiverPoint(pt, properties=props))
out_group[int(fid)] = out_points
feat = None
return out_group
def get_riverpoints(inpath, epsg, attribute_filter=None):
"""[summary]
Args:
inpath ([type]): Path to a ShapeFile
epsg ([type]): Desired output spatial reference
attribute_filter ([type], optional): [description]. Defaults to None.
Returns:
[type]: List of RiverPoint objects
"""
log = Logger('get_riverpoints')
points = []
with get_shp_or_gpkg(inpath) as in_lyr:
_out_spatial_ref, transform = get_transform_from_epsg(
in_lyr.spatial_ref, epsg)
for feat, _counter, progbar in in_lyr.iterate_features(
'Getting points for use in Thiessen',
attribute_filter=attribute_filter):
new_geom = feat.GetGeometryRef()
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning(
'Feature with FID={} has no geometry. Skipping'.format(
feat.GetFID()))
continue
new_geom.Transform(transform)
new_shape = VectorBase.ogr2shapely(new_geom)
if new_shape.type == 'Polygon':
new_shape = MultiPolygon([new_shape])
for poly in new_shape:
# Exterior is the shell and there is only ever 1
for pt in list(poly.exterior.coords):
points.append(RiverPoint(pt, interior=False))
# Now we consider interiors. NB: Interiors are only qualifying islands in this case
for idx, island in enumerate(poly.interiors):
for pt in list(island.coords):
points.append(RiverPoint(pt, interior=True,
island=idx))
return points
def load_geometries(database, target_srs=None, where_clause=None):
transform = None
if target_srs:
db_srs = get_db_srs(database)
# https://github.com/OSGeo/gdal/issues/1546
target_srs.SetAxisMappingStrategy(db_srs.GetAxisMappingStrategy())
transform = osr.CoordinateTransformation(db_srs, target_srs)
conn = sqlite3.connect(database)
curs = conn.execute('SELECT ReachID, Geometry FROM Reaches {}'.format('WHERE {}'.format(where_clause) if where_clause else ''))
reaches = {}
for row in curs.fetchall():
geom = ogr.CreateGeometryFromJson(row[1])
if transform:
geom.Transform(transform)
reaches[row[0]] = VectorBase.ogr2shapely(geom)
return reaches
def get_geometry_union(in_layer_path: str, epsg: int = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""[summary]
Args:
in_layer_path (str): [description]
epsg (int, optional): [description]. Defaults to None.
attribute_filter (str, optional): [description]. Defaults to None.
clip_shape (BaseGeometry, optional): [description]. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_union')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
geom = None
for feature, _counter, progbar in in_layer.iterate_features("Getting geometry union", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
if feature.GetGeometryRef() is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geometry. Skipping'.format(feature.GetFID()))
continue
new_shape = VectorBase.ogr2shapely(feature, transform=transform)
try:
geom = geom.union(new_shape) if geom is not None else new_shape
except Exception:
progbar.erase() # get around the progressbar
log.warning('Union failed for shape with FID={} and will be ignored'.format(feature.GetFID()))
return geom
def calc_max_drainage(huc_search, precip_raster, wbd, bankfull):
""" Temporary script to calculate the BRAT maximum drainage area threshold
Takes HUC8 watershed boundary polygons and finds the mean annual
precipitation then uses the inverted Beechi and Imaki formula
to derive drainage area at the specified constant bankfull width.
Args:
huc_search (str): feature layer attribute filter string (e.g. '17%' for hydro region 17)
precip_raster (str): path to the national PRISM annual precipitation raster
wbd (str): file path to the national watershed boundary dataset (WBD) file geodatabase
bankfull (float): bankfull width at which drainage area threshold is calculated
"""
# open watershed boundary file geodatabase
driver = ogr.GetDriverByName('OpenFileGDB')
data_source = driver.Open(wbd, 0)
wbd_layer = data_source.GetLayer('WBDHU8')
wbd_layer.SetAttributeFilter('HUC8 LIKE \'{}\''.format(huc_search))
# Need to convert watersheds to the PESG:4269 used by the PRISM raster
_srs, transform = get_transform_from_epsg(wbd_layer.GetSpatialRef(), 4269)
watersheds = {}
for feature in wbd_layer:
huc = feature.GetField('HUC8')
states = feature.GetField('states')
if 'cn' not in states.lower():
watersheds[huc] = VectorBase.ogr2shapely(feature, transform)
stats = raster_buffer_stats2(watersheds, precip_raster)
for huc, stat in stats.items():
# PRISM precipitation is in mm but Beechie and Imaki require it in cm
mean_precip_cm = stat['Mean'] / 10.0
max_drain = pow(bankfull / (0.177) / (pow(mean_precip_cm, 0.453)),
1 / 0.397)
print(
"UPDATE watersheds SET max_drainage = {} WHERE watershed_id = '{}' AND ((max_drainage IS NULL) OR (max_drainage = 0)); -- {}"
.format(int(max_drain), huc, mean_precip_cm))
def centerline_vertex_between_distance(in_lines, distance=0.0):
out_group = []
with get_shp_or_gpkg(in_lines) as in_lyr:
for feat, _counter, _progbar in in_lyr.iterate_features(
"Centerline points between distance"):
line = VectorBase.ogr2shapely(feat)
out_points = []
out_points.append(RiverPoint(line.interpolate(distance)))
out_points.append(RiverPoint(line.interpolate(-distance)))
max_distance = line.length - distance
for vertex in list(line.coords):
test_dist = line.project(Point(vertex))
if test_dist > distance and test_dist < max_distance:
out_points.append(RiverPoint(Point(vertex)))
out_group.append(out_points)
feat = None
return out_group
def get_geometry_unary_union(in_layer_path: str, epsg: int = None, spatial_ref: osr.SpatialReference = None,
attribute_filter: str = None,
clip_shape: BaseGeometry = None,
clip_rect: List[float] = None
) -> BaseGeometry:
"""Load all features from a ShapeFile and union them together into a single geometry
Args:
in_layer_path (str): path to layer
epsg (int, optional): EPSG to project to. Defaults to None.
spatial_ref (osr.SpatialReference, optional): Spatial Ref to project to. Defaults to None.
attribute_filter (str, optional): Filter to a set of attributes. Defaults to None.
clip_shape (BaseGeometry, optional): Clip to a specified shape. Defaults to None.
clip_rect (List[double minx, double miny, double maxx, double maxy)]): Iterate over a subset by clipping to a Shapely-ish geometry. Defaults to None.
Raises:
VectorBaseException: [description]
Returns:
BaseGeometry: [description]
"""
log = Logger('get_geometry_unary_union')
if epsg is not None and spatial_ref is not None:
raise VectorBaseException('Specify either an EPSG or a spatial_ref. Not both')
with get_shp_or_gpkg(in_layer_path) as in_layer:
transform = None
if epsg is not None:
_outref, transform = VectorBase.get_transform_from_epsg(in_layer.spatial_ref, epsg)
elif spatial_ref is not None:
transform = in_layer.get_transform(in_layer.spatial_ref, spatial_ref)
geom_list = []
for feature, _counter, progbar in in_layer.iterate_features("Unary Unioning features", attribute_filter=attribute_filter, clip_shape=clip_shape, clip_rect=clip_rect):
new_geom = feature.GetGeometryRef()
geo_type = new_geom.GetGeometryType()
# We can't union non-valid shapes but sometimes a buffer by 0 can help
if not new_geom.IsValid():
progbar.erase() # get around the progressbar
log.warning('Invalid shape with FID={} trying the Buffer0 technique...'.format(feature.GetFID()))
try:
new_geom = new_geom.Buffer(0)
if not new_geom.IsValid():
log.warning(' Still invalid. Skipping this geometry')
continue
except Exception:
log.warning('Exception raised during buffer 0 technique. skipping this file')
continue
if new_geom is None:
progbar.erase() # get around the progressbar
log.warning('Feature with FID={} has no geoemtry. Skipping'.format(feature.GetFID()))
# Filter out zero-length lines
elif geo_type in VectorBase.LINE_TYPES and new_geom.Length() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Length for shape with FID={}'.format(feature.GetFID()))
# Filter out zero-area polys
elif geo_type in VectorBase.POLY_TYPES and new_geom.Area() == 0:
progbar.erase() # get around the progressbar
log.warning('Zero Area for shape with FID={}'.format(feature.GetFID()))
else:
geom_list.append(VectorBase.ogr2shapely(new_geom, transform))
# IF we get past a certain size then run the union
if len(geom_list) >= 500:
geom_list = [unary_union(geom_list)]
new_geom = None
log.debug('finished iterating with list of size: {}'.format(len(geom_list)))
if len(geom_list) > 1:
log.debug('Starting final union of geom_list of size: {}'.format(len(geom_list)))
# Do a final union to clean up anything that might still be in the list
geom_union = unary_union(geom_list)
elif len(geom_list) == 0:
log.warning('No geometry found to union')
return None
else:
log.debug('FINAL Unioning geom_list of size {}'.format(len(geom_list)))
geom_union = geom_list[0]
log.debug(' done')
print_geom_size(log, geom_union)
log.debug('Complete')
# Return a shapely object
return geom_union
def reach_geometry(flow_lines: Path, dem_path: Path, buffer_distance: float):
""" Calculate reach geometry BRAT attributes
Args:
flow_lines (Path): [description]
dem_path (Path): [description]
buffer_distance (float): [description]
"""
log = Logger('Reach Geometry')
# Determine the best projected coordinate system based on the raster
dataset = gdal.Open(dem_path)
geo_transform = dataset.GetGeoTransform()
xcentre = geo_transform[0] + (dataset.RasterXSize * geo_transform[1]) / 2.0
epsg = get_utm_zone_epsg(xcentre)
with rasterio.open(dem_path) as raster:
bounds = raster.bounds
extent = box(*bounds)
# Buffer the start and end point of each reach
line_start_polygons = {}
line_end_polygons = {}
reaches = {}
with get_shp_or_gpkg(flow_lines) as lyr:
# Transformations from original flow line features to metric EPSG, and to raster spatial reference
_srs, transform_to_metres = VectorBase.get_transform_from_epsg(lyr.spatial_ref, epsg)
_srs, transform_to_raster = VectorBase.get_transform_from_raster(lyr.spatial_ref, dem_path)
# Buffer distance converted to the units of the raster spatial reference
vector_buffer = VectorBase.rough_convert_metres_to_raster_units(dem_path, buffer_distance)
for feature, _counter, _progbar in lyr.iterate_features("Processing reaches"):
reach_id = feature.GetFID()
geom = feature.GetGeometryRef()
geom_clone = geom.Clone()
# Calculate the reach length in the output spatial reference
if transform_to_metres is not None:
geom.Transform(transform_to_metres)
reaches[reach_id] = {'iGeo_Len': geom.Length(), 'iGeo_Slope': 0.0, 'iGeo_ElMin': None, 'IGeo_ElMax': None}
if transform_to_raster is not None:
geom_clone.Transform(transform_to_raster)
# Buffer the ends of the reach polyline in the raster spatial reference
pt_start = Point(VectorBase.ogr2shapely(geom_clone, transform_to_raster).coords[0])
pt_end = Point(VectorBase.ogr2shapely(geom_clone, transform_to_raster).coords[-1])
if extent.contains(pt_start) and extent.contains(pt_end):
line_start_polygons[reach_id] = pt_start.buffer(vector_buffer)
line_end_polygons[reach_id] = pt_end.buffer(vector_buffer)
# Retrieve the mean elevation of start and end of point
line_start_elevations = raster_buffer_stats2(line_start_polygons, dem_path)
line_end_elevations = raster_buffer_stats2(line_end_polygons, dem_path)
for reach_id, data in reaches.items():
if reach_id in line_start_elevations and reach_id in line_end_elevations:
sta_data = line_start_elevations[reach_id]
end_data = line_end_elevations[reach_id]
data['iGeo_ElMax'] = _max_ignore_none(sta_data['Maximum'], end_data['Maximum'])
data['iGeo_ElMin'] = _min_ignore_none(sta_data['Minimum'], end_data['Minimum'])
if sta_data['Mean'] is not None and end_data['Mean'] is not None and sta_data['Mean'] != end_data['Mean']:
data['iGeo_Slope'] = abs(sta_data['Mean'] - end_data['Mean']) / data['iGeo_Len']
else:
log.warning('{:,} features skipped because one or both ends of polyline not on DEM raster'.format(reach_id))
write_db_attributes(os.path.dirname(flow_lines), reaches, ['iGeo_Len', 'iGeo_ElMax', 'iGeo_ElMin', 'iGeo_Slope'])