def test_copy_feature_class(self):
in_path = os.path.join(datadir, 'WBDHU12.shp')
out_path = os.path.join(self.outdir, 'WBDHU12_copy.gpkg')
vector_ops.copy_feature_class(in_path,
os.path.join(out_path, 'WBDHU12_no_ref'),
epsg=4326)
with ShapefileLayer(in_path) as in_lyr, GeopackageLayer(
os.path.join(out_path, 'WBDHU12_no_ref')) as out_lyr:
numfeats_orig = in_lyr.ogr_layer.GetFeatureCount()
numfeats1 = out_lyr.ogr_layer.GetFeatureCount()
vector_ops.copy_feature_class(in_path,
os.path.join(out_path, 'WBDHU12_ref'))
with GeopackageLayer(os.path.join(out_path,
'WBDHU12_no_ref')) as out_lyr:
numfeats2 = out_lyr.ogr_layer.GetFeatureCount()
self.assertEqual(numfeats_orig, numfeats1)
self.assertEqual(numfeats_orig, numfeats2)
def confinement(huc: int,
flowlines_orig: Path,
confining_polygon_orig: Path,
output_folder: Path,
buffer_field: str,
confinement_type: str,
reach_codes: List[str],
min_buffer: float = 0.0,
bankfull_expansion_factor: float = 1.0,
debug: bool = False,
meta=None):
"""Generate confinement attribute for a stream network
Args:
huc (integer): Huc identifier
flowlines (path): input flowlines layer
confining_polygon (path): valley bottom or other boundary defining confining margins
output_folder (path): location to store confinement project and output geopackage
buffer_field (string): name of float field with buffer values in meters (i.e. 'BFWidth')
confinement_type (string): name of type of confinement generated
reach_codes (List[int]): NHD reach codes for features to include in outputs
min_buffer (float): minimum bankfull value to use in buffers e.g. raster cell resolution
bankfull_expansion_factor (float): factor to expand bankfull on each side of bank
debug (bool): run tool in debug mode (save intermediate outputs). Default = False
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("Confinement")
log.info(f'Confinement v.{cfg.version}') # .format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
# Make the projectXML
project, _realization, proj_nodes, report_path = create_project(
huc, output_folder, {'ConfinementType': confinement_type})
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
# Copy input shapes to a geopackage
flowlines_path = os.path.join(
output_folder, LayerTypes['INPUTS'].rel_path,
LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
confining_path = os.path.join(
output_folder, LayerTypes['INPUTS'].rel_path,
LayerTypes['INPUTS'].sub_layers['CONFINING_POLYGON'].rel_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(confining_polygon_orig,
confining_path,
epsg=cfg.OUTPUT_EPSG)
_nd, _inputs_gpkg_path, inputs_gpkg_lyrs = project.add_project_geopackage(
proj_nodes['Inputs'], LayerTypes['INPUTS'])
output_gpkg = os.path.join(output_folder,
LayerTypes['CONFINEMENT'].rel_path)
intermediates_gpkg = os.path.join(output_folder,
LayerTypes['INTERMEDIATES'].rel_path)
# Creates an empty geopackage and replaces the old one
GeopackageLayer(output_gpkg, delete_dataset=True)
GeopackageLayer(intermediates_gpkg, delete_dataset=True)
# Add the flowlines file with some metadata
project.add_metadata({'BufferField': buffer_field},
inputs_gpkg_lyrs['FLOWLINES'][0])
# Add the confinement polygon
project.add_project_geopackage(proj_nodes['Intermediates'],
LayerTypes['INTERMEDIATES'])
_nd, _inputs_gpkg_path, out_gpkg_lyrs = project.add_project_geopackage(
proj_nodes['Outputs'], LayerTypes['CONFINEMENT'])
# Additional Metadata
project.add_metadata(
{
'Min Buffer': str(min_buffer),
"Expansion Factor": str(bankfull_expansion_factor)
}, out_gpkg_lyrs['CONFINEMENT_BUFFERS'][0])
# Generate confining margins
log.info(f"Preparing output geopackage: {output_gpkg}")
log.info(f"Generating Confinement from buffer field: {buffer_field}")
# Load input datasets and set the global srs and a meter conversion factor
with GeopackageLayer(flowlines_path) as flw_lyr:
srs = flw_lyr.spatial_ref
meter_conversion = flw_lyr.rough_convert_metres_to_vector_units(1)
geom_confining_polygon = get_geometry_unary_union(confining_path,
cfg.OUTPUT_EPSG)
# Calculate Spatial Constants
# Get a very rough conversion factor for 1m to whatever units the shapefile uses
offset = 0.1 * meter_conversion
selection_buffer = 0.1 * meter_conversion
# Standard Outputs
field_lookup = {
'side': ogr.FieldDefn("Side", ogr.OFTString),
'flowlineID': ogr.FieldDefn(
"NHDPlusID", ogr.OFTString
), # ArcGIS cannot read Int64 and will show up as 0, however data is stored correctly in GPKG
'confinement_type': ogr.FieldDefn("Confinement_Type", ogr.OFTString),
'confinement_ratio': ogr.FieldDefn("Confinement_Ratio", ogr.OFTReal),
'constriction_ratio': ogr.FieldDefn("Constriction_Ratio", ogr.OFTReal),
'length': ogr.FieldDefn("ApproxLeng", ogr.OFTReal),
'confined_length': ogr.FieldDefn("ConfinLeng", ogr.OFTReal),
'constricted_length': ogr.FieldDefn("ConstrLeng", ogr.OFTReal),
'bankfull_width': ogr.FieldDefn("Bankfull_Width", ogr.OFTReal),
'buffer_width': ogr.FieldDefn("Buffer_Width", ogr.OFTReal),
# Couple of Debug fields too
'process': ogr.FieldDefn("ErrorProcess", ogr.OFTString),
'message': ogr.FieldDefn("ErrorMessage", ogr.OFTString)
}
field_lookup['side'].SetWidth(5)
field_lookup['confinement_type'].SetWidth(5)
# Here we open all the necessary output layers and write the fields to them. There's no harm in quickly
# Opening these layers to instantiate them
# Standard Outputs
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_MARGINS"].rel_path,
write=True) as margins_lyr:
margins_lyr.create(ogr.wkbLineString, spatial_ref=srs)
margins_lyr.ogr_layer.CreateField(field_lookup['side'])
margins_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
margins_lyr.ogr_layer.CreateField(field_lookup['length'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_RAW"].rel_path,
write=True) as raw_lyr:
raw_lyr.create(ogr.wkbLineString, spatial_ref=srs)
raw_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
raw_lyr.ogr_layer.CreateField(field_lookup['confinement_type'])
raw_lyr.ogr_layer.CreateField(field_lookup['length'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_RATIO"].rel_path,
write=True) as ratio_lyr:
ratio_lyr.create(ogr.wkbLineString, spatial_ref=srs)
ratio_lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
ratio_lyr.ogr_layer.CreateField(field_lookup['confinement_ratio'])
ratio_lyr.ogr_layer.CreateField(field_lookup['constriction_ratio'])
ratio_lyr.ogr_layer.CreateField(field_lookup['length'])
ratio_lyr.ogr_layer.CreateField(field_lookup['confined_length'])
ratio_lyr.ogr_layer.CreateField(field_lookup['constricted_length'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
lyr.ogr_layer.CreateField(field_lookup['bankfull_width'])
lyr.ogr_layer.CreateField(field_lookup['buffer_width'])
with GeopackageLayer(output_gpkg,
layer_name=LayerTypes['CONFINEMENT'].
sub_layers["CONFINEMENT_BUFFERS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
lyr.ogr_layer.CreateField(field_lookup['bankfull_width'])
lyr.ogr_layer.CreateField(field_lookup['buffer_width'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["SPLIT_POINTS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPoint, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["FLOWLINE_SEGMENTS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbLineString, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['side'])
lyr.ogr_layer.CreateField(field_lookup['flowlineID'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["ERROR_POLYLINES"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbLineString, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['process'])
lyr.ogr_layer.CreateField(field_lookup['message'])
with GeopackageLayer(intermediates_gpkg,
layer_name=LayerTypes['INTERMEDIATES'].
sub_layers["ERROR_POLYGONS"].rel_path,
write=True) as lyr:
lyr.create(ogr.wkbPolygon, spatial_ref=srs)
lyr.ogr_layer.CreateField(field_lookup['process'])
lyr.ogr_layer.CreateField(field_lookup['message'])
# Generate confinement per Flowline
with GeopackageLayer(flowlines_path) as flw_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_MARGINS"].rel_path, write=True) as margins_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RAW"].rel_path, write=True) as raw_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_RATIO"].rel_path, write=True) as ratio_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["SPLIT_POINTS"].rel_path, write=True) as dbg_splitpts_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["FLOWLINE_SEGMENTS"].rel_path, write=True) as dbg_flwseg_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["CONFINEMENT_BUFFER_SPLIT"].rel_path, write=True) as conf_buff_split_lyr, \
GeopackageLayer(output_gpkg, layer_name=LayerTypes['CONFINEMENT'].sub_layers["CONFINEMENT_BUFFERS"].rel_path, write=True) as buff_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYLINES"].rel_path, write=True) as dbg_err_lines_lyr, \
GeopackageLayer(intermediates_gpkg, layer_name=LayerTypes['INTERMEDIATES'].sub_layers["ERROR_POLYGONS"].rel_path, write=True) as dbg_err_polygons_lyr:
err_count = 0
for flowline, _counter, progbar in flw_lyr.iterate_features(
"Generating confinement for flowlines",
attribute_filter="FCode IN ({0})".format(','.join(
[key for key in reach_codes])),
write_layers=[
margins_lyr, raw_lyr, ratio_lyr, dbg_splitpts_lyr,
dbg_flwseg_lyr, buff_lyr, conf_buff_split_lyr,
dbg_err_lines_lyr, dbg_err_polygons_lyr
]):
# Load Flowline
flowlineID = int(flowline.GetFieldAsInteger64("NHDPlusID"))
bankfull_width = flowline.GetField(buffer_field)
buffer_value = max(bankfull_width, min_buffer)
geom_flowline = GeopackageLayer.ogr2shapely(flowline)
if not geom_flowline.is_valid or geom_flowline.is_empty or geom_flowline.length == 0:
progbar.erase()
log.warning("Invalid flowline with id: {}".format(flowlineID))
continue
# Generate buffer on each side of the flowline
geom_buffer = geom_flowline.buffer(
((buffer_value * meter_conversion) / 2) *
bankfull_expansion_factor,
cap_style=2)
# inital cleanup if geom is multipolygon
if geom_buffer.geom_type == "MultiPolygon":
log.warning(f"Cleaning multipolygon for id{flowlineID}")
polys = [g for g in geom_buffer if g.intersects(geom_flowline)]
if len(polys) == 1:
geom_buffer = polys[0]
if not geom_buffer.is_valid or geom_buffer.is_empty or geom_buffer.area == 0 or geom_buffer.geom_type not in [
"Polygon"
]:
progbar.erase()
log.warning("Invalid flowline (after buffering) id: {}".format(
flowlineID))
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess": "Generate Buffer",
"ErrorMessage": "Invalid Buffer"
})
err_count += 1
continue
buff_lyr.create_feature(
geom_buffer, {
"NHDPlusID": flowlineID,
"Buffer_Width": buffer_value,
"Bankfull_Width": bankfull_width
})
# Split the Buffer by the flowline
geom_buffer_splits = split(
geom_buffer, geom_flowline
) # snap(geom, geom_buffer)) <--shapely does not snap vertex to edge. need to make new function for this to ensure more buffers have 2 split polygons
# Process only if 2 buffers exist
if len(geom_buffer_splits) != 2:
# Lets try to split this again by slightly extending the line
geom_newline = scale(geom_flowline, 1.1, 1.1, origin='center')
geom_buffer_splits = split(geom_buffer, geom_newline)
if len(geom_buffer_splits) != 2:
# triage the polygon if still cannot split it
error_message = f"WARNING: Flowline FID {flowline.GetFID()} | Incorrect number of split buffer polygons: {len(geom_buffer_splits)}"
progbar.erase()
log.warning(error_message)
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
err_count += 1
if len(geom_buffer_splits) > 1:
for geom in geom_buffer_splits:
dbg_err_polygons_lyr.create_feature(
geom, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
else:
dbg_err_polygons_lyr.create_feature(
geom_buffer_splits, {
"ErrorProcess": "Buffer Split",
"ErrorMessage": error_message
})
continue
# Generate point to test side of flowline
geom_offset = geom_flowline.parallel_offset(offset, "left")
if not geom_offset.is_valid or geom_offset.is_empty or geom_offset.length == 0:
progbar.erase()
log.warning("Invalid flowline (after offset) id: {}".format(
flowlineID))
err_count += 1
dbg_err_lines_lyr.create_feature(
geom_flowline, {
"ErrorProcess":
"Offset Error",
"ErrorMessage":
"Invalid flowline (after offset) id: {}".format(
flowlineID)
})
continue
geom_side_point = geom_offset.interpolate(0.5, True)
# Store output segements
lgeoms_right_confined_flowline_segments = []
lgeoms_left_confined_flowline_segments = []
for geom_side in geom_buffer_splits:
# Identify side of flowline
side = "LEFT" if geom_side.contains(
geom_side_point) else "RIGHT"
# Save the polygon
conf_buff_split_lyr.create_feature(
geom_side, {
"Side": side,
"NHDPlusID": flowlineID,
"Buffer_Width": buffer_value,
"Bankfull_Width": bankfull_width
})
# Generate Confining margins
geom_confined_margins = geom_confining_polygon.boundary.intersection(
geom_side) # make sure intersection splits lines
if geom_confined_margins.is_empty:
continue
# Multilinestring to individual linestrings
lines = [
line for line in geom_confined_margins
] if geom_confined_margins.geom_type == 'MultiLineString' else [
geom_confined_margins
]
for line in lines:
margins_lyr.create_feature(
line, {
"Side": side,
"NHDPlusID": flowlineID,
"ApproxLeng": line.length / meter_conversion
})
# Split flowline by Near Geometry
pt_start = nearest_points(Point(line.coords[0]),
geom_flowline)[1]
pt_end = nearest_points(Point(line.coords[-1]),
geom_flowline)[1]
for point in [pt_start, pt_end]:
dbg_splitpts_lyr.create_feature(
point, {
"Side": side,
"NHDPlusID": flowlineID
})
distance_sorted = sorted([
geom_flowline.project(pt_start),
geom_flowline.project(pt_end)
])
segment = substring(geom_flowline, distance_sorted[0],
distance_sorted[1])
# Store the segment by flowline side
if segment.is_valid and segment.geom_type in [
"LineString", "MultiLineString"
]:
if side == "LEFT":
lgeoms_left_confined_flowline_segments.append(
segment)
else:
lgeoms_right_confined_flowline_segments.append(
segment)
dbg_flwseg_lyr.create_feature(segment, {
"Side": side,
"NHDPlusID": flowlineID
})
# Raw Confinement Output
# Prepare flowline splits
splitpoints = [
Point(x, y)
for line in lgeoms_left_confined_flowline_segments +
lgeoms_right_confined_flowline_segments for x, y in line.coords
]
cut_distances = sorted(
list(
set([
geom_flowline.project(point) for point in splitpoints
])))
lgeoms_flowlines_split = []
current_line = geom_flowline
cumulative_distance = 0.0
while len(cut_distances) > 0:
distance = cut_distances.pop(0) - cumulative_distance
if not distance == 0.0:
outline = cut(current_line, distance)
if len(outline) == 1:
current_line = outline[0]
else:
current_line = outline[1]
lgeoms_flowlines_split.append(outline[0])
cumulative_distance = cumulative_distance + distance
lgeoms_flowlines_split.append(current_line)
# Confined Segments
lgeoms_confined_left_split = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_left_confined_flowline_segments,
buffer=selection_buffer)
lgeoms_confined_right_split = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_right_confined_flowline_segments,
buffer=selection_buffer)
lgeoms_confined_left = select_geoms_by_intersection(
lgeoms_confined_left_split,
lgeoms_confined_right_split,
buffer=selection_buffer,
inverse=True)
lgeoms_confined_right = select_geoms_by_intersection(
lgeoms_confined_right_split,
lgeoms_confined_left_split,
buffer=selection_buffer,
inverse=True)
geom_confined = unary_union(lgeoms_confined_left_split +
lgeoms_confined_right_split)
# Constricted Segments
lgeoms_constricted_l = select_geoms_by_intersection(
lgeoms_confined_left_split,
lgeoms_confined_right_split,
buffer=selection_buffer)
lgeoms_constrcited_r = select_geoms_by_intersection(
lgeoms_confined_right_split,
lgeoms_confined_left_split,
buffer=selection_buffer)
lgeoms_constricted = []
for geom in lgeoms_constricted_l + lgeoms_constrcited_r:
if not any(g.equals(geom) for g in lgeoms_constricted):
lgeoms_constricted.append(geom)
geom_constricted = MultiLineString(lgeoms_constricted)
# Unconfined Segments
lgeoms_unconfined = select_geoms_by_intersection(
lgeoms_flowlines_split,
lgeoms_confined_left_split + lgeoms_confined_right_split,
buffer=selection_buffer,
inverse=True)
# Save Raw Confinement
for con_type, geoms in zip(["Left", "Right", "Both", "None"], [
lgeoms_confined_left, lgeoms_confined_right,
lgeoms_constricted, lgeoms_unconfined
]):
for g in geoms:
if g.geom_type == "LineString":
raw_lyr.create_feature(
g, {
"NHDPlusID": flowlineID,
"Confinement_Type": con_type,
"ApproxLeng": g.length / meter_conversion
})
elif geoms.geom_type in ["Point", "MultiPoint"]:
progbar.erase()
log.warning(
f"Flowline FID: {flowline.GetFID()} | Point geometry identified generating outputs for Raw Confinement."
)
else:
progbar.erase()
log.warning(
f"Flowline FID: {flowline.GetFID()} | Unknown geometry identified generating outputs for Raw Confinement."
)
# Calculated Confinement per Flowline
confinement_ratio = geom_confined.length / geom_flowline.length if geom_confined else 0.0
constricted_ratio = geom_constricted.length / geom_flowline.length if geom_constricted else 0.0
# Save Confinement Ratio
attributes = {
"NHDPlusID":
flowlineID,
"Confinement_Ratio":
confinement_ratio,
"Constriction_Ratio":
constricted_ratio,
"ApproxLeng":
geom_flowline.length / meter_conversion,
"ConfinLeng":
geom_confined.length /
meter_conversion if geom_confined else 0.0,
"ConstrLeng":
geom_constricted.length /
meter_conversion if geom_constricted else 0.0
}
ratio_lyr.create_feature(geom_flowline, attributes)
# Write a report
report = ConfinementReport(output_gpkg, report_path, project)
report.write()
progbar.finish()
log.info(f"Count of Flowline segments with errors: {err_count}")
log.info('Confinement Finished')
return
def brat_build(huc: int, flowlines: Path, dem: Path, slope: Path, hillshade: Path,
existing_veg: Path, historical_veg: Path, output_folder: Path,
streamside_buffer: float, riparian_buffer: float,
reach_codes: List[str], canal_codes: List[str], peren_codes: List[str],
flow_areas: Path, waterbodies: Path, max_waterbody: float,
valley_bottom: Path, roads: Path, rail: Path, canals: Path, ownership: Path,
elevation_buffer: float, meta: Dict[str, str]):
"""Build a BRAT project by segmenting a reach network and copying
all the necessary layers into the resultant BRAT project
Arguments:
huc {str} -- Watershed identifier
flowlines {str} -- Path to the raw, original polyline flowline ShapeFile
flow_areas {str} -- Path to the polygon ShapeFile that contains large river outlines
waterbodies {str} -- Path to the polygon ShapeFile containing water bodies
max_length {float} -- Maximum allowable flow line segment after segmentation
min_length {float} -- Shortest allowable flow line segment after segmentation
dem {str} -- Path to the DEM raster for the watershed
slope {str} -- Path to the slope raster
hillshade {str} -- Path to the DEM hillshade raster
existing_veg {str} -- Path to the excisting vegetation raster
historical_veg {str} -- Path to the historical vegetation raster
output_folder {str} -- Output folder where the BRAT project will get created
streamside_buffer {float} -- Streamside vegetation buffer (meters)
riparian_buffer {float} -- Riparian vegetation buffer (meters)
intermittent {bool} -- True to keep intermittent streams. False discard them.
ephemeral {bool} -- True to keep ephemeral streams. False to discard them.
max_waterbody {float} -- Area (sqm) of largest waterbody to be retained.
valley_bottom {str} -- Path to valley bottom polygon layer.
roads {str} -- Path to polyline roads ShapeFile
rail {str} -- Path to polyline railway ShapeFile
canals {str} -- Path to polyline canals ShapeFile
ownership {str} -- Path to land ownership polygon ShapeFile
elevation_buffer {float} -- Distance to buffer DEM when sampling elevation
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("BRAT Build")
log.info('HUC: {}'.format(huc))
log.info('EPSG: {}'.format(cfg.OUTPUT_EPSG))
project, _realization, proj_nodes = create_project(huc, output_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
log.info('Adding input rasters to project')
_dem_raster_path_node, dem_raster_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['DEM'], dem)
_existing_path_node, prj_existing_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['EXVEG'], existing_veg)
_historic_path_node, prj_historic_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HISTVEG'], historical_veg)
project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade)
project.add_project_raster(proj_nodes['Inputs'], LayerTypes['SLOPE'], slope)
project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['OUTPUTS'])
inputs_gpkg_path = os.path.join(output_folder, LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path)
outputs_gpkg_path = os.path.join(output_folder, LayerTypes['OUTPUTS'].rel_path)
# Make sure we're starting with empty/fresh geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
GeopackageLayer.delete(outputs_gpkg_path)
# Copy all the original vectors to the inputs geopackage. This will ensure on same spatial reference
source_layers = {
'FLOWLINES': flowlines,
'FLOW_AREA': flow_areas,
'WATERBODIES': waterbodies,
'VALLEY_BOTTOM': valley_bottom,
'ROADS': roads,
'RAIL': rail,
'CANALS': canals
}
input_layers = {}
for input_key, rslayer in LayerTypes['INPUTS'].sub_layers.items():
input_layers[input_key] = os.path.join(inputs_gpkg_path, rslayer.rel_path)
copy_feature_class(source_layers[input_key], input_layers[input_key], cfg.OUTPUT_EPSG)
# Create the output feature class fields. Only those listed here will get copied from the source
with GeopackageLayer(outputs_gpkg_path, layer_name=LayerTypes['OUTPUTS'].sub_layers['BRAT_GEOMETRY'].rel_path, delete_dataset=True) as out_lyr:
out_lyr.create_layer(ogr.wkbMultiLineString, epsg=cfg.OUTPUT_EPSG, options=['FID=ReachID'], fields={
'WatershedID': ogr.OFTString,
'FCode': ogr.OFTInteger,
'TotDASqKm': ogr.OFTReal,
'GNIS_Name': ogr.OFTString,
'NHDPlusID': ogr.OFTReal
})
metadata = {
'BRAT_Build_DateTime': datetime.datetime.now().isoformat(),
'Streamside_Buffer': streamside_buffer,
'Riparian_Buffer': riparian_buffer,
'Reach_Codes': reach_codes,
'Canal_Codes': canal_codes,
'Max_Waterbody': max_waterbody,
'Elevation_Buffer': elevation_buffer
}
# Execute the SQL to create the lookup tables in the output geopackage
watershed_name = create_database(huc, outputs_gpkg_path, metadata, cfg.OUTPUT_EPSG, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'brat_schema.sql'))
project.add_metadata({'Watershed': watershed_name})
# Copy the reaches into the output feature class layer, filtering by reach codes
reach_geometry_path = os.path.join(outputs_gpkg_path, LayerTypes['OUTPUTS'].sub_layers['BRAT_GEOMETRY'].rel_path)
build_network(input_layers['FLOWLINES'], input_layers['FLOW_AREA'], reach_geometry_path, waterbodies_path=input_layers['WATERBODIES'], epsg=cfg.OUTPUT_EPSG, reach_codes=reach_codes, create_layer=False)
with SQLiteCon(outputs_gpkg_path) as database:
# Data preparation SQL statements to handle any weird attributes
database.curs.execute('INSERT INTO ReachAttributes (ReachID, Orig_DA, iGeo_DA, ReachCode, WatershedID, StreamName) SELECT ReachID, TotDASqKm, TotDASqKm, FCode, WatershedID, GNIS_NAME FROM ReachGeometry')
database.curs.execute('UPDATE ReachAttributes SET IsPeren = 1 WHERE (ReachCode IN ({}))'.format(','.join(peren_codes)))
database.curs.execute('UPDATE ReachAttributes SET iGeo_DA = 0 WHERE iGeo_DA IS NULL')
# Register vwReaches as a feature layer as well as its geometry column
database.curs.execute("""INSERT INTO gpkg_contents (table_name, data_type, identifier, min_x, min_y, max_x, max_y, srs_id)
SELECT 'vwReaches', data_type, 'Reaches', min_x, min_y, max_x, max_y, srs_id FROM gpkg_contents WHERE table_name = 'ReachGeometry'""")
database.curs.execute("""INSERT INTO gpkg_geometry_columns (table_name, column_name, geometry_type_name, srs_id, z, m)
SELECT 'vwReaches', column_name, geometry_type_name, srs_id, z, m FROM gpkg_geometry_columns WHERE table_name = 'ReachGeometry'""")
database.conn.commit()
# Calculate the geophysical properties slope, min and max elevations
reach_geometry(reach_geometry_path, dem_raster_path, elevation_buffer)
# Calculate the conflict attributes ready for conservation
conflict_attributes(outputs_gpkg_path, reach_geometry_path,
input_layers['VALLEY_BOTTOM'], input_layers['ROADS'], input_layers['RAIL'], input_layers['CANALS'],
ownership, 30, 5, cfg.OUTPUT_EPSG, canal_codes, intermediates_gpkg_path)
# Calculate the vegetation cell counts for each epoch and buffer
for label, veg_raster in [('Existing Veg', prj_existing_path), ('Historical Veg', prj_historic_path)]:
for buffer in [streamside_buffer, riparian_buffer]:
vegetation_summary(outputs_gpkg_path, '{} {}m'.format(label, buffer), veg_raster, buffer)
log.info('BRAT build completed successfully.')
def rs_context(huc, existing_veg, historic_veg, ownership, fair_market,
ecoregions, prism_folder, output_folder, download_folder,
scratch_dir, parallel, force_download, meta: Dict[str, str]):
"""
Download riverscapes context layers for the specified HUC and organize them as a Riverscapes project
:param huc: Eight, 10 or 12 digit HUC identification number
:param existing_veg: Path to the existing vegetation conditions raster
:param historic_veg: Path to the historical vegetation conditions raster
:param ownership: Path to the national land ownership Shapefile
:param output_folder: Output location for the riverscapes context project
:param download_folder: Temporary folder where downloads are cached. This can be shared between rs_context processes
:param force_download: If false then downloads can be skipped if the files already exist
:param prism_folder: folder containing PRISM rasters in *.bil format
:param meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
:return:
"""
log = Logger("RS Context")
log.info('Starting RSContext v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception(
'Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) in [4, 8, 10, 12]):
raise Exception(
'Invalid HUC identifier. Must be 4, 8, 10 or 12 digit integer')
safe_makedirs(output_folder)
safe_makedirs(download_folder)
# We need a temporary folder for slope rasters, Stitching inputs, intermeditary products, etc.
scratch_dem_folder = os.path.join(scratch_dir, 'rs_context', huc)
safe_makedirs(scratch_dem_folder)
project, realization = create_project(huc, output_folder)
hydrology_gpkg_path = os.path.join(output_folder,
LayerTypes['HYDROLOGY'].rel_path)
dem_node, dem_raster = project.add_project_raster(realization,
LayerTypes['DEM'])
_node, hill_raster = project.add_project_raster(realization,
LayerTypes['HILLSHADE'])
_node, flow_accum = project.add_project_raster(realization,
LayerTypes['FA'])
_node, drain_area = project.add_project_raster(realization,
LayerTypes['DA'])
hand_node, hand_raster = project.add_project_raster(
realization, LayerTypes['HAND'])
_node, slope_raster = project.add_project_raster(realization,
LayerTypes['SLOPE'])
_node, existing_clip = project.add_project_raster(realization,
LayerTypes['EXVEG'])
_node, historic_clip = project.add_project_raster(realization,
LayerTypes['HISTVEG'])
_node, fair_market_clip = project.add_project_raster(
realization, LayerTypes['FAIR_MARKET'])
# Download the four digit NHD archive containing the flow lines and watershed boundaries
log.info('Processing NHD')
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
nhd_download_folder = os.path.join(download_folder, 'nhd', huc[:4])
nhd_unzip_folder = os.path.join(scratch_dir, 'nhd', huc[:4])
nhd, db_path, huc_name, nhd_url = clean_nhd_data(
huc, nhd_download_folder, nhd_unzip_folder,
os.path.join(output_folder, 'hydrology'), cfg.OUTPUT_EPSG, False)
# Clean up the unzipped files. We won't need them again
if parallel:
safe_remove_dir(nhd_unzip_folder)
project.add_metadata({'Watershed': huc_name})
boundary = 'WBDHU{}'.format(len(huc))
# For coarser rasters than the DEM we need to buffer our clip polygon to include enough pixels
# This shouldn't be too much more data because these are usually integer rasters that are much lower res.
buffered_clip_path100 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP100'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path100,
epsg=cfg.OUTPUT_EPSG,
buffer=100)
buffered_clip_path500 = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
copy_feature_class(nhd[boundary],
buffered_clip_path500,
epsg=cfg.OUTPUT_EPSG,
buffer=500)
# PRISM climate rasters
mean_annual_precip = None
bil_files = glob.glob(os.path.join(prism_folder, '**', '*.bil'))
if (len(bil_files) == 0):
raise Exception('Could not find any .bil files in the prism folder')
for ptype in PrismTypes:
try:
# Next should always be guarded
source_raster_path = next(
x for x in bil_files
if ptype.lower() in os.path.basename(x).lower())
except StopIteration:
raise Exception(
'Could not find .bil file corresponding to "{}"'.format(ptype))
_node, project_raster_path = project.add_project_raster(
realization, LayerTypes[ptype])
raster_warp(source_raster_path, project_raster_path, cfg.OUTPUT_EPSG,
buffered_clip_path500, {"cutlineBlend": 1})
# Use the mean annual precipitation to calculate bankfull width
if ptype.lower() == 'ppt':
polygon = get_geometry_unary_union(nhd[boundary],
epsg=cfg.OUTPUT_EPSG)
mean_annual_precip = raster_buffer_stats2(
{1: polygon}, project_raster_path)[1]['Mean']
log.info('Mean annual precipitation for HUC {} is {} mm'.format(
huc, mean_annual_precip))
project.add_metadata(
{'mean_annual_precipitation_mm': str(mean_annual_precip)})
calculate_bankfull_width(nhd['NHDFlowline'], mean_annual_precip)
# Add the DB record to the Project XML
db_lyr = RSLayer('NHD Tables', 'NHDTABLES', 'SQLiteDB',
os.path.relpath(db_path, output_folder))
sqlite_el = project.add_dataset(realization, db_path, db_lyr, 'SQLiteDB')
project.add_metadata({'origin_url': nhd_url}, sqlite_el)
# Add any results to project XML
for name, file_path in nhd.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
vector_nod, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({'origin_url': nhd_url}, vector_nod)
states = get_nhd_states(nhd[boundary])
# Download the NTD archive containing roads and rail
log.info('Processing NTD')
ntd_raw = {}
ntd_unzip_folders = []
ntd_urls = get_ntd_urls(states)
for state, ntd_url in ntd_urls.items():
ntd_download_folder = os.path.join(download_folder, 'ntd',
state.lower())
ntd_unzip_folder = os.path.join(
scratch_dir, 'ntd', state.lower(), 'unzipped'
) # a little awkward but I need a folder for this and this was the best name I could find
ntd_raw[state] = download_shapefile_collection(ntd_url,
ntd_download_folder,
ntd_unzip_folder,
force_download)
ntd_unzip_folders.append(ntd_unzip_folder)
ntd_clean = clean_ntd_data(ntd_raw, nhd['NHDFlowline'], nhd[boundary],
os.path.join(output_folder, 'transportation'),
cfg.OUTPUT_EPSG)
# clean up the NTD Unzip folder. We won't need it again
if parallel:
for unzip_path in ntd_unzip_folders:
safe_remove_dir(unzip_path)
# Write transportation layers to project file
log.info('Write transportation layers to project file')
# Add any results to project XML
for name, file_path in ntd_clean.items():
lyr_obj = RSLayer(name, name, 'Vector',
os.path.relpath(file_path, output_folder))
ntd_node, _fpath = project.add_project_vector(realization, lyr_obj)
project.add_metadata({**ntd_urls}, ntd_node)
# Download the HAND raster
huc6 = huc[0:6]
hand_download_folder = os.path.join(download_folder, 'hand')
_hpath, hand_url = download_hand(huc6,
cfg.OUTPUT_EPSG,
hand_download_folder,
nhd[boundary],
hand_raster,
warp_options={"cutlineBlend": 1})
project.add_metadata({'origin_url': hand_url}, hand_node)
# download contributing DEM rasters, mosaic and reproject into compressed GeoTIF
ned_download_folder = os.path.join(download_folder, 'ned')
ned_unzip_folder = os.path.join(scratch_dir, 'ned')
dem_rasters, urls = download_dem(nhd[boundary], cfg.OUTPUT_EPSG, 0.01,
ned_download_folder, ned_unzip_folder,
force_download)
need_dem_rebuild = force_download or not os.path.exists(dem_raster)
if need_dem_rebuild:
raster_vrt_stitch(dem_rasters,
dem_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
warp_options={"cutlineBlend": 1})
verify_areas(dem_raster, nhd[boundary])
# Calculate slope rasters seperately and then stitch them
slope_parts = []
hillshade_parts = []
need_slope_build = need_dem_rebuild or not os.path.isfile(slope_raster)
need_hs_build = need_dem_rebuild or not os.path.isfile(hill_raster)
project.add_metadata(
{
'num_rasters': str(len(urls)),
'origin_urls': json.dumps(urls)
}, dem_node)
for dem_r in dem_rasters:
slope_part_path = os.path.join(
scratch_dem_folder,
'SLOPE__' + os.path.basename(dem_r).split('.')[0] + '.tif')
hs_part_path = os.path.join(
scratch_dem_folder,
'HS__' + os.path.basename(dem_r).split('.')[0] + '.tif')
slope_parts.append(slope_part_path)
hillshade_parts.append(hs_part_path)
if force_download or need_dem_rebuild or not os.path.exists(
slope_part_path):
gdal_dem_geographic(dem_r, slope_part_path, 'slope')
need_slope_build = True
if force_download or need_dem_rebuild or not os.path.exists(
hs_part_path):
gdal_dem_geographic(dem_r, hs_part_path, 'hillshade')
need_hs_build = True
if need_slope_build:
raster_vrt_stitch(slope_parts,
slope_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(slope_raster, nhd[boundary])
else:
log.info('Skipping slope build because nothing has changed.')
if need_hs_build:
raster_vrt_stitch(hillshade_parts,
hill_raster,
cfg.OUTPUT_EPSG,
clip=nhd[boundary],
clean=parallel,
warp_options={"cutlineBlend": 1})
verify_areas(hill_raster, nhd[boundary])
else:
log.info('Skipping hillshade build because nothing has changed.')
# Remove the unzipped rasters. We won't need them anymore
if parallel:
safe_remove_dir(ned_unzip_folder)
# Calculate flow accumulation raster based on the DEM
log.info('Running flow accumulation and converting to drainage area.')
flow_accumulation(dem_raster, flow_accum, dinfinity=False, pitfill=True)
flow_accum_to_drainage_area(flow_accum, drain_area)
# Clip and re-project the existing and historic vegetation
log.info('Processing existing and historic vegetation rasters.')
clip_vegetation(buffered_clip_path100, existing_veg, existing_clip,
historic_veg, historic_clip, cfg.OUTPUT_EPSG)
log.info('Process the Fair Market Value Raster.')
raster_warp(fair_market,
fair_market_clip,
cfg.OUTPUT_EPSG,
clip=buffered_clip_path500,
warp_options={"cutlineBlend": 1})
# Clip the landownership Shapefile to a 10km buffer around the watershed boundary
own_path = os.path.join(output_folder, LayerTypes['OWNERSHIP'].rel_path)
project.add_dataset(realization, own_path, LayerTypes['OWNERSHIP'],
'Vector')
clip_ownership(nhd[boundary], ownership, own_path, cfg.OUTPUT_EPSG, 10000)
#######################################################
# Segmentation
#######################################################
# For now let's just make a copy of the NHD FLowlines
tmr = Timer()
rs_segmentation(nhd['NHDFlowline'], ntd_clean['Roads'], ntd_clean['Rail'],
own_path, hydrology_gpkg_path, SEGMENTATION['Max'],
SEGMENTATION['Min'], huc)
log.debug('Segmentation done in {:.1f} seconds'.format(tmr.ellapsed()))
project.add_project_geopackage(realization, LayerTypes['HYDROLOGY'])
# Add Bankfull Buffer Polygons
bankfull_path = os.path.join(
hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['BANKFULL_CHANNEL'].rel_path)
bankfull_buffer(
os.path.join(hydrology_gpkg_path,
LayerTypes['HYDROLOGY'].sub_layers['NETWORK'].rel_path),
cfg.OUTPUT_EPSG,
bankfull_path,
)
# TODO Add nhd/bankfull union when merge feature classes in vector.ops works with Geopackage layers
# bankfull_nhd_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# clip_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path)
# bankfull_nhd_area(bankfull_path, nhd['NHDArea'], clip_path, cfg.OUTPUT_EPSG, hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path)
# Filter the ecoregions Shapefile to only include attributes that intersect with our HUC
eco_path = os.path.join(output_folder, 'ecoregions', 'ecoregions.shp')
project.add_dataset(realization, eco_path, LayerTypes['ECOREGIONS'],
'Vector')
filter_ecoregions(nhd[boundary], ecoregions, eco_path, cfg.OUTPUT_EPSG,
10000)
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(realization, LayerTypes['REPORT'], replace=True)
report = RSContextReport(report_path, project, output_folder)
report.write()
log.info('Process completed successfully.')
return {
'DEM': dem_raster,
'Slope': slope_raster,
'ExistingVeg': existing_veg,
'HistoricVeg': historic_veg,
'NHD': nhd
}
def vbet(huc, flowlines_orig, flowareas_orig, orig_slope, json_transforms,
orig_dem, hillshade, max_hand, min_hole_area_m, project_folder,
reach_codes: List[str], meta: Dict[str, str]):
"""[summary]
Args:
huc ([type]): [description]
flowlines_orig ([type]): [description]
flowareas_orig ([type]): [description]
orig_slope ([type]): [description]
json_transforms ([type]): [description]
orig_dem ([type]): [description]
hillshade ([type]): [description]
max_hand ([type]): [description]
min_hole_area_m ([type]): [description]
project_folder ([type]): [description]
reach_codes (List[int]): NHD reach codes for features to include in outputs
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger('VBET')
log.info('Starting VBET v.{}'.format(cfg.version))
project, _realization, proj_nodes = create_project(huc, project_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
# Copy the inp
_proj_slope_node, proj_slope = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['SLOPE_RASTER'], orig_slope)
_proj_dem_node, proj_dem = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['DEM'], orig_dem)
_hillshade_node, hillshade = project.add_project_raster(
proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade)
# Copy input shapes to a geopackage
inputs_gpkg_path = os.path.join(project_folder,
LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(
project_folder, LayerTypes['INTERMEDIATES'].rel_path)
flowlines_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
flowareas_path = os.path.join(
inputs_gpkg_path,
LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
# Make sure we're starting with a fresh slate of new geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(flowareas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Create a copy of the flow lines with just the perennial and also connectors inside flow areas
network_path = os.path.join(
intermediates_gpkg_path,
LayerTypes['INTERMEDIATES'].sub_layers['VBET_NETWORK'].rel_path)
vbet_network(flowlines_path, flowareas_path, network_path, cfg.OUTPUT_EPSG,
reach_codes)
# Generate HAND from dem and vbet_network
# TODO make a place for this temporary folder. it can be removed after hand is generated.
temp_hand_dir = os.path.join(project_folder, "intermediates",
"hand_processing")
safe_makedirs(temp_hand_dir)
hand_raster = os.path.join(project_folder,
LayerTypes['HAND_RASTER'].rel_path)
create_hand_raster(proj_dem, network_path, temp_hand_dir, hand_raster)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['HAND_RASTER'])
# Build Transformation Tables
with sqlite3.connect(intermediates_gpkg_path) as conn:
cursor = conn.cursor()
# Build tables
with open(
os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'database', 'vbet_schema.sql')) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load tables
for sqldata in glob.glob(os.path.join(
os.path.abspath(os.path.dirname(__file__)), '..', 'database',
'data', '**', '*.sql'),
recursive=True):
with open(sqldata) as sqlfile:
sql_commands = sqlfile.read()
cursor.executescript(sql_commands)
conn.commit()
# Load transforms from table
transforms = load_transform_functions(json_transforms,
intermediates_gpkg_path)
# Get raster resolution as min buffer and apply bankfull width buffer to reaches
with rasterio.open(proj_slope) as raster:
t = raster.transform
min_buffer = (t[0] + abs(t[4])) / 2
log.info("Buffering Polyine by bankfull width buffers")
network_path_buffered = os.path.join(
intermediates_gpkg_path, LayerTypes['INTERMEDIATES'].
sub_layers['VBET_NETWORK_BUFFERED'].rel_path)
buffer_by_field(network_path, network_path_buffered, "BFwidth",
cfg.OUTPUT_EPSG, min_buffer)
# Rasterize the channel polygon and write to raster
log.info('Writing channel raster using slope as a template')
flow_area_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_RASTER'].rel_path)
channel_buffer_raster = os.path.join(
project_folder, LayerTypes['CHANNEL_BUFFER_RASTER'].rel_path)
rasterize(network_path_buffered, channel_buffer_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['CHANNEL_BUFFER_RASTER'])
rasterize(flowareas_path, flow_area_raster, proj_slope)
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['FLOW_AREA_RASTER'])
channel_dist_raster = os.path.join(project_folder,
LayerTypes['CHANNEL_DISTANCE'].rel_path)
fa_dist_raster = os.path.join(project_folder,
LayerTypes['FLOW_AREA_DISTANCE'].rel_path)
proximity_raster(channel_buffer_raster, channel_dist_raster)
proximity_raster(flow_area_raster, fa_dist_raster)
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['FLOW_AREA_DISTANCE'])
slope_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_SLOPE'].rel_path)
hand_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_HAND'].rel_path)
chan_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_CHANNEL_DISTANCE'].rel_path)
fa_dist_transform_raster = os.path.join(
project_folder, LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'].rel_path)
topo_evidence_raster = os.path.join(project_folder,
LayerTypes['EVIDENCE_TOPO'].rel_path)
channel_evidence_raster = os.path.join(
project_folder, LayerTypes['EVIDENCE_CHANNEL'].rel_path)
evidence_raster = os.path.join(project_folder,
LayerTypes['VBET_EVIDENCE'].rel_path)
# Open evidence rasters concurrently. We're looping over windows so this shouldn't affect
# memory consumption too much
with rasterio.open(proj_slope) as slp_src, \
rasterio.open(hand_raster) as hand_src, \
rasterio.open(channel_dist_raster) as cdist_src, \
rasterio.open(fa_dist_raster) as fadist_src:
# All 3 rasters should have the same extent and properties. They differ only in dtype
out_meta = slp_src.meta
# Rasterio can't write back to a VRT so rest the driver and number of bands for the output
out_meta['driver'] = 'GTiff'
out_meta['count'] = 1
out_meta['compress'] = 'deflate'
# out_meta['dtype'] = rasterio.uint8
# We use this to buffer the output
cell_size = abs(slp_src.get_transform()[1])
with rasterio.open(evidence_raster, 'w', **out_meta) as dest_evidence, \
rasterio.open(topo_evidence_raster, "w", **out_meta) as dest, \
rasterio.open(channel_evidence_raster, 'w', **out_meta) as dest_channel, \
rasterio.open(slope_transform_raster, "w", **out_meta) as slope_ev_out, \
rasterio.open(hand_transform_raster, 'w', **out_meta) as hand_ev_out, \
rasterio.open(chan_dist_transform_raster, 'w', **out_meta) as chan_dist_ev_out, \
rasterio.open(fa_dist_transform_raster, 'w', **out_meta) as fa_dist_ev_out:
progbar = ProgressBar(len(list(slp_src.block_windows(1))), 50,
"Calculating evidence layer")
counter = 0
# Again, these rasters should be orthogonal so their windows should also line up
for _ji, window in slp_src.block_windows(1):
progbar.update(counter)
counter += 1
slope_data = slp_src.read(1, window=window, masked=True)
hand_data = hand_src.read(1, window=window, masked=True)
cdist_data = cdist_src.read(1, window=window, masked=True)
fadist_data = fadist_src.read(1, window=window, masked=True)
slope_transform = np.ma.MaskedArray(transforms["Slope"](
slope_data.data),
mask=slope_data.mask)
hand_transform = np.ma.MaskedArray(transforms["HAND"](
hand_data.data),
mask=hand_data.mask)
channel_dist_transform = np.ma.MaskedArray(
transforms["Channel"](cdist_data.data),
mask=cdist_data.mask)
fa_dist_transform = np.ma.MaskedArray(transforms["Flow Areas"](
fadist_data.data),
mask=fadist_data.mask)
fvals_topo = slope_transform * hand_transform
fvals_channel = np.maximum(channel_dist_transform,
fa_dist_transform)
fvals_evidence = np.maximum(fvals_topo, fvals_channel)
# Fill the masked values with the appropriate nodata vals
# Unthresholded in the base band (mostly for debugging)
dest.write(np.ma.filled(np.float32(fvals_topo),
out_meta['nodata']),
window=window,
indexes=1)
slope_ev_out.write(slope_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
hand_ev_out.write(hand_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
chan_dist_ev_out.write(
channel_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
fa_dist_ev_out.write(
fa_dist_transform.astype('float32').filled(
out_meta['nodata']),
window=window,
indexes=1)
dest_channel.write(np.ma.filled(np.float32(fvals_channel),
out_meta['nodata']),
window=window,
indexes=1)
dest_evidence.write(np.ma.filled(np.float32(fvals_evidence),
out_meta['nodata']),
window=window,
indexes=1)
progbar.finish()
# The remaining rasters get added to the project
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_SLOPE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_HAND'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_CHANNEL_DISTANCE'])
project.add_project_raster(proj_nodes["Intermediates"],
LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_TOPO'])
project.add_project_raster(proj_nodes['Intermediates'],
LayerTypes['EVIDENCE_CHANNEL'])
project.add_project_raster(proj_nodes['Outputs'],
LayerTypes['VBET_EVIDENCE'])
# Get the length of a meter (roughly)
degree_factor = GeopackageLayer.rough_convert_metres_to_raster_units(
proj_slope, 1)
buff_dist = cell_size
min_hole_degrees = min_hole_area_m * (degree_factor**2)
# Get the full paths to the geopackages
intermed_gpkg_path = os.path.join(project_folder,
LayerTypes['INTERMEDIATES'].rel_path)
vbet_path = os.path.join(project_folder,
LayerTypes['VBET_OUTPUTS'].rel_path)
for str_val, thr_val in thresh_vals.items():
plgnize_id = 'THRESH_{}'.format(str_val)
with TempRaster('vbet_raw_thresh_{}'.format(plgnize_id)) as tmp_raw_thresh, \
TempRaster('vbet_cleaned_thresh_{}'.format(plgnize_id)) as tmp_cleaned_thresh:
log.debug('Temporary threshold raster: {}'.format(
tmp_raw_thresh.filepath))
threshold(evidence_raster, thr_val, tmp_raw_thresh.filepath)
raster_clean(tmp_raw_thresh.filepath,
tmp_cleaned_thresh.filepath,
buffer_pixels=1)
plgnize_lyr = RSLayer('Raw Threshold at {}%'.format(str_val),
plgnize_id, 'Vector', plgnize_id.lower())
# Add a project node for this thresholded vector
LayerTypes['INTERMEDIATES'].add_sub_layer(plgnize_id, plgnize_lyr)
vbet_id = 'VBET_{}'.format(str_val)
vbet_lyr = RSLayer('Threshold at {}%'.format(str_val), vbet_id,
'Vector', vbet_id.lower())
# Add a project node for this thresholded vector
LayerTypes['VBET_OUTPUTS'].add_sub_layer(vbet_id, vbet_lyr)
# Now polygonize the raster
log.info('Polygonizing')
polygonize(
tmp_cleaned_thresh.filepath, 1,
'{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path), cfg.OUTPUT_EPSG)
log.info('Done')
# Now the final sanitization
sanitize(str_val, '{}/{}'.format(intermed_gpkg_path,
plgnize_lyr.rel_path),
'{}/{}'.format(vbet_path,
vbet_lyr.rel_path), buff_dist, network_path)
log.info('Completed thresholding at {}'.format(thr_val))
# Now add our Geopackages to the project XML
project.add_project_geopackage(proj_nodes['Intermediates'],
LayerTypes['INTERMEDIATES'])
project.add_project_geopackage(proj_nodes['Outputs'],
LayerTypes['VBET_OUTPUTS'])
report_path = os.path.join(project.project_dir,
LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'],
LayerTypes['REPORT'],
replace=True)
report = VBETReport(report_path, project)
report.write()
log.info('VBET Completed Successfully')
def rvd(huc: int, flowlines_orig: Path, existing_veg_orig: Path, historic_veg_orig: Path,
valley_bottom_orig: Path, output_folder: Path, reach_codes: List[str], flow_areas_orig: Path, waterbodies_orig: Path, meta=None):
"""[Generate segmented reaches on flowline network and calculate RVD from historic and existing vegetation rasters
Args:
huc (integer): Watershed ID
flowlines_orig (Path): Segmented flowlines feature layer
existing_veg_orig (Path): LANDFIRE version 2.00 evt raster, with adjacent xml metadata file
historic_veg_orig (Path): LANDFIRE version 2.00 bps raster, with adjacent xml metadata file
valley_bottom_orig (Path): Vbet polygon feature layer
output_folder (Path): destination folder for project output
reach_codes (List[int]): NHD reach codes for features to include in outputs
flow_areas_orig (Path): NHD flow area polygon feature layer
waterbodies (Path): NHD waterbodies polygon feature layer
meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs
"""
log = Logger("RVD")
log.info('RVD v.{}'.format(cfg.version))
try:
int(huc)
except ValueError:
raise Exception('Invalid HUC identifier "{}". Must be an integer'.format(huc))
if not (len(huc) == 4 or len(huc) == 8):
raise Exception('Invalid HUC identifier. Must be four digit integer')
safe_makedirs(output_folder)
project, _realization, proj_nodes = create_project(huc, output_folder)
# Incorporate project metadata to the riverscapes project
if meta is not None:
project.add_metadata(meta)
log.info('Adding inputs to project')
_prj_existing_path_node, prj_existing_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['EXVEG'], existing_veg_orig)
_prj_historic_path_node, prj_historic_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HISTVEG'], historic_veg_orig)
# TODO: Don't forget the att_filter
# _prj_flowlines_node, prj_flowlines = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'], flowlines, att_filter="\"ReachCode\" Like '{}%'".format(huc))
# Copy in the vectors we need
inputs_gpkg_path = os.path.join(output_folder, LayerTypes['INPUTS'].rel_path)
intermediates_gpkg_path = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path)
outputs_gpkg_path = os.path.join(output_folder, LayerTypes['OUTPUTS'].rel_path)
# Make sure we're starting with empty/fresh geopackages
GeopackageLayer.delete(inputs_gpkg_path)
GeopackageLayer.delete(intermediates_gpkg_path)
GeopackageLayer.delete(outputs_gpkg_path)
# Copy our input layers and also find the difference in the geometry for the valley bottom
flowlines_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path)
vbottom_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['VALLEY_BOTTOM'].rel_path)
copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG)
copy_feature_class(valley_bottom_orig, vbottom_path, epsg=cfg.OUTPUT_EPSG)
with GeopackageLayer(flowlines_path) as flow_lyr:
# Set the output spatial ref as this for the whole project
out_srs = flow_lyr.spatial_ref
meter_conversion = flow_lyr.rough_convert_metres_to_vector_units(1)
distance_buffer = flow_lyr.rough_convert_metres_to_vector_units(1)
# Transform issues reading 102003 as espg id. Using sr wkt seems to work, however arcgis has problems loading feature classes with this method...
raster_srs = ogr.osr.SpatialReference()
ds = gdal.Open(prj_existing_path, 0)
raster_srs.ImportFromWkt(ds.GetProjectionRef())
raster_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
transform_shp_to_raster = VectorBase.get_transform(out_srs, raster_srs)
gt = ds.GetGeoTransform()
cell_area = ((gt[1] / meter_conversion) * (-gt[5] / meter_conversion))
# Create the output feature class fields
with GeopackageLayer(outputs_gpkg_path, layer_name='ReachGeometry', delete_dataset=True) as out_lyr:
out_lyr.create_layer(ogr.wkbMultiLineString, spatial_ref=out_srs, options=['FID=ReachID'], fields={
'GNIS_NAME': ogr.OFTString,
'ReachCode': ogr.OFTString,
'TotDASqKm': ogr.OFTReal,
'NHDPlusID': ogr.OFTReal,
'WatershedID': ogr.OFTInteger
})
metadata = {
'RVD_DateTime': datetime.datetime.now().isoformat(),
'Reach_Codes': reach_codes
}
# Execute the SQL to create the lookup tables in the RVD geopackage SQLite database
watershed_name = create_database(huc, outputs_gpkg_path, metadata, cfg.OUTPUT_EPSG, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'rvd_schema.sql'))
project.add_metadata({'Watershed': watershed_name})
geom_vbottom = get_geometry_unary_union(vbottom_path, spatial_ref=raster_srs)
flowareas_path = None
if flow_areas_orig:
flowareas_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path)
copy_feature_class(flow_areas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG)
geom_flow_areas = get_geometry_unary_union(flowareas_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_flow_areas)
else:
del LayerTypes['INPUTS'].sub_layers['FLOW_AREA']
waterbodies_path = None
if waterbodies_orig:
waterbodies_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['WATERBODIES'].rel_path)
copy_feature_class(waterbodies_orig, waterbodies_path, epsg=cfg.OUTPUT_EPSG)
geom_waterbodies = get_geometry_unary_union(waterbodies_path)
# Difference with existing vbottom
geom_vbottom = geom_vbottom.difference(geom_waterbodies)
else:
del LayerTypes['INPUTS'].sub_layers['WATERBODIES']
# Add the inputs to the XML
_nd, _in_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'])
# Filter the flow lines to just the required features and then segment to desired length
# TODO: These are brat methods that need to be refactored to use VectorBase layers
cleaned_path = os.path.join(outputs_gpkg_path, 'ReachGeometry')
build_network(flowlines_path, flowareas_path, cleaned_path, waterbodies_path=waterbodies_path, epsg=cfg.OUTPUT_EPSG, reach_codes=reach_codes, create_layer=False)
# Generate Voroni polygons
log.info("Calculating Voronoi Polygons...")
# Add all the points (including islands) to the list
flowline_thiessen_points_groups = centerline_points(cleaned_path, distance_buffer, transform_shp_to_raster)
flowline_thiessen_points = [pt for group in flowline_thiessen_points_groups.values() for pt in group]
simple_save([pt.point for pt in flowline_thiessen_points], ogr.wkbPoint, raster_srs, "Thiessen_Points", intermediates_gpkg_path)
# Exterior is the shell and there is only ever 1
myVorL = NARVoronoi(flowline_thiessen_points)
# Generate Thiessen Polys
myVorL.createshapes()
# Dissolve by flowlines
log.info("Dissolving Thiessen Polygons")
dissolved_polys = myVorL.dissolve_by_property('fid')
# Clip Thiessen Polys
log.info("Clipping Thiessen Polygons to Valley Bottom")
clipped_thiessen = clip_polygons(geom_vbottom, dissolved_polys)
# Save Intermediates
simple_save(clipped_thiessen.values(), ogr.wkbPolygon, raster_srs, "Thiessen", intermediates_gpkg_path)
simple_save(dissolved_polys.values(), ogr.wkbPolygon, raster_srs, "ThiessenPolygonsDissolved", intermediates_gpkg_path)
simple_save(myVorL.polys, ogr.wkbPolygon, raster_srs, "ThiessenPolygonsRaw", intermediates_gpkg_path)
_nd, _inter_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES'])
# OLD METHOD FOR AUDIT
# dissolved_polys2 = dissolve_by_points(flowline_thiessen_points_groups, myVorL.polys)
# simple_save(dissolved_polys2.values(), ogr.wkbPolygon, out_srs, "ThiessenPolygonsDissolved_OLD", intermediates_gpkg_path)
# Load Vegetation Rasters
log.info(f"Loading Existing and Historic Vegetation Rasters")
vegetation = {}
vegetation["EXISTING"] = load_vegetation_raster(prj_existing_path, outputs_gpkg_path, True, output_folder=os.path.join(output_folder, 'Intermediates'))
vegetation["HISTORIC"] = load_vegetation_raster(prj_historic_path, outputs_gpkg_path, False, output_folder=os.path.join(output_folder, 'Intermediates'))
for epoch in vegetation.keys():
for name in vegetation[epoch].keys():
if not f"{epoch}_{name}" == "HISTORIC_LUI":
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[f"{epoch}_{name}"])
if vegetation["EXISTING"]["RAW"].shape != vegetation["HISTORIC"]["RAW"].shape:
raise Exception('Vegetation raster shapes are not equal Existing={} Historic={}. Cannot continue'.format(vegetation["EXISTING"]["RAW"].shape, vegetation["HISTORIC"]["RAW"].shape))
# Vegetation zone calculations
riparian_zone_arrays = {}
riparian_zone_arrays["RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["RIPARIAN"] + vegetation["HISTORIC"]["RIPARIAN"]) > 0) * 1
riparian_zone_arrays["NATIVE_RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["NATIVE_RIPARIAN"] + vegetation["HISTORIC"]["NATIVE_RIPARIAN"]) > 0) * 1
riparian_zone_arrays["VEGETATION_ZONES"] = ((vegetation["EXISTING"]["VEGETATED"] + vegetation["HISTORIC"]["VEGETATED"]) > 0) * 1
# Save Intermediate Rasters
for name, raster in riparian_zone_arrays.items():
save_intarr_to_geotiff(raster, os.path.join(output_folder, "Intermediates", f"{name}.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[name])
# Calculate Riparian Departure per Reach
riparian_arrays = {f"{epoch.capitalize()}{(name.capitalize()).replace('Native_riparian', 'NativeRiparian')}Mean": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name in ["RIPARIAN", "NATIVE_RIPARIAN"]}
# Vegetation Cell Counts
raw_arrays = {f"{epoch}": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name == "RAW"}
# Generate Vegetation Conversions
vegetation_change = (vegetation["HISTORIC"]["CONVERSION"] - vegetation["EXISTING"]["CONVERSION"])
save_intarr_to_geotiff(vegetation_change, os.path.join(output_folder, "Intermediates", "Conversion_Raster.tif"), prj_existing_path)
project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['VEGETATION_CONVERSION'])
# load conversion types dictionary from database
conn = sqlite3.connect(outputs_gpkg_path)
conn.row_factory = dict_factory
curs = conn.cursor()
curs.execute('SELECT * FROM ConversionTypes')
conversion_classifications = curs.fetchall()
curs.execute('SELECT * FROM vwConversions')
conversion_ids = curs.fetchall()
# Split vegetation change classes into binary arrays
vegetation_change_arrays = {
c['FieldName']: (vegetation_change == int(c["TypeValue"])) * 1 if int(c["TypeValue"]) in np.unique(vegetation_change) else None
for c in conversion_classifications
}
# Calcuate average and unique cell counts per reach
progbar = ProgressBar(len(clipped_thiessen.keys()), 50, "Extracting array values by reach...")
counter = 0
discarded = 0
with rasterio.open(prj_existing_path) as dataset:
unique_vegetation_counts = {}
reach_average_riparian = {}
reach_average_change = {}
for reachid, poly in clipped_thiessen.items():
counter += 1
progbar.update(counter)
# we can discount a lot of shapes here.
if not poly.is_valid or poly.is_empty or poly.area == 0 or poly.geom_type not in ["Polygon", "MultiPolygon"]:
discarded += 1
continue
raw_values_unique = {}
change_values_mean = {}
riparian_values_mean = {}
reach_raster = np.ma.masked_invalid(
features.rasterize(
[poly],
out_shape=dataset.shape,
transform=dataset.transform,
all_touched=True,
fill=np.nan))
for raster_name, raster in raw_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
raw_values_unique[raster_name] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True)
else:
raw_values_unique[raster_name] = []
for raster_name, raster in riparian_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
riparian_values_mean[raster_name] = np.ma.mean(current_raster)
else:
riparian_values_mean[raster_name] = 0.0
for raster_name, raster in vegetation_change_arrays.items():
if raster is not None:
current_raster = np.ma.masked_array(raster, mask=reach_raster.mask)
change_values_mean[raster_name] = np.ma.mean(current_raster)
else:
change_values_mean[raster_name] = 0.0
unique_vegetation_counts[reachid] = raw_values_unique
reach_average_riparian[reachid] = riparian_values_mean
reach_average_change[reachid] = change_values_mean
progbar.finish()
with SQLiteCon(outputs_gpkg_path) as gpkg:
# Ensure all reaches are present in the ReachAttributes table before storing RVD output values
gpkg.curs.execute('INSERT INTO ReachAttributes (ReachID) SELECT ReachID FROM ReachGeometry;')
errs = 0
for reachid, epochs in unique_vegetation_counts.items():
for epoch in epochs.values():
insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
try:
gpkg.curs.executemany('''INSERT INTO ReachVegetation (
ReachID,
VegetationID,
Area,
CellCount)
VALUES (?,?,?,?)''', insert_values)
# Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
except sqlite3.IntegrityError as err:
# THis is likely a constraint error.
errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
log.error(errstr)
errs += 1
except sqlite3.Error as err:
# This is any other kind of error
errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
log.error(errstr)
errs += 1
if errs > 0:
raise Exception('Errors were found inserting records into the database. Cannot continue.')
gpkg.conn.commit()
# load RVD departure levels from DepartureLevels database table
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxRVD FROM DepartureLevels ORDER BY MaxRVD ASC')
departure_levels = gpkg.curs.fetchall()
# Calcuate Average Departure for Riparian and Native Riparian
riparian_departure_values = riparian_departure(reach_average_riparian, departure_levels)
write_db_attributes(outputs_gpkg_path, riparian_departure_values, departure_type_columns)
# Add Conversion Code, Type to Vegetation Conversion
with SQLiteCon(outputs_gpkg_path) as gpkg:
gpkg.curs.execute('SELECT LevelID, MaxValue, NAME FROM ConversionLevels ORDER BY MaxValue ASC')
conversion_levels = gpkg.curs.fetchall()
reach_values_with_conversion_codes = classify_conversions(reach_average_change, conversion_ids, conversion_levels)
write_db_attributes(outputs_gpkg_path, reach_values_with_conversion_codes, rvd_columns)
# # Write Output to GPKG table
# log.info('Insert values to GPKG tables')
# # TODO move this to write_attirubtes method
# with get_shp_or_gpkg(outputs_gpkg_path, layer_name='ReachAttributes', write=True, ) as in_layer:
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "FromConifer": ogr.OFTReal,
# "FromDevegetated": ogr.OFTReal,
# "FromGrassShrubland": ogr.OFTReal,
# "FromDeciduous": ogr.OFTReal,
# "NoChange": ogr.OFTReal,
# "Deciduous": ogr.OFTReal,
# "GrassShrubland": ogr.OFTReal,
# "Devegetation": ogr.OFTReal,
# "Conifer": ogr.OFTReal,
# "Invasive": ogr.OFTReal,
# "Development": ogr.OFTReal,
# "Agriculture": ogr.OFTReal,
# "ConversionCode": ogr.OFTInteger,
# "ConversionType": ogr.OFTString}.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in reach_values_with_conversion_codes:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in reach_values_with_conversion_codes[reach]:
# if not reach_values_with_conversion_codes[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, reach_values_with_conversion_codes[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# # Create each field and store the name and index in a list of tuples
# field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in {
# "EXISTING_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_RIPARIAN_MEAN": ogr.OFTReal,
# "RIPARIAN_DEPARTURE": ogr.OFTReal,
# "EXISTING_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "HISTORIC_NATIVE_RIPARIAN_MEAN": ogr.OFTReal,
# "NATIVE_RIPARIAN_DEPARTURE": ogr.OFTReal, }.items()]
# for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]):
# reach = feature.GetFID()
# if reach not in riparian_departure_values:
# continue
# # Set all the field values and then store the feature
# for field, _idx in field_indices:
# if field in riparian_departure_values[reach]:
# if not riparian_departure_values[reach][field]:
# feature.SetField(field, None)
# else:
# feature.SetField(field, riparian_departure_values[reach][field])
# in_layer.ogr_layer.SetFeature(feature)
# with sqlite3.connect(outputs_gpkg_path) as conn:
# cursor = conn.cursor()
# errs = 0
# for reachid, epochs in unique_vegetation_counts.items():
# for epoch in epochs.values():
# insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0]
# try:
# cursor.executemany('''INSERT INTO ReachVegetation (
# ReachID,
# VegetationID,
# Area,
# CellCount)
# VALUES (?,?,?,?)''', insert_values)
# # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is
# except sqlite3.IntegrityError as err:
# # THis is likely a constraint error.
# errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0])))
# log.error(errstr)
# errs += 1
# except sqlite3.Error as err:
# # This is any other kind of error
# errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err))
# log.error(errstr)
# errs += 1
# if errs > 0:
# raise Exception('Errors were found inserting records into the database. Cannot continue.')
# conn.commit()
# Add intermediates and the report to the XML
# project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) already
# added above
project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['OUTPUTS'])
# Add the report to the XML
report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True)
report = RVDReport(report_path, project)
report.write()
log.info('RVD complete')
def rs_segmentation(nhd_flowlines_path: str, roads_path: str,
railways_path: str, ownership_path: str, out_gpkg: str,
interval: float, minimum: float, watershed_id: str):
"""Segment the network in a few different ways
Args:
nhd_flowlines_path (str): Path to shapefile or geopackage containing the original network
roads_path (str): Roads linestring shapefile or geopackage
railways_path (str): Rails lienstring shapefile or geopackage
ownership_path (str): Ownership polygon shapefile or geopackage
out_gpkg (str): Output geopackage for all the output layers
interval (float): Preferred segmentation distance split
minimum (float): Minimum possible segment size
watershed_id (str): Watershed ID
"""
log = Logger('rs_segmentation')
# First make a copy of the network.
# TODO: When we migrate to geopackages we may need to revisit this.
log.info('Copying raw network')
network_copy_path = os.path.join(out_gpkg, 'network')
copy_feature_class(nhd_flowlines_path, network_copy_path)
# Segment the raw network without doing any intersections
log.info('Segmenting the raw network')
segment_network(network_copy_path,
os.path.join(out_gpkg, 'network_300m'),
interval,
minimum,
watershed_id,
create_layer=True)
# If a point needs to be split we store the split pieces here
split_feats = {}
# Intersection points are useful in other tools so we keep them
intersect_pts = {}
log.info('Finding road intersections')
intersect_pts['roads'] = split_geoms(network_copy_path, roads_path,
split_feats)
log.info('Finding rail intersections')
intersect_pts['rail'] = split_geoms(network_copy_path, railways_path,
split_feats)
# With ownership we need to convert polygons to polylines (linestrings) to get the crossing points
# We can't use intersect_geometry_with_feature_class for this so we need to do something a little more manual
log.info('Finding ownership intersections')
ownership_lines_path = os.path.join(out_gpkg, "ownership_lines")
with GeopackageLayer(ownership_lines_path,
write=True) as out_layer, get_shp_or_gpkg(
ownership_path) as own_lyr:
out_layer.create_layer(ogr.wkbLineString,
spatial_ref=own_lyr.spatial_ref)
network_owener_collect = collect_feature_class(network_copy_path)
for feat, _counter, _progbar in own_lyr.iterate_features(
'Converting ownership polygons to polylines',
clip_shape=network_owener_collect):
geom = feat.GetGeometryRef()
# Check that this feature has valid geometry. Really important since ownership shape layers are
# Usually pretty messy.
if geom.IsValid() and not geom.IsEmpty():
# Flatten to 2D first to speed up the potential transform
if geom.IsMeasured() > 0 or geom.Is3D() > 0:
geom.FlattenTo2D()
# Get the boundary linestring
boundary = geom.GetBoundary()
b_type = boundary.GetGeometryType()
# If the boundary is a multilinestring that's fine
if b_type == ogr.wkbMultiLineString:
pass
# if it's just one linestring we make it a multilinestring of one.
elif b_type == ogr.wkbLineString:
boundary = [boundary]
else:
raise Exception('Unsupported type: {}'.format(
ogr.GeometryTypeToName(b_type)))
# Now write each individual linestring back to our output layer
for b_line in boundary:
out_feature = ogr.Feature(out_layer.ogr_layer_def)
out_feature.SetGeometry(b_line)
out_layer.ogr_layer.CreateFeature(out_feature)
# Now, finally, we're ready to do the actual intersection and splitting
intersect_pts['ownership'] = split_geoms(network_copy_path,
ownership_lines_path, split_feats)
# Let's write our crossings to layers for later use. This can be used in BRAT or our other tools
with GeopackageLayer(out_gpkg, layer_name='network_crossings', write=True) as out_lyr, \
GeopackageLayer(network_copy_path) as in_lyr:
out_lyr.create_layer(ogr.wkbPoint,
spatial_ref=in_lyr.spatial_ref,
fields={'type': ogr.OFTString})
for geom_type_name, ogr_geom in intersect_pts.items():
for pt in list(ogr_geom):
out_feature = ogr.Feature(out_lyr.ogr_layer_def)
out_feature.SetGeometry(GeopackageLayer.shapely2ogr(pt))
out_feature.SetField('type', geom_type_name)
out_lyr.ogr_layer.CreateFeature(out_feature)
# We're done with the original. Let that memory go.
intersect_pts = None
# Now, finally, write all the shapes, substituting splits where necessary
network_crossings_path = os.path.join(out_gpkg, 'network_intersected')
with GeopackageLayer(network_crossings_path, write=True) as out_lyr, \
GeopackageLayer(network_copy_path) as net_lyr:
out_lyr.create_layer_from_ref(net_lyr)
fcounter = 0
for feat, _counter, _progbar in net_lyr.iterate_features(
'Writing split features'):
fid = feat.GetFID()
# If a split happened then write the split geometries to the file.
if fid in split_feats:
for split_geom in split_feats[fid]:
new_feat = feat.Clone()
new_feat.SetFID(fcounter)
new_feat.SetGeometry(
GeopackageLayer.shapely2ogr(split_geom))
out_lyr.ogr_layer.CreateFeature(new_feat)
fcounter += 1
# If no split was found, write the feature as-is
else:
new_feat = feat.Clone()
new_feat.SetFID(fcounter)
out_lyr.ogr_layer.CreateFeature(new_feat)
fcounter += 1
# Finally, segment this new layer the same way we did the raw network above.
log.info('Segmenting the intersected network')
segment_network(network_crossings_path,
os.path.join(out_gpkg, 'network_intersected_300m'),
interval,
minimum,
watershed_id,
create_layer=True)
log.info('Segmentation Complete')
def calc_conflict_attributes(flowlines_path, valley_bottom, roads, rail,
canals, ownership, buffer_distance_metres,
cell_size_meters, epsg, canal_codes,
intermediates_gpkg_path):
log = Logger('Conflict')
log.info('Calculating conflict attributes')
# Create union of all reaches and another of the reaches without any canals
reach_union = get_geometry_unary_union(flowlines_path)
if canal_codes is None:
reach_union_no_canals = reach_union
else:
reach_union_no_canals = get_geometry_unary_union(
flowlines_path,
attribute_filter='FCode NOT IN ({})'.format(','.join(canal_codes)))
crossin = intersect_geometry_to_layer(intermediates_gpkg_path,
'road_crossings', ogr.wkbMultiPoint,
reach_union, roads, epsg)
diverts = intersect_geometry_to_layer(intermediates_gpkg_path,
'diversions', ogr.wkbMultiPoint,
reach_union_no_canals, canals, epsg)
road_vb = intersect_to_layer(intermediates_gpkg_path, valley_bottom, roads,
'road_valleybottom', ogr.wkbMultiLineString,
epsg)
rail_vb = intersect_to_layer(intermediates_gpkg_path, valley_bottom, rail,
'rail_valleybottom', ogr.wkbMultiLineString,
epsg)
private = os.path.join(intermediates_gpkg_path, 'private_land')
copy_feature_class(ownership, private, epsg,
"ADMIN_AGEN = 'PVT' OR ADMIN_AGEN = 'UND'")
# Buffer all reaches (being careful to use the units of the Shapefile)
reaches = load_geometries(flowlines_path, epsg=epsg)
with get_shp_or_gpkg(flowlines_path) as lyr:
buffer_distance = lyr.rough_convert_metres_to_vector_units(
buffer_distance_metres)
cell_size = lyr.rough_convert_metres_to_vector_units(cell_size_meters)
geopackage_path = lyr.filepath
polygons = {
reach_id: polyline.buffer(buffer_distance)
for reach_id, polyline in reaches.items()
}
results = {}
tmp_folder = os.path.join(os.path.dirname(intermediates_gpkg_path),
'tmp_conflict')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, road_vb,
'Mean', 'iPC_RoadVB')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, crossin,
'Mean', 'iPC_RoadX')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, diverts,
'Mean', 'iPC_DivPts')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, private,
'Mean', 'iPC_Privat')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, rail_vb,
'Mean', 'iPC_RailVB')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, canals,
'Mean', 'iPC_Canal')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, roads, 'Mean',
'iPC_Road')
distance_from_features(polygons, tmp_folder, reach_union.bounds,
cell_size_meters, cell_size, results, rail, 'Mean',
'iPC_Rail')
# Calculate minimum distance to conflict
min_keys = [
'iPC_Road', 'iPC_RoadX', 'iPC_RoadVB', 'iPC_Rail', 'iPC_RailVB'
]
for values in results.values():
values['oPC_Dist'] = min([values[x] for x in min_keys if x in values])
# Retrieve the agency responsible for administering the land at the midpoint of each reach
admin_agency(geopackage_path, reaches, ownership, results)
log.info('Conflict attribute calculation complete')
# Cleanup temporary feature classes
safe_remove_dir(tmp_folder)
return results