def burn(subregion_ned, nhd_gdb, burnt_out, projection = arcpy.SpatialReference(102039)):
env.snapRaster = subregion_ned
env.outputCoordinateSystem = projection
env.compression = "LZ77" # compress temp tifs for speed
env.workspace = nhd_gdb
# Copy flowlines to shapefile that will inherit environ output coord system
flow_line = "NHDFlowline_Projected"
if not arcpy.Exists(flow_line):
arcpy.FeatureClassToFeatureClass_conversion("NHDFlowline", nhd_gdb, flow_line)
cu.multi_msg("Prepared NHDFlowline for rasterizing.")
# Feature to Raster- rasterize the NHDFlowline
flow_line_raster = "in_memory/flow_line_raster"
arcpy.FeatureToRaster_conversion(flow_line, "OBJECTID", flow_line_raster, "10")
cu.multi_msg("Converted flowlines to raster.")
# Raster Calculator- burns in streams, beveling in from 500m
cu.multi_msg("Burning streams into raster, 10m deep and beveling in from 500m out. This may take a while....")
distance = EucDistance(flow_line, cell_size = "10")
streams = Reclassify(Raster(flow_line_raster) > 0, "Value", "1 1; NoData 0")
burnt = Raster(subregion_ned) - (10 * streams) - (0.02 * (500 - distance) * (distance < 500))
cu.multi_msg("Saving output raster...")
burnt.save(burnt_out)
# Delete intermediate rasters and shapefiles
cu.cleanup([flow_line, flow_line_raster])
cu.multi_msg("Burn process completed")
def unzip_ned(file_id, ned_dir, out_dir):
# clunky but this works in USA: zipped files sometimes called
# something like n36w87 instead of n36w087 so try all 3
filename_variants = [
os.path.join(ned_dir, f) for f in [
file_id + ".zip", file_id[0:4] + file_id[5:] + ".zip", file_id[0] +
file_id[2:] + ".zip"
]
]
filename_to_use = ''
for f in filename_variants:
if not os.path.exists(f):
continue
else:
filename_to_use = f
if filename_to_use:
cu.multi_msg("Unzipping file %s" % filename_to_use)
zf = zipfile.ZipFile(filename_to_use)
zf.extractall(out_dir)
return True
else:
cu.multi_msg(
"ERROR: A tile for %s does not exist in the specified location" %
file_id)
return False
def stats_overlap(non_overlapping_zones_list, zone_field, in_value_raster, out_table, is_thematic):
temp_out_tables = ['in_memory/' + os.path.basename(zfc) + "_temp_table" for zfc in non_overlapping_zones_list]
arcpy.CheckOutExtension("Spatial")
for zones, temp_table in zip(non_overlapping_zones_list, temp_out_tables):
cu.multi_msg('Calculating statistics for layer {0}'.format(zones))
zonal_tabarea.stats_area_table(zones, zone_field, in_value_raster, temp_table, is_thematic)
arcpy.CheckInExtension("Spatial")
# doing this append/copy method instead of merge prevents problems with
# differences in the field length of the zone field created by
# Zonal Statistics As Table, merge doesn't have 'NO_TEST' option.
target_table = temp_out_tables.pop(0)
arcpy.Append_management(temp_out_tables, target_table, 'NO_TEST')
arcpy.CopyRows_management(target_table, out_table)
in_count = 0
for zones in non_overlapping_zones_list:
in_count += int(arcpy.GetCount_management(zones).getOutput(0))
out_count = int(arcpy.GetCount_management(out_table).getOutput(0))
if out_count < in_count:
warn_msg = ("WARNING: {0} features are missing in the output table"
" because they are too small for this raster's"
" resolution. This may be okay depending on your"
" application.").format(in_count - out_count)
arcpy.AddWarning(warn_msg)
print(warn_msg)
for t in temp_out_tables + [target_table]:
arcpy.Delete_management(t)
def streams_in_zones(zones_fc, zone_field, streams_fc, output_table):
arcpy.env.workspace = 'in_memory'
# this bit enforces the correct ftype restriction just in case
# geodata doesn't have this filtered already
# this won't be as good as doing it before hand because it's
# not sophisticated enough to take out the artificial lines going
# through lakes
need_selection = False
with arcpy.da.SearchCursor(streams_fc, ["FType"]) as cursor:
for row in cursor:
if row[0] == 566:
need_selection = True
if need_selection:
whereClause = """"FType" <> 566"""
arcpy.Select_analysis(temp_lakes, "no_coastlines", whereClause)
streams_fc = os.path.join(arcpy.env.workspace, "no_coastlines")
selections = ['',
""""Strahler" <= 3""",
""""Strahler" > 3 AND "Strahler" <= 6""",
""""Strahler" > 6""",
]
temp_tables = ['Streams', 'Headwaters', 'Midreaches', 'Rivers']
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg("Creating temporary table called {0} for streams where {1}".format(temp_table, sel))
LineDensity.line_density(zones_fc, zone_field, streams_fc, temp_table, sel)
new_fields = ['SUM_LengthM', 'Density_MperHA']
for f in new_fields:
cu.rename_field(temp_table, f, temp_table + '_' + f, True)
# join em up and copy to final
temp_tables.remove('Streams')
for t in temp_tables:
try:
arcpy.JoinField_management('Streams', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [temp_table + '_' + f for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Streams', ef, 'Double')
arcpy.CalculateField_management('Streams', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('Streams', zone_field + '_*')]
for f in drop_fields:
arcpy.DeleteField_management('Streams', f)
arcpy.CopyRows_management('Streams', output_table)
# clean up
for item in ['Streams', 'no_coastlines'] + temp_tables:
try:
arcpy.Delete_management(item)
except:
continue
def clip_to_hu8(raster,
nhd_gdb,
out_dir,
projection=arcpy.SpatialReference(102039)):
"""Outputs a series of rasters, each one clipped to a different HU8. """
env.workspace = 'in_memory'
env.outputCoordinateSystem = projection
env.compression = "NONE" # only final tifs are generated
env.snapRaster = raster
env.cellSize = '10'
env.pyramids = "PYRAMIDS -1 SKIP_FIRST"
arcpy.CheckOutExtension("Spatial")
# HUC8 polygons each saved as separate fc inheriting albers from environ
huc8_fc = os.path.join(nhd_gdb, "WBD_HU8")
arcpy.MakeFeatureLayer_management(huc8_fc, "huc8_layer")
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
clips_dir = os.path.join(out_dir, 'huc8clips{0}'.format(huc4_code))
if not os.path.exists(clips_dir):
os.mkdir(clips_dir)
## # add walls
## arcpy.PolygonToLine_management(huc8_fc, 'wall_lines')
## arcpy.AddField_management('wall_lines', "height", "DOUBLE")
## arcpy.CalculateField_management('wall_lines', "height", '500', "PYTHON")
## arcpy.FeatureToRaster_conversion('wall_lines', "height", 'wall_raster')
## wallsObject = Raster('wall_raster')
## elevObject = Raster(raster)
## walled_ned = Con(IsNull(wallsObject), elevObject,
## (wallsObject + elevObject))
# for each HU8 feature in the fc, make a clip
with arcpy.da.SearchCursor(huc8_fc, ["HUC_8"]) as cursor:
for row in cursor:
if row[0].startswith(huc4_code):
whereClause = """"{0}" = '{1}'""".format("HUC_8", row[0])
arcpy.SelectLayerByAttribute_management(
"huc8_layer", 'NEW_SELECTION', whereClause)
arcpy.CopyFeatures_management("huc8_layer", "this_hu8")
# clip the raster
out_raster = os.path.join(clips_dir,
'NED{0}.tif'.format(row[0]))
cu.multi_msg('Creating output {0}'.format(out_raster))
# use a small buffer here because otherwise the walls get
# cut off in slivers
arcpy.Buffer_analysis('this_hu8', 'this_hu8_buffer', 5000)
arcpy.Clip_management(raster, '', out_raster,
'this_hu8_buffer', '#',
'ClippingGeometry')
arcpy.Delete_management('this_hu8')
arcpy.Delete_management('this_hu8_buffer')
arcpy.Delete_management('huc8_layer')
arcpy.ResetEnvironments()
arcpy.CheckInExtension("Spatial")
def clip_to_hu8(raster, nhd_gdb, out_dir,
projection = arcpy.SpatialReference(102039)):
"""Outputs a series of rasters, each one clipped to a different HU8. """
env.workspace = 'in_memory'
env.outputCoordinateSystem = projection
env.compression = "NONE" # only final tifs are generated
env.snapRaster = raster
env.cellSize = '10'
env.pyramids = "PYRAMIDS -1 SKIP_FIRST"
arcpy.CheckOutExtension("Spatial")
# HUC8 polygons each saved as separate fc inheriting albers from environ
huc8_fc = os.path.join(nhd_gdb, "WBD_HU8")
arcpy.MakeFeatureLayer_management(huc8_fc, "huc8_layer")
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
clips_dir = os.path.join(out_dir, 'huc8clips{0}'.format(huc4_code))
if not os.path.exists(clips_dir):
os.mkdir(clips_dir)
## # add walls
## arcpy.PolygonToLine_management(huc8_fc, 'wall_lines')
## arcpy.AddField_management('wall_lines', "height", "DOUBLE")
## arcpy.CalculateField_management('wall_lines', "height", '500', "PYTHON")
## arcpy.FeatureToRaster_conversion('wall_lines', "height", 'wall_raster')
## wallsObject = Raster('wall_raster')
## elevObject = Raster(raster)
## walled_ned = Con(IsNull(wallsObject), elevObject,
## (wallsObject + elevObject))
# for each HU8 feature in the fc, make a clip
with arcpy.da.SearchCursor(huc8_fc, ["HUC_8"]) as cursor:
for row in cursor:
if row[0].startswith(huc4_code):
whereClause = """"{0}" = '{1}'""".format("HUC_8", row[0])
arcpy.SelectLayerByAttribute_management("huc8_layer", 'NEW_SELECTION', whereClause)
arcpy.CopyFeatures_management("huc8_layer", "this_hu8")
# clip the raster
out_raster = os.path.join(clips_dir, 'NED{0}.tif'.format(row[0]))
cu.multi_msg('Creating output {0}'.format(out_raster))
# use a small buffer here because otherwise the walls get
# cut off in slivers
arcpy.Buffer_analysis('this_hu8', 'this_hu8_buffer', 5000)
arcpy.Clip_management(raster, '', out_raster,
'this_hu8_buffer', '#',
'ClippingGeometry')
arcpy.Delete_management('this_hu8')
arcpy.Delete_management('this_hu8_buffer')
arcpy.Delete_management('huc8_layer')
arcpy.ResetEnvironments()
arcpy.CheckInExtension("Spatial")
def wetlands_in_zones(infolder, idfield, wetlands, top_outfolder):
# Create output geodatabase in outfolder
out_gdb = os.path.join(top_outfolder, "WetlandsInZones.gdb")
if not arcpy.Exists(out_gdb):
arcpy.CreateFileGDB_management(top_outfolder, "WetlandsInZones")
# Add WetlandHa field if it doesn't exist
if len(arcpy.ListFields(wetlands, 'WetlandHa')) == 0:
arcpy.AddField_management(wetlands, "WetlandHa", "DOUBLE")
expha = "!shape.area@hectares!"
arcpy.CalculateField_management(wetlands, "WetlandHa", expha, "PYTHON")
# Set in memory as workspace. Intermediate output will be held in RAM.
mem = "in_memory"
arcpy.env.workspace = mem
# Make wetlands in memory.
exp = """"ATTRIBUTE" LIKE 'P%'AND "WETLAND_TY" <> 'Freshwater_Pond'"""
arcpy.Select_analysis(wetlands, "wetlandspoly", exp)
# Convert wetlands to points
arcpy.FeatureToPoint_management("wetlandspoly", "wetlands", "INSIDE")
# List extent feature classes
fcs = []
for root, dirs, files in arcpy.da.Walk(infolder):
for file in files:
fcs.append(os.path.join(root,file))
# Spatial Join the wetlands to each extent
out_fcs = []
for fc in fcs:
cu.multi_msg("Creating results for %s" % fc)
name = os.path.basename(fc)
fms = arcpy.FieldMappings()
fmid = arcpy.FieldMap()
fmha = arcpy.FieldMap()
fmid.addInputField(fc, idfield)
fmha.addInputField("wetlands", "WetlandHa")
fmha.mergeRule = 'Sum'
fms.addFieldMap(fmid)
fms.addFieldMap(fmha)
out_fc = os.path.join(out_gdb, name + "_Wetlands")
arcpy.SpatialJoin_analysis(fc, wetlands, out_fc ,'','',fms)
out_fcs.append(out_fc)
# Export feature classes attribute tables to tables
for f in out_fcs:
arcpy.CopyRows_management(f, os.path.join(out_gdb, "Table" + os.path.basename(f)))
def wetlands_in_zones(infolder, idfield, wetlands, top_outfolder):
# Create output geodatabase in outfolder
out_gdb = os.path.join(top_outfolder, "WetlandsInZones.gdb")
if not arcpy.Exists(out_gdb):
arcpy.CreateFileGDB_management(top_outfolder, "WetlandsInZones")
# Add WetlandHa field if it doesn't exist
if len(arcpy.ListFields(wetlands, 'WetlandHa')) == 0:
arcpy.AddField_management(wetlands, "WetlandHa", "DOUBLE")
expha = "!shape.area@hectares!"
arcpy.CalculateField_management(wetlands, "WetlandHa", expha, "PYTHON")
# Set in memory as workspace. Intermediate output will be held in RAM.
mem = "in_memory"
arcpy.env.workspace = mem
# Make wetlands in memory.
exp = """"ATTRIBUTE" LIKE 'P%'AND "WETLAND_TY" <> 'Freshwater_Pond'"""
arcpy.Select_analysis(wetlands, "wetlandspoly", exp)
# Convert wetlands to points
arcpy.FeatureToPoint_management("wetlandspoly", "wetlands", "INSIDE")
# List extent feature classes
fcs = []
for root, dirs, files in arcpy.da.Walk(infolder):
for file in files:
fcs.append(os.path.join(root, file))
# Spatial Join the wetlands to each extent
out_fcs = []
for fc in fcs:
cu.multi_msg("Creating results for %s" % fc)
name = os.path.basename(fc)
fms = arcpy.FieldMappings()
fmid = arcpy.FieldMap()
fmha = arcpy.FieldMap()
fmid.addInputField(fc, idfield)
fmha.addInputField("wetlands", "WetlandHa")
fmha.mergeRule = 'Sum'
fms.addFieldMap(fmid)
fms.addFieldMap(fmha)
out_fc = os.path.join(out_gdb, name + "_Wetlands")
arcpy.SpatialJoin_analysis(fc, wetlands, out_fc, '', '', fms)
out_fcs.append(out_fc)
# Export feature classes attribute tables to tables
for f in out_fcs:
arcpy.CopyRows_management(
f, os.path.join(out_gdb, "Table" + os.path.basename(f)))
def nhd_merge(gdb_list, example_feature_class_name, out_fc, selection = ''):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039) # USA_Contiguous_Albers_Equal_Area_Conic_USGS_version
arcpy.env.workspace = 'in_memory'
gdb_list = [os.path.join(gdb, os.path.basename(example_feature_class_name)) for gdb in gdb_list]
gdb0 = gdb_list.pop(0)
desc = arcpy.Describe(gdb0)
cu.multi_msg('Merging all features together...')
arcpy.CopyFeatures_management(gdb0, 'temp_merged')
cu.lengthen_field('temp_merged', 'Permanent_Identifier', 255)
cu.merge_many(gdb_list, 'in_memory/the_rest_merged')
arcpy.Append_management('in_memory/the_rest_merged', 'temp_merged', 'NO_TEST')
# use in_memory explicitly here because i haven't figured out how to pass arcgis environments to my functions :(
fc_temp = 'in_memory/temp_merged'
fcount1 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('Before selection and cleaning, feature count is {0}'.format(fcount1))
if selection:
cu.multi_msg('Selecting features...')
arcpy.Select_analysis('temp_merged', 'in_memory/merged_select', selection)
fc_temp = 'in_memory/merged_select'
fcount2 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('After selection and before cleaning, feature count is {0}'.format(fcount2))
arcpy.CopyFeatures_management(fc_temp, out_fc)
def main():
subregion_ned = arcpy.GetParameterAsText(0)
nhd_gdb = arcpy.GetParameterAsText(1)
burnt_out = arcpy.GetParameterAsText(2)
input_burnt = arcpy.GetParameter(3)
arcpy.CheckOutExtension("Spatial")
if not input_burnt:
burn(subregion_ned, nhd_gdb, burnt_out, projection)
input_burnt = burnt_out
try:
clip(input_burnt, nhd_gdb, projection, outfolder)
arcpy.Delete_management(input_burnt)
cu.multi_msg("Complete. HUC8 burned clips are now ready for flow direction.")
except arcpy.ExecuteError:
cu.multi_msg("Clip failed, try again. Mosaic file is %s and burnt NED file is %s" %
(subregion_ned, burnt_ned))
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
cu.multi_msg("Clip failed, try again. Mosaic file is %s and burnt NED file is %s" %
(subregion_ned, burnt_ned))
cu.multi_msg(e.message)
finally:
arcpy.CheckInExtension("Spatial")
def main():
subregion_ned = arcpy.GetParameterAsText(0)
nhd_gdb = arcpy.GetParameterAsText(1)
burnt_out = arcpy.GetParameterAsText(2)
input_burnt = arcpy.GetParameter(3)
arcpy.CheckOutExtension("Spatial")
if not input_burnt:
burn(subregion_ned, nhd_gdb, burnt_out, projection)
input_burnt = burnt_out
try:
clip(input_burnt, nhd_gdb, projection, outfolder)
arcpy.Delete_management(input_burnt)
cu.multi_msg(
"Complete. HUC8 burned clips are now ready for flow direction.")
except arcpy.ExecuteError:
cu.multi_msg(
"Clip failed, try again. Mosaic file is %s and burnt NED file is %s"
% (subregion_ned, burnt_ned))
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
cu.multi_msg(
"Clip failed, try again. Mosaic file is %s and burnt NED file is %s"
% (subregion_ned, burnt_ned))
cu.multi_msg(e.message)
finally:
arcpy.CheckInExtension("Spatial")
def nhd_merge(gdb_list, example_feature_class_name, out_fc, selection=''):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(
102039) # USA_Contiguous_Albers_Equal_Area_Conic_USGS_version
arcpy.env.workspace = 'in_memory'
gdb_list = [
os.path.join(gdb, os.path.basename(example_feature_class_name))
for gdb in gdb_list
]
gdb0 = gdb_list.pop(0)
desc = arcpy.Describe(gdb0)
cu.multi_msg('Merging all features together...')
arcpy.CopyFeatures_management(gdb0, 'temp_merged')
cu.lengthen_field('temp_merged', 'Permanent_Identifier', 255)
cu.merge_many(gdb_list, 'in_memory/the_rest_merged')
arcpy.Append_management('in_memory/the_rest_merged', 'temp_merged',
'NO_TEST')
# use in_memory explicitly here because i haven't figured out how to pass arcgis environments to my functions :(
fc_temp = 'in_memory/temp_merged'
fcount1 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg(
'Before selection and cleaning, feature count is {0}'.format(fcount1))
if selection:
cu.multi_msg('Selecting features...')
arcpy.Select_analysis('temp_merged', 'in_memory/merged_select',
selection)
fc_temp = 'in_memory/merged_select'
fcount2 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg(
'After selection and before cleaning, feature count is {0}'.format(
fcount2))
arcpy.CopyFeatures_management(fc_temp, out_fc)
def burn_streams(subregion_ned,
nhd_gdb,
burnt_out,
projection=arcpy.SpatialReference(102039)):
env.snapRaster = subregion_ned
env.outputCoordinateSystem = projection
env.compression = "LZ77" # compress temp tifs for speed
env.extent = subregion_ned
env.workspace = 'in_memory'
flowline = os.path.join(nhd_gdb, 'NHDFlowline')
# Copy flowlines to shapefile that will inherit environ output coord system
# just easier to have a copy in the correct projection later
arcpy.CopyFeatures_management(flowline, 'flowline_proj')
## arcpy.FeatureClassToFeatureClass_conversion("NHDFlowline", "in_memory", flow_line)
cu.multi_msg("Prepared NHDFlowline for rasterizing.")
# Feature to Raster- rasterize the NHDFlowline
# will inherit grid from env.snapRaster
arcpy.FeatureToRaster_conversion('flowline_proj', "OBJECTID",
'flowline_raster', "10")
cu.multi_msg("Converted flowlines to raster.")
# Raster Calculator- burns in streams, beveling in from 500m
cu.multi_msg(
"Burning streams into raster, 10m deep and beveling in from 500m out. This may take a while...."
)
arcpy.CheckOutExtension("Spatial")
distance = EucDistance('flowline_proj', cell_size="10")
streams = Reclassify(
Raster('flowline_raster') > 0, "Value", "1 1; NoData 0")
burnt = Raster(subregion_ned) - (10 * streams) - (0.02 * (500 - distance) *
(distance < 500))
cu.multi_msg("Saving output raster...")
burnt.save(burnt_out)
# Delete intermediate rasters and shapefiles
for item in ['flowline_proj', 'flowline_raster']:
arcpy.Delete_management(item)
arcpy.CheckInExtension("Spatial")
cu.multi_msg("Burn process completed")
arcpy.ResetEnvironments()
def unzip_ned(file_id, ned_dir, out_dir):
# clunky but this works in USA: zipped files sometimes called
# something like n36w87 instead of n36w087 so try all 3
filename_variants = [os.path.join(ned_dir, f) for f in [file_id + ".zip",
file_id[0:4] + file_id[5:] + ".zip",
file_id[0] + file_id[2:] + ".zip"]]
filename_to_use = ''
for f in filename_variants:
if not os.path.exists(f):
continue
else:
filename_to_use = f
if filename_to_use:
cu.multi_msg("Unzipping file %s" % filename_to_use)
zf = zipfile.ZipFile(filename_to_use)
zf.extractall(out_dir)
return True
else:
cu.multi_msg("ERROR: A tile for %s does not exist in the specified location" % file_id)
return False
def InsideState(state, nwi, lakes, outfc):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
# Select only wetlands with their center inside this state
# This way all wetlands will only be represented once when we merge all states
arcpy.env.workspace = 'in_memory'
arcpy.MakeFeatureLayer_management(nwi, "nwi_lyr")
cu.multi_msg('Selecting wetlands with their center in the state.')
arcpy.SelectLayerByLocation_management("nwi_lyr", 'HAVE_THEIR_CENTER_IN', state,'','NEW_SELECTION')
# Two things to make wetlands conform to the CSI definition
# Select only palustrine systems that aren't freshwater ponds
# and make it impossible for wetlands to be inside lakes
wetland_type_field = arcpy.ListFields("nwi_lyr", "WETLAND_TY*")[0].name
filter = """"ATTRIBUTE" LIKE 'P%' AND {} <> 'Freshwater Pond'""".format(wetland_type_field)
cu.multi_msg("Selecting only palustrine wetlands...")
arcpy.SelectLayerByAttribute_management("nwi_lyr", "SUBSET_SELECTION", filter)
cu.multi_msg('Erasing lakes from wetlands layer.')
arcpy.Erase_analysis("nwi_lyr", lakes, outfc)
# Add two fields we will use, an ID field and the area in hectares
arcpy.AddField_management(outfc, "WET_ID", "LONG")
arcpy.CalculateField_management(outfc, "WET_ID", "!OBJECTID!", "PYTHON")
arcpy.AddField_management(outfc, "AreaHa", "DOUBLE")
arcpy.CalculateField_management(outfc, "AreaHa", "!shape.area@hectares!", "PYTHON")
def InsideState(state, nwi, lakes, outfc):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
# Select only wetlands with their center inside this state
# This way all wetlands will only be represented once when we merge all states
arcpy.env.workspace = 'in_memory'
arcpy.MakeFeatureLayer_management(nwi, "nwi_lyr")
cu.multi_msg('Selecting wetlands with their center in the state.')
arcpy.SelectLayerByLocation_management("nwi_lyr", 'HAVE_THEIR_CENTER_IN',
state, '', 'NEW_SELECTION')
# Two things to make wetlands conform to the CSI definition
# Select only palustrine systems that aren't freshwater ponds
# and make it impossible for wetlands to be inside lakes
wetland_type_field = arcpy.ListFields("nwi_lyr", "WETLAND_TY*")[0].name
filter = """"ATTRIBUTE" LIKE 'P%' AND {} <> 'Freshwater Pond'""".format(
wetland_type_field)
cu.multi_msg("Selecting only palustrine wetlands...")
arcpy.SelectLayerByAttribute_management("nwi_lyr", "SUBSET_SELECTION",
filter)
cu.multi_msg('Erasing lakes from wetlands layer.')
arcpy.Erase_analysis("nwi_lyr", lakes, outfc)
# Add two fields we will use, an ID field and the area in hectares
arcpy.AddField_management(outfc, "WET_ID", "LONG")
arcpy.CalculateField_management(outfc, "WET_ID", "!OBJECTID!", "PYTHON")
arcpy.AddField_management(outfc, "AreaHa", "DOUBLE")
arcpy.CalculateField_management(outfc, "AreaHa", "!shape.area@hectares!",
"PYTHON")
def burn(subregion_ned,
nhd_gdb,
burnt_out,
projection=arcpy.SpatialReference(102039)):
env.snapRaster = subregion_ned
env.outputCoordinateSystem = projection
env.compression = "LZ77" # compress temp tifs for speed
env.workspace = nhd_gdb
# Copy flowlines to shapefile that will inherit environ output coord system
flow_line = "NHDFlowline_Projected"
if not arcpy.Exists(flow_line):
arcpy.FeatureClassToFeatureClass_conversion("NHDFlowline", nhd_gdb,
flow_line)
cu.multi_msg("Prepared NHDFlowline for rasterizing.")
# Feature to Raster- rasterize the NHDFlowline
flow_line_raster = "in_memory/flow_line_raster"
arcpy.FeatureToRaster_conversion(flow_line, "OBJECTID", flow_line_raster,
"10")
cu.multi_msg("Converted flowlines to raster.")
# Raster Calculator- burns in streams, beveling in from 500m
cu.multi_msg(
"Burning streams into raster, 10m deep and beveling in from 500m out. This may take a while...."
)
distance = EucDistance(flow_line, cell_size="10")
streams = Reclassify(
Raster(flow_line_raster) > 0, "Value", "1 1; NoData 0")
burnt = Raster(subregion_ned) - (10 * streams) - (0.02 * (500 - distance) *
(distance < 500))
cu.multi_msg("Saving output raster...")
burnt.save(burnt_out)
# Delete intermediate rasters and shapefiles
cu.cleanup([flow_line, flow_line_raster])
cu.multi_msg("Burn process completed")
def nhd_merge(gdb_list, example_feature_class_name, out_fc, selection = ''):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.env.workspace = 'in_memory'
gdb_list = [os.path.join(gdb, os.path.basename(example_feature_class_name)) for gdb in gdb_list]
gdb0 = gdb_list.pop(0)
desc = arcpy.Describe(gdb0)
cu.multi_msg('Merging all features together...')
arcpy.CopyFeatures_management(gdb0, 'temp_merged')
cu.lengthen_field('temp_merged', 'Permanent_Identifier', 255)
arcpy.Append_management(gdb_list, 'temp_merged', 'NO_TEST')
# use in_memory explicitly here because i haven't figured out how to pass arcgis environments to my functions :(
if selection:
cu.multi_msg('Selecting features...')
arcpy.Select_analysis('temp_merged', 'in_memory/merged_select', selection)
cu.multi_msg('Removing ID duplicates...')
assumptions.remove_nhd_duplicates('in_memory/merged_select', 'Permanent_Identifier', 'in_memory/no_id_dupes')
if desc.shapeType == 'Polygon':
cu.multi_msg('Removing geographic duplicates and substantially overlapping features...')
assumptions.remove_geographic_doubles('in_memory/no_id_dupes', out_fc, 'Permanent_Identifier', percent_overlap_allowed = 10)
cu.multi_msg('nhd_merge complete.')
def burn_streams(subregion_ned, nhd_gdb, burnt_out,
projection = arcpy.SpatialReference(102039)):
env.snapRaster = subregion_ned
env.outputCoordinateSystem = projection
env.compression = "LZ77" # compress temp tifs for speed
env.extent = subregion_ned
env.workspace = 'in_memory'
flowline = os.path.join(nhd_gdb, 'NHDFlowline')
# Copy flowlines to shapefile that will inherit environ output coord system
# just easier to have a copy in the correct projection later
arcpy.CopyFeatures_management(flowline, 'flowline_proj')
## arcpy.FeatureClassToFeatureClass_conversion("NHDFlowline", "in_memory", flow_line)
cu.multi_msg("Prepared NHDFlowline for rasterizing.")
# Feature to Raster- rasterize the NHDFlowline
# will inherit grid from env.snapRaster
arcpy.FeatureToRaster_conversion('flowline_proj', "OBJECTID",
'flowline_raster', "10")
cu.multi_msg("Converted flowlines to raster.")
# Raster Calculator- burns in streams, beveling in from 500m
cu.multi_msg("Burning streams into raster, 10m deep and beveling in from 500m out. This may take a while....")
arcpy.CheckOutExtension("Spatial")
distance = EucDistance('flowline_proj', cell_size = "10")
streams = Reclassify(Raster('flowline_raster') > 0, "Value", "1 1; NoData 0")
burnt = Raster(subregion_ned) - (10 * streams) - (0.02 * (500 - distance) * (distance < 500))
cu.multi_msg("Saving output raster...")
burnt.save(burnt_out)
# Delete intermediate rasters and shapefiles
for item in ['flowline_proj', 'flowline_raster']:
arcpy.Delete_management(item)
arcpy.CheckInExtension("Spatial")
cu.multi_msg("Burn process completed")
arcpy.ResetEnvironments()
def clip(raster, nhd_gdb, projection, outfolder):
env.workspace = nhd_gdb
env.outputCoordinateSystem = projection
env.compression = "NONE" # only final tifs are generated
env.pyramid = "NONE"
# Create a feature dataset in NHD file geodatabase named "HUC8_Albers" in Albers projection
out_feature_dataset = "HUC8_Albers"
arcpy.CreateFeatureDataset_management(env.workspace, out_feature_dataset,
projection)
arcpy.RefreshCatalog(nhd)
# HUC8 polygons each saved as separate fc inheriting albers from environ
huc8_fc = "WBD_HU8"
field = "HUC_8"
arcpy.MakeFeatureLayer_management(huc8_fc, "huc8_layer")
with arcpy.da.SearchCursor(huc8_fc, field) as cursor:
for row in cursor:
if row[0].startswith(nhdsubregion):
whereClause = ''' "%s" = '%s' ''' % (field, row[0])
arcpy.SelectLayerByAttribute_management(
"huc8_layer", 'NEW_SELECTION', whereClause)
arcpy.CopyFeatures_management(
"huc8_layer",
os.path.join(out_feature_dataset, "HUC" + row[0]))
#retrieve only the single huc8 fcs and not the one with all of them
fcs = arcpy.ListFeatureClasses("HUC%s*" % nhdsubregion, "Polygon",
out_feature_dataset)
fcs_buffered = [
os.path.join(out_feature_dataset, fc + "_buffer") for fc in fcs
]
out_clips = [
os.path.join(outfolder, "huc8clips" + nhdsubregion,
"NED" + fc[3:] + ".tif") for fc in fcs
]
# Buffer HUC8 feature classes by 5000m
for fc, fc_buffered in zip(fcs, fcs_buffered):
arcpy.Buffer_analysis(fc, fc_buffered, "5000 meters")
cu.multi_msg("Created HUC8 buffers.")
arcpy.RefreshCatalog(nhd)
# Clips rasters
cu.multi_msg("Starting HUC8 clips...")
for fc_buffered, out_clip in zip(fcs_buffered, out_clips):
arcpy.Clip_management(raster, '', out_clip, fc_buffered, "0",
"ClippingGeometry")
arcpy.Compact_management(nhd)
cu.multi_msg("Clipping complete.")
def clip(raster, nhd_gdb, projection, outfolder):
env.workspace = nhd_gdb
env.outputCoordinateSystem = projection
env.compression = "NONE" # only final tifs are generated
env.pyramid = "NONE"
# Create a feature dataset in NHD file geodatabase named "HUC8_Albers" in Albers projection
out_feature_dataset = "HUC8_Albers"
arcpy.CreateFeatureDataset_management(env.workspace, out_feature_dataset, projection)
arcpy.RefreshCatalog(nhd)
# HUC8 polygons each saved as separate fc inheriting albers from environ
huc8_fc = "WBD_HU8"
field = "HUC_8"
arcpy.MakeFeatureLayer_management(huc8_fc, "huc8_layer")
with arcpy.da.SearchCursor(huc8_fc, field) as cursor:
for row in cursor:
if row[0].startswith(nhdsubregion):
whereClause = ''' "%s" = '%s' ''' % (field, row[0])
arcpy.SelectLayerByAttribute_management("huc8_layer", 'NEW_SELECTION', whereClause)
arcpy.CopyFeatures_management("huc8_layer", os.path.join(out_feature_dataset, "HUC" + row[0]))
#retrieve only the single huc8 fcs and not the one with all of them
fcs = arcpy.ListFeatureClasses("HUC%s*" % nhdsubregion, "Polygon", out_feature_dataset)
fcs_buffered = [os.path.join(out_feature_dataset, fc + "_buffer") for fc in fcs]
out_clips = [os.path.join(outfolder, "huc8clips" + nhdsubregion,
"NED" + fc[3:] + ".tif") for fc in fcs]
# Buffer HUC8 feature classes by 5000m
for fc, fc_buffered in zip(fcs, fcs_buffered):
arcpy.Buffer_analysis(fc, fc_buffered, "5000 meters")
cu.multi_msg("Created HUC8 buffers.")
arcpy.RefreshCatalog(nhd)
# Clips rasters
cu.multi_msg("Starting HUC8 clips...")
for fc_buffered, out_clip in zip(fcs_buffered, out_clips):
arcpy.Clip_management(raster, '', out_clip, fc_buffered, "0", "ClippingGeometry")
arcpy.Compact_management(nhd)
cu.multi_msg("Clipping complete.")
def split_strahler(stream_area_fc, streams, out_area_fc):
"""This function splits up the NHDArea feature class, which does not
start and stop polygons at confluences, by creating break points near the
confluences to split up the polygons. Then, it adds the Strahler value from
the stream centerline."""
# 1) Generate euclidean allocation raster from streams (use OBJECTID)
# 2) Convert euclidean allocation raster to polygons
# 3) Join allocation polygons "gridcode" to streams "OBJECTID" so that
# Strahler value is attached to allocation polygon
# 4) Use identity function to split up the StreamRiver polygons at the
# allocation polygon boundaries, and add the Strahler values
old_workspace = env.workspace
env.workspace = 'in_memory'
cu.multi_msg("Splitting stream area polygons between confluences and joining 1) Strahler order to them...")
cu.multi_msg('next messages for testing')
arcpy.CheckOutExtension('Spatial')
cu.multi_msg('euc')
euc = EucAllocation(streams, cell_size = '50', source_field = 'OBJECTID')
arcpy.CheckInExtension('Spatial')
cu.multi_msg('conversion')
arcpy.RasterToPolygon_conversion(euc, 'allocation_polys')
stream_id_field = arcpy.ListFields(streams, 'Permanent_')[0].name
cu.multi_msg('join')
arcpy.JoinField_management('allocation_polys', 'grid_code', streams, 'OBJECTID', ['Strahler', 'LengthKm', stream_id_field])
cu.multi_msg('identity')
arcpy.Identity_analysis(stream_area_fc, 'allocation_polys', out_area_fc)
env.workspace = old_workspace
cu.multi_msg("Splitting strema area polygons finished.")
def mosaic(in_workspace,
out_dir,
available_ram=4,
projection=arcpy.SpatialReference(102039)):
# Set up environments
env.terrainMemoryUsage = True
env.compression = "LZ77" # compress temp tifs for speed
env.pyramids = "NONE" # for intermediates only
env.outputCoordinateSystem = projection
env.workspace = in_workspace
huc4_code = re.search('\d{4}', os.path.basename(in_workspace)).group()
nhd_gdb = arcpy.ListWorkspaces()[0]
# Select the right HUC4 from WBD_HU4 and make it it's own layer.
wbd_hu4 = os.path.join(nhd_gdb, "WBD_HU4")
if not arcpy.Exists(wbd_hu4):
wbd_hu4 = os.path.join(nhd_gdb, "WBDHU4")
arcpy.AddMessage(wbd_hu4)
arcpy.AddMessage(arcpy.Exists(wbd_hu4))
field_name = (arcpy.ListFields(wbd_hu4, "HU*4"))[0].name
whereClause = """{0} = '{1}'""".format(
arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc4_code)
arcpy.MakeFeatureLayer_management(wbd_hu4, "Subregion", whereClause)
# Apply a 5000 meter buffer around subregion
subregion_buffer = os.path.join(nhd_gdb, "Subregion_Buffered_5000m")
arcpy.Buffer_analysis("Subregion", subregion_buffer, "5000 meters")
cu.multi_msg("Buffered subregion.")
# Walk through the folder with NEDs to make a list of rasters
in_workspace = r'D:\Continental_Limnology\Data_Working\Test_Watersheds\NHD0503'
mosaic_rasters = []
for dirpath, dirnames, filenames in arcpy.da.Walk(
in_workspace, datatype="RasterDataset"):
for filename in filenames:
print(filename)
if not '.jpg' in filename:
name = os.path.join(dirpath, filename)
mosaic_rasters.append(name)
cu.multi_msg("Found NED rasters.")
# Update environments
env.extent = subregion_buffer
approx_size_dir_GB = len(mosaic_rasters) * .5
if approx_size_dir_GB < .5 * int(available_ram):
env.workspace = 'in_memory'
memory_msg = (
"Attempting to use in_memory workspace. If you" +
" experience problems during the execution of this tool, " +
"try running it again with a lower value " +
"entered for 'Available RAM'.")
cu.multi_msg(memory_msg)
else:
env.workspace = out_dir
env.outputCoordinateSystem = mosaic_rasters[0]
# Assign names to intermediate outputs in outfolder
mosaic_unproj = "mosaic_t1"
mosaic_proj = "mosaic_t2"
# Mosaic, then project
# Cannot do this in one step using MosaicToNewRaster's projection parameter
# because you cannot set the cell size correctly
cu.multi_msg("Creating initial mosaic. This may take a while...")
arcpy.MosaicToNewRaster_management(mosaic_rasters, env.workspace,
mosaic_unproj, "", "32_BIT_FLOAT", "",
"1", "LAST")
cu.multi_msg("Projecting mosaic...")
arcpy.ProjectRaster_management(mosaic_unproj, mosaic_proj, projection,
"BILINEAR", "10")
#final mosaic environs
env.pyramids = "PYRAMIDS -1 SKIP_FIRST" # need to check outputs efficiently
env.outputCoordinateSystem = projection
cu.multi_msg("Clipping final mosaic...")
out_mosaic = os.path.join(out_dir, "NED13_%s.tif" % huc4_code)
arcpy.Clip_management(mosaic_proj, '', out_mosaic, subregion_buffer, "0",
"ClippingGeometry")
# Clean up
for item in [mosaic_unproj, mosaic_proj]:
arcpy.Delete_management(item)
cu.multi_msg("Mosaicked NED tiles and clipped to HUC4 extent.")
for raster in mosaic_rasters:
arcpy.Delete_management(raster)
def stage_files(nhd_gdb, ned_dir, ned_footprints_fc, out_dir, is_zipped):
env.workspace = 'in_memory'
#####################################################################
cu.multi_msg("1) Creating Directory Structure and Copying NHD Geodatabase")
#####################################################################
#finds the 4-digit huc code in the filename
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
out_subdir = os.path.join(out_dir, "NHD" + huc4_code)
if not os.path.exists(out_subdir):
os.mkdir(out_subdir)
nhd_gdb_copy = os.path.join(out_subdir, os.path.basename(nhd_gdb))
arcpy.Copy_management(nhd_gdb, nhd_gdb_copy)
####################################################################
cu.multi_msg("2) Getting WBD Poly For Subregion and Buffering by 5000m...")
#####################################################################
#select only this subregion from the wbd layer in the nhd_gdb (bordering
# subregions are included in there too) and buffer it
wbd_hu4 = os.path.join(nhd_gdb_copy, "WBD_HU4")
if not arcpy.Exists(wbd_hu4):
wbd_hu4 = os.path.join(nhd_gdb_copy, "WBDHU4")
field_name = (arcpy.ListFields(wbd_hu4, "HU*4"))[0].name
whereClause = """{0} = '{1}'""".format(
arcpy.AddFieldDelimiters(nhd_gdb_copy, field_name), huc4_code)
arcpy.MakeFeatureLayer_management(wbd_hu4, "wbd_poly", whereClause)
arcpy.Buffer_analysis("wbd_poly", "wbd_buf", "5000 meters")
#####################################################################
cu.multi_msg(
"3) Clipping NED Tile polys from Buffered NHD Subregion From WBD...")
#####################################################################
arcpy.Clip_analysis(ned_footprints_fc, "wbd_buf", "ned_clip")
#####################################################################
cu.multi_msg(
"4) Getting File_ID of clipped NED footprint polys and copying NED data to output location"
)
#####################################################################
missing_NED_list = []
with arcpy.da.SearchCursor("ned_clip", ["FILE_ID"]) as cursor:
for row in cursor:
file_id = row[0].replace("g", "")
# unzipping if needed
if is_zipped:
unzipped_file = unzip_ned(file_id, ned_dir, out_subdir)
if not unzipped_file:
missing_NED_list.append(file_id)
else:
# copy ned tiles to output location
ned_source = os.path.join(ned_dir, file_id)
ned_destination = os.path.join(out_subdir, file_id)
if not os.path.exists(ned_source):
cu.multi_msg(
"ERROR: Tile %s does not exist in the specified location"
% file_id)
missing_NED_list.append(file_id)
else:
if not os.path.exists(ned_destination):
shutil.copytree(ned_source, ned_destination)
else:
cu.multi_msg(
"Output folder for this NED tile already exists.")
if missing_NED_list:
warning_text = "WARNING: NED tiles did not exist for the following: %s" % ','.join(
missing_NED_list)
arcpy.AddWarning(warning_text)
print(warning_text)
for item in ["wbd_buf", "ned_clip"]:
arcpy.Delete_management(item)
return out_subdir
def nhd_merge(gdb_list, example_feature_class_name, out_fc, selection = ''):
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039) # USA_Contiguous_Albers_Equal_Area_Conic_USGS_version
arcpy.env.workspace = 'in_memory'
gdb_list = [os.path.join(gdb, os.path.basename(example_feature_class_name)) for gdb in gdb_list]
gdb0 = gdb_list.pop(0)
desc = arcpy.Describe(gdb0)
cu.multi_msg('Merging all features together...')
arcpy.CopyFeatures_management(gdb0, 'temp_merged')
cu.lengthen_field('temp_merged', 'Permanent_Identifier', 255)
cu.merge_many(gdb_list, 'in_memory/the_rest_merged')
arcpy.Append_management('in_memory/the_rest_merged', 'temp_merged', 'NO_TEST')
# use in_memory explicitly here because i haven't figured out how to pass arcgis environments to my functions :(
fc_temp = 'in_memory/temp_merged'
fcount1 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('Before selection and cleaning, feature count is {0}'.format(fcount1))
if selection:
cu.multi_msg('Selecting features...')
arcpy.Select_analysis('temp_merged', 'in_memory/merged_select', selection)
fc_temp = 'in_memory/merged_select'
fcount2 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('After selection and before cleaning, feature count is {0}'.format(fcount2))
cu.multi_msg('Removing complete duplicates...')
fc_temp_fields = [f.name for f in arcpy.ListFields(fc_temp) if f.type <> 'OID']
arcpy.DeleteIdentical_management(fc_temp, fields = [f.name for f in arcpy.ListFields(fc_temp) if f.type <> 'OID'])
fcount3 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('After removing complete duplicates only, feature count is {0}'.format(fcount3))
cu.multi_msg('Removing remaining ID duplicates...')
assumptions.remove_nhd_duplicates(fc_temp, 'Permanent_Identifier', 'in_memory/no_id_dupes')
fcount4 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('After removing all ID duplicates, feature count is {0}'.format(fcount4))
if desc.shapeType == 'Polygon':
cu.multi_msg('Removing geographic duplicates and substantially overlapping features...')
assumptions.remove_geographic_doubles('in_memory/no_id_dupes', out_fc, 'Permanent_Identifier', percent_overlap_allowed = 10)
cu.multi_msg('nhd_merge complete.')
fcount5 = int(arcpy.GetCount_management(fc_temp).getOutput(0))
cu.multi_msg('Final feature count is {0}'.format(fcount5))
def mosaic(in_workspace, out_dir, available_ram = 4, projection = arcpy.SpatialReference(102039)) :
# Set up environments
env.terrainMemoryUsage = True
env.compression = "LZ77" # compress temp tifs for speed
env.pyramids = "NONE" # for intermediates only
env.outputCoordinateSystem = projection
env.workspace = in_workspace
huc4_code = re.search('\d{4}', os.path.basename(in_workspace)).group()
nhd_gdb = os.path.join(in_workspace, 'NHDH%s.gdb' % huc4_code)
# Select the right HUC4 from WBD_HU4 and make it it's own layer.
wbd_hu4 = os.path.join(nhd_gdb, "WBD_HU4")
field_name = (arcpy.ListFields(wbd_hu4, "HU*4"))[0].name
whereClause = """{0} = '{1}'""".format(arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc4_code)
arcpy.MakeFeatureLayer_management(wbd_hu4, "Subregion", whereClause)
# Apply a 5000 meter buffer around subregion
subregion_buffer = os.path.join(nhd_gdb, "Subregion_Buffered_5000m")
arcpy.Buffer_analysis("Subregion", subregion_buffer, "5000 meters")
cu.multi_msg("Buffered subregion.")
# Walk through the folder with NEDs to make a list of rasters
mosaic_rasters = []
for dirpath, dirnames, filenames in arcpy.da.Walk(in_workspace, datatype="RasterDataset"):
for filename in filenames:
print(filename)
if filename.startswith('grd'):
name = os.path.join(dirpath, filename)
mosaic_rasters.append(name)
cu.multi_msg("Found NED ArcGrids.")
# Update environments
env.extent = subregion_buffer
approx_size_dir_GB = len(mosaic_rasters) * .5
if approx_size_dir_GB < .5 * int(available_ram):
env.workspace = 'in_memory'
memory_msg = ("Attempting to use in_memory workspace. If you " +
" experience problems during the exectuion of this tool, " +
"try running it again with a lower value " +
"entered for 'Available RAM'.")
cu.multi_msg(memory_msg)
else:
env.workspace = out_dir
env.outputCoordinateSystem = mosaic_rasters[0]
# Assign names to intermediate outputs in outfolder
mosaic_unproj = "mosaic_t1"
mosaic_proj = "mosaic_t2"
# Mosaic, then project
# Cannot do this in one step using MosaicToNewRaster's projection parameter
# because you cannot set the cell size correctly
cu.multi_msg("Creating initial mosaic. This may take a while...")
arcpy.MosaicToNewRaster_management(mosaic_rasters, env.workspace,
mosaic_unproj, "", "32_BIT_FLOAT", "", "1", "LAST")
cu.multi_msg("Projecting mosaic...")
arcpy.ProjectRaster_management(mosaic_unproj, mosaic_proj,
projection, "BILINEAR", "10")
#final mosaic environs
env.pyramids = "PYRAMIDS -1 SKIP_FIRST" # need to check outputs efficiently
env.outputCoordinateSystem = projection
cu.multi_msg("Clipping final mosaic...")
out_mosaic = os.path.join(out_dir, "NED13_%s.tif" % huc4_code)
arcpy.Clip_management(mosaic_proj, '', out_mosaic, subregion_buffer,
"0", "ClippingGeometry")
# Clean up
for item in [mosaic_unproj, mosaic_proj]:
arcpy.Delete_management(item)
cu.multi_msg("Mosaicked NED tiles and clipped to HUC4 extent.")
def polygons_in_zones(zone_fc,
zone_field,
polygons_of_interest,
output_table,
interest_selection_expr,
contrib_area=True):
old_workspace = arcpy.env.workspace
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
temp_polyzones = cu.create_temp_GDB('temp_polyzones')
selected_polys = os.path.join(temp_polyzones, 'selected_polys')
cu.multi_msg('Copying/selecting polygon features...')
if interest_selection_expr:
arcpy.Select_analysis(polygons_of_interest, selected_polys,
interest_selection_expr)
else:
arcpy.CopyFeatures_management(polygons_of_interest, selected_polys)
arcpy.AddField_management(selected_polys, 'POLYAREA_ha', 'DOUBLE')
arcpy.CalculateField_management(selected_polys, 'POLYAREA_ha',
'!shape.area@hectares!', 'PYTHON')
# use tabulate intersection for the areas overlapping
tab_table = 'tabulate_intersection_table'
cu.multi_msg('Tabulating intersection between zones and polygons...')
arcpy.TabulateIntersection_analysis(zone_fc, zone_field, selected_polys,
tab_table)
# area was calculated in map units which was m2 so convert to hectares
arcpy.AddField_management(tab_table, 'Poly_Overlapping_AREA_ha', 'DOUBLE')
arcpy.CalculateField_management(tab_table, 'Poly_Overlapping_AREA_ha',
'!AREA!/10000', 'PYTHON')
# just change the name of the percent field
cu.rename_field(tab_table, 'PERCENTAGE', 'Poly_Overlapping_AREA_pct', True)
spjoin_fc = 'spatial_join_output'
# Spatial join for the count and contributing area
fms = arcpy.FieldMappings()
fm_zone_id = arcpy.FieldMap()
fm_zone_id.addInputField(zone_fc, zone_field)
fm_count = arcpy.FieldMap()
fm_count.addInputField(selected_polys, 'POLYAREA_ha')
count_name = fm_count.outputField
count_name.name = 'Poly_Count'
count_name.alias = 'Poly_Count'
fm_count.outputField = count_name
fm_count.mergeRule = 'Count'
fm_contrib_area = arcpy.FieldMap()
fm_contrib_area.addInputField(selected_polys, 'POLYAREA_ha')
contrib_area_name = fm_contrib_area.outputField
contrib_area_name.name = 'Poly_Contributing_AREA_ha'
contrib_area_name.alias = 'Poly_Contributing_AREA_ha'
fm_contrib_area.outputField = contrib_area_name
fm_contrib_area.mergeRule = 'Sum'
fms.addFieldMap(fm_zone_id)
fms.addFieldMap(fm_count)
fms.addFieldMap(fm_contrib_area)
cu.multi_msg('Spatial join between zones and wetlands...')
arcpy.SpatialJoin_analysis(zone_fc, selected_polys, spjoin_fc,
"JOIN_ONE_TO_ONE", "KEEP_ALL", fms, "INTERSECT")
cu.multi_msg('Refining output...')
arcpy.JoinField_management(tab_table, zone_field, spjoin_fc, zone_field,
["Poly_Count", "Poly_Contributing_AREA_ha"])
final_fields = [
'Poly_Overlapping_AREA_ha', 'Poly_Overlapping_AREA_pct', 'Poly_Count',
'Poly_Contributing_AREA_ha'
]
# make output nice
cu.one_in_one_out(tab_table, final_fields, zone_fc, zone_field,
output_table)
cu.redefine_nulls(output_table, final_fields, [0, 0, 0, 0])
# clean up
for item in [selected_polys, tab_table, spjoin_fc]:
arcpy.Delete_management(item)
arcpy.Delete_management(temp_polyzones)
arcpy.env.workspace = old_workspace
cu.multi_msg('Polygons in zones tool complete.')
def stats_area_table(zone_fc, zone_field, in_value_raster, out_table, is_thematic):
arcpy.CheckOutExtension("Spatial")
cu.multi_msg("Calculating zonal statistics...")
temp_zonal_table = 'in_memory/zonal_stats_temp'
temp_entire_table = 'in_memory/temp_entire_table'
# calculate/doit
env.snapRaster = in_value_raster
env.cellSize = in_value_raster
# this has to be on disk for some reason to avoid background processing
# errors thrown up at random
# hence we get the following awkward horribleness
use_convert_raster = False
try:
arcpy.sa.ZonalStatisticsAsTable(zone_fc, zone_field, in_value_raster,
temp_zonal_table, 'DATA', 'ALL')
# with Permanent_Identifier as the zone_field, background processing errors
# and another error get thrown up at random
# it's faster to do zonal stats as above but if it fails (which it does
# pretty quickly, usually), do this way which always works but takes
# twice as long on large rasters
except:
use_convert_raster = True
temp_workspace = cu.create_temp_GDB('temp_zonal')
convert_raster = os.path.join(temp_workspace,
cu.shortname(zone_fc) + '_converted')
cu.multi_msg('Creating raster {0}'.format(convert_raster))
arcpy.PolygonToRaster_conversion(zone_fc, zone_field, convert_raster)
arcpy.sa.ZonalStatisticsAsTable(convert_raster, zone_field, in_value_raster,
temp_zonal_table, "DATA", "ALL")
if is_thematic:
#for some reason env.celLSize doesn't work
desc = arcpy.Describe(in_value_raster)
cell_size = desc.meanCelLHeight
# calculate/doit
temp_area_table = 'in_memory/tab_area_temp'
cu.multi_msg("Tabulating areas...")
if use_convert_raster:
arcpy.sa.TabulateArea(convert_raster, zone_field, in_value_raster,
'Value', temp_area_table, cell_size)
else:
arcpy.sa.TabulateArea(zone_fc, zone_field, in_value_raster,
'Value', temp_area_table, cell_size)
# making the output table
arcpy.CopyRows_management(temp_area_table, temp_entire_table)
zonal_stats_fields = ['AREA']
arcpy.JoinField_management(temp_entire_table, zone_field, temp_zonal_table, zone_field, zonal_stats_fields)
# cleanup
arcpy.Delete_management(temp_area_table)
if not is_thematic:
# making the output table
arcpy.CopyRows_management(temp_zonal_table, temp_entire_table)
cu.multi_msg("Refining output table...")
refine_zonal_output(temp_entire_table, is_thematic)
#final table gets a record even for no-data zones
keep_fields = [f.name for f in arcpy.ListFields(temp_entire_table)]
if zone_field.upper() in keep_fields:
keep_fields.remove(zone_field.upper())
if zone_field in keep_fields:
keep_fields.remove(zone_field)
cu.one_in_one_out(temp_entire_table, keep_fields, zone_fc, zone_field, out_table)
## cu.redefine_nulls(out_table, keep_fields, ["NA"]* len(keep_fields))
# count whether all zones got an output record or not)
out_count = int(arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
if out_count < in_count:
warn_msg = ("WARNING: {0} features are missing in the output table"
" because they are too small for this raster's"
" resolution. This may be okay depending on your"
" application.").format(in_count - out_count)
arcpy.AddWarning(warn_msg)
print(warn_msg)
# cleanup
arcpy.Delete_management(temp_zonal_table)
arcpy.Delete_management(temp_entire_table)
if use_convert_raster:
arcpy.Delete_management(os.path.dirname(temp_workspace))
arcpy.CheckInExtension("Spatial")
def line_density(zones, zonefield, lines, out_table, interest_selection_expr):
# Make output folder
## name = "LineDensity_" + os.path.splitext(os.path.basename(zones))[0]
## outfolder = os.path.join(topoutfolder, name)
## if not os.path.exists(outfolder):
## os.mkdir(outfolder)
# Environmental Settings
ws = "in_memory"
if interest_selection_expr:
arcpy.MakeFeatureLayer_management(lines, "selected_lines", interest_selection_expr)
else:
arcpy.MakeFeatureLayer_management(lines, "selected_lines")
arcpy.env.workspace = ws
albers = arcpy.SpatialReference(102039)
arcpy.env.outputCoordinateSystem = albers
arcpy.env.extent = zones
# Zones will be coerced to albers, have to check lines though
arcpy.CopyFeatures_management(zones, "zones_temp")
if arcpy.Describe(lines).spatialReference.factoryCode != albers.factoryCode:
arcpy.AddError("Lines feature class does not have desired projection (Albers USGS). Re-project to factory code 102039 and try again.")
sys.exit(1)
# Add hectares field to zones
arcpy.AddField_management("zones_temp", "ZoneAreaHa", "DOUBLE")
arcpy.CalculateField_management("zones_temp", "ZoneAreaHa", "!shape.area@hectares!", "PYTHON")
# Perform identity analysis to join fields and crack lines at polygon boundaries
cu.multi_msg("Cracking lines at polygon boundaries...")
arcpy.Identity_analysis("selected_lines", "zones_temp", "lines_identity")
cu.multi_msg("Cracking lines complete.")
# Recalculate lengths
arcpy.AddField_management("lines_identity", "LengthM", "DOUBLE")
arcpy.CalculateField_management("lines_identity", "LengthM", '!shape.length@meters!', "PYTHON")
# Summarize statistics by zone
arcpy.Statistics_analysis("lines_identity", "length_in_zone", "LengthM SUM", zonefield)
# Join ZoneAreaHa to table
arcpy.JoinField_management("length_in_zone", zonefield, "zones_temp" , zonefield, "ZoneAreaHa")
# Delete rows in table with zero for zone area
## with arcpy.da.UpdateCursor("length_in_zone", "ZoneAreaHa") as cursor:
## for row in cursor:
## if row[0] is None:
## cursor.deleteRow()
# Add Density field and calc
arcpy.AddField_management("length_in_zone", "Density_MperHA", "DOUBLE",'','','','',"NULLABLE")
exp = "!SUM_LengthM! / !ZONEAREAHA!"
arcpy.CalculateField_management("length_in_zone", "Density_MperHA", exp, "PYTHON")
cu.one_in_one_out("length_in_zone", ['SUM_LengthM', 'Density_MperHA'], zones, zonefield, out_table)
cu.redefine_nulls(out_table, ['SUM_LengthM', 'Density_MperHA'], [0, 0])
## # Join to the original table
## keep_fields = ["ZoneID", "SUM_LengthM", "Density_MperHA"]
## arcpy.JoinField_management('zones_temp', zonefield, "length_in_zone", zonefield, keep_fields[1:])
##
## # Select only null records and change to 0
## arcpy.MakeFeatureLayer_management('zones_temp', 'zones_temp_lyr')
## arcpy.SelectLayerByAttribute_management('zones_temp_lyr', "NEW_SELECTION", '''"SUM_LengthM" is null''')
## fields_to_calc = ["SUM_LengthM", "Density_MperHA"]
## for f in fields_to_calc:
## arcpy.CalculateField_management('zones_temp_lyr', f, 0, "PYTHON")
##
## #Delete all the fields that aren't the ones I need
## keep_fields = ["ZoneID", "SUM_LengthM", "Density_MperHA"]
## all_fields = [f.name for f in arcpy.ListFields('zones_temp_lyr')]
## for f in all_fields:
## if f not in keep_fields:
## try:
## arcpy.DeleteField_management('zones_temp_lyr', f)
## except:
## continue
## arcpy.SelectLayerByAttribute_management('zones_temp_lyr', 'CLEAR_SELECTION')
##
## arcpy.CopyRows_management('zones_temp_lyr', out_table)
for tempitem in ['zones_temp', 'lines_identity', 'length_in_zone']:
arcpy.Delete_management(tempitem)
return out_table
def upstream_lakes(nhd_gdb, output_table):
#define paths in nhd geodatabase
nhd_waterbody = os.path.join(nhd_gdb, 'NHDWaterbody')
nhd_flowline = os.path.join(nhd_gdb, 'NHDFlowline')
nhd_junctions = os.path.join(nhd_gdb, 'HYDRO_NET_Junctions')
hydro_net = os.path.join(nhd_gdb, 'Hydrography', 'HYDRO_NET')
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.MakeFeatureLayer_management(nhd_junctions, 'junctions')
cu.multi_msg('Preparing layers and fields for calculations....')
# select only lakes as defined in our project: waterbodies with one of these
# types and greater than 4ha, and certain reservoirs greater than 10ha
fcodes = (39000, 39004, 39009, 39010, 39011, 39012, 43600, 43613, 43615,
43617, 43618, 43619, 43621)
gte_4ha_lakes_query = '''("AreaSqKm" >=0.04 AND "FCode" IN %s) OR ("FCode" = 43601 AND "AreaSqKm" >= 0.1)''' % (
fcodes, )
gte_10ha_lakes_query = '''("AreaSqKm" >=0.1 AND "FCode" IN %s) OR ("FCode" = 43601 AND "AreaSqKm" >= 0.1)''' % (
fcodes, )
arcpy.MakeFeatureLayer_management(nhd_waterbody, 'gte_4ha_lakes',
gte_4ha_lakes_query)
arcpy.MakeFeatureLayer_management(nhd_waterbody, 'gte_10ha_lakes',
gte_10ha_lakes_query)
arcpy.CopyRows_management('gte_4ha_lakes', 'output_table')
# (need this for line 61 make feature layer to work right!)
arcpy.CopyFeatures_management('gte_4ha_lakes', 'gte_4ha_lakes_DISK')
# add the new fields we'll calculate: count and area
count_fields = ['Upstream_Lakes_4ha_Count', 'Upstream_Lakes_10ha_Count']
area_fields = ['Upstream_Lakes_4ha_Area_ha', 'Upstream_Lakes_10ha_Area_ha']
for cf in count_fields:
arcpy.AddField_management('output_table', cf, 'LONG')
for af in area_fields:
arcpy.AddField_management('output_table', af, 'DOUBLE')
new_fields = count_fields + area_fields
# for each lake, use its junctions as input flags to the upstream trace, then
# evalute the traced network for how many lakes are in it
cu.multi_msg(
"Calculating upstream network for each lake. This may take a few minutes, or a few hours..."
)
with arcpy.da.UpdateCursor('output_table', ['Permanent_Identifier'] +
new_fields) as cursor:
for row in cursor:
id = row[0]
# get the junction points on top of this lake, can be 0 or several
where_clause = """"{0}" = '{1}'""".format('Permanent_Identifier',
id)
arcpy.MakeFeatureLayer_management('gte_4ha_lakes_DISK',
"this_lake", where_clause)
arcpy.SelectLayerByLocation_management('junctions', "INTERSECT",
"this_lake", "1 Meters")
count_jxns = int(
arcpy.GetCount_management('junctions').getOutput(0))
# if the lake has no upstream connectivity whatsoever, it will have
# no junctions on top of it. assign 0s
# to the counts of upstream lakes. This can either be an "isolated"
# lake or a "headwater" lake
if count_jxns <= 0:
row[1:] = [0, 0, 0, 0]
# if there are any junctions, go ahead and trace upstream
else:
arcpy.CopyFeatures_management("junctions", 'this_lake_jxns')
arcpy.TraceGeometricNetwork_management(hydro_net, "upstream",
'this_lake_jxns',
"TRACE_UPSTREAM")
arcpy.SelectLayerByLocation_management('gte_4ha_lakes',
"INTERSECT",
"upstream/NHDFlowline",
'1 Meters',
'NEW_SELECTION')
# Remove this lake from the selection!!
arcpy.SelectLayerByAttribute_management(
'gte_4ha_lakes', 'REMOVE_FROM_SELECTION', where_clause)
count_4ha_lakes = int(
arcpy.GetCount_management('gte_4ha_lakes').getOutput(0))
# if there is upstream connectivity but no countable lakes
# assign 0s for all the counts/areas and do not try to copy rows
if count_4ha_lakes <= 0:
row[1:] = [0, 0, 0, 0]
# if there are any lakes upstream, copy rows before doing
# counts so we can use a search cursor on just these lakes
else:
arcpy.CopyRows_management('gte_4ha_lakes',
'these_4ha_lakes')
# row[1] is 4ha_Upstream_Lakes_Count
# row [3] is 4ha_Upstream_Lakes_Area
row[1] = int(
arcpy.GetCount_management('these_4ha_lakes').getOutput(
0))
total_area = 0
with arcpy.da.SearchCursor('these_4ha_lakes',
['AreaSqKm']) as area4_cursor:
for area4_row in area4_cursor:
total_area += area4_row[0] * 100
row[3] = total_area
# same but for 10ha
arcpy.SelectLayerByLocation_management(
'gte_10ha_lakes', "INTERSECT", "upstream/NHDFlowline",
'1 Meters', 'NEW_SELECTION')
arcpy.SelectLayerByAttribute_management(
'gte_10ha_lakes', 'REMOVE_FROM_SELECTION',
where_clause)
count_10ha_lakes = int(
arcpy.GetCount_management('gte_10ha_lakes').getOutput(
0))
# don't try to copy rows if selection is empty
if count_10ha_lakes <= 0:
row[2] = 0
row[4] = 0
# if there are features selected, copy rows so we
# can use a search cursor
else:
arcpy.CopyRows_management('gte_10ha_lakes',
'these_10ha_lakes')
row[2] = int(
arcpy.GetCount_management(
'these_10ha_lakes').getOutput(0))
total_area = 0
with arcpy.da.SearchCursor(
'these_4ha_lakes',
['AreaSqKm']) as area10_cursor:
for area10_row in area10_cursor:
total_area += area10_row[0] * 100
row[4] = total_area
cursor.updateRow(row)
# delete intermediates before next iteration
for item in [
'this_lake', 'this_lake_jxns', 'upstream',
'these_4ha_lakes', 'these_10ha_lakes'
]:
try:
arcpy.Delete_management(item)
except:
continue
# clean up the output table
all_fields = [f.name for f in arcpy.ListFields('output_table')]
for f in all_fields:
if f not in ['Permanent_Identifier'] + new_fields:
try:
arcpy.DeleteField_management('output_table', f)
except:
continue
# write out the final file and clean up intermediates
arcpy.CopyRows_management('output_table', output_table)
for item in [
'junctions', 'gte_4ha_lakes', 'gte_10ha_lakes', 'output_table'
]:
arcpy.Delete_management(item)
def streams_in_zones(zones_fc, zone_field, streams_fc, output_table):
arcpy.env.workspace = 'in_memory'
# this bit enforces the correct ftype restriction just in case
# geodata doesn't have this filtered already
# this won't be as good as doing it before hand because it's
# not sophisticated enough to take out the artificial lines going
# through lakes
need_selection = False
with arcpy.da.SearchCursor(streams_fc, ["FType"]) as cursor:
for row in cursor:
if row[0] == 566:
need_selection = True
if need_selection:
whereClause = """"FType" <> 566"""
arcpy.Select_analysis(temp_lakes, "no_coastlines", whereClause)
streams_fc = os.path.join(arcpy.env.workspace, "no_coastlines")
selections = [
'',
""""Strahler" <= 3""",
""""Strahler" > 3 AND "Strahler" <= 6""",
""""Strahler" > 6""",
]
temp_tables = ['Streams', 'Headwaters', 'Midreaches', 'Rivers']
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg(
"Creating temporary table called {0} for streams where {1}".format(
temp_table, sel))
LineDensity.line_density(zones_fc, zone_field, streams_fc, temp_table,
sel)
new_fields = ['SUM_LengthM', 'Density_MperHA']
for f in new_fields:
cu.rename_field(temp_table, f, temp_table + '_' + f, True)
# join em up and copy to final
temp_tables.remove('Streams')
for t in temp_tables:
try:
arcpy.JoinField_management('Streams', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [temp_table + '_' + f for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Streams', ef, 'Double')
arcpy.CalculateField_management('Streams', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [
f.name for f in arcpy.ListFields('Streams', zone_field + '_*')
]
for f in drop_fields:
arcpy.DeleteField_management('Streams', f)
arcpy.CopyRows_management('Streams', output_table)
# clean up
for item in ['Streams', 'no_coastlines'] + temp_tables:
try:
arcpy.Delete_management(item)
except:
continue
def create_csi_watersheds(flowdir, pour_dir, nhd_gdb, out_gdb):
# Starting environmental variables:
env.extent = flowdir
env.snapRaster = flowdir
env.cellSize = 10
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.CheckOutExtension('Spatial')
huc8_code = re.search('\d{8}', os.path.basename(flowdir)).group()
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
# create temp directory because we need shape geometry
temp_gdb = cu.create_temp_GDB('watersheds' + huc4_code)
cu.multi_msg("Temp geodatabase is located at {}".format(temp_gdb))
env.workspace = temp_gdb
wbd_hu8 = os.path.join(nhd_gdb, "WBD_HU8")
field_name = (arcpy.ListFields(wbd_hu8, "HU*8"))[0].name
whereClause8 = """{0} = '{1}'""".format(
arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc8_code)
arcpy.Select_analysis(wbd_hu8, "hu8", whereClause8)
arcpy.Buffer_analysis("hu8", "hu8_buffered", "100 meters")
# Create the basic watersheds
pour_points = os.path.join(pour_dir, 'pour_points.tif')
arcpy.Clip_management(pour_points, '', "pour_points_clipped",
"hu8_buffered", '0', 'ClippingGeometry')
raw_watersheds = os.path.join(
out_gdb, 'huc{}_watersheds_precursors'.format(huc8_code))
cu.multi_msg("Calculating preliminary watersheds...")
outWatershed = Watershed(flowdir, "pour_points_clipped")
outWatershed.save(raw_watersheds)
cu.multi_msg(
"Clipping watersheds to subregion boundaries and filtering spurious watersheds..."
)
# Watershed raster to polygons
arcpy.RasterToPolygon_conversion(raw_watersheds, "wspoly1", 'NO_SIMPLIFY',
"Value")
# Clip watershed polygons to subregion polys.
arcpy.Clip_analysis("wspoly1", "hu8", "wsclip1")
# Clip watershed
## # Calculate hectares
## arcpy.AddField_management("wsclip1", "HA", "DOUBLE")
## arcpy.CalculateField_management("wsclip1", "HA", '''!shape.area@hectares!''', "PYTHON")
# Create fc of watershed polygons >= 1 ha that coincide with seed lines and polys.
seedline = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_flowlines')
seedpoly = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_lakes')
arcpy.MakeFeatureLayer_management("wsclip1", "wsclip1_lyr")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT",
seedline, '', "NEW_SELECTION")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT",
seedpoly, '', "ADD_TO_SELECTION")
arcpy.SelectLayerByAttribute_management("wsclip1_lyr", "SUBSET_SELECTION",
'''"Shape_Area" >= 10000''')
cu.multi_msg("Reshaping watersheds...")
# Polygon back to raster
grid_code = arcpy.ListFields("wsclip1_lyr", "grid*")[0].name
arcpy.PolygonToRaster_conversion("wsclip1_lyr", grid_code, "ws_legit_ras")
arcpy.Clip_management("ws_legit_ras", '', "ws_legit_clipped_ras", "hu8",
"0", "ClippingGeometry")
# Make a raster from the subregion (boundary) polygon with zero for cell values.
arcpy.AddField_management("hu8", "Value", "SHORT")
arcpy.CalculateField_management("hu8", "Value", "0", "PYTHON")
arcpy.PolygonToRaster_conversion("hu8", "Value", "boundary_raster")
arcpy.Clip_management("boundary_raster", '', "boundary_raster_clip", "hu8",
'', "ClippingGeometry")
# Fill NoData in watersheds with the zero values from the subregion raster's cells.
composite = Con(IsNull("ws_legit_clipped_ras"), "boundary_raster_clip",
"ws_legit_clipped_ras")
composite.save("composite_raster")
arcpy.Clip_management("composite_raster", '', "composite_raster_clip",
"hu8", '0', "ClippingGeometry")
# Make a mask of zero cells. NoData cells are the actual mask for nibble.
premask = Con(IsNull("composite_raster_clip"), "composite_raster_clip", 0)
premask.save("premask")
arcpy.Clip_management("premask", '', "mask", "hu8", '', "ClippingGeometry")
# Set Null to 1.
pre_watersheds = Con(IsNull("composite_raster_clip"), 1,
"composite_raster_clip")
pre_watersheds.save("pre_watersheds") # does this speed things up?
## prews.save("prews.tif")
# Nibble masked values (null values along boundary).
cu.multi_msg('Nibbling watersheds as part of reshaping...')
nibble = Nibble("pre_watersheds", "mask", "DATA_ONLY")
nibble.save("nibble")
# Use HU8 buffer so that watersheds will overrun HU8 boundaries and get
# clipped without weird slivers later
arcpy.Clip_management("nibble", "", "watersheds_ras", "hu8_buffered",
"NoData", "ClippingGeometry")
# Convert watershed raster to polygon.
# Setting simplify keyword to TRUE in RasterToPolygon_conversion
# is not working reliably so need to do this in two steps, unfortunately
cu.multi_msg(
"Converted reshaped watersheds raster to polygons. If you experience problems with this step, please read Known and Outstanding Bugs.txt"
)
arcpy.RasterToPolygon_conversion("watersheds_ras", "nibble_sheds",
'SIMPLIFY', "Value") #simplify okay
## # I'm using 15 as the tolerance
## # here because the diagonal of a 10x10 pixel is 14.14 m and
## # I'm okay with a vertex moving as far as it can on the edges of the pixel
## # This also produces results very similar to using the simplify setting
## # on RasterToPolygon_conversion, when it works.
## arcpy.SimplifyPolygon_cartography("nibble_sheds_unsimple",
## "nibble_sheds_simplify", "POINT_REMOVE", "15 Meters", "0 SquareMeters",
## "RESOLVE_ERRORS", "NO_KEEP")
arcpy.Clip_analysis("nibble_sheds", "hu8", "final_watersheds")
# Join Permanent ID from Waterbody seed shapefile
final_watersheds_out = os.path.join(
out_gdb, 'huc{}_final_watersheds'.format(huc8_code))
arcpy.JoinField_management("final_watersheds", grid_code, seedpoly,
'POUR_ID', ['Permanent_Identifier'])
# this block bumps out sheds so that they fully contain their own lakes
# sometimes a little bit of another shed is overlapping the lake simply
# due to the raster/polygon differences
# 1) delete fields so watersheds and seedpoly share schema
# 2) update features, keeping borders
# 3) instead of lots of nulls make unique dissolve_id for all so that nulls aren't dissolved into one
# 4) dissolve features on dissolve_id keeping the Permanent_Identifier field
arcpy.CopyFeatures_management(seedpoly, 'lakes_nofields')
for fc in ['lakes_nofields', 'final_watersheds']:
fields = arcpy.ListFields(fc)
for f in fields:
if f != 'Permanent_Identifier':
try:
arcpy.DeleteField_management(fc, f)
except:
continue
arcpy.Update_analysis("final_watersheds", 'lakes_nofields', 'update_fc')
arcpy.AddField_management('update_fc', 'dissolve_id', 'TEXT', 255)
arcpy.MakeFeatureLayer_management('update_fc', 'update_lyr')
arcpy.SelectLayerByAttribute_management(
'update_lyr', 'NEW_SELECTION',
""""Permanent_Identifier" is not null""")
arcpy.CalculateField_management('update_lyr', 'dissolve_id',
'!Permanent_Identifier!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'SWITCH_SELECTION')
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!OBJECTID!',
'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'CLEAR_SELECTION')
arcpy.Dissolve_management('update_lyr', "final_watersheds_bumped",
'dissolve_id', 'Permanent_Identifier FIRST')
cu.rename_field("final_watersheds_bumped",
"FIRST_Permanent_Identifier",
"Permanent_Identifier",
deleteOld=True)
arcpy.DeleteField_management('final_watersheds_bumped', 'dissolve_id')
arcpy.Clip_analysis('final_watersheds_bumped', 'hu8',
'final_watersheds_clipped')
arcpy.CopyFeatures_management("final_watersheds_clipped",
final_watersheds_out)
temp_items = arcpy.ListRasters() + arcpy.ListFeatureClasses() + [temp_gdb]
for item in temp_items:
try:
arcpy.Delete_management(item)
except:
continue
arcpy.ResetEnvironments()
arcpy.CheckInExtension('Spatial')
cu.multi_msg("Complete.")
def split_strahler(stream_area_fc, streams, out_area_fc):
"""This function splits up the NHDArea feature class, which does not
start and stop polygons at confluences, by creating break points near the
confluences to split up the polygons. Then, it adds the Strahler value from
the stream centerline."""
# 1) Generate euclidean allocation raster from streams (use OBJECTID)
# 2) Convert euclidean allocation raster to polygons
# 3) Join allocation polygons "gridcode" to streams "OBJECTID" so that
# Strahler value is attached to allocation polygon
# 4) Use identity function to split up the StreamRiver polygons at the
# allocation polygon boundaries, and add the Strahler values
old_workspace = env.workspace
env.workspace = 'in_memory'
cu.multi_msg(
"Splitting stream area polygons between confluences and joining 1) Strahler order to them..."
)
cu.multi_msg('next messages for testing')
arcpy.CheckOutExtension('Spatial')
cu.multi_msg('euc')
euc = EucAllocation(streams, cell_size='50', source_field='OBJECTID')
arcpy.CheckInExtension('Spatial')
cu.multi_msg('conversion')
arcpy.RasterToPolygon_conversion(euc, 'allocation_polys')
stream_id_field = arcpy.ListFields(streams, 'Permanent_')[0].name
cu.multi_msg('join')
arcpy.JoinField_management('allocation_polys', 'grid_code', streams,
'OBJECTID',
['Strahler', 'LengthKm', stream_id_field])
cu.multi_msg('identity')
arcpy.Identity_analysis(stream_area_fc, 'allocation_polys', out_area_fc)
env.workspace = old_workspace
cu.multi_msg("Splitting strema area polygons finished.")
def wetland_order(rivex, stream_area_fc, nwi, out_fc):
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.env.extent = nwi
# Buffer the wetland perimeters by 30 meters
cu.multi_msg('Creating 30m wetland buffers...')
arcpy.Buffer_analysis(nwi, "wetland_buffers", "30 meters", "OUTSIDE_ONLY")
arcpy.env.extent = "wetland_buffers"
cu.multi_msg('Preparing for river line and area merge...')
arcpy.CopyFeatures_management(rivex, 'rivex_extent')
arcpy.CopyFeatures_management(stream_area_fc, 'stream_area_extent')
arcpy.MakeFeatureLayer_management('rivex_extent', 'rivex_lyr')
arcpy.SelectLayerByLocation_management('rivex_lyr', 'COMPLETELY_WITHIN', stream_area_fc)
arcpy.CopyFeatures_management('rivex_lyr', 'rivex_for_splitting')
arcpy.SelectLayerByAttribute_management('rivex_lyr', 'SWITCH_SELECTION')
arcpy.CopyFeatures_management('rivex_lyr', 'rivex_not_areas')
split_strahler('stream_area_extent', 'rivex_for_splitting', 'stream_area_split')
# areas TO lines
arcpy.PolygonToLine_management('stream_area_split', 'streamarea_to_line', False)
# Merge features together
arcpy.Merge_management(['streamarea_to_line', 'rivex_not_areas'], 'merged_rivers', 'NO_TEST')
# FOR THE LINE-BASED PORTION
# Spatial join connected wetlands and streams
##################Field Maps########################
fms = arcpy.FieldMappings()
fm_strahlermax = arcpy.FieldMap()
fm_strahlersum = arcpy.FieldMap()
fm_lengthkm = arcpy.FieldMap()
fm_wetid = arcpy.FieldMap()
fm_strahlermax.addInputField('merged_rivers', "Strahler")
fm_strahlersum.addInputField('merged_rivers', "Strahler")
fm_lengthkm.addInputField('merged_rivers', "LengthKm")
fm_wetid.addInputField("wetland_buffers", "WET_ID")
fm_lengthkm.mergeRule = 'Sum'
fm_strahlermax.mergeRule = 'Max'
fm_strahlersum.mergeRule = 'Sum'
lengthkm_name = fm_lengthkm.outputField
lengthkm_name.name = 'StreamKm'
lengthkm_name.aliasName = 'StreamKm'
fm_lengthkm.outputField = lengthkm_name
strahlermax_name = fm_strahlermax.outputField
strahlermax_name.name = 'StrOrdMax'
strahlermax_name.aliasName = 'StrOrdMax'
fm_strahlermax.outputField = strahlermax_name
strahlersum_name = fm_strahlersum.outputField
strahlersum_name.name = 'StrOrdSum'
strahlersum_name.aliasName = 'StrOrdSum'
fm_strahlersum.outputField = strahlersum_name
fms.addFieldMap(fm_strahlermax)
fms.addFieldMap(fm_strahlersum)
fms.addFieldMap(fm_lengthkm)
fms.addFieldMap(fm_wetid)
#####################################################
arcpy.SpatialJoin_analysis("wetland_buffers", 'merged_rivers', "wetland_spjoin_streams", '', '', fms)
# Get the stream count from the join count
cu.rename_field("wetland_spjoin_streams", 'Join_Count', "StreamCnt", True)
# Join the new fields back to the original feature class based on WET_ID
join_fields = ['StrOrdMax', 'StrOrdSum', 'StreamKm', 'StreamCnt']
arcpy.CopyFeatures_management(nwi, out_fc)
arcpy.JoinField_management(out_fc, 'WET_ID', 'wetland_spjoin_streams', 'WET_ID', join_fields)
# Set these to 0 where there is no connection
cu.redefine_nulls(out_fc, join_fields, [0,0,0,0])
# Classify VegType: 4 options based on class code in ATTRIBUTE field
arcpy.AddField_management(out_fc, "VegType", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["ATTRIBUTE", "VegType"]) as cursor:
for row in cursor:
attr_abbv = row[0][:3]
if attr_abbv == "PEM" or attr_abbv == "PAB":
row[1] = "PEMorPAB"
elif attr_abbv == "PFO":
row[1] = "PFO"
elif attr_abbv == "PSS":
row[1] = "PSS"
else:
row[1] = "Other"
cursor.updateRow(row)
# Determine the regime from the letter code. Examples: PSS1E ---> E,
# PEM1/SS1Fb --> F
class_codes = 'RB UB AB US ML EM SS FO'.split()
regime_codes = 'A B C E F G H J K'.split()
arcpy.AddField_management(out_fc, "Regime", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["ATTRIBUTE", "Regime"]) as cursor:
for row in cursor:
# All the wetlands are already palustrine, so if we remove the class
# codes, any capital letters left besides the P in front
# are the regime code
# example codes: PEM1E, PSS1/EM1E, PEM1/5C, PUSA, PSSf
# If you ever can figure out the regex for this instead, go ahead.
code_value = row[0]
regime_value = 'unknown'
# this bit searches for the class codes and replaces them with nothing
# this is necessary because meaning of A, B, E, F is context dependent
for class_code in class_codes:
if class_code in code_value:
code_value = code_value.replace(class_code, '')
for char in code_value:
if char in regime_codes:
regime_value = char
row[1] = regime_value
cursor.updateRow(row)
# Calculate WetOrder from StrOrdSum
arcpy.AddField_management(out_fc,"WetOrder", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["StrOrdSum", "WetOrder"]) as cursor:
for row in cursor:
if row[0] == 0:
row[1] = "Isolated"
elif row[0] == 1:
row[1] = "Single"
elif row[0] == None:
row[1] = "Isolated"
else:
row[1] = "Connected"
cursor.updateRow(row)
arcpy.Delete_management('in_memory')
def upstream_lakes(nhd_gdb, output_table):
#define paths in nhd geodatabase
nhd_waterbody = os.path.join(nhd_gdb, 'NHDWaterbody')
nhd_flowline = os.path.join(nhd_gdb, 'NHDFlowline')
nhd_junctions = os.path.join(nhd_gdb, 'HYDRO_NET_Junctions')
hydro_net = os.path.join(nhd_gdb, 'Hydrography', 'HYDRO_NET')
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.MakeFeatureLayer_management(nhd_junctions, 'junctions')
cu.multi_msg('Preparing layers and fields for calculations....')
# select only lakes as defined in our project: waterbodies with one of these
# types and greater than 4ha, and certain reservoirs greater than 10ha
fcodes = (39000, 39004, 39009, 39010, 39011, 39012, 43600, 43613, 43615, 43617, 43618, 43619, 43621)
gte_4ha_lakes_query = '''("AreaSqKm" >=0.04 AND "FCode" IN %s) OR ("FCode" = 43601 AND "AreaSqKm" >= 0.1)''' % (fcodes,)
gte_10ha_lakes_query = '''("AreaSqKm" >=0.1 AND "FCode" IN %s) OR ("FCode" = 43601 AND "AreaSqKm" >= 0.1)''' % (fcodes,)
arcpy.MakeFeatureLayer_management(nhd_waterbody, 'gte_4ha_lakes', gte_4ha_lakes_query)
arcpy.MakeFeatureLayer_management(nhd_waterbody, 'gte_10ha_lakes', gte_10ha_lakes_query)
arcpy.CopyRows_management('gte_4ha_lakes', 'output_table')
# (need this for line 61 make feature layer to work right!)
arcpy.CopyFeatures_management('gte_4ha_lakes', 'gte_4ha_lakes_DISK')
# add the new fields we'll calculate: count and area
count_fields = ['Upstream_Lakes_4ha_Count', 'Upstream_Lakes_10ha_Count']
area_fields = ['Upstream_Lakes_4ha_Area_ha', 'Upstream_Lakes_10ha_Area_ha']
for cf in count_fields:
arcpy.AddField_management('output_table', cf, 'LONG')
for af in area_fields:
arcpy.AddField_management('output_table', af, 'DOUBLE')
new_fields= count_fields + area_fields
# for each lake, use its junctions as input flags to the upstream trace, then
# evalute the traced network for how many lakes are in it
cu.multi_msg("Calculating upstream network for each lake. This may take a few minutes, or a few hours...")
with arcpy.da.UpdateCursor('output_table', ['Permanent_Identifier'] + new_fields) as cursor:
for row in cursor:
id = row[0]
# get the junction points on top of this lake, can be 0 or several
where_clause = """"{0}" = '{1}'""".format('Permanent_Identifier', id)
arcpy.MakeFeatureLayer_management('gte_4ha_lakes_DISK', "this_lake",
where_clause)
arcpy.SelectLayerByLocation_management('junctions', "INTERSECT",
"this_lake", "1 Meters")
count_jxns = int(arcpy.GetCount_management('junctions').getOutput(0))
# if the lake has no upstream connectivity whatsoever, it will have
# no junctions on top of it. assign 0s
# to the counts of upstream lakes. This can either be an "isolated"
# lake or a "headwater" lake
if count_jxns <= 0:
row[1:] = [0, 0, 0, 0]
# if there are any junctions, go ahead and trace upstream
else:
arcpy.CopyFeatures_management("junctions", 'this_lake_jxns')
arcpy.TraceGeometricNetwork_management(hydro_net, "upstream",
'this_lake_jxns', "TRACE_UPSTREAM")
arcpy.SelectLayerByLocation_management('gte_4ha_lakes', "INTERSECT",
"upstream/NHDFlowline", '1 Meters', 'NEW_SELECTION')
# Remove this lake from the selection!!
arcpy.SelectLayerByAttribute_management('gte_4ha_lakes',
'REMOVE_FROM_SELECTION', where_clause)
count_4ha_lakes = int(arcpy.GetCount_management('gte_4ha_lakes').getOutput(0))
# if there is upstream connectivity but no countable lakes
# assign 0s for all the counts/areas and do not try to copy rows
if count_4ha_lakes <= 0:
row[1:] = [0, 0, 0, 0]
# if there are any lakes upstream, copy rows before doing
# counts so we can use a search cursor on just these lakes
else:
arcpy.CopyRows_management('gte_4ha_lakes', 'these_4ha_lakes')
# row[1] is 4ha_Upstream_Lakes_Count
# row [3] is 4ha_Upstream_Lakes_Area
row[1] = int(arcpy.GetCount_management('these_4ha_lakes').getOutput(0))
total_area = 0
with arcpy.da.SearchCursor('these_4ha_lakes', ['AreaSqKm']) as area4_cursor:
for area4_row in area4_cursor:
total_area += area4_row[0] * 100
row[3] = total_area
# same but for 10ha
arcpy.SelectLayerByLocation_management('gte_10ha_lakes', "INTERSECT",
"upstream/NHDFlowline", '1 Meters', 'NEW_SELECTION')
arcpy.SelectLayerByAttribute_management('gte_10ha_lakes',
'REMOVE_FROM_SELECTION', where_clause)
count_10ha_lakes = int(arcpy.GetCount_management('gte_10ha_lakes').getOutput(0))
# don't try to copy rows if selection is empty
if count_10ha_lakes <= 0:
row[2] = 0
row[4] = 0
# if there are features selected, copy rows so we
# can use a search cursor
else:
arcpy.CopyRows_management('gte_10ha_lakes', 'these_10ha_lakes')
row[2] = int(arcpy.GetCount_management('these_10ha_lakes').getOutput(0))
total_area = 0
with arcpy.da.SearchCursor('these_4ha_lakes', ['AreaSqKm']) as area10_cursor:
for area10_row in area10_cursor:
total_area += area10_row[0] * 100
row[4] = total_area
cursor.updateRow(row)
# delete intermediates before next iteration
for item in ['this_lake', 'this_lake_jxns', 'upstream', 'these_4ha_lakes', 'these_10ha_lakes']:
try:
arcpy.Delete_management(item)
except:
continue
# clean up the output table
all_fields = [f.name for f in arcpy.ListFields('output_table')]
for f in all_fields:
if f not in ['Permanent_Identifier'] + new_fields:
try:
arcpy.DeleteField_management('output_table', f)
except:
continue
# write out the final file and clean up intermediates
arcpy.CopyRows_management('output_table', output_table)
for item in ['junctions', 'gte_4ha_lakes', 'gte_10ha_lakes', 'output_table']:
arcpy.Delete_management(item)
def lakes_in_zones(zones_fc, zone_field, lakes_fc, output_table):
# make sure we're only using the right types of lakes, our feature
# class excludes everything else but this is a guarantee this will
# get checked at some point
arcpy.env.workspace = 'in_memory'
# using in_memory workspace means no SHAPE@AREA attribute later so I
# need to calculate another field with the area using temp on disk
# then go ahead and copy to memory, delete temp
temp_workspace = cu.create_temp_GDB('lakezone')
temp_lakes = os.path.join(temp_workspace, 'temp_lakes')
arcpy.CopyFeatures_management(lakes_fc, temp_lakes)
arcpy.AddField_management(temp_lakes, 'Area_ha', 'DOUBLE')
arcpy.CalculateField_management(temp_lakes, 'Area_ha',
'!shape.area@hectares!', 'PYTHON')
# this bit enforces the correct lake type/size restriction just in case
# geodata doesn't have this filtered already
need_selection = False
fcodes = (39000, 39004, 39009, 39010, 39011, 39012, 43600, 43613, 43615,
43617, 43618, 43619, 43621)
with arcpy.da.SearchCursor(temp_lakes, ["FCode"]) as cursor:
for row in cursor:
if row[0] not in fcodes + (43601, ):
need_selection = True
if need_selection:
whereClause = '''
("AreaSqKm" >=0.04 AND "FCode" IN %s)\
OR ("AreaSqKm" >= 0.1 AND "FCode" = 43601)''' % (fcodes, )
arcpy.Select_analysis(temp_lakes, "lakes_4ha", whereClause)
temp_lakes = os.path.join(arcpy.env.workspace, "lakes_4ha")
selections = [
""""Area_ha" >= 4""",
""""Area_ha" >= 4 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 10""",
""""Area_ha" >= 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_LakeStream'"""
]
temp_tables = [
'Lakes4ha', 'Lakes4ha_Isolated', 'Lakes4ha_Headwater',
'Lakes4ha_DRStream', 'Lakes4ha_DRLakeStream', 'Lakes4to10ha',
'Lakes4to10ha_Isolated', 'Lakes4to10ha_Headwater',
'Lakes4to10ha_DRStream', 'Lakes4to10ha_DRLakeStream', 'Lakes10ha',
'Lakes10ha_Isolated', 'Lakes10ha_Headwater', 'Lakes10ha_DRStream',
'Lakes10ha_DRLakeStream'
]
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg(
"Creating temporary table called {0} for lakes where {1}".format(
temp_table, sel))
polygons_in_zones.polygons_in_zones(zones_fc, zone_field, temp_lakes,
temp_table, sel)
new_fields = [
'Poly_Overlapping_AREA_ha', 'Poly_Contributing_AREA_ha',
'Poly_Overlapping_AREA_pct', 'Poly_Count'
]
avg_size_field = temp_table + '_AvgSize_ha'
arcpy.AddField_management(temp_table, avg_size_field, 'DOUBLE')
arcpy.CalculateField_management(
temp_table, avg_size_field,
'!Poly_Contributing_AREA_ha!/!Poly_Count!', 'PYTHON')
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# join em up and copy to final
temp_tables.remove('Lakes4ha')
for t in temp_tables:
try:
arcpy.JoinField_management('Lakes4ha', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Lakes4ha', ef, 'Double')
arcpy.CalculateField_management('Lakes4ha', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('Lakes4ha', 'ZoneID_*')]
for f in drop_fields:
arcpy.DeleteField_management('Lakes4ha', f)
arcpy.CopyRows_management('Lakes4ha', output_table)
# clean up
for item in ['Lakes4ha', temp_lakes,
os.path.dirname(temp_workspace)] + temp_tables:
arcpy.Delete_management(item)
def percent_geographic_doubles(polygon_fc,
unique_id,
percent_overlap_allowed=10,
keep_fc='',
keep_field=''):
neighbor_table = 'in_memory/neighbortable'
cu.multi_msg('Calculating neighbor table...')
arcpy.PolygonNeighbors_analysis(polygon_fc, neighbor_table, unique_id,
'AREA_OVERLAP', 'NO_BOTH_SIDES')
# need these to avoid some naming problems that arise from trying to
# do this directly
src_field = arcpy.ListFields(neighbor_table, 'src*')[0].name
nbr_field = arcpy.ListFields(neighbor_table, 'nbr*')[0].name
arcpy.CopyFeatures_management(polygon_fc, 'in_memory/fc')
fdate_field = arcpy.ListFields('in_memory/fc', '*FDate*')[0].name
cu.multi_msg('Joining neighbor table to feature class...')
arcpy.JoinField_management('in_memory/fc', unique_id, neighbor_table,
src_field)
cursor_fields = ['AREA', 'SHAPE@AREA', unique_id, nbr_field, fdate_field]
## print([f.name for f in arcpy.ListFields('in_memory/fc')])
if keep_fc:
keep_ids = [
row[0] for row in arcpy.da.SearchCursor(keep_fc, keep_field)
]
else:
keep_ids = []
with arcpy.da.SearchCursor('in_memory/fc', cursor_fields) as cursor:
for row in cursor:
# If this row represents a duplicate-type overlap
if row[0] >= row[1] * (100 - percent_overlap_allowed) / 100:
src_value = row[2]
nbr_value = row[3]
cu.multi_msg("testing. pair: %s and %s" %
(src_value, nbr_value))
# Then lookup both rows in the overlap and delete the one with
# the oldest FDate
where_clause = '''"%s" = '%s' OR "%s" = '%s' ''' % (
unique_id, src_value, unique_id, nbr_value)
dates = [
row[4] for row in arcpy.da.SearchCursor(
'in_memory/fc', cursor_fields, where_clause)
]
cu.multi_msg("testing. dates %s and %s not in order" %
(dates[0], dates[1]))
with arcpy.da.UpdateCursor('in_memory/fc', cursor_fields,
where_clause) as c:
for r in c:
if r[4] == min(dates) and r[4] == max(dates):
cu.multi_msg(
"PROBLEM! Same date. Resolve pair %s, %s manually."
% (src_value, nbr_value))
if r[4] == min(dates) and r[4] != max(dates):
if r[4] not in keep_ids:
cu.multi_msg(
"%s has the older date (%s) and would be removed."
% (r[2], r[4]))
if r[4] in keep_ids:
cu.multi_msg(
"You're going to have to write this condition..."
)
else:
continue
else:
continue
for item in [neighbor_table, 'in_memory/fc']:
arcpy.Delete_management(item)
def create_csi_watersheds(flowdir, pour_dir, nhd_gdb, out_gdb):
# Starting environmental variables:
env.extent = flowdir
env.snapRaster = flowdir
env.cellSize = 10
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.CheckOutExtension('Spatial')
huc8_code = re.search('\d{8}', os.path.basename(flowdir)).group()
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
# create temp directory because we need shape geometry
temp_gdb = cu.create_temp_GDB('watersheds' + huc4_code)
print temp_gdb
env.workspace = temp_gdb
wbd_hu8 = os.path.join(nhd_gdb, "WBD_HU8")
field_name = (arcpy.ListFields(wbd_hu8, "HU*8"))[0].name
whereClause8 = """{0} = '{1}'""".format(arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc8_code)
arcpy.Select_analysis(wbd_hu8, "hu8", whereClause8)
arcpy.Buffer_analysis("hu8", "hu8_buffered", "100 meters")
# Create the basic watersheds
pour_points = os.path.join(pour_dir, 'pour_points.tif')
arcpy.Clip_management(pour_points, '', "pour_points_clipped", "hu8_buffered", '0', 'ClippingGeometry')
raw_watersheds = os.path.join(out_gdb, 'huc{}_watersheds_precursors'.format(huc8_code))
cu.multi_msg("Calculating preliminary watersheds...")
outWatershed = Watershed(flowdir, "pour_points_clipped")
outWatershed.save(raw_watersheds)
cu.multi_msg("Clipping watersheds to subregion boundaries and filtering spurious watersheds...")
# Watershed raster to polygons
arcpy.RasterToPolygon_conversion(raw_watersheds, "wspoly1", 'NO_SIMPLIFY', "Value")
# Clip watershed polygons to subregion polys.
arcpy.Clip_analysis("wspoly1", "hu8", "wsclip1")
# Clip watershed
## # Calculate hectares
## arcpy.AddField_management("wsclip1", "HA", "DOUBLE")
## arcpy.CalculateField_management("wsclip1", "HA", '''!shape.area@hectares!''', "PYTHON")
# Create fc of watershed polygons >= 1 ha that coincide with seed lines and polys.
seedline = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_flowlines')
seedpoly = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_lakes')
arcpy.MakeFeatureLayer_management("wsclip1", "wsclip1_lyr")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT", seedline, '', "NEW_SELECTION")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT", seedpoly, '', "ADD_TO_SELECTION")
arcpy.SelectLayerByAttribute_management("wsclip1_lyr", "SUBSET_SELECTION",'''"Shape_Area" >= 10000''')
cu.multi_msg("Reshaping watersheds...")
# Polygon back to raster
arcpy.PolygonToRaster_conversion("wsclip1_lyr", "grid_code", "ws_legit_ras")
arcpy.Clip_management("ws_legit_ras", '', "ws_legit_clipped_ras",
"hu8", "0", "ClippingGeometry")
# Make a raster from the subregion (boundary) polygon with zero for cell values.
arcpy.AddField_management("hu8", "Value", "SHORT")
arcpy.CalculateField_management("hu8", "Value", "0", "PYTHON")
arcpy.PolygonToRaster_conversion("hu8", "Value","boundary_raster")
arcpy.Clip_management("boundary_raster", '', "boundary_raster_clip", "hu8", '', "ClippingGeometry")
# Fill NoData in watersheds with the zero values from the subregion raster's cells.
composite = Con(IsNull("ws_legit_clipped_ras"), "boundary_raster_clip", "ws_legit_clipped_ras")
composite.save("composite_raster")
arcpy.Clip_management("composite_raster", '', "composite_raster_clip", "hu8", '0', "ClippingGeometry")
# Make a mask of zero cells. NoData cells are the actual mask for nibble.
premask = Con(IsNull("composite_raster_clip"), "composite_raster_clip", 0)
premask.save("premask")
arcpy.Clip_management("premask",'', "mask", "hu8", '', "ClippingGeometry")
# Set Null to 1.
pre_watersheds = Con(IsNull("composite_raster_clip"), 1, "composite_raster_clip")
pre_watersheds.save("pre_watersheds") # does this speed things up?
## prews.save("prews.tif")
# Nibble masked values (null values along boundary).
cu.multi_msg('Nibbling watersheds as part of reshaping...')
nibble = Nibble("pre_watersheds", "mask", "DATA_ONLY")
nibble.save("nibble")
# Use HU8 buffer so that watersheds will overrun HU8 boundaries and get
# clipped without weird slivers later
arcpy.Clip_management("nibble", "", "watersheds_ras", "hu8_buffered", "NoData","ClippingGeometry")
# Convert watershed raster to polygon.
# Setting simplify keyword to TRUE in RasterToPolygon_conversion
# is not working reliably so need to do this in two steps, unfortunately
cu.multi_msg("Converted reshaped watersheds raster to polygons. If you experience problems with this step, please read Known and Outstanding Bugs.txt")
arcpy.RasterToPolygon_conversion("watersheds_ras", "nibble_sheds",'SIMPLIFY', "Value") #simplify okay
## # I'm using 15 as the tolerance
## # here because the diagonal of a 10x10 pixel is 14.14 m and
## # I'm okay with a vertex moving as far as it can on the edges of the pixel
## # This also produces results very similar to using the simplify setting
## # on RasterToPolygon_conversion, when it works.
## arcpy.SimplifyPolygon_cartography("nibble_sheds_unsimple",
## "nibble_sheds_simplify", "POINT_REMOVE", "15 Meters", "0 SquareMeters",
## "RESOLVE_ERRORS", "NO_KEEP")
arcpy.Clip_analysis("nibble_sheds", "hu8", "final_watersheds")
# Join Permanent ID from Waterbody seed shapefile
final_watersheds_out = os.path.join(out_gdb, 'huc{}_final_watersheds'.format(huc8_code))
arcpy.JoinField_management("final_watersheds", 'grid_code', seedpoly, 'POUR_ID', ['Permanent_Identifier'])
# this block bumps out sheds so that they fully contain their own lakes
# sometimes a little bit of another shed is overlapping the lake simply
# due to the raster/polygon differences
# 1) delete fields so watersheds and seedpoly share schema
# 2) update features, keeping borders
# 3) instead of lots of nulls make unique dissolve_id for all so that nulls aren't dissolved into one
# 4) dissolve features on dissolve_id keeping the Permanent_Identifier field
arcpy.CopyFeatures_management(seedpoly, 'lakes_nofields')
for fc in ['lakes_nofields', 'final_watersheds']:
fields = arcpy.ListFields(fc)
for f in fields:
if f != 'Permanent_Identifier':
try:
arcpy.DeleteField_management(fc, f)
except:
continue
arcpy.Update_analysis("final_watersheds", 'lakes_nofields', 'update_fc')
arcpy.AddField_management('update_fc', 'dissolve_id', 'TEXT', 255)
arcpy.MakeFeatureLayer_management('update_fc', 'update_lyr')
arcpy.SelectLayerByAttribute_management('update_lyr', 'NEW_SELECTION', """"Permanent_Identifier" is not null""")
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!Permanent_Identifier!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'SWITCH_SELECTION')
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!OBJECTID!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'CLEAR_SELECTION')
arcpy.Dissolve_management('update_lyr', "final_watersheds_bumped", 'dissolve_id', 'Permanent_Identifier FIRST')
cu.rename_field("final_watersheds_bumped", "FIRST_Permanent_Identifier", "Permanent_Identifier", deleteOld = True)
arcpy.DeleteField_management('final_watersheds_bumped', 'dissolve_id')
arcpy.Clip_analysis('final_watersheds_bumped', 'hu8', 'final_watersheds_clipped')
arcpy.CopyFeatures_management("final_watersheds_clipped", final_watersheds_out)
temp_items = arcpy.ListRasters() + arcpy.ListFeatureClasses() + [temp_gdb]
for item in temp_items:
try:
arcpy.Delete_management(item)
except:
continue
arcpy.ResetEnvironments()
arcpy.CheckInExtension('Spatial')
cu.multi_msg("Complete.")
def stage_files(nhd_gdb, ned_dir, ned_footprints_fc, out_dir, is_zipped):
env.workspace = 'in_memory'
#####################################################################
cu.multi_msg("1) Creating Directory Structure and Copying NHD Geodatabase")
#####################################################################
#finds the 4-digit huc code in the filename
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
out_subdir = os.path.join(out_dir, "NHD" + huc4_code)
if not os.path.exists(out_subdir):
os.mkdir(out_subdir)
nhd_gdb_copy = os.path.join(out_subdir, os.path.basename(nhd_gdb))
arcpy.Copy_management(nhd_gdb,nhd_gdb_copy)
####################################################################
cu.multi_msg("2) Getting WBD Poly For Subregion and Buffering by 5000m...")
#####################################################################
#select only this subregion from the wbd layer in the nhd_gdb (bordering
# subregions are included in there too) and buffer it
wbd_hu4 = os.path.join(nhd_gdb_copy, "WBD_HU4")
field_name = (arcpy.ListFields(wbd_hu4, "HU*4"))[0].name
whereClause = """{0} = '{1}'""".format(arcpy.AddFieldDelimiters(nhd_gdb_copy, field_name), huc4_code)
arcpy.MakeFeatureLayer_management(wbd_hu4, "wbd_poly", whereClause)
arcpy.Buffer_analysis("wbd_poly", "wbd_buf", "5000 meters")
#####################################################################
cu.multi_msg("3) Clipping NED Tile polys from Buffered NHD Subregion From WBD...")
#####################################################################
arcpy.Clip_analysis(ned_footprints_fc, "wbd_buf", "ned_clip")
#####################################################################
cu.multi_msg("4) Getting File_ID of clipped NED footprint polys and copying NED data to output location")
#####################################################################
missing_NED_list = []
with arcpy.da.SearchCursor("ned_clip", ["FILE_ID"]) as cursor:
for row in cursor:
file_id = row[0].replace("g","")
# unzipping if needed
if is_zipped:
unzipped_file = unzip_ned(file_id, ned_dir, out_subdir)
if not unzipped_file:
missing_NED_list.append(file_id)
else:
# copy ned tiles to output location
ned_source = os.path.join(ned_dir, file_id)
ned_destination = os.path.join(out_subdir, file_id)
if not os.path.exists(ned_source):
cu.multi_msg("ERROR: Tile %s does not exist in the specified location" % file_id)
missing_NED_list.append(file_id)
else:
if not os.path.exists(ned_destination):
shutil.copytree(ned_source, ned_destination)
else:
cu.multi_msg("Output folder for this NED tile already exists.")
if missing_NED_list:
warning_text = "WARNING: NED tiles did not exist for the following: %s" % ','.join(missing_NED_list)
arcpy.AddWarning(warning_text)
print(warning_text)
for item in ["wbd_buf", "ned_clip"]:
arcpy.Delete_management(item)
return out_subdir
def remove_nhd_duplicates(in_fc, unique_id, out_fc):
"""This function will resolve identifier duplications from any NHD-based
layer. Use when you need to be sure the identifier values are unique. Some
processing chains will result in duplications where edits to the feature
are apparently saved twice. This function will save the feature with the
newest time stamp and drop all the rest of the records with the same value
in the field you specify as unique_id. If there are no duplications, the
function will exit safely.
in_fc: the feature class that might have duplicates that is derived from an
NHD (National Hydrological Dataset) layer such as NHDWaterbodies or
NHDFlowlines
unique_id: The field you want to use as a primary key, or unique identifier
of features. Any value appearing more than once in this input will be
resolved in the output so that it only appears once. With the NHD, this will
usually be the 'Permanent_Identifier' column, which you may have already
renamed.
be the 'Permanent_Identifier' column
out_fc: the desired path for the output feature class"""
ws = 'in_memory'
env.workspace = ws
cu.multi_msg("Creating frequency table...")
arcpy.Frequency_analysis(in_fc, "freqtable", unique_id)
arcpy.TableSelect_analysis("freqtable", "dupeslist", '''"FREQUENCY" > 1''')
# Make a list of truly unique ids for use in selecting groups of identical ids
select_ids = [
row[0] for row in arcpy.da.SearchCursor("dupeslist", (unique_id))
]
# Pick out just the records we need to work with, the rest will be
# saved to the output and we will merge in the results from our working
# set later
# cu.multi_msg("test1")
# arcpy.MakeFeatureLayer_management(in_fc, "fc_lyr")
#
# cu.multi_msg("test2")
# arcpy.AddJoin_management("fc_lyr", unique_id, "dupeslist", unique_id, "KEEP_COMMON")
# arcpy.SelectLayerByAttribute_management('fc_lyr', 'NEW_SELECTION', '''"FREQUENCY" = 1''')
# arcpy.RemoveJoin_management("fc_lyr")
# arcpy.CopyFeatures_management("fc_lyr", "unique")
#
# cu.multi_msg("test3")
# arcpy.AddJoin_management("fc_lyr", unique_id, "freqtable", unique_id)
# arcpy.SelectLayerByAttribute_management('fc_lyr', 'NEW_SELECTION', '''"FREQUENCY" > 1''')
# arcpy.RemoveJoin_management("fc_lyr")
# arcpy.CopyFeatures_management("fc_lyr", "fc_temp")
arcpy.CopyFeatures_management(in_fc, "fc_temp")
#
# cu.multi_msg("test4")
arcpy.AddField_management("fc_temp", "NewestFDate", "SHORT")
arcpy.CalculateField_management("fc_temp", "NewestFDate", 1, 'PYTHON')
fdate_field = arcpy.ListFields('fc_temp', '*FDate', 'Date')[0].name
unique_id = arcpy.ListFields('fc_temp', '*Permanent_Identifier',
'Text')[0].name
#
# cu.multi_msg("test5")
# arcpy.TableSelect_analysis("freqtable", "dupetable", '''"FREQUENCY" > 1''')
count_dupes = int(arcpy.GetCount_management("dupeslist").getOutput(0))
if count_dupes > 0:
cu.multi_msg("Number of records in duplicates table is %d" %
count_dupes)
# # Make a list of truly unique ids for use in selecting groups of identical ids
# select_ids = [row[0] for row in arcpy.da.SearchCursor("dupetable", (unique_id))]
# for each id select its group of duplicates
for s_id in select_ids:
# fdate_field = arcpy.ListFields('fc_temp', '*FDate', 'Date')[0].name
# unique_id = arcpy.ListFields('fc_temp', '*Permanent_Identifier', 'Text')[0].name
whereClause = ''' "%s" = '%s' ''' % (unique_id, s_id)
# update values to 0, we will change only one to 1 later
# and get a list of all the dates
dates = [
row[0]
for row in arcpy.da.SearchCursor("fc_temp", (
fdate_field), whereClause)
]
print("ID group %s" % s_id)
# print("Date object values: %s" % dates)
# print("Newest date: %s\n" % max(dates))
# just using "Fdate" = max(dates) logic DOES NOT WORK CORRECTLY
# sometimes more than one record with max date but
# the following allows us to use "NewestFDate" = 1 later to
# select ONLY ONE to keep
# Update 2017-06-22: If you just run Delete Identical on full identicals, you actually could use that
# but I'm not going to change this now.
with arcpy.da.UpdateCursor("fc_temp", (fdate_field, "NewestFDate"),
whereClause) as cursor:
i = 1
for row in cursor:
if row[0] == max(dates):
row[1] = i
i += 1
else:
row[1] = 0
cursor.updateRow(row)
# create a new, distinct output rather than updating table in place
# arcpy.Select_analysis("fc_temp", "newest_only", ''' "NewestFDate" = 1 ''')
arcpy.Select_analysis("fc_temp", out_fc, ''' "NewestFDate" = 1 ''')
arcpy.DeleteField_management(out_fc, "NewestFDate")
# arcpy.Merge_management(["newest_only", "unique"], out_fc)
## for intermediate in ["freqtable", "dupetable", "fc_temp", "newest_only"]:
## arcpy.Delete_management(intermediate)
else:
cu.multi_msg(
"There were no remaining duplicates with differing FDates.")
arcpy.CopyFeatures_management(in_fc, out_fc)
def lakes_in_zones(zones_fc, zone_field, lakes_fc, output_table):
# make sure we're only using the right types of lakes, our feature
# class excludes everything else but this is a guarantee this will
# get checked at some point
arcpy.env.workspace = 'in_memory'
# using in_memory workspace means no SHAPE@AREA attribute later so I
# need to calculate another field with the area using temp on disk
# then go ahead and copy to memory, delete temp
temp_workspace = cu.create_temp_GDB('lakezone')
temp_lakes = os.path.join(temp_workspace, 'temp_lakes')
arcpy.CopyFeatures_management(lakes_fc, temp_lakes)
arcpy.AddField_management(temp_lakes, 'Area_ha', 'DOUBLE')
arcpy.CalculateField_management(temp_lakes, 'Area_ha', '!shape.area@hectares!', 'PYTHON')
# this bit enforces the correct lake type/size restriction just in case
# geodata doesn't have this filtered already
need_selection = False
fcodes = (39000, 39004, 39009, 39010, 39011, 39012,
43600, 43613, 43615, 43617, 43618, 43619, 43621)
with arcpy.da.SearchCursor(temp_lakes, ["FCode"]) as cursor:
for row in cursor:
if row[0] not in fcodes + (43601,):
need_selection = True
if need_selection:
whereClause = '''
("AreaSqKm" >=0.04 AND "FCode" IN %s)\
OR ("AreaSqKm" >= 0.1 AND "FCode" = 43601)''' % (fcodes,)
arcpy.Select_analysis(temp_lakes, "lakes_4ha", whereClause)
temp_lakes = os.path.join(arcpy.env.workspace, "lakes_4ha")
selections = [""""Area_ha" >= 4""",
""""Area_ha" >= 4 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 10""",
""""Area_ha" >= 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_LakeStream'"""
]
temp_tables = ['Lakes4ha',
'Lakes4ha_Isolated',
'Lakes4ha_Headwater',
'Lakes4ha_DRStream',
'Lakes4ha_DRLakeStream',
'Lakes4to10ha',
'Lakes4to10ha_Isolated',
'Lakes4to10ha_Headwater',
'Lakes4to10ha_DRStream',
'Lakes4to10ha_DRLakeStream',
'Lakes10ha',
'Lakes10ha_Isolated',
'Lakes10ha_Headwater',
'Lakes10ha_DRStream',
'Lakes10ha_DRLakeStream'
]
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg("Creating temporary table called {0} for lakes where {1}".format(temp_table, sel))
polygons_in_zones.polygons_in_zones(zones_fc, zone_field, temp_lakes, temp_table, sel)
new_fields = ['Poly_Overlapping_AREA_ha', 'Poly_Contributing_AREA_ha', 'Poly_Overlapping_AREA_pct', 'Poly_Count']
avg_size_field = temp_table + '_AvgSize_ha'
arcpy.AddField_management(temp_table, avg_size_field , 'DOUBLE')
arcpy.CalculateField_management(temp_table, avg_size_field, '!Poly_Contributing_AREA_ha!/!Poly_Count!', 'PYTHON')
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# join em up and copy to final
temp_tables.remove('Lakes4ha')
for t in temp_tables:
try:
arcpy.JoinField_management('Lakes4ha', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Lakes4ha', ef, 'Double')
arcpy.CalculateField_management('Lakes4ha', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('Lakes4ha', 'ZoneID_*')]
for f in drop_fields:
arcpy.DeleteField_management('Lakes4ha', f)
arcpy.CopyRows_management('Lakes4ha', output_table)
# clean up
for item in ['Lakes4ha', temp_lakes, os.path.dirname(temp_workspace)] + temp_tables:
arcpy.Delete_management(item)
def colorPolygons(feature_class, feature_field, out_gdb):
arcpy.env.overwriteOutput = True
## # Create temporary directory
## temp_dir = os.path.join(tempfile.gettempdir(), 'zonal')
## index = 0
## while os.path.exists(temp_dir):
## temp_dir = os.path.join(tempfile.gettempdir(), 'zonal%d' % index)
## index += 1
## os.mkdir(temp_dir)
# Initialize variables
temp_features = 'in_memory/dissolve'
bldissolved = False
# Dissolve on non-ObjectID field
desc = arcpy.Describe(feature_class)
cu.multi_msg("Dissolving features.")
if hasattr(desc, "OIDFieldName"):
if feature_field != desc.OIDFieldName:
arcpy.Dissolve_management(feature_class, temp_features, \
feature_field)
bldissolved = True
else:
temp_features = feature_class
else:
arcpy.Dissolve_management(feature_class, temp_features, \
feature_field)
bldissolved = True
# Get ObjectID field from dissolved
if bldissolved:
desc = arcpy.Describe(temp_features)
oid_field = desc.OIDFieldName
else:
oid_field = feature_field
# Calculate polygon contiguity
cu.multi_msg("Identifying overlapping polygons...")
arcpy.env.outputMFlag = "Disabled"
result = arcpy.PolygonNeighbors_analysis(temp_features,
'in_memory/neighbors', oid_field, "AREA_OVERLAP", "BOTH_SIDES")
if 'WARNING 000117:' in result.getMessages(1):
arcpy.AddError("Input feature zone data: {} does not contain "
"overlapping features.".format(temp_features))
sys.exit(1)
cu.multi_msg("Identified overlapping polygons.")
cu.multi_msg("Calculating feature subsets without overlaps...")
# Retrieve as array with columns src_FID and nbr_FID
arr = arcpy.da.TableToNumPyArray('in_memory/neighbors',
['src_%s' % oid_field, 'nbr_%s' % oid_field])
arr = numpy.array(arr.tolist())
# Retrieves the colors of the neighboring nodes
def get_colors(nodes, neighbors):
colors = set()
for neighbor in neighbors:
colors.add(nodes[neighbor][0])
colors.difference([0])
return colors
# Create a new color
def get_new_color(colors):
return max(colors)+1 if len(colors) > 0 else 1
# Chooses from existing colors randomly
def choose_color(colors):
return random.choice(list(colors))
# Sort source FIDs in descending order by number of neighbors
arr_uniq = numpy.unique(arr[:,0])
arr_count = numpy.zeros_like(arr_uniq)
for index in range(arr_uniq.size):
arr_count[index] = numpy.count_nonzero(arr[:, 0] == arr_uniq[index])
arr_ind = numpy.argsort(arr_count)[::-1]
# Initialize node dictionary --
# where key := FID of feature (integer)
# where value[0] := color of feature (integer)
# where value[1] := FIDs of neighboring features (set)
nodes = collections.OrderedDict()
for item in arr_uniq[arr_ind]:
nodes[item] = [0, set()]
# Populate neighbors
for index in range(arr.shape[0]):
nodes[arr[index, 0]][1].add(arr[index, 1])
# Color nodes --
colors = set()
for node in nodes:
# Get colors of neighboring nodes
nbr_colors = get_colors(nodes, nodes[node][1])
# Search for a color not among those colors
choices = colors.difference(nbr_colors)
# Assign the node that color or create it when necessary
if len(choices) == 0:
new_color = get_new_color(colors)
colors.add(new_color)
nodes[node][0] = new_color
else:
nodes[node][0] = choose_color(choices)
# Classify nodes by colors --
classes = {}
for node in nodes:
color = nodes[node][0]
if color in classes:
classes[color].add(node)
else:
classes[color] = set([node])
# Get set of all FIDs
all_fids = set()
with arcpy.da.SearchCursor(temp_features, oid_field) as cursor:
for row in cursor:
all_fids.add(row[0])
# Add disjoint FIDs to new class if necessary
disjoint_fids = all_fids.difference(set(nodes.keys()))
if len(disjoint_fids) > 0:
new_color = get_new_color(colors)
classes[new_color] = disjoint_fids
# Calculate number of classes
num_classes = len(classes)
# Save each class
temp_lyr = "temp_layer"
cl_separator = ' OR \"%s\" = ' % oid_field
for index, cl in enumerate(classes):
## arcpy.SetProgressorLabel(
## "Processing layer %d of %d..." % (index+1, num_classes))
test = tuple(map(int, classes[cl]))
where_clause = '\"%s\" IN %s' % (oid_field, \
test)
arcpy.MakeFeatureLayer_management(temp_features, temp_lyr, \
where_clause)
# This part added by Nicole
# Save polygon layers to output gdb
outLayerBase = os.path.splitext(os.path.basename(feature_class))[0]
outLayerName = arcpy.CreateUniqueName(outLayerBase + "_NoOverlap", out_gdb)
cu.multi_msg("Saving feature class %s of %s with name %s" % (str(index + 1),
str(num_classes), outLayerName))
## arcpy.Select_analysis(temp_features, outLayerName, where_clause)
count = arcpy.GetCount_management(temp_lyr).getOutput(0)
cu.multi_msg("count of features: " + str(count))
arcpy.CopyFeatures_management(temp_lyr, outLayerName)
arcpy.Delete_management(temp_lyr)
return out_gdb
def wetland_order(rivex, stream_area_fc, nwi, out_fc):
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.env.extent = nwi
# Buffer the wetland perimeters by 30 meters
cu.multi_msg('Creating 30m wetland buffers...')
arcpy.Buffer_analysis(nwi, "wetland_buffers", "30 meters", "OUTSIDE_ONLY")
arcpy.env.extent = "wetland_buffers"
cu.multi_msg('Preparing for river line and area merge...')
arcpy.CopyFeatures_management(rivex, 'rivex_extent')
arcpy.CopyFeatures_management(stream_area_fc, 'stream_area_extent')
arcpy.MakeFeatureLayer_management('rivex_extent', 'rivex_lyr')
arcpy.SelectLayerByLocation_management('rivex_lyr', 'COMPLETELY_WITHIN',
stream_area_fc)
arcpy.CopyFeatures_management('rivex_lyr', 'rivex_for_splitting')
arcpy.SelectLayerByAttribute_management('rivex_lyr', 'SWITCH_SELECTION')
arcpy.CopyFeatures_management('rivex_lyr', 'rivex_not_areas')
split_strahler('stream_area_extent', 'rivex_for_splitting',
'stream_area_split')
# areas TO lines
arcpy.PolygonToLine_management('stream_area_split', 'streamarea_to_line',
False)
# Merge features together
arcpy.Merge_management(['streamarea_to_line', 'rivex_not_areas'],
'merged_rivers', 'NO_TEST')
# FOR THE LINE-BASED PORTION
# Spatial join connected wetlands and streams
##################Field Maps########################
fms = arcpy.FieldMappings()
fm_strahlermax = arcpy.FieldMap()
fm_strahlersum = arcpy.FieldMap()
fm_lengthkm = arcpy.FieldMap()
fm_wetid = arcpy.FieldMap()
fm_strahlermax.addInputField('merged_rivers', "Strahler")
fm_strahlersum.addInputField('merged_rivers', "Strahler")
fm_lengthkm.addInputField('merged_rivers', "LengthKm")
fm_wetid.addInputField("wetland_buffers", "WET_ID")
fm_lengthkm.mergeRule = 'Sum'
fm_strahlermax.mergeRule = 'Max'
fm_strahlersum.mergeRule = 'Sum'
lengthkm_name = fm_lengthkm.outputField
lengthkm_name.name = 'StreamKm'
lengthkm_name.aliasName = 'StreamKm'
fm_lengthkm.outputField = lengthkm_name
strahlermax_name = fm_strahlermax.outputField
strahlermax_name.name = 'StrOrdMax'
strahlermax_name.aliasName = 'StrOrdMax'
fm_strahlermax.outputField = strahlermax_name
strahlersum_name = fm_strahlersum.outputField
strahlersum_name.name = 'StrOrdSum'
strahlersum_name.aliasName = 'StrOrdSum'
fm_strahlersum.outputField = strahlersum_name
fms.addFieldMap(fm_strahlermax)
fms.addFieldMap(fm_strahlersum)
fms.addFieldMap(fm_lengthkm)
fms.addFieldMap(fm_wetid)
#####################################################
arcpy.SpatialJoin_analysis("wetland_buffers", 'merged_rivers',
"wetland_spjoin_streams", '', '', fms)
# Get the stream count from the join count
cu.rename_field("wetland_spjoin_streams", 'Join_Count', "StreamCnt", True)
# Join the new fields back to the original feature class based on WET_ID
join_fields = ['StrOrdMax', 'StrOrdSum', 'StreamKm', 'StreamCnt']
arcpy.CopyFeatures_management(nwi, out_fc)
arcpy.JoinField_management(out_fc, 'WET_ID', 'wetland_spjoin_streams',
'WET_ID', join_fields)
# Set these to 0 where there is no connection
cu.redefine_nulls(out_fc, join_fields, [0, 0, 0, 0])
# Classify VegType: 4 options based on class code in ATTRIBUTE field
arcpy.AddField_management(out_fc, "VegType", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["ATTRIBUTE", "VegType"]) as cursor:
for row in cursor:
attr_abbv = row[0][:3]
if attr_abbv == "PEM" or attr_abbv == "PAB":
row[1] = "PEMorPAB"
elif attr_abbv == "PFO":
row[1] = "PFO"
elif attr_abbv == "PSS":
row[1] = "PSS"
else:
row[1] = "Other"
cursor.updateRow(row)
# Determine the regime from the letter code. Examples: PSS1E ---> E,
# PEM1/SS1Fb --> F
class_codes = 'RB UB AB US ML EM SS FO'.split()
regime_codes = 'A B C E F G H J K'.split()
arcpy.AddField_management(out_fc, "Regime", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["ATTRIBUTE", "Regime"]) as cursor:
for row in cursor:
# All the wetlands are already palustrine, so if we remove the class
# codes, any capital letters left besides the P in front
# are the regime code
# example codes: PEM1E, PSS1/EM1E, PEM1/5C, PUSA, PSSf
# If you ever can figure out the regex for this instead, go ahead.
code_value = row[0]
regime_value = 'unknown'
# this bit searches for the class codes and replaces them with nothing
# this is necessary because meaning of A, B, E, F is context dependent
for class_code in class_codes:
if class_code in code_value:
code_value = code_value.replace(class_code, '')
for char in code_value:
if char in regime_codes:
regime_value = char
row[1] = regime_value
cursor.updateRow(row)
# Calculate WetOrder from StrOrdSum
arcpy.AddField_management(out_fc, "WetOrder", "TEXT")
with arcpy.da.UpdateCursor(out_fc, ["StrOrdSum", "WetOrder"]) as cursor:
for row in cursor:
if row[0] == 0:
row[1] = "Isolated"
elif row[0] == 1:
row[1] = "Single"
elif row[0] == None:
row[1] = "Isolated"
else:
row[1] = "Connected"
cursor.updateRow(row)
arcpy.Delete_management('in_memory')
def line_density(zones, zonefield, lines, out_table, interest_selection_expr):
# Make output folder
## name = "LineDensity_" + os.path.splitext(os.path.basename(zones))[0]
## outfolder = os.path.join(topoutfolder, name)
## if not os.path.exists(outfolder):
## os.mkdir(outfolder)
# Environmental Settings
ws = "in_memory"
if interest_selection_expr:
arcpy.MakeFeatureLayer_management(lines, "selected_lines",
interest_selection_expr)
else:
arcpy.MakeFeatureLayer_management(lines, "selected_lines")
arcpy.env.workspace = ws
albers = arcpy.SpatialReference(102039)
arcpy.env.outputCoordinateSystem = albers
arcpy.env.extent = zones
# Zones will be coerced to albers, have to check lines though
arcpy.CopyFeatures_management(zones, "zones_temp")
if arcpy.Describe(
lines).spatialReference.factoryCode != albers.factoryCode:
arcpy.AddError(
"Lines feature class does not have desired projection (Albers USGS). Re-project to factory code 102039 and try again."
)
sys.exit(1)
# Add hectares field to zones
arcpy.AddField_management("zones_temp", "ZoneAreaHa", "DOUBLE")
arcpy.CalculateField_management("zones_temp", "ZoneAreaHa",
"!shape.area@hectares!", "PYTHON")
# Perform identity analysis to join fields and crack lines at polygon boundaries
cu.multi_msg("Cracking lines at polygon boundaries...")
arcpy.Identity_analysis("selected_lines", "zones_temp", "lines_identity")
cu.multi_msg("Cracking lines complete.")
# Recalculate lengths
arcpy.AddField_management("lines_identity", "LengthM", "DOUBLE")
arcpy.CalculateField_management("lines_identity", "LengthM",
'!shape.length@meters!', "PYTHON")
# Summarize statistics by zone
arcpy.Statistics_analysis("lines_identity", "length_in_zone",
"LengthM SUM", zonefield)
# Join ZoneAreaHa to table
arcpy.JoinField_management("length_in_zone", zonefield, "zones_temp",
zonefield, "ZoneAreaHa")
# Delete rows in table with zero for zone area
## with arcpy.da.UpdateCursor("length_in_zone", "ZoneAreaHa") as cursor:
## for row in cursor:
## if row[0] is None:
## cursor.deleteRow()
# Add Density field and calc
arcpy.AddField_management("length_in_zone", "Density_MperHA", "DOUBLE", '',
'', '', '', "NULLABLE")
exp = "!SUM_LengthM! / !ZONEAREAHA!"
arcpy.CalculateField_management("length_in_zone", "Density_MperHA", exp,
"PYTHON")
cu.one_in_one_out("length_in_zone", ['SUM_LengthM', 'Density_MperHA'],
zones, zonefield, out_table)
cu.redefine_nulls(out_table, ['SUM_LengthM', 'Density_MperHA'], [0, 0])
## # Join to the original table
## keep_fields = ["ZoneID", "SUM_LengthM", "Density_MperHA"]
## arcpy.JoinField_management('zones_temp', zonefield, "length_in_zone", zonefield, keep_fields[1:])
##
## # Select only null records and change to 0
## arcpy.MakeFeatureLayer_management('zones_temp', 'zones_temp_lyr')
## arcpy.SelectLayerByAttribute_management('zones_temp_lyr', "NEW_SELECTION", '''"SUM_LengthM" is null''')
## fields_to_calc = ["SUM_LengthM", "Density_MperHA"]
## for f in fields_to_calc:
## arcpy.CalculateField_management('zones_temp_lyr', f, 0, "PYTHON")
##
## #Delete all the fields that aren't the ones I need
## keep_fields = ["ZoneID", "SUM_LengthM", "Density_MperHA"]
## all_fields = [f.name for f in arcpy.ListFields('zones_temp_lyr')]
## for f in all_fields:
## if f not in keep_fields:
## try:
## arcpy.DeleteField_management('zones_temp_lyr', f)
## except:
## continue
## arcpy.SelectLayerByAttribute_management('zones_temp_lyr', 'CLEAR_SELECTION')
##
## arcpy.CopyRows_management('zones_temp_lyr', out_table)
for tempitem in ['zones_temp', 'lines_identity', 'length_in_zone']:
arcpy.Delete_management(tempitem)
return out_table
