def lakes_in_zones(zones_fc, zone_field, lakes_fc, output_table):
# make sure we're only using the right types of lakes, our feature
# class excludes everything else but this is a guarantee this will
# get checked at some point
arcpy.env.workspace = 'in_memory'
# using in_memory workspace means no SHAPE@AREA attribute later so I
# need to calculate another field with the area using temp on disk
# then go ahead and copy to memory, delete temp
temp_workspace = cu.create_temp_GDB('lakezone')
temp_lakes = os.path.join(temp_workspace, 'temp_lakes')
arcpy.CopyFeatures_management(lakes_fc, temp_lakes)
arcpy.AddField_management(temp_lakes, 'Area_ha', 'DOUBLE')
arcpy.CalculateField_management(temp_lakes, 'Area_ha', '!shape.area@hectares!', 'PYTHON')
# this bit enforces the correct lake type/size restriction just in case
# geodata doesn't have this filtered already
need_selection = False
fcodes = (39000, 39004, 39009, 39010, 39011, 39012,
43600, 43613, 43615, 43617, 43618, 43619, 43621)
with arcpy.da.SearchCursor(temp_lakes, ["FCode"]) as cursor:
for row in cursor:
if row[0] not in fcodes + (43601,):
need_selection = True
if need_selection:
whereClause = '''
("AreaSqKm" >=0.04 AND "FCode" IN %s)\
OR ("AreaSqKm" >= 0.1 AND "FCode" = 43601)''' % (fcodes,)
arcpy.Select_analysis(temp_lakes, "lakes_4ha", whereClause)
temp_lakes = os.path.join(arcpy.env.workspace, "lakes_4ha")
selections = [""""Area_ha" >= 4""",
""""Area_ha" >= 4 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 10""",
""""Area_ha" >= 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_LakeStream'"""
]
temp_tables = ['Lakes4ha',
'Lakes4ha_Isolated',
'Lakes4ha_Headwater',
'Lakes4ha_DRStream',
'Lakes4ha_DRLakeStream',
'Lakes4to10ha',
'Lakes4to10ha_Isolated',
'Lakes4to10ha_Headwater',
'Lakes4to10ha_DRStream',
'Lakes4to10ha_DRLakeStream',
'Lakes10ha',
'Lakes10ha_Isolated',
'Lakes10ha_Headwater',
'Lakes10ha_DRStream',
'Lakes10ha_DRLakeStream'
]
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg("Creating temporary table called {0} for lakes where {1}".format(temp_table, sel))
polygons_in_zones.polygons_in_zones(zones_fc, zone_field, temp_lakes, temp_table, sel)
new_fields = ['Poly_Overlapping_AREA_ha', 'Poly_Contributing_AREA_ha', 'Poly_Overlapping_AREA_pct', 'Poly_Count']
avg_size_field = temp_table + '_AvgSize_ha'
arcpy.AddField_management(temp_table, avg_size_field , 'DOUBLE')
arcpy.CalculateField_management(temp_table, avg_size_field, '!Poly_Contributing_AREA_ha!/!Poly_Count!', 'PYTHON')
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# join em up and copy to final
temp_tables.remove('Lakes4ha')
for t in temp_tables:
try:
arcpy.JoinField_management('Lakes4ha', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Lakes4ha', ef, 'Double')
arcpy.CalculateField_management('Lakes4ha', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('Lakes4ha', 'ZoneID_*')]
for f in drop_fields:
arcpy.DeleteField_management('Lakes4ha', f)
arcpy.CopyRows_management('Lakes4ha', output_table)
# clean up
for item in ['Lakes4ha', temp_lakes, os.path.dirname(temp_workspace)] + temp_tables:
arcpy.Delete_management(item)
def create_csi_watersheds(flowdir, pour_dir, nhd_gdb, out_gdb):
# Starting environmental variables:
env.extent = flowdir
env.snapRaster = flowdir
env.cellSize = 10
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.CheckOutExtension('Spatial')
huc8_code = re.search('\d{8}', os.path.basename(flowdir)).group()
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
# create temp directory because we need shape geometry
temp_gdb = cu.create_temp_GDB('watersheds' + huc4_code)
print temp_gdb
env.workspace = temp_gdb
wbd_hu8 = os.path.join(nhd_gdb, "WBD_HU8")
field_name = (arcpy.ListFields(wbd_hu8, "HU*8"))[0].name
whereClause8 = """{0} = '{1}'""".format(arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc8_code)
arcpy.Select_analysis(wbd_hu8, "hu8", whereClause8)
arcpy.Buffer_analysis("hu8", "hu8_buffered", "100 meters")
# Create the basic watersheds
pour_points = os.path.join(pour_dir, 'pour_points.tif')
arcpy.Clip_management(pour_points, '', "pour_points_clipped", "hu8_buffered", '0', 'ClippingGeometry')
raw_watersheds = os.path.join(out_gdb, 'huc{}_watersheds_precursors'.format(huc8_code))
cu.multi_msg("Calculating preliminary watersheds...")
outWatershed = Watershed(flowdir, "pour_points_clipped")
outWatershed.save(raw_watersheds)
cu.multi_msg("Clipping watersheds to subregion boundaries and filtering spurious watersheds...")
# Watershed raster to polygons
arcpy.RasterToPolygon_conversion(raw_watersheds, "wspoly1", 'NO_SIMPLIFY', "Value")
# Clip watershed polygons to subregion polys.
arcpy.Clip_analysis("wspoly1", "hu8", "wsclip1")
# Clip watershed
## # Calculate hectares
## arcpy.AddField_management("wsclip1", "HA", "DOUBLE")
## arcpy.CalculateField_management("wsclip1", "HA", '''!shape.area@hectares!''', "PYTHON")
# Create fc of watershed polygons >= 1 ha that coincide with seed lines and polys.
seedline = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_flowlines')
seedpoly = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_lakes')
arcpy.MakeFeatureLayer_management("wsclip1", "wsclip1_lyr")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT", seedline, '', "NEW_SELECTION")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT", seedpoly, '', "ADD_TO_SELECTION")
arcpy.SelectLayerByAttribute_management("wsclip1_lyr", "SUBSET_SELECTION",'''"Shape_Area" >= 10000''')
cu.multi_msg("Reshaping watersheds...")
# Polygon back to raster
arcpy.PolygonToRaster_conversion("wsclip1_lyr", "grid_code", "ws_legit_ras")
arcpy.Clip_management("ws_legit_ras", '', "ws_legit_clipped_ras",
"hu8", "0", "ClippingGeometry")
# Make a raster from the subregion (boundary) polygon with zero for cell values.
arcpy.AddField_management("hu8", "Value", "SHORT")
arcpy.CalculateField_management("hu8", "Value", "0", "PYTHON")
arcpy.PolygonToRaster_conversion("hu8", "Value","boundary_raster")
arcpy.Clip_management("boundary_raster", '', "boundary_raster_clip", "hu8", '', "ClippingGeometry")
# Fill NoData in watersheds with the zero values from the subregion raster's cells.
composite = Con(IsNull("ws_legit_clipped_ras"), "boundary_raster_clip", "ws_legit_clipped_ras")
composite.save("composite_raster")
arcpy.Clip_management("composite_raster", '', "composite_raster_clip", "hu8", '0', "ClippingGeometry")
# Make a mask of zero cells. NoData cells are the actual mask for nibble.
premask = Con(IsNull("composite_raster_clip"), "composite_raster_clip", 0)
premask.save("premask")
arcpy.Clip_management("premask",'', "mask", "hu8", '', "ClippingGeometry")
# Set Null to 1.
pre_watersheds = Con(IsNull("composite_raster_clip"), 1, "composite_raster_clip")
pre_watersheds.save("pre_watersheds") # does this speed things up?
## prews.save("prews.tif")
# Nibble masked values (null values along boundary).
cu.multi_msg('Nibbling watersheds as part of reshaping...')
nibble = Nibble("pre_watersheds", "mask", "DATA_ONLY")
nibble.save("nibble")
# Use HU8 buffer so that watersheds will overrun HU8 boundaries and get
# clipped without weird slivers later
arcpy.Clip_management("nibble", "", "watersheds_ras", "hu8_buffered", "NoData","ClippingGeometry")
# Convert watershed raster to polygon.
# Setting simplify keyword to TRUE in RasterToPolygon_conversion
# is not working reliably so need to do this in two steps, unfortunately
cu.multi_msg("Converted reshaped watersheds raster to polygons. If you experience problems with this step, please read Known and Outstanding Bugs.txt")
arcpy.RasterToPolygon_conversion("watersheds_ras", "nibble_sheds",'SIMPLIFY', "Value") #simplify okay
## # I'm using 15 as the tolerance
## # here because the diagonal of a 10x10 pixel is 14.14 m and
## # I'm okay with a vertex moving as far as it can on the edges of the pixel
## # This also produces results very similar to using the simplify setting
## # on RasterToPolygon_conversion, when it works.
## arcpy.SimplifyPolygon_cartography("nibble_sheds_unsimple",
## "nibble_sheds_simplify", "POINT_REMOVE", "15 Meters", "0 SquareMeters",
## "RESOLVE_ERRORS", "NO_KEEP")
arcpy.Clip_analysis("nibble_sheds", "hu8", "final_watersheds")
# Join Permanent ID from Waterbody seed shapefile
final_watersheds_out = os.path.join(out_gdb, 'huc{}_final_watersheds'.format(huc8_code))
arcpy.JoinField_management("final_watersheds", 'grid_code', seedpoly, 'POUR_ID', ['Permanent_Identifier'])
# this block bumps out sheds so that they fully contain their own lakes
# sometimes a little bit of another shed is overlapping the lake simply
# due to the raster/polygon differences
# 1) delete fields so watersheds and seedpoly share schema
# 2) update features, keeping borders
# 3) instead of lots of nulls make unique dissolve_id for all so that nulls aren't dissolved into one
# 4) dissolve features on dissolve_id keeping the Permanent_Identifier field
arcpy.CopyFeatures_management(seedpoly, 'lakes_nofields')
for fc in ['lakes_nofields', 'final_watersheds']:
fields = arcpy.ListFields(fc)
for f in fields:
if f != 'Permanent_Identifier':
try:
arcpy.DeleteField_management(fc, f)
except:
continue
arcpy.Update_analysis("final_watersheds", 'lakes_nofields', 'update_fc')
arcpy.AddField_management('update_fc', 'dissolve_id', 'TEXT', 255)
arcpy.MakeFeatureLayer_management('update_fc', 'update_lyr')
arcpy.SelectLayerByAttribute_management('update_lyr', 'NEW_SELECTION', """"Permanent_Identifier" is not null""")
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!Permanent_Identifier!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'SWITCH_SELECTION')
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!OBJECTID!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'CLEAR_SELECTION')
arcpy.Dissolve_management('update_lyr', "final_watersheds_bumped", 'dissolve_id', 'Permanent_Identifier FIRST')
cu.rename_field("final_watersheds_bumped", "FIRST_Permanent_Identifier", "Permanent_Identifier", deleteOld = True)
arcpy.DeleteField_management('final_watersheds_bumped', 'dissolve_id')
arcpy.Clip_analysis('final_watersheds_bumped', 'hu8', 'final_watersheds_clipped')
arcpy.CopyFeatures_management("final_watersheds_clipped", final_watersheds_out)
temp_items = arcpy.ListRasters() + arcpy.ListFeatureClasses() + [temp_gdb]
for item in temp_items:
try:
arcpy.Delete_management(item)
except:
continue
arcpy.ResetEnvironments()
arcpy.CheckInExtension('Spatial')
cu.multi_msg("Complete.")
def connected_wetlands(lakes_fc, lake_id_field, wetlands_fc, out_table):
env.workspace = 'in_memory'
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.Buffer_analysis(lakes_fc, 'lakes_30m', '30 meters')
arcpy.FeatureToLine_management('lakes_30m', 'shorelines')
# 3 selections for the wetlands types we want to look at
openwater_exp = """"VegType" = 'PEMorPAB'"""
forested_exp = """"VegType" = 'PFO'"""
scrubshrub_exp = """"VegType" = 'PSS'"""
other_exp = """"VegType" = 'Other'"""
all_exp = ''
selections = [
all_exp, forested_exp, scrubshrub_exp, openwater_exp, other_exp
]
temp_tables = [
'AllWetlands', 'ForestedWetlands', 'ScrubShrubWetlands',
'OpenWaterWetlands', 'OtherWetlands'
]
# for each wetland type, get the count of intersection wetlands, and the length of the lake
# shoreline that is within a wetland polygon
for sel, temp_table in zip(selections, temp_tables):
print("Creating temporary table for wetlands where {0}".format(sel))
# this function adds the count and the area using the lake as the zone
polygons_in_zones('lakes_30m',
lake_id_field,
wetlands_fc,
temp_table,
sel,
contrib_area=True)
# make good field names now rather than later
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# shoreline calculation
# using the Shape_Length field so can't do this part in memory
shoreline_gdb = cu.create_temp_GDB('shoreline')
selected_wetlands = os.path.join(shoreline_gdb, 'wetlands')
arcpy.Select_analysis(wetlands_fc, selected_wetlands, sel)
intersect_output = os.path.join(shoreline_gdb, "intersect")
arcpy.Intersect_analysis(['shorelines', selected_wetlands],
intersect_output)
arcpy.Statistics_analysis(intersect_output, 'intersect_stats',
[['Shape_Length', 'SUM']], lake_id_field)
cu.one_in_one_out('intersect_stats', ['SUM_Shape_Length'], lakes_fc,
lake_id_field, 'temp_shoreline_table')
cu.redefine_nulls('temp_shoreline_table', ['SUM_Shape_Length'], [0])
shoreline_field = temp_table + "_Shoreline_Km"
arcpy.AddField_management('temp_shoreline_table', shoreline_field,
'DOUBLE')
arcpy.CalculateField_management('temp_shoreline_table',
shoreline_field,
'!SUM_Shape_Length!/1000', 'PYTHON')
# join the shoreline value to the temp_table
arcpy.JoinField_management(temp_table, lake_id_field,
'temp_shoreline_table', lake_id_field,
shoreline_field)
# clean up shoreline intermediates
for item in [shoreline_gdb, 'intersect_stats', 'temp_shoreline_table']:
arcpy.Delete_management(item)
# join em up and copy to final
temp_tables.remove('AllWetlands')
for t in temp_tables:
try:
arcpy.JoinField_management('AllWetlands', lake_id_field, t,
lake_id_field)
# sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('AllWetlands', ef, 'Double')
arcpy.CalculateField_management('AllWetlands', ef, '0',
'PYTHON')
continue
# remove all the extra zone fields, which have underscore in name
drop_fields = [
f.name
for f in arcpy.ListFields('AllWetlands', 'Permanent_Identifier_*')
]
for f in drop_fields:
arcpy.DeleteField_management('AllWetlands', f)
# remove all the overlapping metrics, which do not apply by definition
fields = [f.name for f in arcpy.ListFields('AlLWetlands')]
for f in fields:
if 'Overlapping' in f:
arcpy.DeleteField_management('AllWetlands', f)
arcpy.CopyRows_management('AllWetlands', out_table)
for item in ['AllWetlands'] + temp_tables:
try:
arcpy.Delete_management(item)
except:
continue
def lakes_in_zones(zones_fc, zone_field, lakes_fc, output_table):
# make sure we're only using the right types of lakes, our feature
# class excludes everything else but this is a guarantee this will
# get checked at some point
arcpy.env.workspace = 'in_memory'
# using in_memory workspace means no SHAPE@AREA attribute later so I
# need to calculate another field with the area using temp on disk
# then go ahead and copy to memory, delete temp
temp_workspace = cu.create_temp_GDB('lakezone')
temp_lakes = os.path.join(temp_workspace, 'temp_lakes')
arcpy.CopyFeatures_management(lakes_fc, temp_lakes)
arcpy.AddField_management(temp_lakes, 'Area_ha', 'DOUBLE')
arcpy.CalculateField_management(temp_lakes, 'Area_ha',
'!shape.area@hectares!', 'PYTHON')
# this bit enforces the correct lake type/size restriction just in case
# geodata doesn't have this filtered already
need_selection = False
fcodes = (39000, 39004, 39009, 39010, 39011, 39012, 43600, 43613, 43615,
43617, 43618, 43619, 43621)
with arcpy.da.SearchCursor(temp_lakes, ["FCode"]) as cursor:
for row in cursor:
if row[0] not in fcodes + (43601, ):
need_selection = True
if need_selection:
whereClause = '''
("AreaSqKm" >=0.04 AND "FCode" IN %s)\
OR ("AreaSqKm" >= 0.1 AND "FCode" = 43601)''' % (fcodes, )
arcpy.Select_analysis(temp_lakes, "lakes_4ha", whereClause)
temp_lakes = os.path.join(arcpy.env.workspace, "lakes_4ha")
selections = [
""""Area_ha" >= 4""",
""""Area_ha" >= 4 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 4 AND "Area_ha" < 10 AND "Connection" = 'DR_LakeStream'""",
""""Area_ha" >= 10""",
""""Area_ha" >= 10 AND "Connection" = 'Isolated'""",
""""Area_ha" >= 10 AND "Connection" = 'Headwater'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_Stream'""",
""""Area_ha" >= 10 AND "Connection" = 'DR_LakeStream'"""
]
temp_tables = [
'Lakes4ha', 'Lakes4ha_Isolated', 'Lakes4ha_Headwater',
'Lakes4ha_DRStream', 'Lakes4ha_DRLakeStream', 'Lakes4to10ha',
'Lakes4to10ha_Isolated', 'Lakes4to10ha_Headwater',
'Lakes4to10ha_DRStream', 'Lakes4to10ha_DRLakeStream', 'Lakes10ha',
'Lakes10ha_Isolated', 'Lakes10ha_Headwater', 'Lakes10ha_DRStream',
'Lakes10ha_DRLakeStream'
]
for sel, temp_table in zip(selections, temp_tables):
cu.multi_msg(
"Creating temporary table called {0} for lakes where {1}".format(
temp_table, sel))
polygons_in_zones.polygons_in_zones(zones_fc, zone_field, temp_lakes,
temp_table, sel)
new_fields = [
'Poly_Overlapping_AREA_ha', 'Poly_Contributing_AREA_ha',
'Poly_Overlapping_AREA_pct', 'Poly_Count'
]
avg_size_field = temp_table + '_AvgSize_ha'
arcpy.AddField_management(temp_table, avg_size_field, 'DOUBLE')
arcpy.CalculateField_management(
temp_table, avg_size_field,
'!Poly_Contributing_AREA_ha!/!Poly_Count!', 'PYTHON')
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# join em up and copy to final
temp_tables.remove('Lakes4ha')
for t in temp_tables:
try:
arcpy.JoinField_management('Lakes4ha', zone_field, t, zone_field)
#sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('Lakes4ha', ef, 'Double')
arcpy.CalculateField_management('Lakes4ha', ef, '0', 'PYTHON')
continue
# remove all the extra zoneID fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('Lakes4ha', 'ZoneID_*')]
for f in drop_fields:
arcpy.DeleteField_management('Lakes4ha', f)
arcpy.CopyRows_management('Lakes4ha', output_table)
# clean up
for item in ['Lakes4ha', temp_lakes,
os.path.dirname(temp_workspace)] + temp_tables:
arcpy.Delete_management(item)
def stats_area_table(zone_fc, zone_field, in_value_raster, out_table, is_thematic, debug_mode = False):
orig_env = arcpy.env.workspace
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('zonal_tabarea')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
arcpy.CheckOutExtension("Spatial")
# Set up environments for alignment between zone raster and theme raster
this_files_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(this_files_dir)
common_grid = os.path.abspath('../common_grid.tif')
env.snapRaster = common_grid
env.cellSize = common_grid
CELL_SIZE = 30
env.extent = zone_fc
zone_desc = arcpy.Describe(zone_fc)
zone_raster = 'convertraster'
if zone_desc.dataType != 'RasterDataset':
arcpy.PolygonToRaster_conversion(zone_fc, zone_field, zone_raster, 'CELL_CENTER', cellsize = CELL_SIZE)
else:
zone_raster = zone_fc
# I tested and there is no need to resample the raster being summarized. It will be resampled correctly
# internally in the following tool given that the necessary environments are set above (cell size, snap).
#in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE)
if not is_thematic:
arcpy.AddMessage("Calculating Zonal Statistics...")
temp_entire_table = arcpy.sa.ZonalStatisticsAsTable(zone_raster, zone_field, in_value_raster, 'temp_zonal_table', 'DATA', 'MIN_MAX_MEAN')
if is_thematic:
#for some reason env.cellSize doesn't work
# calculate/doit
arcpy.AddMessage("Tabulating areas...")
temp_entire_table = arcpy.sa.TabulateArea(zone_raster, zone_field, in_value_raster, 'Value', 'temp_area_table', CELL_SIZE)
# replaces join to Zonal Stats in previous versions of tool
# no joining, just calculate the area/count from what's produced by TabulateArea
arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE')
cursor_fields = ['AREA', 'COUNT']
value_fields = [f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*')]
cursor_fields.extend(value_fields)
with arcpy.da.UpdateCursor(temp_entire_table, cursor_fields) as uCursor:
for uRow in uCursor:
area, count, value_fields = uRow[0], uRow[1], uRow[2:]
area = sum(value_fields)
count = round(area/(CELL_SIZE*CELL_SIZE), 0)
new_row = [area, count] + value_fields
uCursor.updateRow(new_row)
arcpy.AddMessage("Refining output table...")
arcpy.AddField_management(temp_entire_table, 'DataCoverage_pct', 'DOUBLE')
# calculate DataCoverage_pct by comparing to original areas in zone raster
# alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count
zone_raster_dict = {row[0]:row[1] for row in arcpy.da.SearchCursor(zone_raster, [zone_field, 'Count'])}
temp_entire_table_dict = {row[0]:row[1] for row in arcpy.da.SearchCursor(temp_entire_table, [zone_field, 'COUNT'])}
with arcpy.da.UpdateCursor(temp_entire_table, [zone_field, 'DataCoverage_Pct']) as cursor:
for uRow in cursor:
key_value, data_pct = uRow
count_orig = zone_raster_dict[key_value]
if key_value in temp_entire_table_dict:
count_summarized = temp_entire_table_dict[key_value]
data_pct = 100*float(count_summarized/count_orig)
else:
data_pct = None
cursor.updateRow((key_value, data_pct))
# Refine the output
refine_zonal_output(temp_entire_table, zone_field, is_thematic)
# final table gets a record even for no-data zones
keep_fields = [f.name for f in arcpy.ListFields(temp_entire_table)]
if zone_field.upper() in keep_fields:
keep_fields.remove(zone_field.upper())
zone_field = zone_field.upper()
if zone_field in keep_fields:
keep_fields.remove(zone_field)
# not needed as long we are working only with rasters
# in order to add vector capabilities back, need to do something with this
# right now we just can't fill in polygon zones that didn't convert to raster in our system
cu.one_in_one_out(temp_entire_table, keep_fields, zone_fc, zone_field, out_table)
# Convert "DataCoverage_pct" values to 0 for zones with no metrics calculated
codeblock = """def convert_pct(arg1):
if arg1 is None:
return float(0)
else:
return arg1"""
arcpy.CalculateField_management(out_table, 'DataCoverage_pct', 'convert_pct(!DataCoverage_pct!)', 'PYTHON_9.3', codeblock)
# count whether all zones got an output record or not)
out_count = int(arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
count_diff = in_count - out_count
# cleanup
if not debug_mode:
for item in ['temp_zonal_table', 'temp_entire_table', 'in_memory', 'zones_VAT']:
arcpy.Delete_management(item)
arcpy.ResetEnvironments()
arcpy.env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch
arcpy.CheckInExtension("Spatial")
return [out_table, count_diff]
def connected_wetlands(lakes_fc, lake_id_field, wetlands_fc, out_table):
env.workspace = 'in_memory'
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.Buffer_analysis(lakes_fc, 'lakes_30m', '30 meters')
arcpy.FeatureToLine_management('lakes_30m', 'shorelines')
# 3 selections for the wetlands types we want to look at
openwater_exp = """"VegType" = 'PEMorPAB'"""
forested_exp = """"VegType" = 'PFO'"""
scrubshrub_exp = """"VegType" = 'PSS'"""
other_exp = """"VegType" = 'Other'"""
all_exp = ''
selections = [all_exp, forested_exp, scrubshrub_exp, openwater_exp, other_exp]
temp_tables = ['AllWetlands', 'ForestedWetlands', 'ScrubShrubWetlands', 'OpenWaterWetlands', 'OtherWetlands']
# for each wetland type, get the count of intersection wetlands, and the length of the lake
# shoreline that is within a wetland polygon
for sel, temp_table in zip(selections, temp_tables):
print("Creating temporary table for wetlands where {0}".format(sel))
# this function adds the count and the area using the lake as the zone
polygons_in_zones('lakes_30m', lake_id_field, wetlands_fc, temp_table, sel, contrib_area = True)
# make good field names now rather than later
for f in new_fields:
cu.rename_field(temp_table, f, f.replace('Poly', temp_table), True)
# shoreline calculation
# using the Shape_Length field so can't do this part in memory
shoreline_gdb = cu.create_temp_GDB('shoreline')
selected_wetlands = os.path.join(shoreline_gdb, 'wetlands')
arcpy.Select_analysis(wetlands_fc, selected_wetlands, sel)
intersect_output = os.path.join(shoreline_gdb, "intersect")
arcpy.Intersect_analysis(['shorelines', selected_wetlands], intersect_output)
arcpy.Statistics_analysis(intersect_output, 'intersect_stats', [['Shape_Length', 'SUM']], lake_id_field)
cu.one_in_one_out('intersect_stats', ['SUM_Shape_Length'], lakes_fc, lake_id_field, 'temp_shoreline_table')
cu.redefine_nulls('temp_shoreline_table', ['SUM_Shape_Length'], [0])
shoreline_field = temp_table + "_Shoreline_Km"
arcpy.AddField_management('temp_shoreline_table', shoreline_field, 'DOUBLE')
arcpy.CalculateField_management('temp_shoreline_table', shoreline_field, '!SUM_Shape_Length!/1000', 'PYTHON')
# join the shoreline value to the temp_table
arcpy.JoinField_management(temp_table, lake_id_field, 'temp_shoreline_table', lake_id_field, shoreline_field)
# clean up shoreline intermediates
for item in [shoreline_gdb, 'intersect_stats', 'temp_shoreline_table']:
arcpy.Delete_management(item)
# join em up and copy to final
temp_tables.remove('AllWetlands')
for t in temp_tables:
try:
arcpy.JoinField_management('AllWetlands', lake_id_field, t, lake_id_field)
# sometimes there's no table if it was an empty selection
except:
empty_fields = [f.replace('Poly', t) for f in new_fields]
for ef in empty_fields:
arcpy.AddField_management('AllWetlands', ef, 'Double')
arcpy.CalculateField_management('AllWetlands', ef, '0', 'PYTHON')
continue
# remove all the extra zone fields, which have underscore in name
drop_fields = [f.name for f in arcpy.ListFields('AllWetlands', 'Permanent_Identifier_*')]
for f in drop_fields:
arcpy.DeleteField_management('AllWetlands', f)
# remove all the overlapping metrics, which do not apply by definition
fields = [f.name for f in arcpy.ListFields('AlLWetlands')]
for f in fields:
if 'Overlapping' in f:
arcpy.DeleteField_management('AllWetlands', f)
arcpy.CopyRows_management('AllWetlands', out_table)
for item in ['AllWetlands'] + temp_tables:
try:
arcpy.Delete_management(item)
except:
continue
def stats_area_table(zone_fc, zone_field, in_value_raster, out_table, is_thematic):
arcpy.CheckOutExtension("Spatial")
cu.multi_msg("Calculating zonal statistics...")
temp_zonal_table = 'in_memory/zonal_stats_temp'
temp_entire_table = 'in_memory/temp_entire_table'
# calculate/doit
env.snapRaster = in_value_raster
env.cellSize = in_value_raster
# this has to be on disk for some reason to avoid background processing
# errors thrown up at random
# hence we get the following awkward horribleness
use_convert_raster = False
try:
arcpy.sa.ZonalStatisticsAsTable(zone_fc, zone_field, in_value_raster,
temp_zonal_table, 'DATA', 'ALL')
# with Permanent_Identifier as the zone_field, background processing errors
# and another error get thrown up at random
# it's faster to do zonal stats as above but if it fails (which it does
# pretty quickly, usually), do this way which always works but takes
# twice as long on large rasters
except:
use_convert_raster = True
temp_workspace = cu.create_temp_GDB('temp_zonal')
convert_raster = os.path.join(temp_workspace,
cu.shortname(zone_fc) + '_converted')
cu.multi_msg('Creating raster {0}'.format(convert_raster))
arcpy.PolygonToRaster_conversion(zone_fc, zone_field, convert_raster)
arcpy.sa.ZonalStatisticsAsTable(convert_raster, zone_field, in_value_raster,
temp_zonal_table, "DATA", "ALL")
if is_thematic:
#for some reason env.celLSize doesn't work
desc = arcpy.Describe(in_value_raster)
cell_size = desc.meanCelLHeight
# calculate/doit
temp_area_table = 'in_memory/tab_area_temp'
cu.multi_msg("Tabulating areas...")
if use_convert_raster:
arcpy.sa.TabulateArea(convert_raster, zone_field, in_value_raster,
'Value', temp_area_table, cell_size)
else:
arcpy.sa.TabulateArea(zone_fc, zone_field, in_value_raster,
'Value', temp_area_table, cell_size)
# making the output table
arcpy.CopyRows_management(temp_area_table, temp_entire_table)
zonal_stats_fields = ['AREA']
arcpy.JoinField_management(temp_entire_table, zone_field, temp_zonal_table, zone_field, zonal_stats_fields)
# cleanup
arcpy.Delete_management(temp_area_table)
if not is_thematic:
# making the output table
arcpy.CopyRows_management(temp_zonal_table, temp_entire_table)
cu.multi_msg("Refining output table...")
refine_zonal_output(temp_entire_table, is_thematic)
#final table gets a record even for no-data zones
keep_fields = [f.name for f in arcpy.ListFields(temp_entire_table)]
if zone_field.upper() in keep_fields:
keep_fields.remove(zone_field.upper())
if zone_field in keep_fields:
keep_fields.remove(zone_field)
cu.one_in_one_out(temp_entire_table, keep_fields, zone_fc, zone_field, out_table)
## cu.redefine_nulls(out_table, keep_fields, ["NA"]* len(keep_fields))
# count whether all zones got an output record or not)
out_count = int(arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
if out_count < in_count:
warn_msg = ("WARNING: {0} features are missing in the output table"
" because they are too small for this raster's"
" resolution. This may be okay depending on your"
" application.").format(in_count - out_count)
arcpy.AddWarning(warn_msg)
print(warn_msg)
# cleanup
arcpy.Delete_management(temp_zonal_table)
arcpy.Delete_management(temp_entire_table)
if use_convert_raster:
arcpy.Delete_management(os.path.dirname(temp_workspace))
arcpy.CheckInExtension("Spatial")
def stats_area_table(zone_fc,
zone_field,
in_value_raster,
out_table,
is_thematic,
debug_mode=False):
orig_env = arcpy.env.workspace
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('zonal_tabarea')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
arcpy.CheckOutExtension("Spatial")
# Set up environments for alignment between zone raster and theme raster
this_files_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(this_files_dir)
common_grid = os.path.abspath('../common_grid.tif')
env.snapRaster = common_grid
env.cellSize = common_grid
CELL_SIZE = 30
env.extent = zone_fc
zone_desc = arcpy.Describe(zone_fc)
zone_raster = 'convertraster'
if zone_desc.dataType != 'RasterDataset':
arcpy.PolygonToRaster_conversion(zone_fc,
zone_field,
zone_raster,
'CELL_CENTER',
cellsize=CELL_SIZE)
else:
zone_raster = zone_fc
# I tested and there is no need to resample the raster being summarized. It will be resampled correctly
# internally in the following tool given that the necessary environments are set above (cell size, snap).
#in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE)
if not is_thematic:
arcpy.AddMessage("Calculating Zonal Statistics...")
temp_entire_table = arcpy.sa.ZonalStatisticsAsTable(
zone_raster, zone_field, in_value_raster, 'temp_zonal_table',
'DATA', 'MIN_MAX_MEAN')
if is_thematic:
#for some reason env.cellSize doesn't work
# calculate/doit
arcpy.AddMessage("Tabulating areas...")
temp_entire_table = arcpy.sa.TabulateArea(zone_raster, zone_field,
in_value_raster, 'Value',
'temp_area_table', CELL_SIZE)
# replaces join to Zonal Stats in previous versions of tool
# no joining, just calculate the area/count from what's produced by TabulateArea
arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE')
cursor_fields = ['AREA', 'COUNT']
value_fields = [
f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*')
]
cursor_fields.extend(value_fields)
with arcpy.da.UpdateCursor(temp_entire_table,
cursor_fields) as uCursor:
for uRow in uCursor:
area, count, value_fields = uRow[0], uRow[1], uRow[2:]
area = sum(value_fields)
count = round(area / (CELL_SIZE * CELL_SIZE), 0)
new_row = [area, count] + value_fields
uCursor.updateRow(new_row)
arcpy.AddMessage("Refining output table...")
arcpy.AddField_management(temp_entire_table, 'DataCoverage_pct', 'DOUBLE')
# calculate DataCoverage_pct by comparing to original areas in zone raster
# alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count
zone_raster_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(zone_raster, [zone_field, 'Count'])
}
temp_entire_table_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(temp_entire_table,
[zone_field, 'COUNT'])
}
with arcpy.da.UpdateCursor(temp_entire_table,
[zone_field, 'DataCoverage_Pct']) as cursor:
for uRow in cursor:
key_value, data_pct = uRow
count_orig = zone_raster_dict[key_value]
if key_value in temp_entire_table_dict:
count_summarized = temp_entire_table_dict[key_value]
data_pct = 100 * float(count_summarized / count_orig)
else:
data_pct = None
cursor.updateRow((key_value, data_pct))
# Refine the output
refine_zonal_output(temp_entire_table, zone_field, is_thematic)
# final table gets a record even for no-data zones
keep_fields = [f.name for f in arcpy.ListFields(temp_entire_table)]
if zone_field.upper() in keep_fields:
keep_fields.remove(zone_field.upper())
zone_field = zone_field.upper()
if zone_field in keep_fields:
keep_fields.remove(zone_field)
# not needed as long we are working only with rasters
# in order to add vector capabilities back, need to do something with this
# right now we just can't fill in polygon zones that didn't convert to raster in our system
cu.one_in_one_out(temp_entire_table, keep_fields, zone_fc, zone_field,
out_table)
# Convert "DataCoverage_pct" values to 0 for zones with no metrics calculated
codeblock = """def convert_pct(arg1):
if arg1 is None:
return float(0)
else:
return arg1"""
arcpy.CalculateField_management(out_table, 'DataCoverage_pct',
'convert_pct(!DataCoverage_pct!)',
'PYTHON_9.3', codeblock)
# count whether all zones got an output record or not)
out_count = int(arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
count_diff = in_count - out_count
# cleanup
if not debug_mode:
for item in [
'temp_zonal_table', 'temp_entire_table', 'in_memory',
'zones_VAT'
]:
arcpy.Delete_management(item)
arcpy.ResetEnvironments()
arcpy.env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch
arcpy.CheckInExtension("Spatial")
return [out_table, count_diff]
def classify_lake_connectivity(nhd,
out_feature_class,
exclude_intermit_flowlines=False,
debug_mode=False):
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('classify_lake_connectivity')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
layers_list = []
temp_feature_class = "in_memory/temp_fc"
# Local variables:
nhdflowline = os.path.join(nhd, "Hydrography", "NHDFLowline")
nhdjunction = os.path.join(nhd, "Hydrography", "HYDRO_NET_Junctions")
nhdwaterbody = os.path.join(nhd, "Hydrography", "NHDWaterbody")
network = os.path.join(nhd, "Hydrography", "HYDRO_NET")
# Get lakes, ponds and reservoirs over a hectare.
csi_population_filter = '''"AreaSqKm" >=0.01 AND\
"FCode" IN (39000,39004,39009,39010,39011,39012,43600,43613,43615,43617,43618,43619,43621)'''
all_lakes_reservoirs_filter = '''"FType" IN (390, 436)'''
# Can't see why we shouldn't just attribute all lakes and reservoirs
# arcpy.Select_analysis(nhdwaterbody, "csiwaterbody", lake_population_filter)
arcpy.AddMessage("Initializing output.")
if exclude_intermit_flowlines:
arcpy.CopyFeatures_management(out_feature_class, temp_feature_class)
else:
arcpy.Select_analysis(nhdwaterbody, temp_feature_class,
all_lakes_reservoirs_filter)
# Get lakes, ponds and reservoirs over 10 hectares.
lakes_10ha_filter = '''"AreaSqKm" >= 0.1 AND "FType" IN (390, 436)'''
arcpy.Select_analysis(nhdwaterbody, "csiwaterbody_10ha", lakes_10ha_filter)
# Exclude intermittent flowlines, if requested
if exclude_intermit_flowlines:
flowline_where_clause = '''"FCode" NOT IN (46003,46007)'''
nhdflowline = arcpy.Select_analysis(nhdflowline,
"nhdflowline_filtered",
flowline_where_clause)
# Make dangle points at end of nhdflowline
arcpy.FeatureVerticesToPoints_management(nhdflowline, "dangles", "DANGLE")
layers_list.append(
arcpy.MakeFeatureLayer_management("dangles", "dangles_lyr"))
# Isolate start dangles from end dangles.
arcpy.FeatureVerticesToPoints_management(nhdflowline, "start", "START")
arcpy.FeatureVerticesToPoints_management(nhdflowline, "end", "END")
arcpy.SelectLayerByLocation_management("dangles_lyr", "ARE_IDENTICAL_TO",
"start")
arcpy.CopyFeatures_management("dangles_lyr", "startdangles")
arcpy.SelectLayerByLocation_management("dangles_lyr", "ARE_IDENTICAL_TO",
"end")
arcpy.CopyFeatures_management("dangles_lyr", "enddangles")
# Special handling for lakes that have some intermittent flow in and some permanent
if exclude_intermit_flowlines:
layers_list.append(
arcpy.MakeFeatureLayer_management(nhdflowline, "nhdflowline_lyr"))
arcpy.SelectLayerByAttribute_management(
"nhdflowline_lyr", "NEW_SELECTION",
'''"WBArea_Permanent_Identifier" is null''')
arcpy.FeatureVerticesToPoints_management("nhdflowline_lyr",
"non_artificial_end", "END")
arcpy.SelectLayerByAttribute_management("nhdflowline_lyr",
"CLEAR_SELECTION")
arcpy.AddMessage("Found source area nodes.")
# Get junctions from lakes >= 10 hectares.
layers_list.append(
arcpy.MakeFeatureLayer_management(nhdjunction, "junction_lyr"))
arcpy.SelectLayerByLocation_management("junction_lyr", "INTERSECT",
"csiwaterbody_10ha", XY_TOLERANCE,
"NEW_SELECTION")
arcpy.CopyFeatures_management("junction_lyr", "flags_10ha_lake_junctions")
arcpy.AddMessage("Found lakes >= 10 ha.")
# Make points shapefile and layer at flowline vertices to act as potential flags and/or barriers.
arcpy.AddMessage("Tracing...")
arcpy.FeatureVerticesToPoints_management(nhdflowline, "midvertices", "MID")
layers_list.append(
arcpy.MakeFeatureLayer_management("midvertices", "midvertices_lyr"))
# Get vertices that are not coincident with 10 hectare lake junctions.
arcpy.SelectLayerByLocation_management("midvertices_lyr", "INTERSECT",
"flags_10ha_lake_junctions", "",
"NEW_SELECTION")
arcpy.SelectLayerByLocation_management("midvertices_lyr", "INTERSECT",
"flags_10ha_lake_junctions", "",
"SWITCH_SELECTION")
arcpy.CopyFeatures_management("midvertices_lyr", "non10vertices")
# Get junctions that are not coincident with 10 hectare lake junctions.
arcpy.SelectLayerByLocation_management("junction_lyr", "INTERSECT",
"flags_10ha_lake_junctions", "",
"NEW_SELECTION")
arcpy.SelectLayerByLocation_management("junction_lyr", "INTERSECT",
"flags_10ha_lake_junctions", "",
"SWITCH_SELECTION")
arcpy.CopyFeatures_management("junction_lyr", "non10junctions")
# Merge non10vertices with non10junctions
arcpy.Merge_management(
["non10junctions", "non10vertices"],
"all_non_flag_points") # inputs both point fc in_memory
layers_list.append(
arcpy.MakeFeatureLayer_management("all_non_flag_points",
"all_non_flag_points_lyr"))
# Tests the counts...for some reason I'm not getting stable behavior from the merge.
mid_n = int(arcpy.GetCount_management("non10vertices").getOutput(0))
jxn_n = int(arcpy.GetCount_management("non10junctions").getOutput(0))
merge_n = int(
arcpy.GetCount_management("all_non_flag_points").getOutput(0))
if merge_n < mid_n + jxn_n:
arcpy.AddWarning(
"The total number of flags ({0}) is less than the sum of the input junctions ({1}) "
"and input midpoints ({2})".format(merge_n, jxn_n, mid_n))
# For tracing barriers, select all_non_flag_points points that intersect a 10 ha lake.
arcpy.SelectLayerByLocation_management("all_non_flag_points_lyr",
"INTERSECT", "csiwaterbody_10ha",
XY_TOLERANCE, "NEW_SELECTION")
arcpy.CopyFeatures_management("all_non_flag_points_lyr", "barriers")
# Trace1-Trace downstream to first barrier (junctions+midvertices in 10 ha lake) starting from flags_10ha_lake_junctions flag points.
arcpy.TraceGeometricNetwork_management(network, "trace1",
"flags_10ha_lake_junctions",
"TRACE_DOWNSTREAM", "barriers")
# Save trace1 flowlines and junctions to layers on disk.
arcpy.CopyFeatures_management("trace1\HYDRO_NET_Junctions",
"trace1_junctions") # extra for debugging
arcpy.CopyFeatures_management("trace1\NHDFlowline", "trace1_flowline")
# Select vertice midpoints that intersect trace1 flowlines selection for new flags for trace2.
layers_list.append(
arcpy.MakeFeatureLayer_management("non10vertices",
"non10vertices_lyr"))
arcpy.SelectLayerByLocation_management("non10vertices_lyr", "INTERSECT",
"trace1_flowline", "",
"NEW_SELECTION")
# Trace2-Trace downstream from midpoints of flowlines that intersect the selected flowlines from trace1.
arcpy.TraceGeometricNetwork_management(network, "trace2",
"non10vertices_lyr",
"TRACE_DOWNSTREAM")
# Save trace1 flowlines and junctions to layers and then shapes on disk.
arcpy.CopyFeatures_management("trace2\HYDRO_NET_Junctions",
"trace2junctions")
arcpy.CopyFeatures_management("trace2\NHDFlowline",
"trace2_flowline") # extra for debugging
arcpy.AddMessage("Done tracing.")
# Make shapefile for seepage lakes. (Ones that don't intersect flowlines)
if exclude_intermit_flowlines:
class_field_name = "Permanent_Lake_Connectivity"
else:
class_field_name = "Maximum_Lake_Connectivity"
arcpy.AddField_management(temp_feature_class,
class_field_name,
"TEXT",
field_length=13)
layers_list.append(
arcpy.MakeFeatureLayer_management(temp_feature_class, "out_fc_lyr"))
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
nhdflowline, XY_TOLERANCE,
"NEW_SELECTION")
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
nhdflowline, "", "SWITCH_SELECTION")
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'Isolated'""", "PYTHON")
# New type of "Isolated" classification, mostly for "permanent" but there were some oddballs in "maximum" too
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
"startdangles", XY_TOLERANCE,
"NEW_SELECTION")
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
"enddangles", XY_TOLERANCE,
"SUBSET_SELECTION")
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'Isolated'""", "PYTHON")
# Get headwater lakes.
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
"startdangles", XY_TOLERANCE,
"NEW_SELECTION")
arcpy.SelectLayerByAttribute_management(
"out_fc_lyr", "REMOVE_FROM_SELECTION",
'''"{}" = 'Isolated' '''.format(class_field_name))
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'Headwater'""", "PYTHON")
# Select csiwaterbody that intersect trace2junctions
arcpy.AddMessage("Beginning connectivity attribution...")
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
"trace2junctions", XY_TOLERANCE,
"NEW_SELECTION")
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'DR_LakeStream'""", "PYTHON")
# Get stream drainage lakes. Either unassigned so far or convert "Headwater" if a permanent stream flows into it,
# which is detected with "non_artificial_end"
arcpy.SelectLayerByAttribute_management(
"out_fc_lyr", "NEW_SELECTION",
'''"{}" IS NULL'''.format(class_field_name))
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'DR_Stream'""", "PYTHON")
if exclude_intermit_flowlines:
arcpy.SelectLayerByAttribute_management(
"out_fc_lyr", "NEW_SELECTION",
'''"{}" = 'Headwater' '''.format(class_field_name))
arcpy.SelectLayerByLocation_management("out_fc_lyr", "INTERSECT",
"non_artificial_end",
XY_TOLERANCE,
"SUBSET_SELECTION")
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'DR_Stream'""", "PYTHON")
# Prevent 'upgrades' due to very odd flow situations, better to stick with calling both classes 'Headwater'
arcpy.SelectLayerByAttribute_management(
"out_fc_lyr", "NEW_SELECTION",
'''"Maximum_Lake_Connectivity" = 'Headwater' AND "Permanent_Lake_Connectivity" = 'DR_Stream' '''
)
arcpy.CalculateField_management("out_fc_lyr", class_field_name,
"""'Headwater'""", "PYTHON")
# Project output once done with both. Switching CRS earlier causes trace problems.
if not exclude_intermit_flowlines:
arcpy.CopyFeatures_management(temp_feature_class, out_feature_class)
else:
arcpy.Project_management(temp_feature_class, out_feature_class,
arcpy.SpatialReference(102039))
# Clean up
for layer in layers_list:
arcpy.Delete_management(layer)
else:
arcpy.Delete_management("in_memory")
arcpy.AddMessage("{} classification is complete.".format(class_field_name))
from arcpy import management as DM
from arcpy import analysis as AN
from csiutils import create_temp_GDB
import lagosGIS
# files accessed by this script
MASTER_CLIPPING_POLY = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\US_Countybased_Clip_Polygon'
LAGOSNE_GDB = r'C:\Users\smithn78\Dropbox\CSI\CSI_LAGOS-exports\LAGOS-NE-EDI\LAGOS-NE-GIS\FileGDB\LAGOS_NE_GIS_Data_v1.0.gdb'
LAND_BORDER = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\Derived_Land_Borders'
COASTLINE = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\TIGER_Coastline'
STATE_FC = r'D:\Continental_Limnology\Data_Downloaded\LAGOS_ZONES_ALL\TIGER_Boundaries\Unzipped Original\tl_2016_us_state.shp'
LAGOS_LAKES = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\LAGOS_US_All_Lakes_1ha'
# files to control this script
ZONE_CONTROL_CSV = r"C:\Users\smithn78\Dropbox\CL_HUB_GEO\Reprocessing_LAGOS_Zones.csv"
TEMP_GDB = create_temp_GDB('process_zones')
arcpy.env.workspace = TEMP_GDB
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(5070)
# read the control file
with open(ZONE_CONTROL_CSV) as csv_file:
reader = csv.DictReader(csv_file)
lines = [line for line in reader]
HU8_OUTPUT = [line for line in lines if line['LAGOS Zone Name'] == 'hu8'][0]['Output']
# Define the processing steps. 1) Dissolve on ID. 2) Clip to USA 3) Calculate original area. 4) Generate unique zone IDs.
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field, other_keep_fields, clip_hu8, lagosne_name):
# dissolve fields by the field that zone_id is based on (the field that identifies a unique zone)
dissolve_fields = [f for f in "{}, {}, {}".format(
from arcpy import management as DM
from arcpy import analysis as AN
from csiutils import create_temp_GDB
import lagosGIS
# files accessed by this script
MASTER_CLIPPING_POLY = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\US_Countybased_Clip_Polygon'
LAGOSNE_GDB = r'C:\Users\smithn78\Dropbox\CSI\CSI_LAGOS-exports\LAGOS-NE-EDI\LAGOS-NE-GIS\FileGDB\LAGOS_NE_GIS_Data_v1.0.gdb'
LAND_BORDER = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\Derived_Land_Borders'
COASTLINE = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\NonPublished\TIGER_Coastline'
STATE_FC = r'D:\Continental_Limnology\Data_Downloaded\LAGOS_ZONES_ALL\TIGER_Boundaries\Unzipped Original\tl_2016_us_state.shp'
LAGOS_LAKES = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.5.gdb\LAGOS_US_All_Lakes_1ha'
# files to control this script
ZONE_CONTROL_CSV = r"C:\Users\smithn78\Dropbox\CL_HUB_GEO\Reprocessing_LAGOS_Zones.csv"
TEMP_GDB = create_temp_GDB('process_zones')
arcpy.env.workspace = TEMP_GDB
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(5070)
# read the control file
with open(ZONE_CONTROL_CSV) as csv_file:
reader = csv.DictReader(csv_file)
lines = [line for line in reader]
HU8_OUTPUT = [line for line in lines
if line['LAGOS Zone Name'] == 'hu8'][0]['Output']
# Define the processing steps. 1) Dissolve on ID. 2) Clip to USA 3) Calculate original area. 4) Generate unique zone IDs.
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field,
def aggregate_watersheds(catchments_fc, nhdplus_gdb, eligible_lakes_fc, output_fc,
mode = ['interlake', 'network', 'both']):
"""
Accumulate upstream watersheds for all eligible lakes in this subregion and save result as a feature class.
Interlake watersheds identify upstream lakes greater than 10 hectares in size as sinks and the accumulated
watershed does not include upstream flow "sunk" into those other lakes.
Network watersheds accumulate all upstream flow with no barriers, until the HU4 boundary. LAGOS network watersheds
do not extend across HU4 boundaries even though water physically flows from the upstream HU4 into this one.
Only lakes found in both the "eligible" lakes feature class and the NHD geodatabase will have watersheds
delineated in the result. (Permanent_Identifier for the lake in both data sources.)
:param catchments_fc: The result of nhdplushr_tools.delineate_catchments().
:param nhdplus_gdb: The NHDPlus HR geodatabase for which watershed accumulations are needed.
:param eligible_lakes_fc: The input lakes for which watershed accumulations are needed.
:param output_fc: A feature class containing the (overlapping) accumulated watersheds results
:param mode: Options = 'network', 'interlake', or 'both. For'interlake' (and 'both'), upstream 10ha+ lakes will
act as sinks in the focal lake's accumulated watershed. 'both' option will output two feature classes, one
with name ending 'interlake', and one with name ending 'network'
:return: ArcGIS Result object(s) for output(s)
"""
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(5070)
arcpy.env.workspace = 'in_memory'
temp_gdb = csiutils.create_temp_GDB('aggregate_watersheds')
# get this huc4
huc4_code = re.search('\d{4}', os.path.basename(nhdplus_gdb)).group()
wbd_hu4 = os.path.join(nhdplus_gdb, "WBDHU4")
field_name = (arcpy.ListFields(wbd_hu4, "HU*4"))[0].name
whereClause4 = """{0} = '{1}'""".format(arcpy.AddFieldDelimiters(nhdplus_gdb, field_name), huc4_code)
hu4 = arcpy.Select_analysis(wbd_hu4, "hu4", whereClause4)
# initialize the network object and build a dictionary we'll use
arcpy.AddMessage("Preparing network traces...")
nhd_network = NHDNetwork(nhdplus_gdb)
# use temp gdb to make indexed, island-free copy of lakes
temp_waterbodies = os.path.join(temp_gdb, 'waterbody_holeless')
# so there's something stupid where ArcGIS doesn't like fields after shape_* but there's no way to move them
# except to do this
waterbody_mem = DM.CopyFeatures(nhd_network.waterbody, 'waterbody_mem')
waterbody_holeless = DM.EliminatePolygonPart(waterbody_mem, temp_waterbodies,
'PERCENT', part_area_percent = '99')
DM.AddIndex(waterbody_holeless, 'Permanent_Identifier', 'permid_idx')
waterbody_lyr = DM.MakeFeatureLayer(waterbody_holeless)
# have to send watersheds to a temp gdb so we can use an index
if not nhd_network.nhdpid_flowline:
nhd_network.map_nhdpid_to_flowlines()
if not nhd_network.nhdpid_waterbody:
nhd_network.map_waterbody_to_nhdpids()
watersheds_fc_copy = DM.CopyFeatures(catchments_fc, 'watersheds_fc_copy')
DM.AddField(watersheds_fc_copy, 'Permanent_Identifier', 'TEXT', field_length = 40)
# add permids to watersheds
with arcpy.da.UpdateCursor(watersheds_fc_copy, ['NHDPlusID', 'Permanent_Identifier']) as u_cursor:
for row in u_cursor:
nhdpid, permid = row
if nhdpid in nhd_network.nhdpid_flowline:
permid = nhd_network.nhdpid_flowline[nhdpid]
elif nhdpid in nhd_network.nhdpid_waterbody:
permid = nhd_network.nhdpid_waterbody[nhdpid]
else:
permid = None # sinks, no permanent identifiers, can't be traced, which is fine.
u_cursor.updateRow((nhdpid, permid))
# dropping extra watersheds fields speeds up dissolve 6X, which we NEED
temp_gdb_watersheds_path = os.path.join(temp_gdb, 'watersheds_simple')
watersheds_simple = lagosGIS.select_fields(watersheds_fc_copy, temp_gdb_watersheds_path, ['Permanent_Identifier'])
DM.AddIndex(watersheds_simple, 'Permanent_Identifier', 'permid_idx')
watersheds_lyr = DM.MakeFeatureLayer(watersheds_simple, 'watersheds_lyr')
# Step 1: intersect eligible_lakes and lakes for this NHD gdb (eligible_lakes can have much larger spatial extent)
# any lake id that doesn't intersect/inner join will be DROPPED and will not get a watershed traced
gdb_wb_permids = {row[0] for row in arcpy.da.SearchCursor(nhd_network.waterbody, 'Permanent_Identifier') if row[0]}
eligible_lake_ids = {row[0] for row in arcpy.da.SearchCursor(eligible_lakes_fc, 'Permanent_Identifier')}
matching_ids = list(gdb_wb_permids.intersection(eligible_lake_ids))
# Step 2: If interlake, get traces for all 10ha+ lakes, so they can be identified and
# erased from interlake watersheds for other (focal) lakes later. If network, do nothing.
if mode in ('interlake', 'both'):
# get list of 10ha+ lakes and get their NETWORK traces
nhd_network.activate_10ha_lake_stops() # get the stop ids
tenha_start_ids = nhd_network.waterbody_stop_ids
nhd_network.set_start_ids(tenha_start_ids)
nhd_network.deactivate_stops()
tenha_traces_no_stops = nhd_network.trace_up_from_waterbody_starts() # with no stops!
# get network traces for all lakes in order to identify "contained" 10ha+ lake subnetworks in step 4
nhd_network.set_start_ids(matching_ids)
traces_no_stops = nhd_network.trace_up_from_waterbody_starts()
# Step 3: run the desired traces according to the mode. trace[id] = list of all flowline IDS in trace
nhd_network.set_start_ids(matching_ids)
traces = nhd_network.trace_up_from_waterbody_starts()
# Step 4: If interlake, identify the 10ha+ lake subnetworks to be erased from the dissolved (holeless) interlake
# watershed, which are the intersection of this lake's full network trace and any upstream 10ha+ lake traces
if mode in ('interlake', 'both'):
minus_traces = dict()
for k, v in traces_no_stops.items():
this_lake_id = k
full_network = set(v) # still missing sinks
# determine if this lake is upstream of any
reset_kv = []
if k in tenha_traces_no_stops:
reset_kv = tenha_traces_no_stops[k]
del tenha_traces_no_stops[k] # take this lake's network out temporarily
# if this lake id in the trace, then the dict entry is for a downstream lake. keep only upstream.
upstream_tenha_subnets = [v1 for k1, v1 in tenha_traces_no_stops.items() if this_lake_id not in v1]
upstream_subnet_ids = {id for id_list in upstream_tenha_subnets for id in id_list}
if reset_kv:
tenha_traces_no_stops[k] = reset_kv # restore this lake's network to the dict
# to be erased: subnetworks
minus_traces[k] = full_network.intersection(upstream_subnet_ids)
# Identify inlets
inlets = set(nhd_network.identify_inlets())
partial_test = {k:set(v).intersection(inlets) for k,v in traces.items()}
arcpy.AddMessage("Accumulating watersheds according to traces...")
counter = 0
single_catchment_ids = []
# The specific recipe for speed in this loop (about 0.8 seconds per loop/drainage lake):
# 1) The watersheds dataset being queried must have an index. (use temp_gdb instead of in_memory above)
# 2) It is faster to AN.Select from a feature layer of the indexed lake/watershed dataset than the dataset itself.
# 3) Dissolve must work on something in_memory (not a selected layer on disk) for a big speed increase.
# 4) Extraneous fields are NOT ignored by Dissolve and slow it down, so they are removed earlier.
# 5) Spatial queries were actually quite fast but picked up extraneous catchments, so we will not use that method.
# 6) Deletions in the loop waste time (1/3 second per loop) and overwriting causes no problems.
# 7) Avoid processing
arcpy.env.overwriteOutput = True
for lake_id in matching_ids:
# Loop Step 2: Determine if the lake is on the network. If not, skip accumulation.
trace_permids = traces[lake_id]
if len(trace_permids) <= 1:
single_catchment_ids.append(lake_id)
else:
# print updates roughly every 5 minutes
counter += 1
if counter % 500 == 0:
print("{} of {} lakes completed...".format(counter, len(matching_ids)))
# Loop Step 1: Fetch this lake
this_lake = AN.Select(waterbody_lyr, 'this_lake', "Permanent_Identifier = '{}'".format(lake_id))
# Loop Step 2: Select catchments with their Permanent_Identifier in the trace
flowline_query = 'Permanent_Identifier IN ({})'.format(','.join(['\'{}\''.format(id)
for id in trace_permids]))
selected_watersheds = AN.Select(watersheds_lyr, 'selected_watersheds', flowline_query)
# Loop Step 3: If lake has upstream connectivity, dissolve catchments and eliminate holes.
# Calculate some metrics of the change.
this_watershed_holes = DM.Dissolve(selected_watersheds, 'this_watershed_holes') # sheds has selection
DM.AddField(this_watershed_holes, 'Permanent_Identifier', 'TEXT', field_length=40)
DM.AddField(this_watershed_holes, 'Sink_Count', 'LONG')
DM.AddField(this_watershed_holes, 'Sink_Area', 'DOUBLE')
DM.AddField(this_watershed_holes, 'Partial_Watershed', 'TEXT', field_length = 1)
# calculations of the fields just added, area is only a placeholder
with arcpy.da.UpdateCursor(this_watershed_holes, ['Permanent_Identifier', 'Sink_Count', 'Sink_Area',
'Partial_Watershed', 'SHAPE@']) as u_cursor:
for row in u_cursor:
permid, count, sink_area, partial, shape = row
permid = lake_id
count = shape.boundary().partCount - shape.partCount
# if inlets are in the trace, the watershed is only partial
if partial_test[this_lake_id]:
partial = 'Y'
else:
partial = 'N'
sink_area = shape.area # temp value, will subtract later
u_cursor.updateRow((permid, count, sink_area, partial, shape))
no_holes = DM.EliminatePolygonPart(this_watershed_holes, 'no_holes', 'PERCENT', part_area_percent='99.999')
DM.CalculateField(no_holes, 'Sink_Area', '(!shape.area!-!Sink_Area!)/10000', 'PYTHON') # subtract / ha
# Loop Step 4: Erase the lake from its own shed
lakeless_watershed = arcpy.Erase_analysis(no_holes, this_lake, 'lakeless_watershed')
# Loop Step 5: If interlake and there are things to erase, erase upstream 10ha+ lake subnetworks
# (after dissolving them to remove holes). This erasing pattern allows us to remove other sinks only.
if mode in ('interlake', 'both') and minus_traces[lake_id]:
#select matching watersheds
tenha_subnetworks_query = 'Permanent_Identifier IN ({})'.format(','.join(['\'{}\''.format(id)
for id in minus_traces[lake_id]]))
other_tenha = AN.Select(watersheds_lyr, 'other_tenha', tenha_subnetworks_query)
# since subnetworks can be hole-y due to sinks also, need to dissolve before erasing
other_tenha_dissolved = DM.Dissolve(other_tenha, 'other_tenha_dissolved')
other_tenha_holeless = DM.EliminatePolygonPart(other_tenha_dissolved, 'other_tenha_holeless', 'PERCENT',
part_area_percent='99.999')
this_watershed = arcpy.Erase_analysis(lakeless_watershed, other_tenha_holeless, 'this_watershed')
else:
this_watershed = lakeless_watershed
# Loop Step 7: Save to results
if not arcpy.Exists('merged_fc'):
merged_fc = DM.CopyFeatures(this_watershed, 'merged_fc')
# to avoid append mismatch due to permanent_identifier
DM.AlterField(merged_fc, 'Permanent_Identifier', field_length=40)
else:
DM.Append(this_watershed, merged_fc, 'NO_TEST')
if mode == 'both':
# then lakeless_watershed contains the network shed
if not arcpy.Exists('merged_fc_both_network'):
merged_fc_both_network = DM.CopyFeatures(lakeless_watershed, 'merged_fc_both_network')
# to avoid append mismatch due to permanent_identifier
DM.AlterField(merged_fc_both_network, 'Permanent_Identifier', field_length=40)
else:
DM.Append(lakeless_watershed, merged_fc_both_network)
arcpy.env.overwriteOutput = False
# Step 5: For all isolated lakes, select the correct catchments and erase focal lakes from their own sheds
# in one operation (to save time in loop above)
arcpy.AddMessage("Batch processing remaining lakes...")
if single_catchment_ids:
waterbodies_query = 'Permanent_Identifier IN ({})'.format(','.join(['\'{}\''.format(id) for id in single_catchment_ids]))
these_lakes = AN.Select(waterbody_lyr, 'these_lakes', waterbodies_query)
watersheds_query = 'Permanent_Identifier IN ({})'.format(','.join(['\'{}\''.format(id)
for id in single_catchment_ids]))
these_watersheds = AN.Select(watersheds_simple, 'these_watersheds', watersheds_query)
lakeless_watersheds = AN.Erase(these_watersheds, these_lakes, 'lakeless_watersheds')
DM.Append(lakeless_watersheds, merged_fc, 'NO_TEST')
if mode == 'both':
DM.Append(lakeless_watersheds, merged_fc_both_network, 'NO_TEST')
if mode in ('interlake', 'both'):
# Remove isolated 10 ha sinks--union and select out any where permids unequal (holes). Dissolve back to one.
# Roundabout way of saying erase 10ha catchments from me EXCEPT my own catchment.
tenha_local_query = 'Permanent_Identifier IN ({})'.format(','.join(['\'{}\''.format(id)
for id in tenha_start_ids]))
tenha_watersheds = AN.Select(watersheds_simple, 'tenha_watersheds', tenha_local_query)
union = AN.Union([merged_fc, tenha_watersheds], 'union')
sink_drop = AN.Select(union, 'sink_drop', "Permanent_Identifier = Permanent_Identifier_1 OR Permanent_Identifier_1 = ''")
dissolve_fields = ['Permanent_Identifier',
'Sink_Count',
'Sink_Area',
'Partial_Watershed']
sink_drop_dissolved = DM.Dissolve(sink_drop, 'sink_drop_dissolved', dissolve_fields)
merged_fc = sink_drop_dissolved
for item in [tenha_watersheds, union, sink_drop]:
DM.Delete(item)
result_fcs = [merged_fc]
if mode == 'both':
result_fcs.append(merged_fc_both_network)
# calculate the iws = network flag: Y if areas are equal, N if not equal.
DM.AddField(merged_fc, 'Equals_Network_Watershed', 'TEXT', field_length=1)
network_areas = {r[0]:r[1] for r in arcpy.da.SearchCursor(
merged_fc_both_network, ['Permanent_Identifier', 'SHAPE@area'])}
with arcpy.da.UpdateCursor(merged_fc, ['Permanent_Identifier',
'SHAPE@area', 'Equals_Network_Watershed']) as u_cursor:
for row in u_cursor:
permid, area, flag = row
if abs(network_areas[permid] - area) < .5: # if areas are equal
flag = 'Y'
else:
flag = 'N'
u_cursor.updateRow((permid, area, flag))
# Fill in all the missing flag values
for fc in result_fcs:
with arcpy.da.UpdateCursor(fc, ['Permanent_Identifier', 'Sink_Count',
'Sink_Area', 'Partial_Watershed']) as u_cursor:
for row in u_cursor:
permid, count, area, partial = row
if not count:
count = 0
area = 0
if not partial:
partial = 'N'
u_cursor.updateRow((permid, count, area, partial))
DM.DeleteField(fc, 'ORIG_FID')
# Clean up results a bit and output. Eliminate slivers smaller than NHD raster cell, clip to HU4, output
refined1 = DM.EliminatePolygonPart(merged_fc, 'refined1', 'AREA', part_area='99', part_option='ANY')
result1 = AN.Clip(refined1, hu4, output_fc)
if mode == 'both':
refined2 = DM.EliminatePolygonPart(merged_fc_both_network, 'refined2', 'AREA',
part_area='99', part_option='ANY')
result1 = DM.Rename(output_fc, output_fc + '_interlake')
result2 = AN.Clip(merged_fc_both_network, hu4, output_fc + '_network')
for item in [merged_fc_both_network, refined2]:
DM.Delete(item)
# Delete work: first fcs to free up temp_gdb, then temp_gdb
for item in [hu4, waterbody_holeless, watersheds_simple, watersheds_fc_copy, watersheds_lyr, merged_fc]:
DM.Delete(item)
DM.Delete(temp_gdb)
if mode == 'both':
return (result1, result2)
else:
return result1
def classify_lakes(nhd,
out_feature_class,
exclude_intermit_flowlines=False,
debug_mode=False):
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('classify_lake_connectivity')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
if arcpy.Exists("temp_fc"):
print("There is a problem here.")
raise Exception
# Tool temporary feature classes
temp_fc = "temp_fc"
csiwaterbody_10ha = "csiwaterbody_10ha"
nhdflowline_filtered = "nhdflowline_filtered"
dangles = "dangles"
start = "start"
end = "end"
startdangles = "startdangles"
enddangles = "enddangles"
non_artificial_end = "non_artificial_end"
flags_10ha_lake_junctions = "flags_10ha_lake_junctions"
midvertices = "midvertices"
non10vertices = "non10vertices"
non10junctions = "non10junctions"
all_non_flag_points = "all_non_flag_points"
barriers = "barriers"
trace1_junctions = "trace1_junctions"
trace1_flowline = "trace1_flowline"
trace2_junctions = "trace2junctions"
trace2_flowline = "trace2_flowline"
# Clean up workspace in case of bad exit from prior run in same session.
this_tool_layers = [
"dangles_lyr", "nhdflowline_lyr", "junction_lyr", "midvertices_lyr",
"all_non_flag_points_lyr", "non10vertices_lyr", "out_fc_lyr", "trace1",
"trace2"
]
this_tool_temp = [
temp_fc, csiwaterbody_10ha, nhdflowline_filtered, dangles, start, end,
startdangles, enddangles, non_artificial_end,
flags_10ha_lake_junctions, midvertices, non10vertices, non10junctions,
all_non_flag_points, barriers, trace1_junctions, trace1_flowline,
trace2_junctions, trace2_flowline
]
for item in this_tool_layers + this_tool_temp:
try:
DM.Delete(item)
except:
pass
# Local variables:
nhdflowline = os.path.join(nhd, "Hydrography", "NHDFLowline")
nhdjunction = os.path.join(nhd, "Hydrography", "HYDRO_NET_Junctions")
nhdwaterbody = os.path.join(nhd, "Hydrography", "NHDWaterbody")
network = os.path.join(nhd, "Hydrography", "HYDRO_NET")
# Get lakes, ponds and reservoirs over a hectare.
#csi_population_filter = '''"AreaSqKm" >=0.01 AND\
#"FCode" IN (39000,39004,39009,39010,39011,39012,43600,43613,43615,43617,43618,43619,43621)'''
all_lakes_reservoirs_filter = '''"FType" IN (390, 436)'''
# Can't see why we shouldn't just attribute all lakes and reservoirs
# arcpy.Select_analysis(nhdwaterbody, "csiwaterbody", lake_population_filter)
arcpy.AddMessage("Initializing output.")
if exclude_intermit_flowlines:
DM.CopyFeatures(out_feature_class, temp_fc)
DM.Delete(out_feature_class)
else:
arcpy.Select_analysis(nhdwaterbody, temp_fc,
all_lakes_reservoirs_filter)
# Get lakes, ponds and reservoirs over 10 hectares.
lakes_10ha_filter = '''"AreaSqKm" >= 0.1 AND "FType" IN (390, 436)'''
arcpy.Select_analysis(nhdwaterbody, csiwaterbody_10ha, lakes_10ha_filter)
# Exclude intermittent flowlines, if requested
if exclude_intermit_flowlines:
flowline_where_clause = '''"FCode" NOT IN (46003,46007)'''
nhdflowline = arcpy.Select_analysis(nhdflowline, nhdflowline_filtered,
flowline_where_clause)
# Make dangle points at end of nhdflowline
DM.FeatureVerticesToPoints(nhdflowline, dangles, "DANGLE")
DM.MakeFeatureLayer(dangles, "dangles_lyr")
# Isolate start dangles from end dangles.
DM.FeatureVerticesToPoints(nhdflowline, start, "START")
DM.FeatureVerticesToPoints(nhdflowline, end, "END")
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", start)
DM.CopyFeatures("dangles_lyr", startdangles)
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", end)
DM.CopyFeatures("dangles_lyr", enddangles)
# Special handling for lakes that have some intermittent flow in and some permanent
if exclude_intermit_flowlines:
DM.MakeFeatureLayer(nhdflowline, "nhdflowline_lyr")
DM.SelectLayerByAttribute("nhdflowline_lyr", "NEW_SELECTION",
'''"WBArea_Permanent_Identifier" is null''')
DM.FeatureVerticesToPoints("nhdflowline_lyr", non_artificial_end,
"END")
DM.SelectLayerByAttribute("nhdflowline_lyr", "CLEAR_SELECTION")
arcpy.AddMessage("Found source area nodes.")
# Get junctions from lakes >= 10 hectares.
DM.MakeFeatureLayer(nhdjunction, "junction_lyr")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT", csiwaterbody_10ha,
XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("junction_lyr", flags_10ha_lake_junctions)
arcpy.AddMessage("Found lakes >= 10 ha.")
# Make points shapefile and layer at flowline vertices to act as potential flags and/or barriers.
arcpy.AddMessage("Tracing...")
DM.FeatureVerticesToPoints(nhdflowline, midvertices, "MID")
DM.MakeFeatureLayer(midvertices, "midvertices_lyr")
# Get vertices that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("midvertices_lyr", non10vertices)
# Get junctions that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("junction_lyr", non10junctions)
# Merge non10vertices with non10junctions
DM.Merge([non10junctions, non10vertices],
all_non_flag_points) # inputs both point fc in_memory
DM.MakeFeatureLayer(all_non_flag_points, "all_non_flag_points_lyr")
# Tests the counts...for some reason I'm not getting stable behavior from the merge.
mid_n = int(DM.GetCount(non10vertices).getOutput(0))
jxn_n = int(DM.GetCount(non10junctions).getOutput(0))
merge_n = int(DM.GetCount(all_non_flag_points).getOutput(0))
if merge_n < mid_n + jxn_n:
arcpy.AddWarning(
"The total number of flags ({0}) is less than the sum of the input junctions ({1}) "
"and input midpoints ({2})".format(merge_n, jxn_n, mid_n))
# For tracing barriers, select all_non_flag_points points that intersect a 10 ha lake.
DM.SelectLayerByLocation("all_non_flag_points_lyr", "INTERSECT",
csiwaterbody_10ha, XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("all_non_flag_points_lyr", barriers)
# Trace1-Trace downstream to first barrier (junctions+midvertices in 10 ha lake) starting from flags_10ha_lake_junctions flag points.
DM.TraceGeometricNetwork(network, "trace1", flags_10ha_lake_junctions,
"TRACE_DOWNSTREAM", barriers)
# Save trace1 flowlines and junctions to layers on disk.
DM.CopyFeatures("trace1\HYDRO_NET_Junctions",
trace1_junctions) # extra for debugging
DM.CopyFeatures("trace1\NHDFlowline", trace1_flowline)
# Select vertice midpoints that intersect trace1 flowlines selection for new flags for trace2.
DM.MakeFeatureLayer(non10vertices, "non10vertices_lyr")
DM.SelectLayerByLocation("non10vertices_lyr", "INTERSECT", trace1_flowline,
"", "NEW_SELECTION")
# Trace2-Trace downstream from midpoints of flowlines that intersect the selected flowlines from trace1.
DM.TraceGeometricNetwork(network, "trace2", "non10vertices_lyr",
"TRACE_DOWNSTREAM")
# Save trace1 flowlines and junctions to layers and then shapes on disk.
DM.CopyFeatures("trace2\HYDRO_NET_Junctions", trace2_junctions)
DM.CopyFeatures("trace2\NHDFlowline",
trace2_flowline) # extra for debugging
arcpy.AddMessage("Done tracing.")
# Make shapefile for seepage lakes. (Ones that don't intersect flowlines)
if exclude_intermit_flowlines:
class_field_name = "Lake_Connectivity_Permanent"
else:
class_field_name = "Lake_Connectivity_Class"
DM.AddField(temp_fc, class_field_name, "TEXT", field_length=13)
DM.MakeFeatureLayer(temp_fc, "out_fc_lyr")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline, "",
"SWITCH_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# New type of "Isolated" classification, mostly for "permanent" but there were some oddballs in "maximum" too
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", enddangles,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# Get headwater lakes.
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByAttribute(
"out_fc_lyr", "REMOVE_FROM_SELECTION",
'''"{}" = 'Isolated' '''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# Select csiwaterbody that intersect trace2junctions
arcpy.AddMessage("Beginning connectivity attribution...")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", trace2_junctions,
XY_TOLERANCE, "NEW_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'DrainageLk'""",
"PYTHON")
# Get stream drainage lakes. Either unassigned so far or convert "Headwater" if a permanent stream flows into it,
# which is detected with "non_artificial_end"
DM.SelectLayerByAttribute("out_fc_lyr", "NEW_SELECTION",
'''"{}" IS NULL'''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
if exclude_intermit_flowlines:
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"{}" = 'Headwater' '''.format(class_field_name))
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", non_artificial_end,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
# Prevent 'upgrades' due to very odd flow situations and artifacts of bad digitization. The effects of these
# are varied--to avoid confusion, just keep the class assigned with all flowlines
# 1--Purely hypothetical, not seen in testing
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Isolated' AND "Lake_Connectivity_Permanent" <> 'Isolated' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# 2--Headwater to Drainage upgrade seen in testing with odd multi-inlet flow situation
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Headwater' AND "Lake_Connectivity_Permanent" IN ('Drainage', 'DrainageLk') '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# 3--Drainage to DrainageLk upgrade seen in testing when intermittent stream segments were used
# erroneously instead of artificial paths
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Drainage' AND "Lake_Connectivity_Permanent" = 'DrainageLk' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
DM.SelectLayerByAttribute("out_fc_lyr", "CLEAR_SELECTION")
# Add change flag for users
DM.AddField(temp_fc,
"Lake_Connectivity_Fluctuates",
"Text",
field_length="1")
flag_codeblock = """def flag_calculate(arg1, arg2):
if arg1 == arg2:
return 'N'
else:
return 'Y'"""
expression = 'flag_calculate(!Lake_Connectivity_Class!, !Lake_Connectivity_Permanent!)'
DM.CalculateField(temp_fc, "Lake_Connectivity_Fluctuates", expression,
"PYTHON", flag_codeblock)
# Project output once done with both. Switching CRS earlier causes trace problems.
if not exclude_intermit_flowlines:
DM.CopyFeatures(temp_fc, out_feature_class)
else:
DM.Project(temp_fc, out_feature_class, arcpy.SpatialReference(102039))
# Clean up
if not debug_mode:
for item in this_tool_layers + this_tool_temp:
if arcpy.Exists(item):
DM.Delete(item)
if not debug_mode:
DM.Delete("trace1")
DM.Delete("trace2")
arcpy.AddMessage("{} classification is complete.".format(class_field_name))
def handle_overlaps(zone_fc,
zone_field,
in_value_raster,
out_table,
is_thematic,
unflat_table='',
rename_tag='',
units='',
debug_mode=False):
orig_env = env.workspace
if debug_mode:
env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('zonal_tabarea')
env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
env.workspace = 'in_memory'
arcpy.SetLogHistory(False)
arcpy.CheckOutExtension("Spatial")
def stats_area_table(zone_fc=zone_fc,
zone_field=zone_field,
in_value_raster=in_value_raster,
out_table=out_table,
is_thematic=is_thematic):
def refine_zonal_output(t):
"""Makes a nicer output for this tool. Rename some fields, drop unwanted
ones, calculate percentages using raster AREA before deleting that
field."""
if is_thematic:
value_fields = arcpy.ListFields(t, "VALUE*")
pct_fields = [
'{}_pct'.format(f.name) for f in value_fields
] # VALUE_41_pct, etc. Field can't start with number.
# add all the new fields needed
for f, pct_field in zip(value_fields, pct_fields):
arcpy.AddField_management(t, pct_field, f.type)
# calculate the percents
cursor_fields = ['AREA'] + [f.name
for f in value_fields] + pct_fields
uCursor = arcpy.da.UpdateCursor(t, cursor_fields)
for uRow in uCursor:
# unpacks area + 3 tuples of the right fields for each, no matter how many there are
vf_i_end = len(value_fields) + 1
pf_i_end = vf_i_end + len(pct_fields)
# pct_values and ha_values are both null at this point but unpack for clarity
area, value_values, pct_values = uRow[0], uRow[
1:vf_i_end], uRow[vf_i_end:pf_i_end]
new_pct_values = [100 * vv / area for vv in value_values]
new_row = [area] + value_values + new_pct_values
uCursor.updateRow(new_row)
for vf in value_fields:
arcpy.DeleteField_management(t, vf.name)
arcpy.AlterField_management(t, 'COUNT', 'CELL_COUNT')
drop_fields = ['ZONE_CODE', 'COUNT', 'AREA']
if not debug_mode:
for df in drop_fields:
try:
arcpy.DeleteField_management(t, df)
except:
continue
# Set up environments for alignment between zone raster and theme raster
if isinstance(zone_fc, arcpy.Result):
zone_fc = zone_fc.getOutput(0)
this_files_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(this_files_dir)
common_grid = os.path.abspath('../common_grid.tif')
env.snapRaster = common_grid
env.cellSize = common_grid
env.extent = zone_fc
zone_desc = arcpy.Describe(zone_fc)
zone_raster = 'convertraster'
if zone_desc.dataType not in ['RasterDataset', 'RasterLayer']:
zone_raster = arcpy.PolygonToRaster_conversion(
zone_fc,
zone_field,
zone_raster,
'CELL_CENTER',
cellsize=env.cellSize)
print('cell size is {}'.format(env.cellSize))
zone_size = int(env.cellSize)
else:
zone_raster = zone_fc
zone_size = min(
arcpy.Describe(zone_raster).meanCellHeight,
arcpy.Describe(zone_raster).meanCellWidth)
raster_size = min(
arcpy.Describe(in_value_raster).meanCellHeight,
arcpy.Describe(in_value_raster).meanCellWidth)
env.cellSize = min([zone_size, raster_size])
print('cell size is {}'.format(env.cellSize))
# I tested and there is no need to resample the raster being summarized. It will be resampled correctly
# internally in the following tool given that the necessary environments are set above (cell size, snap).
# # in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE)
if not is_thematic:
arcpy.AddMessage("Calculating Zonal Statistics...")
temp_entire_table = arcpy.sa.ZonalStatisticsAsTable(
zone_raster, zone_field, in_value_raster, 'temp_zonal_table',
'DATA', 'MEAN')
if is_thematic:
# for some reason env.cellSize doesn't work
# calculate/doit
arcpy.AddMessage("Tabulating areas...")
temp_entire_table = arcpy.sa.TabulateArea(
zone_raster,
zone_field,
in_value_raster,
'Value',
'temp_area_table',
processing_cell_size=env.cellSize)
# TabulateArea capitalizes the zone for some annoying reason and ArcGIS is case-insensitive to field names
# so we have this work-around:
zone_field_t = '{}_t'.format(zone_field)
DM.AddField(temp_entire_table,
zone_field_t,
'TEXT',
field_length=20)
expr = '!{}!'.format(zone_field.upper())
DM.CalculateField(temp_entire_table, zone_field_t, expr, 'PYTHON')
DM.DeleteField(temp_entire_table, zone_field.upper())
DM.AlterField(temp_entire_table,
zone_field_t,
zone_field,
clear_field_alias=True)
# replaces join to Zonal Stats in previous versions of tool
# no joining, just calculate the area/count from what's produced by TabulateArea
arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE')
cursor_fields = ['AREA', 'COUNT']
value_fields = [
f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*')
]
cursor_fields.extend(value_fields)
with arcpy.da.UpdateCursor(temp_entire_table,
cursor_fields) as uCursor:
for uRow in uCursor:
area, count, value_fields = uRow[0], uRow[1], uRow[2:]
area = sum(value_fields)
count = round(
area / (int(env.cellSize) * int(env.cellSize)), 0)
new_row = [area, count] + value_fields
uCursor.updateRow(new_row)
arcpy.AddMessage("Refining output table...")
arcpy.AddField_management(temp_entire_table, 'datacoveragepct',
'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'ORIGINAL_COUNT', 'LONG')
# calculate datacoveragepct by comparing to original areas in zone raster
# alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count
zone_raster_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(zone_raster,
[zone_field, 'Count'])
}
temp_entire_table_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(temp_entire_table,
[zone_field, 'COUNT'])
}
sum_cell_area = float(env.cellSize) * float(env.cellSize)
orig_cell_area = zone_size * zone_size
with arcpy.da.UpdateCursor(
temp_entire_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT']) as cursor:
for uRow in cursor:
key_value, data_pct, count_orig = uRow
count_orig = zone_raster_dict[key_value]
if key_value in temp_entire_table_dict:
count_summarized = temp_entire_table_dict[key_value]
data_pct = 100 * float((count_summarized * sum_cell_area) /
(count_orig * orig_cell_area))
else:
data_pct = None
cursor.updateRow((key_value, data_pct, count_orig))
# Refine the output
refine_zonal_output(temp_entire_table)
# in order to add vector capabilities back, need to do something with this
# right now we just can't fill in polygon zones that didn't convert to raster in our system
stats_result = cu.one_in_one_out(temp_entire_table, zone_fc,
zone_field, out_table)
# Convert "datacoveragepct" and "ORIGINAL_COUNT" values to 0 for zones with no metrics calculated
with arcpy.da.UpdateCursor(
out_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT', 'CELL_COUNT'
]) as u_cursor:
for row in u_cursor:
# data_coverage pct to 0
if row[1] is None:
row[1] = 0
# original count filled in if a) zone outside raster bounds or b) zone too small to be rasterized
if row[2] is None:
if row[0] in zone_raster_dict:
row[2] = zone_raster_dict[row[0]]
else:
row[2] = 0
# cell count set to 0
if row[3] is None:
row[3] = 0
u_cursor.updateRow(row)
# count whether all zones got an output record or not)
out_count = int(
arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
count_diff = in_count - out_count
# cleanup
if not debug_mode:
for item in [
'temp_zonal_table', temp_entire_table, 'convertraster'
]: # don't add zone_raster, orig
arcpy.Delete_management(item)
arcpy.ResetEnvironments()
env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch
arcpy.CheckInExtension("Spatial")
return [stats_result, count_diff]
def unflatten(intermediate_table):
flat_zoneid = zone_field
unflat_zoneid = zone_field.replace('flat', '')
zone_type = [f.type for f in arcpy.ListFields(zone_fc, flat_zoneid)][0]
# Set up the output table (can't do this until the prior tool is run)
# if os.path.dirname(out_table):
# out_path = os.path.dirname(out_table)
# else:
# out_path = orig_env
unflat_result = DM.CreateTable('in_memory',
os.path.basename(out_table))
# get the fields to add to the table
editable_fields = [
f for f in arcpy.ListFields(intermediate_table)
if f.editable and f.name.lower() != flat_zoneid.lower()
]
# populate the new table schema
DM.AddField(unflat_result, unflat_zoneid, zone_type)
for f in editable_fields:
DM.AddField(unflat_result, f.name, f.type, field_length=f.length)
# map original zone ids to new zone ids
original_flat = defaultdict(list)
with arcpy.da.SearchCursor(unflat_table,
[unflat_zoneid, flat_zoneid]) as cursor:
for row in cursor:
if row[1] not in original_flat[row[0]]:
original_flat[row[0]].append(row[1])
# Use CELL_COUNT as weight for means to calculate final values for each zone.
fixed_fields = [
unflat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
]
other_field_names = [
f.name for f in editable_fields if f.name not in fixed_fields
]
i_cursor = arcpy.da.InsertCursor(
unflat_result,
fixed_fields + other_field_names) # open output table cursor
flat_stats = {
r[0]: r[1:]
for r in arcpy.da.SearchCursor(intermediate_table, [
flat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
] + other_field_names)
}
count_diff = 0
for zid, unflat_ids in original_flat.items():
valid_unflat_ids = [id for id in unflat_ids if id in flat_stats
] # skip flatpolys not rasterized
area_vec = [flat_stats[id][0] for id in valid_unflat_ids
] # ORIGINAL_COUNT specified in 0 index earlier
cell_vec = [flat_stats[id][1] for id in valid_unflat_ids]
coverage_vec = [flat_stats[id][2] for id in valid_unflat_ids
] # datacoveragepct special handling
stat_vectors_by_id = [
flat_stats[id][3:] for id in valid_unflat_ids
] # "the rest", list of lists
# calc the new summarized values
original_count = sum(
filter(None, area_vec)
) # None area is functionally equivalent to 0, all Nones = 0 too
cell_count = sum(filter(None, cell_vec))
if cell_count > 0:
weighted_coverage = sum(
[a * b
for a, b in zip(area_vec, coverage_vec)]) / original_count
# this calculation accounts for fractional missing values, both kinds (whole zone is no data, or zone
# was missing some data and had data coverage % < 100). This is done by converting None to 0
# and by using the cell_count (count of cells with data present)
# instead of the full zone original_count. You have to do both or the mean will be distorted.
# hand-verification that this works as intended using test GIS data on was completed 2019-11-01 by NJS
crossprods = []
for i in range(0, len(valid_unflat_ids)):
crossprods.append([
cell_vec[i] * float(s or 0)
for s in stat_vectors_by_id[i]
])
weighted_stat_means = []
for i in range(0, len(other_field_names)):
weighted_stat_means.append(
sum(zip(*crossprods)[i]) / cell_count)
else:
weighted_coverage = 0
weighted_stat_means = [None] * len(other_field_names)
count_diff += 1
new_row = [zid, original_count, cell_count, weighted_coverage
] + weighted_stat_means
i_cursor.insertRow(new_row)
del i_cursor
DM.Delete(intermediate_table)
return [unflat_result, count_diff]
def rename_to_standard(table):
arcpy.AddMessage("Renaming.")
# datacoverage just gets tag
new_datacov_name = '{}_datacoveragepct'.format(rename_tag)
cu.rename_field(table,
'datacoveragepct',
new_datacov_name,
deleteOld=True)
# DM.AlterField(out_table, 'datacoveragepct', new_datacov_name, clear_field_alias=True)
if not is_thematic:
new_mean_name = '{}_{}'.format(rename_tag, units).rstrip(
'_') # if no units, just rename_tag
cu.rename_field(table, 'MEAN', new_mean_name, deleteOld=True)
# DM.AlterField(out_table, 'MEAN', new_mean_name, clear_field_alias=True)
else:
# look up the values based on the rename tag
geo_file = os.path.abspath('../geo_metric_provenance.csv')
with open(geo_file) as csv_file:
reader = csv.DictReader(csv_file)
mapping = {
row['subgroup_original_code']: row['subgroup']
for row in reader if row['main_feature']
and row['main_feature'] in rename_tag
}
print(mapping)
# update them
for old, new in mapping.items():
old_fname = 'VALUE_{}_pct'.format(old)
new_fname = '{}_{}_pct'.format(rename_tag, new)
if arcpy.ListFields(table, old_fname):
try:
# same problem with AlterField limit of 31 characters here.
DM.AlterField(table,
old_fname,
new_fname,
clear_field_alias=True)
except:
cu.rename_field(table,
old_fname,
new_fname,
deleteOld=True)
return table
if unflat_table:
if not arcpy.Exists(unflat_table):
raise Exception('Unflat_table must exist.')
intermediate_stats = stats_area_table(out_table='intermediate_stats')
named_as_original = unflatten(intermediate_stats[0])
else:
named_as_original = stats_area_table(out_table='named_as_original')
if rename_tag:
named_as_standard = rename_to_standard(named_as_original[0])
out_table = DM.CopyRows(named_as_standard, out_table)
else:
out_table = DM.CopyRows(named_as_original[0], out_table)
total_count_diff = named_as_original[1]
if total_count_diff > 0:
warn_msg = (
"WARNING: {0} zones have null zonal statistics. There are 2 possible reasons:\n"
"1) Presence of zones that are fully outside the extent of the raster summarized.\n"
"2) Zones are too small relative to the raster resolution.".format(
total_count_diff))
arcpy.AddWarning(warn_msg)
arcpy.SetLogHistory(True)
return out_table
def polygons_in_zones(zone_fc,
zone_field,
polygons_of_interest,
output_table,
interest_selection_expr,
contrib_area=True):
old_workspace = arcpy.env.workspace
arcpy.env.workspace = 'in_memory'
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039)
temp_polyzones = cu.create_temp_GDB('temp_polyzones')
selected_polys = os.path.join(temp_polyzones, 'selected_polys')
cu.multi_msg('Copying/selecting polygon features...')
if interest_selection_expr:
arcpy.Select_analysis(polygons_of_interest, selected_polys,
interest_selection_expr)
else:
arcpy.CopyFeatures_management(polygons_of_interest, selected_polys)
arcpy.AddField_management(selected_polys, 'POLYAREA_ha', 'DOUBLE')
arcpy.CalculateField_management(selected_polys, 'POLYAREA_ha',
'!shape.area@hectares!', 'PYTHON')
# use tabulate intersection for the areas overlapping
tab_table = 'tabulate_intersection_table'
cu.multi_msg('Tabulating intersection between zones and polygons...')
arcpy.TabulateIntersection_analysis(zone_fc, zone_field, selected_polys,
tab_table)
# area was calculated in map units which was m2 so convert to hectares
arcpy.AddField_management(tab_table, 'Poly_Overlapping_AREA_ha', 'DOUBLE')
arcpy.CalculateField_management(tab_table, 'Poly_Overlapping_AREA_ha',
'!AREA!/10000', 'PYTHON')
# just change the name of the percent field
cu.rename_field(tab_table, 'PERCENTAGE', 'Poly_Overlapping_AREA_pct', True)
spjoin_fc = 'spatial_join_output'
# Spatial join for the count and contributing area
fms = arcpy.FieldMappings()
fm_zone_id = arcpy.FieldMap()
fm_zone_id.addInputField(zone_fc, zone_field)
fm_count = arcpy.FieldMap()
fm_count.addInputField(selected_polys, 'POLYAREA_ha')
count_name = fm_count.outputField
count_name.name = 'Poly_Count'
count_name.alias = 'Poly_Count'
fm_count.outputField = count_name
fm_count.mergeRule = 'Count'
fm_contrib_area = arcpy.FieldMap()
fm_contrib_area.addInputField(selected_polys, 'POLYAREA_ha')
contrib_area_name = fm_contrib_area.outputField
contrib_area_name.name = 'Poly_Contributing_AREA_ha'
contrib_area_name.alias = 'Poly_Contributing_AREA_ha'
fm_contrib_area.outputField = contrib_area_name
fm_contrib_area.mergeRule = 'Sum'
fms.addFieldMap(fm_zone_id)
fms.addFieldMap(fm_count)
fms.addFieldMap(fm_contrib_area)
cu.multi_msg('Spatial join between zones and wetlands...')
arcpy.SpatialJoin_analysis(zone_fc, selected_polys, spjoin_fc,
"JOIN_ONE_TO_ONE", "KEEP_ALL", fms, "INTERSECT")
cu.multi_msg('Refining output...')
arcpy.JoinField_management(tab_table, zone_field, spjoin_fc, zone_field,
["Poly_Count", "Poly_Contributing_AREA_ha"])
final_fields = [
'Poly_Overlapping_AREA_ha', 'Poly_Overlapping_AREA_pct', 'Poly_Count',
'Poly_Contributing_AREA_ha'
]
# make output nice
cu.one_in_one_out(tab_table, final_fields, zone_fc, zone_field,
output_table)
cu.redefine_nulls(output_table, final_fields, [0, 0, 0, 0])
# clean up
for item in [selected_polys, tab_table, spjoin_fc]:
arcpy.Delete_management(item)
arcpy.Delete_management(temp_polyzones)
arcpy.env.workspace = old_workspace
cu.multi_msg('Polygons in zones tool complete.')
def create_csi_watersheds(flowdir, pour_dir, nhd_gdb, out_gdb):
# Starting environmental variables:
env.extent = flowdir
env.snapRaster = flowdir
env.cellSize = 10
env.outputCoordinateSystem = arcpy.SpatialReference(102039)
arcpy.CheckOutExtension('Spatial')
huc8_code = re.search('\d{8}', os.path.basename(flowdir)).group()
huc4_code = re.search('\d{4}', os.path.basename(nhd_gdb)).group()
# create temp directory because we need shape geometry
temp_gdb = cu.create_temp_GDB('watersheds' + huc4_code)
cu.multi_msg("Temp geodatabase is located at {}".format(temp_gdb))
env.workspace = temp_gdb
wbd_hu8 = os.path.join(nhd_gdb, "WBD_HU8")
field_name = (arcpy.ListFields(wbd_hu8, "HU*8"))[0].name
whereClause8 = """{0} = '{1}'""".format(
arcpy.AddFieldDelimiters(nhd_gdb, field_name), huc8_code)
arcpy.Select_analysis(wbd_hu8, "hu8", whereClause8)
arcpy.Buffer_analysis("hu8", "hu8_buffered", "100 meters")
# Create the basic watersheds
pour_points = os.path.join(pour_dir, 'pour_points.tif')
arcpy.Clip_management(pour_points, '', "pour_points_clipped",
"hu8_buffered", '0', 'ClippingGeometry')
raw_watersheds = os.path.join(
out_gdb, 'huc{}_watersheds_precursors'.format(huc8_code))
cu.multi_msg("Calculating preliminary watersheds...")
outWatershed = Watershed(flowdir, "pour_points_clipped")
outWatershed.save(raw_watersheds)
cu.multi_msg(
"Clipping watersheds to subregion boundaries and filtering spurious watersheds..."
)
# Watershed raster to polygons
arcpy.RasterToPolygon_conversion(raw_watersheds, "wspoly1", 'NO_SIMPLIFY',
"Value")
# Clip watershed polygons to subregion polys.
arcpy.Clip_analysis("wspoly1", "hu8", "wsclip1")
# Clip watershed
## # Calculate hectares
## arcpy.AddField_management("wsclip1", "HA", "DOUBLE")
## arcpy.CalculateField_management("wsclip1", "HA", '''!shape.area@hectares!''', "PYTHON")
# Create fc of watershed polygons >= 1 ha that coincide with seed lines and polys.
seedline = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_flowlines')
seedpoly = os.path.join(pour_dir, 'pourpoints.gdb', 'eligible_lakes')
arcpy.MakeFeatureLayer_management("wsclip1", "wsclip1_lyr")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT",
seedline, '', "NEW_SELECTION")
arcpy.SelectLayerByLocation_management("wsclip1_lyr", "INTERSECT",
seedpoly, '', "ADD_TO_SELECTION")
arcpy.SelectLayerByAttribute_management("wsclip1_lyr", "SUBSET_SELECTION",
'''"Shape_Area" >= 10000''')
cu.multi_msg("Reshaping watersheds...")
# Polygon back to raster
grid_code = arcpy.ListFields("wsclip1_lyr", "grid*")[0].name
arcpy.PolygonToRaster_conversion("wsclip1_lyr", grid_code, "ws_legit_ras")
arcpy.Clip_management("ws_legit_ras", '', "ws_legit_clipped_ras", "hu8",
"0", "ClippingGeometry")
# Make a raster from the subregion (boundary) polygon with zero for cell values.
arcpy.AddField_management("hu8", "Value", "SHORT")
arcpy.CalculateField_management("hu8", "Value", "0", "PYTHON")
arcpy.PolygonToRaster_conversion("hu8", "Value", "boundary_raster")
arcpy.Clip_management("boundary_raster", '', "boundary_raster_clip", "hu8",
'', "ClippingGeometry")
# Fill NoData in watersheds with the zero values from the subregion raster's cells.
composite = Con(IsNull("ws_legit_clipped_ras"), "boundary_raster_clip",
"ws_legit_clipped_ras")
composite.save("composite_raster")
arcpy.Clip_management("composite_raster", '', "composite_raster_clip",
"hu8", '0', "ClippingGeometry")
# Make a mask of zero cells. NoData cells are the actual mask for nibble.
premask = Con(IsNull("composite_raster_clip"), "composite_raster_clip", 0)
premask.save("premask")
arcpy.Clip_management("premask", '', "mask", "hu8", '', "ClippingGeometry")
# Set Null to 1.
pre_watersheds = Con(IsNull("composite_raster_clip"), 1,
"composite_raster_clip")
pre_watersheds.save("pre_watersheds") # does this speed things up?
## prews.save("prews.tif")
# Nibble masked values (null values along boundary).
cu.multi_msg('Nibbling watersheds as part of reshaping...')
nibble = Nibble("pre_watersheds", "mask", "DATA_ONLY")
nibble.save("nibble")
# Use HU8 buffer so that watersheds will overrun HU8 boundaries and get
# clipped without weird slivers later
arcpy.Clip_management("nibble", "", "watersheds_ras", "hu8_buffered",
"NoData", "ClippingGeometry")
# Convert watershed raster to polygon.
# Setting simplify keyword to TRUE in RasterToPolygon_conversion
# is not working reliably so need to do this in two steps, unfortunately
cu.multi_msg(
"Converted reshaped watersheds raster to polygons. If you experience problems with this step, please read Known and Outstanding Bugs.txt"
)
arcpy.RasterToPolygon_conversion("watersheds_ras", "nibble_sheds",
'SIMPLIFY', "Value") #simplify okay
## # I'm using 15 as the tolerance
## # here because the diagonal of a 10x10 pixel is 14.14 m and
## # I'm okay with a vertex moving as far as it can on the edges of the pixel
## # This also produces results very similar to using the simplify setting
## # on RasterToPolygon_conversion, when it works.
## arcpy.SimplifyPolygon_cartography("nibble_sheds_unsimple",
## "nibble_sheds_simplify", "POINT_REMOVE", "15 Meters", "0 SquareMeters",
## "RESOLVE_ERRORS", "NO_KEEP")
arcpy.Clip_analysis("nibble_sheds", "hu8", "final_watersheds")
# Join Permanent ID from Waterbody seed shapefile
final_watersheds_out = os.path.join(
out_gdb, 'huc{}_final_watersheds'.format(huc8_code))
arcpy.JoinField_management("final_watersheds", grid_code, seedpoly,
'POUR_ID', ['Permanent_Identifier'])
# this block bumps out sheds so that they fully contain their own lakes
# sometimes a little bit of another shed is overlapping the lake simply
# due to the raster/polygon differences
# 1) delete fields so watersheds and seedpoly share schema
# 2) update features, keeping borders
# 3) instead of lots of nulls make unique dissolve_id for all so that nulls aren't dissolved into one
# 4) dissolve features on dissolve_id keeping the Permanent_Identifier field
arcpy.CopyFeatures_management(seedpoly, 'lakes_nofields')
for fc in ['lakes_nofields', 'final_watersheds']:
fields = arcpy.ListFields(fc)
for f in fields:
if f != 'Permanent_Identifier':
try:
arcpy.DeleteField_management(fc, f)
except:
continue
arcpy.Update_analysis("final_watersheds", 'lakes_nofields', 'update_fc')
arcpy.AddField_management('update_fc', 'dissolve_id', 'TEXT', 255)
arcpy.MakeFeatureLayer_management('update_fc', 'update_lyr')
arcpy.SelectLayerByAttribute_management(
'update_lyr', 'NEW_SELECTION',
""""Permanent_Identifier" is not null""")
arcpy.CalculateField_management('update_lyr', 'dissolve_id',
'!Permanent_Identifier!', 'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'SWITCH_SELECTION')
arcpy.CalculateField_management('update_lyr', 'dissolve_id', '!OBJECTID!',
'PYTHON')
arcpy.SelectLayerByAttribute_management('update_lyr', 'CLEAR_SELECTION')
arcpy.Dissolve_management('update_lyr', "final_watersheds_bumped",
'dissolve_id', 'Permanent_Identifier FIRST')
cu.rename_field("final_watersheds_bumped",
"FIRST_Permanent_Identifier",
"Permanent_Identifier",
deleteOld=True)
arcpy.DeleteField_management('final_watersheds_bumped', 'dissolve_id')
arcpy.Clip_analysis('final_watersheds_bumped', 'hu8',
'final_watersheds_clipped')
arcpy.CopyFeatures_management("final_watersheds_clipped",
final_watersheds_out)
temp_items = arcpy.ListRasters() + arcpy.ListFeatureClasses() + [temp_gdb]
for item in temp_items:
try:
arcpy.Delete_management(item)
except:
continue
arcpy.ResetEnvironments()
arcpy.CheckInExtension('Spatial')
cu.multi_msg("Complete.")