def createPMPfc():
arcpy.AddMessage(
"\nCreating feature class: 'PMP_Points' in Scratch.gdb...")
dm.MakeFeatureLayer(
home + "\\Input\Non_Storm_Data.gdb\Vector_Grid",
"vgLayer") # make a feature layer of vector grid cells
dm.SelectLayerByLocation(
"vgLayer", "INTERSECT", aoiBasin
) # select the vector grid cells that intersect the aoiBasin polygon
dm.MakeFeatureLayer(home + "\\Input\Non_Storm_Data.gdb\Grid_Points",
"gpLayer") # make a feature layer of grid points
dm.SelectLayerByLocation(
"gpLayer", "HAVE_THEIR_CENTER_IN", "vgLayer"
) # select the grid points within the vector grid selection
con.FeatureClassToFeatureClass(
"gpLayer", env.scratchGDB,
"PMP_Points") # save feature layer as "PMP_Points" feature class
arcpy.AddMessage("(" + str(dm.GetCount("gpLayer")) +
" grid points will be analyzed)\n")
# Add PMP Fields
for dur in durList:
arcpy.AddMessage("\t...adding field: PMP_" + str(dur))
dm.AddField(env.scratchGDB + "\\PMP_Points", "PMP_" + dur,
"DOUBLE")
# Add STORM Fields (this string values identifies the driving storm by SPAS ID number)
for dur in durList:
arcpy.AddMessage("\t...adding field: STORM_" + str(dur))
dm.AddField(env.scratchGDB + "\\PMP_Points", "STORM_" + dur,
"TEXT", "", "", 16)
return
def process_ws(ws_fc, zone_name):
# generate new zone ids
DM.AddField(ws_fc, 'zoneid', 'TEXT', field_length=10)
DM.CalculateField(ws_fc, 'zoneid', '!lagoslakeid!', 'PYTHON')
ws_fc_lyr = DM.MakeFeatureLayer(ws_fc)
# multipart
DM.AddField(ws_fc, 'ismultipart', 'TEXT', field_length=2)
with arcpy.da.UpdateCursor(ws_fc, ['ismultipart', 'SHAPE@']) as u_cursor:
for row in u_cursor:
if row[1].isMultipart:
row[0] = 'Y'
else:
row[0] = 'N'
u_cursor.updateRow(row)
print("Edge flags...")
# add flag fields
DM.AddField(ws_fc, 'onlandborder', 'TEXT', field_length = 2)
DM.AddField(ws_fc, 'oncoast', 'TEXT', field_length = 2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(ws_fc_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'onlandborder' ,"'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(ws_fc_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'oncoast' ,"'N'", 'PYTHON')
print("State assignment...")
# States
state_geo = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Spatial_Classifications\state'
find_states(ws_fc, STATES_GEO)
# glaciation status?
calc_glaciation(ws_fc, 'zoneid')
# preface the names with the zones
DM.DeleteField(ws_fc, 'ORIG_FID')
fields = [f.name for f in arcpy.ListFields(ws_fc, '*') if f.type not in ('OID', 'Geometry') and not f.name.startswith('Shape_')]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig = f).lower()
try:
DM.AlterField(ws_fc, f, new_fname, clear_field_alias = 'TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
# cleanup
lyr_objects = [lyr_object for var_name, lyr_object in locals().items() if var_name.endswith('lyr')]
for l in lyr_objects:
DM.Delete(l)
def outputPMP(type, area, outPath):
desc = arcpy.Describe(basin)
basinName = desc.baseName
pmpPoints = env.scratchGDB + "\\PMP_Points" # Location of 'PMP_Points' feature class which will provide data for output
outType = type[:1]
outArea = str(int(round(area, 0))) + "sqmi"
outFC = outType + "_" + outArea #I don't think I need this.....
arcpy.AddMessage("\nCopying PMP_Points feature class to " + outFC +
"...") #outFC might be replaced with outpath...
dm.Merge(
pmpPoints, outPath
) # merge the scratch feature layer(s) of vector grid cells into the outputs
arcpy.AddMessage("\nCreating Basin Summary Table...")
tableName = type + "_PMP_Basin_Average" + "_" + outArea
tablePath = env.scratchGDB + "\\" + tableName
dm.CreateTable(env.scratchGDB, tableName) # Create blank table
cursor = arcpy.da.InsertCursor(
tablePath,
"*") # Create Insert cursor and add a blank row to the table
cursor.insertRow([0])
del cursor
dm.AddField(tablePath, "STORM_TYPE", "TEXT", "", "", 10,
"Storm Type") # Create "Storm Type" field
dm.CalculateField(tablePath, "STORM_TYPE", "'" + type + "'",
"PYTHON_9.3") # populate storm type field
i = 0
for field in arcpy.ListFields(
pmpPoints, "PMP_*"
): # Add fields for each PMP duration and calculate the basin average
fieldName = field.name
fieldAve = basinAve(
basin, fieldName
) # Calls the basinAve() function - returns the average (weighted or not)
dm.AddField(tablePath, fieldName, "DOUBLE", "",
2) # Add duration field
dm.CalculateField(tablePath, fieldName, fieldAve,
"PYTHON_9.3") # Assigns the basin average
i += 1
arcpy.AddMessage("\nSummary table complete.")
basAveTables.append(tablePath)
return
def createBoundingBoxPolygon(mxd_path, bkmk_name, out_fc): """Create a polygon that from the coordinates of a bookmark bounding box""" geom_type = 'POLYGON' oregon_spn = arcpy.SpatialReference(2913) management.CreateFeatureclass(os.path.dirname(out_fc), os.path.basename(out_fc), geom_type, spatial_reference=oregon_spn) name_field, f_type = 'name', 'TEXT' management.AddField(out_fc, name_field, f_type) # drop defualt field drop_field = 'Id' management.DeleteField(out_fc, drop_field) i_fields = ['Shape@', name_field] i_cursor = da.InsertCursor(out_fc, i_fields) mxd = mapping.MapDocument(mxd_path) for bkmk in arcpy.mapping.ListBookmarks(mxd, bkmk_name): extent = bkmk.extent pt_array = arcpy.Array() pt_array.add(arcpy.Point(extent.XMin, extent.YMin)) pt_array.add(arcpy.Point(extent.XMin, extent.YMax)) pt_array.add(arcpy.Point(extent.XMax, extent.YMax)) pt_array.add(arcpy.Point(extent.XMax, extent.YMin)) # add first point again to close polygon pt_array.add(arcpy.Point(extent.XMin, extent.YMin)) i_cursor.insertRow((arcpy.Polygon(pt_array), bkmk.name)) del i_cursor
def create_points_feature_class(fc, sr=None):
arcpy.env.addOutputsToMap = False
sr = sr or arcpy.env.outputCoordinateSystem
if sr is None:
arcpy.AddError('No spatial reference system.')
return None
scratch_fc = os.path.join(arcpy.env.scratchWorkspace, os.path.basename(fc))
mgmt.CreateFeatureclass(*os.path.split(scratch_fc),
'POINT',
spatial_reference=sr)
mgmt.AddField(scratch_fc, 'ELEVATION', 'DOUBLE')
mgmt.AddField(scratch_fc, 'TIME', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'NAME', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'DESCRIPTION', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'SYMBOL', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'TYPE', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'SAMPLES', 'LONG')
if fc != scratch_fc:
mgmt.Copy(scratch_fc, fc)
mgmt.Delete(scratch_fc)
return fc
def deduplicate(merged_file, rule_dictionary, unique_id='lagoslakeid'):
order_fields = []
sort_fields = []
for i in range(1, len(rule_dictionary) + 1): # priority order
rule = rule_dictionary[i]
if rule['rule'] == 'min':
order_fields.append('{} asc'.format(rule['field']))
elif rule['rule'] == 'max':
order_fields.append('{} desc'.format(rule['field']))
else:
sort_field = '{}_SORT'.format(rule['field'])
sort_fields.append(sort_field)
if not arcpy.ListFields(merged_file, sort_field):
DM.AddField(merged_file, sort_field, 'SHORT')
# Calculate new sort field with numeric order based on custom sort order
with arcpy.da.UpdateCursor(merged_file,
[rule['field'], sort_field]) as cursor:
for row in cursor:
row[1] = rule['sort'].index(row[0])
cursor.updateRow(row)
order_fields.append('{} asc'.format(sort_field))
order_by_clause = 'ORDER BY {}'.format(', '.join(order_fields))
print(order_by_clause)
print("Finding duplicate ids...")
freq = arcpy.Frequency_analysis(merged_file, 'in_memory/freq', unique_id)
dupe_ids = [
row[0] for row in arcpy.da.SearchCursor(freq, unique_id,
'''"FREQUENCY" > 1''')
]
for id in dupe_ids:
if arcpy.ListFields(merged_file,
'{}*'.format(unique_id))[0].type == 'String':
filter = '''{} = '{}' '''.format(unique_id, id)
else:
filter = '''{} = {} '''.format(unique_id, id)
with arcpy.da.UpdateCursor(
merged_file, '*', filter,
sql_clause=(None, order_by_clause)) as dupes_cursor:
counter = 0
# Deletes all but the first sorted row.
for dupe_row in dupes_cursor:
print(dupe_row)
time.sleep(.1)
if counter != 0:
print("DUPLICATE")
dupes_cursor.deleteRow()
counter += 1
print(' ')
arcpy.Delete_management('in_memory/freq')
for f in sort_fields:
DM.DeleteField(merged_file, f)
def educ_to_xy(layer, out_dir, scaling_factor):
levels = ["less_than_hs", "high_school", "some_college", "bachelors", "graduate"]
for level in levels:
trunc_level = level[0:10]
# Add and calculate the new field
field_name = level + str(scaling_factor)
mg.AddField(layer, field_name, "SHORT")
expr = "!{}! / {}".format(trunc_level, scaling_factor)
mg.CalculateField(layer, field_name, expr)
# Generate the dots
out_dots = level + str(scaling_factor)
mg.CreateRandomPoints(out_path = out_dir, out_name = out_dots,
constraining_feature_class = layer,
number_of_points_or_field = field_name)
def createExtentFeatureClass():
"""Create a feature class to the hold the map extent geometries"""
geom_type = 'POLYGON'
oregon_spn = arcpy.SpatialReference(2913)
management.CreateFeatureclass(path.dirname(pylon_extents),
path.basename(pylon_extents),
geom_type,
spatial_reference=oregon_spn)
f_type = 'TEXT'
management.AddField(pylon_extents, name_field, f_type)
drop_field = 'Id'
management.DeleteField(pylon_extents, drop_field)
def generateMatchCode():
"""Generate a code for the collapse dual carriageway tool that will indicate if two
segments are eligible to be snapped to each other"""
service_dict = {'frequent': 1, 'standard': 2, 'rush-hour': 3}
# create a copy of service_level_routes.shp so that the original is not modified
management.CopyFeatures(serv_level_routes_src, serv_level_routes)
merge_field, f_type = 'merge_id', 'LONG'
management.AddField(serv_level_routes, merge_field, f_type)
u_fields = ['serv_level', 'route_type', merge_field]
with da.UpdateCursor(serv_level_routes, u_fields) as u_cursor:
for service, r_type, merge in u_cursor:
# match field must be of type int
merge = int(str(service_dict[service]) + str(int(r_type)))
u_cursor.updateRow((service, r_type, merge))
def generateMatchCode():
"""Generate a code for the collapse dual carriageway tool that will indicate if two
segments are eligible to be snapped to each other"""
service_dict = {'frequent': 1, 'standard': 2, 'rush-hour': 3}
# create a copy of distinct_routes.shp so that the original is not modified
management.CopyFeatures(distinct_routes_src, distinct_routes)
merge_field, f_type = 'merge_id', 'LONG'
management.AddField(distinct_routes, merge_field, f_type)
u_fields = ['route_id', 'serv_level', merge_field]
with da.UpdateCursor(distinct_routes, u_fields) as u_cursor:
for route, service, merge in u_cursor:
# create a unique id based on frequency and route that is an integer
merge = int(str(int(route)) + '000' + str(service_dict[service]))
u_cursor.updateRow((route, service, merge))
def validateGeometry():
"""Check for geometry errors and multipart features that may have been introduced
in the manual editing process of creating the offsets"""
# Check for geometry errors, this tool doesn't flag a lot of the aspects I'm
# interested in, thus the other steps below
error_table = os.path.join(temp_dir, 'carto_errors.dbf')
management.CheckGeometry(offset_routes, error_table)
# Identify any multipart features
multipart_dump = os.path.join(temp_dir, 'multipart_dump.shp')
management.MultipartToSinglepart(offset_routes, multipart_dump)
multipart_dict = {}
dump_fields = ['OID@', 'ORIG_FID']
with da.SearchCursor(multipart_dump, dump_fields) as s_cursor:
for oid, orig_id in s_cursor:
if orig_id not in multipart_dict:
multipart_dict[orig_id] = 1
else:
multipart_dict[orig_id] += 1
print "Features with the following fid's are multipart:"
for orig_id, count in multipart_dict.iteritems():
if count > 1:
print orig_id
# Find other errors like shared geometries and deadends using the merge divided
# roads tool, I'm not actually interested in the output of this tool but rather the
# validation output that it generates
merge_field, field_type = 'merge_id', 'LONG'
management.AddField(offset_routes, merge_field, field_type)
# validation output will be logged here (not that user name portion may be variable):
# C:\Users\humphrig\AppData\Local\ESRI\Geoprocessing
merge_distance = 100 # feet
validation_merge = os.path.join('in_memory', 'validation_merge')
cartography.MergeDividedRoads(offset_routes, merge_field, merge_distance,
validation_merge)
# drop the merge field as it no longer needed
management.DeleteField(offset_routes, merge_field)
def calc_glaciation(fc, zone_field):
# tab area
g_field = '{}_glaciatedlatewisc'.format(os.path.basename(fc))
AN.TabulateIntersection(fc, zone_field, GLACIAL_EXTENT, 'in_memory/glacial_tab')
glacial_pct = {r[0]:r[1] for r in arcpy.da.SearchCursor('in_memory/glacial_tab', [zone_field, 'PERCENTAGE'])}
DM.AddField(fc, g_field, 'TEXT', field_length=20)
with arcpy.da.UpdateCursor(fc, [zone_field, g_field]) as u_cursor:
for row in u_cursor:
zoneid, glaciation = row
if zoneid not in glacial_pct:
glaciation = 'Not_Glaciated'
else:
if glacial_pct[zoneid] >=99.99:
glaciation = 'Glaciated'
elif glacial_pct[zoneid] < 0.01:
glaciation = 'Not_Glaciated'
else:
glaciation = 'Partially_Glaciated'
u_cursor.updateRow((zoneid, glaciation))
DM.Delete('in_memory/glacial_tab')
def createInsetBox():
"""The bus mall inset covers a portion of the city center map so that
needs to be reflected in the inset box, using the inflection point and the
city center bound box create an fc that contains the inset box"""
inflect_pt = {'x': 7649075, 'y': 686384}
bkmk_dict = getBookmarkBbox(city_center_mxd, city_center_bkmk)
geom_type = 'POLYGON'
oregon_spn = arcpy.SpatialReference(2913)
management.CreateFeatureclass(os.path.dirname(inset_box),
os.path.basename(inset_box),
geom_type,
spatial_reference=oregon_spn)
f_name, f_type = 'name', 'TEXT'
management.AddField(inset_box, f_name, f_type)
drop_field = 'Id'
arcpy.management.DeleteField(inset_box, drop_field)
i_fields = ['Shape@', f_name]
i_cursor = da.InsertCursor(inset_box, i_fields)
ap_array = arcpy.Array()
ap_array.add(arcpy.Point(bkmk_dict['x-min'], bkmk_dict['y-min']))
ap_array.add(arcpy.Point(bkmk_dict['x-min'], bkmk_dict['y-max']))
ap_array.add(arcpy.Point(bkmk_dict['x-max'], bkmk_dict['y-max']))
ap_array.add(arcpy.Point(bkmk_dict['x-max'], inflect_pt['y']))
ap_array.add(arcpy.Point(inflect_pt['x'], inflect_pt['y']))
ap_array.add(arcpy.Point(inflect_pt['x'], bkmk_dict['y-min']))
# add first point again to close polygon
ap_array.add(arcpy.Point(bkmk_dict['x-min'], bkmk_dict['y-min']))
i_cursor.insertRow((arcpy.Polygon(ap_array), 'Portland City Center'))
del i_cursor
def lake_from_to(nhd_subregion_gdb, output_table):
arcpy.env.workspace = 'in_memory'
waterbody0 = os.path.join(nhd_subregion_gdb, 'NHDWaterbody')
network = os.path.join(nhd_subregion_gdb, 'Hydrography', 'HYDRO_NET')
junctions0 = os.path.join(nhd_subregion_gdb, 'HYDRO_NET_Junctions')
# use layers for selections. We will only work with lakes over 1 hectare for this tool.
waterbody = DM.MakeFeatureLayer(waterbody0,
'waterbody',
where_clause=LAGOS_LAKE_FILTER)
num_wbs = int(arcpy.GetCount_management(waterbody).getOutput(0))
junctions = DM.MakeFeatureLayer(junctions0, 'junctions')
DM.SelectLayerByLocation(junctions, 'INTERSECT', waterbody, '1 Meters',
'NEW_SELECTION')
junctions_1ha = DM.MakeFeatureLayer(junctions, 'junctions_1ha')
# insert results into output table
DM.CreateTable(os.path.dirname(output_table),
os.path.basename(output_table))
DM.AddField(output_table, 'FROM_PERMANENT_ID', 'TEXT', field_length=40)
DM.AddField(output_table, 'TO_PERMANENT_ID', 'TEXT', field_length=40)
# create a dictionary to hold results in memory
results = []
counter = 0
progress = .01
arcpy.AddMessage("Starting network tracing...")
with arcpy.da.SearchCursor(waterbody, 'Permanent_Identifier') as cursor:
for row in cursor:
# set up a progress printer
counter += 1
if counter >= float(num_wbs) * progress:
progress += .01
arcpy.AddMessage("{}% complete...".format(
round(progress * 100), 1))
# select this lake
id = row[0]
where_clause = """"{0}" = '{1}'""".format('Permanent_Identifier',
id)
this_waterbody = DM.MakeFeatureLayer(waterbody, 'this_waterbody',
where_clause)
# select junctions overlapping this lake. only the downstream one matters, rest have no effect
DM.SelectLayerByLocation(junctions_1ha, 'INTERSECT',
this_waterbody, '1 Meters')
count_junctions = int(
arcpy.GetCount_management(junctions_1ha).getOutput(0))
if count_junctions == 0:
# add a row with no "TO" lake to the results
results.append({'FROM': id, 'TO': None})
else:
# copy with selection on
this_junctions = DM.MakeFeatureLayer(junctions_1ha,
'this_junctions')
DM.TraceGeometricNetwork(network, 'downstream', this_junctions,
'TRACE_DOWNSTREAM')
# select lakes that intersect the downstream network with a tolerance of 1 meters
DM.SelectLayerByLocation(waterbody, 'INTERSECT',
'downstream/NHDFlowline', '1 Meters',
'NEW_SELECTION')
# remove this lake
DM.SelectLayerByAttribute(waterbody, 'REMOVE_FROM_SELECTION',
where_clause)
# get the count, if it's 0 then there should be no table entry or something?
count_waterbody = int(
arcpy.GetCount_management(waterbody).getOutput(0))
# copy those into the table that you're storing stuff in
if count_waterbody == 0:
# add a row with no "TO" lake to the results
results.append({'FROM': id, 'TO': None})
else:
# for each ID, how am I getting those
to_ids = [
row[0] for row in arcpy.da.SearchCursor(
waterbody, 'Permanent_Identifier')
]
for to_id in to_ids:
result = {'FROM': id, 'TO': to_id}
results.append(result)
# delete all the intermediates
DM.SelectLayerByAttribute(waterbody, 'CLEAR_SELECTION')
for item in [this_waterbody, this_junctions, 'downstream']:
DM.Delete(item)
# insert the results in the table
insert_cursor = arcpy.da.InsertCursor(
output_table, ['FROM_PERMANENT_ID', 'TO_PERMANENT_ID'])
for result in results:
insert_cursor.insertRow([result['FROM'], result['TO']])
# delete everything
for item in [waterbody, junctions, junctions_1ha, 'in_memory']:
DM.Delete(item)
arcpy.AddMessage("Completed.")
def georeference_lakes(
lake_points_fc,
out_fc,
lake_id_field,
lake_name_field,
lake_county_field='',
state='',
master_gdb=r'C:\Users\smithn78\Dropbox\CL_HUB_GEO\Lake_Georeferencing\Masters_for_georef.gdb'
):
"""
Evaluate water quality sampling point locations and either assign the point to a lake polygon or flag the
point for manual review.
:param lake_points_fc:
:param out_fc:
:param lake_id_field:
:param lake_name_field:
:param lake_county_field:
:param state:
:param master_gdb: Location of master geodatabase used for linking
:return:
"""
master_lakes_fc = os.path.join(master_gdb, MASTER_LAKES_FC)
master_lakes_lines = os.path.join(master_gdb, MASTER_LAKES_LINES)
master_streams_fc = os.path.join(master_gdb, MASTER_STREAMS_FC)
master_xwalk = os.path.join(master_gdb, MASTER_XWALK)
# setup
arcpy.AddMessage("Joining...")
state = state.upper()
if state not in STATES:
raise ValueError('Use the 2-letter state code abbreviation')
arcpy.env.workspace = 'in_memory'
out_short = os.path.splitext(os.path.basename(out_fc))[0]
join1 = '{}_1'.format(out_short)
join2 = '{}_2'.format(out_short)
join3 = '{}_3'.format(out_short)
join3_select = join3 + '_select'
join4 = '{}_4'.format(out_short)
join5 = '{}_5'.format(out_short)
joinx = '{}_x'.format(out_short)
county_name_results = arcpy.ListFields(
lake_points_fc, '{}*'.format(lake_county_field))[0].name
if lake_county_field and not lake_county_field in county_name_results:
print('{} field does not exist in dataset.'.format(lake_county_field))
raise Exception
point_fields = [f.name for f in arcpy.ListFields(lake_points_fc)]
# update the lake id to a text field if not already
lake_id_field_type = arcpy.ListFields(lake_points_fc,
lake_id_field)[0].type
if lake_id_field_type != 'String':
temp_id_field = '{}_t'.format(lake_id_field)
arcpy.AddField_management(lake_points_fc, '{}_t'.format(lake_id_field),
'TEXT', '255')
expr = '!{}!'.format(lake_id_field)
arcpy.CalculateField_management(lake_points_fc, temp_id_field, expr,
'PYTHON')
arcpy.DeleteField_management(lake_points_fc, lake_id_field)
arcpy.AlterField_management(lake_points_fc,
temp_id_field,
new_field_name=lake_id_field)
# Try to make some spatial connections and fulfill some logic to assign a link
join1 = AN.SpatialJoin(lake_points_fc,
master_lakes_fc,
join1,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join2 = AN.SpatialJoin(join1,
master_streams_fc,
join2,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join3 = AN.SpatialJoin(join2,
master_lakes_fc,
join3,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='10 meters')
join4 = AN.SpatialJoin(join3,
master_lakes_fc,
join4,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='100 meters')
# setup for editing lake assignment values
DM.AddField(join4, 'Auto_Comment', 'TEXT', field_length=100)
DM.AddField(join4, 'Manual_Review', 'SHORT')
DM.AddField(join4, 'Shared_Words', 'TEXT', field_length=100)
DM.AddField(join4, 'Linked_lagoslakeid', 'LONG')
DM.AddField(join4, 'GEO_Discovered_Name', 'TEXT', field_length=255)
DM.AddField(join4, 'Duplicate_Candidate', 'TEXT', field_length=1)
DM.AddField(join4, 'Is_Legacy_Link', 'TEXT', field_length=1)
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID,
MASTER_GNIS_NAME, # 0m match
'PERMANENT_IDENTIFIER_1',
'GNIS_NAME_1', # stream match
MASTER_LAKE_ID + '_1',
MASTER_GNIS_NAME + '_12', # 10m match
MASTER_LAKE_ID + '_12',
MASTER_GNIS_NAME + '_12_13', # 100m match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid'
]
# use a cursor to go through each point and evaluate its assignment
cursor = arcpy.da.UpdateCursor(join4, update_fields)
arcpy.AddMessage("Calculating link status...")
for row in cursor:
id, name, mid_0, mname_0, stream_id, streamname_0, mid_10, mname_10, mid_100, mname_100, comment, review, words, lagosid = row
if mid_0 is not None: # if the point is directly in a polygon
if name and mname_0:
words = lagosGIS.list_shared_words(name,
mname_0,
exclude_lake_words=False)
comment = 'Exact location link'
lagosid = mid_0
review = -1
elif mid_0 is None and mid_10 is not None: # if the point is only within 10m of a lake
if name and mname_10:
words = lagosGIS.list_shared_words(name,
mname_10,
exclude_lake_words=False)
if words:
comment = 'Linked by common name and location'
lagosid = mid_10
review = -1
else:
comment = 'Linked by common location'
lagosid = mid_10
review = 1
elif mid_0 is None and mid_10 is None:
if stream_id is not None: # if there is a stream match
comment = 'Not linked because represented as river in NHD'
review = 2
else:
if mid_100 is not None: # if the point is only within 100m of lake(s)
if name and mname_100:
words = lagosGIS.list_shared_words(
name, mname_100, exclude_lake_words=True)
# TODO: Frequency check
if words:
comment = 'Linked by common name and location'
lagosid = mid_100
review = 1
else:
comment = 'Linked by common location'
lagosid = mid_100
review = 2
cursor.updateRow(
(id, name, mid_0, mname_0, stream_id, streamname_0, mid_10,
mname_10, mid_100, mname_100, comment, review, words, lagosid))
# # So I haven't been able to get the county logic to work and it hasn't been that important yet, ignore for now
# Select down to a minimum set because we're about to join on county, which will create lots of duplicate matches
# Then join calculated results back to full set
# if lake_county_field:
# join5 = AN.Select(join4, join5, 'Manual_Review IS NULL')
# lakes_state = AN.Select(MASTER_LAKES_FC, 'lakes_state', "{0} = '{1}'".format(MASTER_STATE_NAME, state))
# lakes_state_lyr = DM.MakeFeatureLayer(lakes_state, 'lakes_state_lyr')
# join5_lyr = DM.MakeFeatureLayer(join5, 'join5_lyr')
# DM.AddJoin(join5_lyr, lake_county_field, lakes_state_lyr, MASTER_COUNTY_NAME)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_cty')
# j5 = 'DEDUPED_CA_SWAMP_data_linked_5.'
#
# county_update_fields = [j5 + lake_id_field, j5 + lake_name_field, j5 + lake_county_field,
# 'lakes_state.' + MASTER_LAKE_ID, 'lakes_state.' + MASTER_GNIS_NAME, 'lakes_state.' + MASTER_COUNTY_NAME,
# j5 + 'Auto_Comment', j5 + 'Manual_Review', j5 + 'Shared_Words',
# j5 + 'Linked_lagoslakeid']
# with arcpy.da.UpdateCursor(join5_lyr, county_update_fields) as cursor:
# for row in cursor:
# id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid = row
# if county is not None and mcounty is not None:
# if name and mname_cty:
# words = lagosGIS.list_shared_words(name, mname_cty, exclude_lake_words=True)
# if words:
# comment = 'PRELIMINARY: Linked by common name and location'
# lagosid = mid_cty
# review = 2
# cursor.updateRow((id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid))
# DM.RemoveJoin(join5_lyr)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_county')
#
# # join5 = DM.JoinField(join5, lake_county_field, lakes_state, MASTER_COUNTY_NAME,
# fields = [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME])
#
# # This is a long way to make a join
# join_dict = {}
# with arcpy.da.SearchCursor(lakes_state, [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME]) as cursor:
# for row in cursor:
# join_value, val1, val2 = row
# join_dict[join_value] = [val1, val2]
#
# arcpy.AddField_management(join5, MASTER_LAKE_ID + 'cntyj', 'LONG')
# arcpy.AddField_management(join5, MASTER_GNIS_NAME + 'cntyj', 'TEXT', 255)
#
# with arcpy.da.SearchCursor(join5, [lake_county_field, MASTER_LAKE_ID + 'cntyj', MASTER_GNIS_NAME + 'cntyj']) as cursor:
# for row in cursor:
# key_value = row[0]
# words = lagosGIS.list_shared_words()
# if join_dict.has_key(key_value):
# row[1] = join_dict[key_value][0]
# row[2] = join_dict[key_value][1]
# else:
# row[1] = None
# row[2] = None
# cursor.updateRow(row)
#
#
# county_update_fields = [lake_id_field, lake_name_field, lake_county_field,
# MASTER_LAKE_ID + '_12_13_14', MASTER_GNIS_NAME + '_12_13', MASTER_COUNTY_NAME + '_12_13', # county
# 'Auto_Comment', 'Manual_Review', 'Shared_Words',
# 'Linked_lagoslakeid']
# cursor = arcpy.da.UpdateCursor(join5, county_update_fields)
# for row in cursor:
# id, name, county, lagosid_cty, lagosname_cty, mcounty, comment, mreview, words, linked_lagosid = row
# if mcounty is not None:
# words = lagosGIS.list_shared_words()
# else:
# join5 = join4
#
if state in LAGOSNE_STATES:
DM.JoinField(join4, lake_id_field, master_xwalk, 'lagosne_legacyid',
['lagoslakeid', 'lagos_lakename', 'lagos_state'])
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID + '_12_13',
'lagos_lakename',
'lagos_state', # crosswalk match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid',
'Is_Legacy_Link'
]
with arcpy.da.UpdateCursor(join4, update_fields) as uCursor:
for uRow in uCursor:
id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag = uRow
# fields are populated already from links above. Revise only if legacy links
if mid_x is not None:
if state == state_x:
legacy_flag = 'Y' # set to Y regardless of whether using legacy comment if state matches
if comment != 'Exact location link':
review = 1
if state != state_x:
review = 3 # downgrade if states mismatch--border lakes OK, random common IDs NOT. Check.
legacy_flag = 'Y'
comment = 'LAGOS-NE legacy link' # only comment non-exact location matches
lagosid = mid_x
if name and mname_x:
words = lagosGIS.list_shared_words(
name,
mname_x) # update words only if legacy comment
new_row = id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag
uCursor.updateRow(new_row)
# # Undo the next line if you ever bring this chunk back.
join5 = join4
# then re-code the no matches as a 3 and copy comments to the editable field
# compress the joined lake ids into one field
# having two fields lets us keep track of how many of the auto matches are bad
if arcpy.ListFields(join5, 'Comment'):
comment_field_name = 'Comment_LAGOS'
else:
comment_field_name = 'Comment'
DM.AddField(join5, comment_field_name, 'TEXT', field_length=100)
with arcpy.da.UpdateCursor(
join5, ['Manual_Review', 'Auto_Comment', 'Comment']) as cursor:
for flag, ac, comment in cursor:
if flag is None:
flag = 3
ac = 'Not linked'
comment = ac
cursor.updateRow((flag, ac, comment))
# Re-code points more than 100m into the polygon of the lake as no need to check
DM.MakeFeatureLayer(join5, 'join5_lyr')
DM.MakeFeatureLayer(master_lakes_lines, 'lake_lines_lyr')
DM.SelectLayerByAttribute('join5_lyr', 'NEW_SELECTION',
"Auto_Comment = 'Exact location link'")
DM.SelectLayerByLocation('join5_lyr', 'INTERSECT', 'lake_lines_lyr',
'100 meters', 'SUBSET_SELECTION', 'INVERT')
DM.CalculateField('join5_lyr', 'Manual_Review', '-2', 'PYTHON')
DM.Delete('join5_lyr', 'lake_lines_lyr')
# Then make sure to only keep the fields necessary when you write to an output
copy_fields = point_fields + [
'Linked_lagoslakeid', 'Auto_Comment', 'Manual_Review',
'Is_Legacy_Link', 'Shared_Words', 'Comment', 'Duplicate_Candidate',
'GEO_Discovered_Name'
]
copy_fields.remove('Shape')
copy_fields.remove('OBJECTID')
lagosGIS.select_fields(join5, out_fc, copy_fields)
DM.AssignDomainToField(out_fc, 'Comment', 'Comment')
DM.AddField(out_fc, 'Total_points_in_lake_poly', 'Short')
# Remove any duplicates. (These originate from the join3/join4 transition because a point can be both
# within 10m and 100m of lakes, this code takes the closest lake as true for my current sanity.)
# Or, in other words, this is a hack solution.
out_fc_fields = [
f.name for f in arcpy.ListFields(out_fc) if f.name != 'OBJECTID'
]
DM.DeleteIdentical(out_fc, out_fc_fields)
# Get the join_count for each limno lake ID
# De-dupe anything resulting from limno ID duplicates first before counting
id_pairs = list(
set(
arcpy.da.SearchCursor(out_fc,
[lake_id_field, 'Linked_lagoslakeid'])))
# THEN pull out LAGOS id. Any duplicate now are only due to multiple distinct points within lake
lagos_ids = [ids[1] for ids in id_pairs]
sample_ids = [ids[0] for ids in id_pairs]
lagos_lake_counts = Counter(lagos_ids)
linked_multiple_lake_counts = Counter(sample_ids)
# Get the count of points in the polygon
with arcpy.da.UpdateCursor(
out_fc,
['Linked_lagoslakeid', 'Total_points_in_lake_poly']) as cursor:
for lagos_id, join_count in cursor:
join_count = lagos_lake_counts[lagos_id]
cursor.updateRow((lagos_id, join_count))
# Mark any samples linked to more than one lake so that the analyst can select the correct lake in the
# manual process
with arcpy.da.UpdateCursor(
out_fc, [lake_id_field, 'Duplicate_Candidate']) as cursor:
for sample_id, duplicate_flag in cursor:
duplicate_count = linked_multiple_lake_counts[sample_id]
if duplicate_count > 1:
duplicate_flag = "Y"
else:
duplicate_flag = "N"
cursor.updateRow((sample_id, duplicate_flag))
# clean up
DM.AddField(out_fc, 'Note', 'TEXT', field_length=140)
DM.Delete('in_memory')
arcpy.AddMessage('Completed.')
fields = arcpy.ListFields(in_file)
layer = "feat_layer"
arcmgt.MakeFeatureLayer(in_file, layer)
desc = arcpy.Describe(layer)
fld_names = []
for fld in desc.fields:
fld_names.append(fld.name)
try:
fields = ["PATH_FROM", "PATH_TO", "PATH_DIST"]
#arcmgt.DeleteField(layer, fields)
#arcmgt.DeleteField(layer, "FROM_")
for fld in fields:
if not fld in fld_names:
arcmgt.AddField(table_view, fld,
"DOUBLE") # SHOULD GET TYPE FROM target_fld
except Exception as e:
print e.message
arcpy.AddError(e.message)
raise
rows = arcpy.UpdateCursor(table_view)
last_target = None
for row in rows:
if last_target is None:
last_target = row.getValue(target_fld)
continue
def import_gpx(gpx_file, wpt_fc, trk_fc):
GCS_WGS_84 = arcpy.SpatialReference(4326)
GCS_TRANSFORMS = 'WGS_1984_(ITRF08)_To_NAD_1983_2011; NAD_1927_To_NAD_1983_NADCON'
arcpy.env.geographicTransformations = arcpy.env.geographicTransformations or GCS_TRANSFORMS
arcpy.AddMessage('Geographic Transformations: %s' % arcpy.env.geographicTransformations)
scratch = arcpy.env.scratchWorkspace
arcpy.env.addOutputsToMap = False
WPT_FIELDS = [
('ELEVATION', 'gpx:ele'),
('TIME', 'gpx:time'),
('NAME', 'gpx:name'),
('DESCRIPTION', 'gpx:desc'),
('SYMBOL', 'gpx:sym'),
('TYPE', 'gpx:type'),
('SAMPLES', 'gpx:extensions/wptx1:WaypointExtension/wptx1:Samples')
]
ns = {
'gpx': 'http://www.topografix.com/GPX/1/1',
'gpxx': 'http://www.garmin.com/xmlschemas/GpxExtensions/v3',
'wptx1': 'http://www.garmin.com/xmlschemas/WaypointExtension/v1',
'ctx': 'http://www.garmin.com/xmlschemas/CreationTimeExtension/v1',
}
etree.register_namespace('', 'http://www.topografix.com/GPX/1/1')
etree.register_namespace('gpxx', 'http://www.garmin.com/xmlschemas/GpxExtensions/v3')
etree.register_namespace('wptx1', 'http://www.garmin.com/xmlschemas/WaypointExtension/v1')
etree.register_namespace('ctx', 'http://www.garmin.com/xmlschemas/CreationTimeExtension/v1')
gpx = etree.parse(gpx_file).getroot()
sr = arcpy.env.outputCoordinateSystem
if wpt_fc:
create_points_feature_class(wpt_fc, sr)
waypoints = []
for wpt in gpx.findall('gpx:wpt', ns):
x, y = wpt.get('lon'), wpt.get('lat')
row = [arcpy.PointGeometry(arcpy.Point(x, y), GCS_WGS_84).projectAs(sr)]
for field, tag in WPT_FIELDS:
elem = wpt.find(tag, ns)
if elem is None:
row.append(None)
elif field == 'ELEVATION':
row.append('%0.4f' % (float(elem.text) / sr.metersPerUnit))
elif field == 'NAME' and elem.text.isdigit():
row.append('%d' % int(elem.text))
else:
row.append(elem.text)
waypoints.append(row)
if waypoints:
fields = ['SHAPE@'] + [f[0] for f in WPT_FIELDS]
cur = arcpy.da.InsertCursor(wpt_fc, fields)
for row in waypoints:
cur.insertRow(row)
del cur
if trk_fc:
# idle time between trkpts to start a new track segment
TRKSEG_IDLE_SECS = 600
tracks = []
track_num = 0
for trk in gpx.findall('gpx:trk', ns):
track_num += 1
elem = trk.find('gpx:name', ns)
if elem is None:
track_name = 'track-%04d' % track_num
else:
track_name = elem.text
track_pts = []
dt_last = None
segment_num = 0
for trkpt in trk.findall('./gpx:trkseg/gpx:trkpt', ns):
x, y = trkpt.get('lon'), trkpt.get('lat')
pt = arcpy.PointGeometry(arcpy.Point(x, y), GCS_WGS_84).projectAs(sr).firstPoint
# See if there's a track point time
elem = trkpt.find('gpx:time', ns)
if elem is None:
dt_last = None
else:
dt = utils.default_tzinfo(parser.parse(elem.text), tz.UTC)
if dt_last and (dt - dt_last).seconds > TRKSEG_IDLE_SECS:
# start a new segment
if len(track_pts) > 1:
segment_num += 1
if segment_num > 1:
segment_name = '%s SEG-%04d' % (track_name, segment_num)
else:
segment_name = track_name
geom = arcpy.Polyline(arcpy.Array(track_pts), sr)
tracks.append([geom , segment_name, len(track_pts)])
else:
arcpy.AddMessage('Skipping track "%s": track_pts=%d' % (track_name, len(track_pts)))
track_pts = []
dt_last = dt
track_pts.append(pt)
if len(track_pts) > 1:
segment_num += 1
if segment_num > 1:
segment_name = '%s SEG-%04d' % (track_name, segment_num)
else:
segment_name = track_name
geom = arcpy.Polyline(arcpy.Array(track_pts), sr)
tracks.append([geom, segment_name, len(track_pts)])
else:
arcpy.AddMessage('Skipping track "%s": track_pts=%d' % (track_name, len(track_pts)))
if tracks:
temp_fc = os.path.join(scratch, os.path.basename(trk_fc) + '_Temp')
if sr is None:
arcpy.AddError('Geoprocessing environment not set: outputCoordinateSystem')
return None
fc = mgmt.CreateFeatureclass(*os.path.split(temp_fc), geometry_type='POLYLINE', spatial_reference=sr)
mgmt.AddField(fc, 'NAME', 'TEXT', field_length=64)
mgmt.AddField(fc, 'POINTS', 'LONG')
cur = arcpy.da.InsertCursor(fc, ('SHAPE@', 'NAME', 'POINTS'))
for row in tracks:
cur.insertRow(row)
del cur
mgmt.CopyFeatures(temp_fc, trk_fc)
del fc
def classify_lakes(nhd,
out_feature_class,
exclude_intermit_flowlines=False,
debug_mode=False):
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('classify_lake_connectivity')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
if arcpy.Exists("temp_fc"):
print("There is a problem here.")
raise Exception
# Tool temporary feature classes
temp_fc = "temp_fc"
csiwaterbody_10ha = "csiwaterbody_10ha"
nhdflowline_filtered = "nhdflowline_filtered"
dangles = "dangles"
start = "start"
end = "end"
startdangles = "startdangles"
enddangles = "enddangles"
non_artificial_end = "non_artificial_end"
flags_10ha_lake_junctions = "flags_10ha_lake_junctions"
midvertices = "midvertices"
non10vertices = "non10vertices"
non10junctions = "non10junctions"
all_non_flag_points = "all_non_flag_points"
barriers = "barriers"
trace1_junctions = "trace1_junctions"
trace1_flowline = "trace1_flowline"
trace2_junctions = "trace2junctions"
trace2_flowline = "trace2_flowline"
# Clean up workspace in case of bad exit from prior run in same session.
this_tool_layers = [
"dangles_lyr", "nhdflowline_lyr", "junction_lyr", "midvertices_lyr",
"all_non_flag_points_lyr", "non10vertices_lyr", "out_fc_lyr", "trace1",
"trace2"
]
this_tool_temp = [
temp_fc, csiwaterbody_10ha, nhdflowline_filtered, dangles, start, end,
startdangles, enddangles, non_artificial_end,
flags_10ha_lake_junctions, midvertices, non10vertices, non10junctions,
all_non_flag_points, barriers, trace1_junctions, trace1_flowline,
trace2_junctions, trace2_flowline
]
for item in this_tool_layers + this_tool_temp:
try:
DM.Delete(item)
except:
pass
# Local variables:
nhdflowline = os.path.join(nhd, "Hydrography", "NHDFLowline")
nhdjunction = os.path.join(nhd, "Hydrography", "HYDRO_NET_Junctions")
nhdwaterbody = os.path.join(nhd, "Hydrography", "NHDWaterbody")
network = os.path.join(nhd, "Hydrography", "HYDRO_NET")
# Get lakes, ponds and reservoirs over a hectare.
#csi_population_filter = '''"AreaSqKm" >=0.01 AND\
#"FCode" IN (39000,39004,39009,39010,39011,39012,43600,43613,43615,43617,43618,43619,43621)'''
all_lakes_reservoirs_filter = '''"FType" IN (390, 436)'''
# Can't see why we shouldn't just attribute all lakes and reservoirs
# arcpy.Select_analysis(nhdwaterbody, "csiwaterbody", lake_population_filter)
arcpy.AddMessage("Initializing output.")
if exclude_intermit_flowlines:
DM.CopyFeatures(out_feature_class, temp_fc)
DM.Delete(out_feature_class)
else:
arcpy.Select_analysis(nhdwaterbody, temp_fc,
all_lakes_reservoirs_filter)
# Get lakes, ponds and reservoirs over 10 hectares.
lakes_10ha_filter = '''"AreaSqKm" >= 0.1 AND "FType" IN (390, 436)'''
arcpy.Select_analysis(nhdwaterbody, csiwaterbody_10ha, lakes_10ha_filter)
# Exclude intermittent flowlines, if requested
if exclude_intermit_flowlines:
flowline_where_clause = '''"FCode" NOT IN (46003,46007)'''
nhdflowline = arcpy.Select_analysis(nhdflowline, nhdflowline_filtered,
flowline_where_clause)
# Make dangle points at end of nhdflowline
DM.FeatureVerticesToPoints(nhdflowline, dangles, "DANGLE")
DM.MakeFeatureLayer(dangles, "dangles_lyr")
# Isolate start dangles from end dangles.
DM.FeatureVerticesToPoints(nhdflowline, start, "START")
DM.FeatureVerticesToPoints(nhdflowline, end, "END")
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", start)
DM.CopyFeatures("dangles_lyr", startdangles)
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", end)
DM.CopyFeatures("dangles_lyr", enddangles)
# Special handling for lakes that have some intermittent flow in and some permanent
if exclude_intermit_flowlines:
DM.MakeFeatureLayer(nhdflowline, "nhdflowline_lyr")
DM.SelectLayerByAttribute("nhdflowline_lyr", "NEW_SELECTION",
'''"WBArea_Permanent_Identifier" is null''')
DM.FeatureVerticesToPoints("nhdflowline_lyr", non_artificial_end,
"END")
DM.SelectLayerByAttribute("nhdflowline_lyr", "CLEAR_SELECTION")
arcpy.AddMessage("Found source area nodes.")
# Get junctions from lakes >= 10 hectares.
DM.MakeFeatureLayer(nhdjunction, "junction_lyr")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT", csiwaterbody_10ha,
XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("junction_lyr", flags_10ha_lake_junctions)
arcpy.AddMessage("Found lakes >= 10 ha.")
# Make points shapefile and layer at flowline vertices to act as potential flags and/or barriers.
arcpy.AddMessage("Tracing...")
DM.FeatureVerticesToPoints(nhdflowline, midvertices, "MID")
DM.MakeFeatureLayer(midvertices, "midvertices_lyr")
# Get vertices that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("midvertices_lyr", non10vertices)
# Get junctions that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("junction_lyr", non10junctions)
# Merge non10vertices with non10junctions
DM.Merge([non10junctions, non10vertices],
all_non_flag_points) # inputs both point fc in_memory
DM.MakeFeatureLayer(all_non_flag_points, "all_non_flag_points_lyr")
# Tests the counts...for some reason I'm not getting stable behavior from the merge.
mid_n = int(DM.GetCount(non10vertices).getOutput(0))
jxn_n = int(DM.GetCount(non10junctions).getOutput(0))
merge_n = int(DM.GetCount(all_non_flag_points).getOutput(0))
if merge_n < mid_n + jxn_n:
arcpy.AddWarning(
"The total number of flags ({0}) is less than the sum of the input junctions ({1}) "
"and input midpoints ({2})".format(merge_n, jxn_n, mid_n))
# For tracing barriers, select all_non_flag_points points that intersect a 10 ha lake.
DM.SelectLayerByLocation("all_non_flag_points_lyr", "INTERSECT",
csiwaterbody_10ha, XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("all_non_flag_points_lyr", barriers)
# Trace1-Trace downstream to first barrier (junctions+midvertices in 10 ha lake) starting from flags_10ha_lake_junctions flag points.
DM.TraceGeometricNetwork(network, "trace1", flags_10ha_lake_junctions,
"TRACE_DOWNSTREAM", barriers)
# Save trace1 flowlines and junctions to layers on disk.
DM.CopyFeatures("trace1\HYDRO_NET_Junctions",
trace1_junctions) # extra for debugging
DM.CopyFeatures("trace1\NHDFlowline", trace1_flowline)
# Select vertice midpoints that intersect trace1 flowlines selection for new flags for trace2.
DM.MakeFeatureLayer(non10vertices, "non10vertices_lyr")
DM.SelectLayerByLocation("non10vertices_lyr", "INTERSECT", trace1_flowline,
"", "NEW_SELECTION")
# Trace2-Trace downstream from midpoints of flowlines that intersect the selected flowlines from trace1.
DM.TraceGeometricNetwork(network, "trace2", "non10vertices_lyr",
"TRACE_DOWNSTREAM")
# Save trace1 flowlines and junctions to layers and then shapes on disk.
DM.CopyFeatures("trace2\HYDRO_NET_Junctions", trace2_junctions)
DM.CopyFeatures("trace2\NHDFlowline",
trace2_flowline) # extra for debugging
arcpy.AddMessage("Done tracing.")
# Make shapefile for seepage lakes. (Ones that don't intersect flowlines)
if exclude_intermit_flowlines:
class_field_name = "Lake_Connectivity_Permanent"
else:
class_field_name = "Lake_Connectivity_Class"
DM.AddField(temp_fc, class_field_name, "TEXT", field_length=13)
DM.MakeFeatureLayer(temp_fc, "out_fc_lyr")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline, "",
"SWITCH_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# New type of "Isolated" classification, mostly for "permanent" but there were some oddballs in "maximum" too
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", enddangles,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# Get headwater lakes.
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByAttribute(
"out_fc_lyr", "REMOVE_FROM_SELECTION",
'''"{}" = 'Isolated' '''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# Select csiwaterbody that intersect trace2junctions
arcpy.AddMessage("Beginning connectivity attribution...")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", trace2_junctions,
XY_TOLERANCE, "NEW_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'DrainageLk'""",
"PYTHON")
# Get stream drainage lakes. Either unassigned so far or convert "Headwater" if a permanent stream flows into it,
# which is detected with "non_artificial_end"
DM.SelectLayerByAttribute("out_fc_lyr", "NEW_SELECTION",
'''"{}" IS NULL'''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
if exclude_intermit_flowlines:
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"{}" = 'Headwater' '''.format(class_field_name))
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", non_artificial_end,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
# Prevent 'upgrades' due to very odd flow situations and artifacts of bad digitization. The effects of these
# are varied--to avoid confusion, just keep the class assigned with all flowlines
# 1--Purely hypothetical, not seen in testing
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Isolated' AND "Lake_Connectivity_Permanent" <> 'Isolated' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# 2--Headwater to Drainage upgrade seen in testing with odd multi-inlet flow situation
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Headwater' AND "Lake_Connectivity_Permanent" IN ('Drainage', 'DrainageLk') '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# 3--Drainage to DrainageLk upgrade seen in testing when intermittent stream segments were used
# erroneously instead of artificial paths
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Drainage' AND "Lake_Connectivity_Permanent" = 'DrainageLk' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
DM.SelectLayerByAttribute("out_fc_lyr", "CLEAR_SELECTION")
# Add change flag for users
DM.AddField(temp_fc,
"Lake_Connectivity_Fluctuates",
"Text",
field_length="1")
flag_codeblock = """def flag_calculate(arg1, arg2):
if arg1 == arg2:
return 'N'
else:
return 'Y'"""
expression = 'flag_calculate(!Lake_Connectivity_Class!, !Lake_Connectivity_Permanent!)'
DM.CalculateField(temp_fc, "Lake_Connectivity_Fluctuates", expression,
"PYTHON", flag_codeblock)
# Project output once done with both. Switching CRS earlier causes trace problems.
if not exclude_intermit_flowlines:
DM.CopyFeatures(temp_fc, out_feature_class)
else:
DM.Project(temp_fc, out_feature_class, arcpy.SpatialReference(102039))
# Clean up
if not debug_mode:
for item in this_tool_layers + this_tool_temp:
if arcpy.Exists(item):
DM.Delete(item)
if not debug_mode:
DM.Delete("trace1")
DM.Delete("trace2")
arcpy.AddMessage("{} classification is complete.".format(class_field_name))
sciNames[i['code']] = i['sciname']
commNames = {}
for i in lst:
commNames[i['code']] = i['commname']
habitat_Use = {}
for i in lst:
habitat_Use[i['code']] = i['Loc_Use']
# cycle through each zone raster
for i in range(len(polZ)):
print("working on " + polZ[i])
# don't assume i is the class level -- extract class here
classLevel = polZ[i][-1:]
curZo = "zon_Joined_final_C" + classLevel
fldCodeList = [f.name for f in arcpy.ListFields(curZo, "codeList", "Text")]
if not fldCodeList:
man.AddField(curZo, "codeList", "TEXT", "", "", "500")
fldSciNmList = [
f.name for f in arcpy.ListFields(curZo, "SciNames", "Text")
]
if not fldSciNmList:
man.AddField(curZo, "SciNames", "TEXT", "", "", "550")
fldCmNmList = [
f.name for f in arcpy.ListFields(curZo, "CommNames", "Text")
]
if not fldCmNmList:
man.AddField(curZo, "CommNames", "TEXT", "", "", "500")
fldHabUseList = [
f.name for f in arcpy.ListFields(curZo, "HabitatUse", "Text")
]
if not fldHabUseList:
man.AddField(curZo, "HabitatUse", "TEXT", "", "", "500")
start = 0
#####
# loop through all binary (0/1) grids, build the hypergrid
# with info stored in a single text column
for i in range(start, len(codeL)):
elem = codeL[i]
rasName = elem + "_c.tif"
if rasName in rasL:
if i == 0:
inRas = inPath + "/" + rasName
curHyp = wrk + "/hyp" + str(i)
print("working on " + rasName)
man.CopyRaster(inRas, curHyp)
man.BuildRasterAttributeTable(curHyp)
man.AddField(curHyp, "spp0", "TEXT", "", "", 251)
man.AddField(curHyp, "temp", "SHORT", 1)
expr = "str( !Value! )"
man.CalculateField(curHyp, "spp0", expr, "PYTHON")
else:
iminus = i - 1
prevHyp = wrk + "/hyp" + str(iminus)
print("working on " + elem + ", " + str(i) + " of " + str(listLen))
curHyp = Combine([prevHyp, rasName])
curHyp.save(wrk + "/hyp" + str(i))
man.AddField(curHyp, "spp0", "TEXT", "", "", 251)
jval = "hyp" + str(iminus)
man.JoinField(curHyp, jval, prevHyp, "VALUE", ["spp0"])
rasNoDot = rasName[0:rasName.find(".")]
newCol = rasNoDot[0:11].upper()
expr = "str(!spp0_1!) + str(!" + newCol + "!)"
def flatten_overlaps(zone_fc,
zone_field,
output_fc,
output_table,
cluster_tolerance=' 3 Meters'):
orig_env = arcpy.env.workspace
arcpy.env.workspace = 'in_memory'
objectid = [f.name for f in arcpy.ListFields(zone_fc)
if f.type == 'OID'][0]
zone_type = [f.type for f in arcpy.ListFields(zone_fc, zone_field)][0]
fid1 = 'FID_{}'.format(os.path.basename(zone_fc))
flat_zoneid = 'flat{}'.format(zone_field)
flat_zoneid_prefix = 'flat{}_'.format(zone_field.replace('_zoneid', ''))
# Union with FID_Only (A)
arcpy.AddMessage("Splitting overlaps in polygons...")
zoneid_dict = {
r[0]: r[1]
for r in arcpy.da.SearchCursor(zone_fc, [objectid, zone_field])
}
self_union = AN.Union([zone_fc],
'self_union',
'ONLY_FID',
cluster_tolerance=cluster_tolerance)
# #If you don't run this section, Find Identical fails with error 999999. Seems to have to do with small slivers
# #having 3 vertices and/or only circular arcs in the geometry.
arcpy.AddMessage("Repairing self-union geometries...")
# DM.AddGeometryAttributes(self_union, 'POINT_COUNT; AREA')
# union_fix = DM.MakeFeatureLayer(self_union, 'union_fix', where_clause='PNT_COUNT <= 10 OR POLY_AREA < 5000')
# arcpy.Densify_edit(union_fix, 'DISTANCE', distance = '1 Meters', max_deviation='1 Meters') # selection ON, edits self_union disk
DM.RepairGeometry(
self_union, 'DELETE_NULL'
) # eliminate empty geoms. selection ON, edits self_union disk
# for field in ['PNT_COUNT', 'POLY_AREA']:
# DM.DeleteField(self_union, field)
# Find Identical by Shape (B)
if arcpy.Exists('identical_shapes'):
DM.Delete(
'identical_shapes'
) # causes failure in FindIdentical even when overwrite is allowed
identical_shapes = DM.FindIdentical(self_union, 'identical_shapes',
'Shape')
# Join A to B and calc flat[zone]_zoneid = FEAT_SEQ (C)
DM.AddField(self_union, flat_zoneid, 'TEXT', field_length=20)
union_oid = [
f.name for f in arcpy.ListFields(self_union) if f.type == 'OID'
][0]
identical_shapes_dict = {
r[0]: r[1]
for r in arcpy.da.SearchCursor(identical_shapes,
['IN_FID', 'FEAT_SEQ'])
}
with arcpy.da.UpdateCursor(self_union,
[union_oid, flat_zoneid]) as u_cursor:
for row in u_cursor:
row[1] = '{}{}'.format(flat_zoneid_prefix,
identical_shapes_dict[row[0]])
u_cursor.updateRow(row)
# Add the original zone ids and save to table (E)
arcpy.AddMessage("Assigning temporary IDs to split polygons...")
unflat_table = DM.CopyRows(self_union, 'unflat_table')
DM.AddField(unflat_table, zone_field,
zone_type) # default text length of 50 is fine if needed
with arcpy.da.UpdateCursor(unflat_table, [fid1, zone_field]) as u_cursor:
for row in u_cursor:
row[1] = zoneid_dict[row[0]] # assign zone id
u_cursor.updateRow(row)
# Delete Identical (C) (save as flat[zone])
with arcpy.da.UpdateCursor(self_union, 'OID@') as cursor:
visited = []
for row in cursor:
feat_seq = identical_shapes_dict[row[0]]
if feat_seq in visited:
cursor.deleteRow()
visited.append(feat_seq)
DM.DeleteField(self_union, fid1)
DM.DeleteField(unflat_table, fid1)
# save outputs
output_fc = DM.CopyFeatures(self_union, output_fc)
output_table = DM.CopyRows(unflat_table, output_table)
# cleanup
for item in [self_union, identical_shapes, unflat_table]:
DM.Delete(item)
arcpy.env.workspace = orig_env
return output_fc
def unflatten(intermediate_table):
flat_zoneid = zone_field
unflat_zoneid = zone_field.replace('flat', '')
zone_type = [f.type for f in arcpy.ListFields(zone_fc, flat_zoneid)][0]
# Set up the output table (can't do this until the prior tool is run)
# if os.path.dirname(out_table):
# out_path = os.path.dirname(out_table)
# else:
# out_path = orig_env
unflat_result = DM.CreateTable('in_memory',
os.path.basename(out_table))
# get the fields to add to the table
editable_fields = [
f for f in arcpy.ListFields(intermediate_table)
if f.editable and f.name.lower() != flat_zoneid.lower()
]
# populate the new table schema
DM.AddField(unflat_result, unflat_zoneid, zone_type)
for f in editable_fields:
DM.AddField(unflat_result, f.name, f.type, field_length=f.length)
# map original zone ids to new zone ids
original_flat = defaultdict(list)
with arcpy.da.SearchCursor(unflat_table,
[unflat_zoneid, flat_zoneid]) as cursor:
for row in cursor:
if row[1] not in original_flat[row[0]]:
original_flat[row[0]].append(row[1])
# Use CELL_COUNT as weight for means to calculate final values for each zone.
fixed_fields = [
unflat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
]
other_field_names = [
f.name for f in editable_fields if f.name not in fixed_fields
]
i_cursor = arcpy.da.InsertCursor(
unflat_result,
fixed_fields + other_field_names) # open output table cursor
flat_stats = {
r[0]: r[1:]
for r in arcpy.da.SearchCursor(intermediate_table, [
flat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
] + other_field_names)
}
count_diff = 0
for zid, unflat_ids in original_flat.items():
valid_unflat_ids = [id for id in unflat_ids if id in flat_stats
] # skip flatpolys not rasterized
area_vec = [flat_stats[id][0] for id in valid_unflat_ids
] # ORIGINAL_COUNT specified in 0 index earlier
cell_vec = [flat_stats[id][1] for id in valid_unflat_ids]
coverage_vec = [flat_stats[id][2] for id in valid_unflat_ids
] # datacoveragepct special handling
stat_vectors_by_id = [
flat_stats[id][3:] for id in valid_unflat_ids
] # "the rest", list of lists
# calc the new summarized values
original_count = sum(
filter(None, area_vec)
) # None area is functionally equivalent to 0, all Nones = 0 too
cell_count = sum(filter(None, cell_vec))
if cell_count > 0:
weighted_coverage = sum(
[a * b
for a, b in zip(area_vec, coverage_vec)]) / original_count
# this calculation accounts for fractional missing values, both kinds (whole zone is no data, or zone
# was missing some data and had data coverage % < 100). This is done by converting None to 0
# and by using the cell_count (count of cells with data present)
# instead of the full zone original_count. You have to do both or the mean will be distorted.
# hand-verification that this works as intended using test GIS data on was completed 2019-11-01 by NJS
crossprods = []
for i in range(0, len(valid_unflat_ids)):
crossprods.append([
cell_vec[i] * float(s or 0)
for s in stat_vectors_by_id[i]
])
weighted_stat_means = []
for i in range(0, len(other_field_names)):
weighted_stat_means.append(
sum(zip(*crossprods)[i]) / cell_count)
else:
weighted_coverage = 0
weighted_stat_means = [None] * len(other_field_names)
count_diff += 1
new_row = [zid, original_count, cell_count, weighted_coverage
] + weighted_stat_means
i_cursor.insertRow(new_row)
del i_cursor
DM.Delete(intermediate_table)
return [unflat_result, count_diff]
def basinAve(aoiBasin, pmpField):
pmpPoints = env.scratchGDB + "\\PMP_Points" # Path of 'PMP_Points' scratch feature class
if weightedAve:
arcpy.AddMessage("\tCalculating basin average for " + pmpField +
"(weighted)...")
vectorGridClip = env.scratchGDB + "\\VectorGridClip" # Path of 'PMP_Points' scratch feature class
sumstats = env.scratchGDB + "\\SummaryStats"
dm.MakeFeatureLayer(
home + "\\Input\Non_Storm_Data.gdb\\Vector_Grid",
"vgLayer") # make a feature layer of vector grid cells
dm.SelectLayerByLocation(
"vgLayer", "INTERSECT", aoiBasin
) # select the vector grid cells that intersect the aoiBasin polygon
an.Clip("vgLayer", aoiBasin,
vectorGridClip) # clips aoi vector grid to basin
dm.AddField(
pmpPoints, "WEIGHT", "DOUBLE"
) # adds 'WEIGHT' field to PMP_Points scratch feature class
dm.MakeFeatureLayer(
vectorGridClip, "vgClipLayer"
) # make a feature layer of basin clipped vector grid cells
dm.MakeFeatureLayer(
pmpPoints, "pmpPointsLayer"
) # make a feature layer of PMP_Points feature class
dm.AddJoin("pmpPointsLayer", "ID", "vgClipLayer",
"ID") # joins PMP_Points and vectorGridBasin tables
dm.CalculateField(
"pmpPointsLayer", "WEIGHT", "!vectorGridClip.Shape_Area!",
"PYTHON_9.3"
) # Calculates basin area proportion to use as weight for each grid cell.
dm.RemoveJoin("pmpPointsLayer", "vectorGridClip")
an.Statistics(pmpPoints, sumstats, [["WEIGHT", "SUM"]], "")
stats = arcpy.SearchCursor(sumstats)
pmpWgtAve = pmpField + "_WgtAve"
for row in stats:
calc = row.getValue("SUM_WEIGHT")
express = "(!WEIGHT!/{})* !{}!".format(calc, pmpField)
i = 0
for field in arcpy.ListFields(pmpPoints, pmpField):
dm.AddField(pmpPoints, pmpWgtAve, "DOUBLE", 2)
dm.CalculateField(pmpPoints, pmpWgtAve, express,
"PYTHON_9.3")
i += 1
del stats, row
an.Statistics(pmpPoints, sumstats, [[pmpWgtAve, "SUM"]], "")
sumwgtave = "SUM_" + pmpWgtAve
with arcpy.da.SearchCursor(sumstats, sumwgtave) as stats:
for row in stats:
wgtAve = row[0]
return round(wgtAve, 2)
## na = arcpy.da.TableToNumPyArray(pmpPoints,(pmpField, 'WEIGHT')) # Assign pmpPoints values and weights to Numpy array (na)
## wgtAve = numpy.average(na[pmpField], weights=na['WEIGHT']) # Calculate weighted average with Numpy average
## del na
## return round(wgtAve, 2)
else:
arcpy.AddMessage("\tCalculating basin average for " + pmpField +
"(not weighted)...")
sumstats = env.scratchGDB + "\\SummaryStats"
an.Statistics(pmpPoints, sumstats, [[pmpField, "MEAN"]], "")
mean = "MEAN_" + pmpField
with arcpy.da.SearchCursor(sumstats, mean) as stats:
for row in stats:
fieldAve = row[0]
return round(fieldAve, 2)
def stats_area_table(zone_fc=zone_fc,
zone_field=zone_field,
in_value_raster=in_value_raster,
out_table=out_table,
is_thematic=is_thematic):
def refine_zonal_output(t):
"""Makes a nicer output for this tool. Rename some fields, drop unwanted
ones, calculate percentages using raster AREA before deleting that
field."""
if is_thematic:
value_fields = arcpy.ListFields(t, "VALUE*")
pct_fields = [
'{}_pct'.format(f.name) for f in value_fields
] # VALUE_41_pct, etc. Field can't start with number.
# add all the new fields needed
for f, pct_field in zip(value_fields, pct_fields):
arcpy.AddField_management(t, pct_field, f.type)
# calculate the percents
cursor_fields = ['AREA'] + [f.name
for f in value_fields] + pct_fields
uCursor = arcpy.da.UpdateCursor(t, cursor_fields)
for uRow in uCursor:
# unpacks area + 3 tuples of the right fields for each, no matter how many there are
vf_i_end = len(value_fields) + 1
pf_i_end = vf_i_end + len(pct_fields)
# pct_values and ha_values are both null at this point but unpack for clarity
area, value_values, pct_values = uRow[0], uRow[
1:vf_i_end], uRow[vf_i_end:pf_i_end]
new_pct_values = [100 * vv / area for vv in value_values]
new_row = [area] + value_values + new_pct_values
uCursor.updateRow(new_row)
for vf in value_fields:
arcpy.DeleteField_management(t, vf.name)
arcpy.AlterField_management(t, 'COUNT', 'CELL_COUNT')
drop_fields = ['ZONE_CODE', 'COUNT', 'AREA']
if not debug_mode:
for df in drop_fields:
try:
arcpy.DeleteField_management(t, df)
except:
continue
# Set up environments for alignment between zone raster and theme raster
if isinstance(zone_fc, arcpy.Result):
zone_fc = zone_fc.getOutput(0)
this_files_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(this_files_dir)
common_grid = os.path.abspath('../common_grid.tif')
env.snapRaster = common_grid
env.cellSize = common_grid
env.extent = zone_fc
zone_desc = arcpy.Describe(zone_fc)
zone_raster = 'convertraster'
if zone_desc.dataType not in ['RasterDataset', 'RasterLayer']:
zone_raster = arcpy.PolygonToRaster_conversion(
zone_fc,
zone_field,
zone_raster,
'CELL_CENTER',
cellsize=env.cellSize)
print('cell size is {}'.format(env.cellSize))
zone_size = int(env.cellSize)
else:
zone_raster = zone_fc
zone_size = min(
arcpy.Describe(zone_raster).meanCellHeight,
arcpy.Describe(zone_raster).meanCellWidth)
raster_size = min(
arcpy.Describe(in_value_raster).meanCellHeight,
arcpy.Describe(in_value_raster).meanCellWidth)
env.cellSize = min([zone_size, raster_size])
print('cell size is {}'.format(env.cellSize))
# I tested and there is no need to resample the raster being summarized. It will be resampled correctly
# internally in the following tool given that the necessary environments are set above (cell size, snap).
# # in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE)
if not is_thematic:
arcpy.AddMessage("Calculating Zonal Statistics...")
temp_entire_table = arcpy.sa.ZonalStatisticsAsTable(
zone_raster, zone_field, in_value_raster, 'temp_zonal_table',
'DATA', 'MEAN')
if is_thematic:
# for some reason env.cellSize doesn't work
# calculate/doit
arcpy.AddMessage("Tabulating areas...")
temp_entire_table = arcpy.sa.TabulateArea(
zone_raster,
zone_field,
in_value_raster,
'Value',
'temp_area_table',
processing_cell_size=env.cellSize)
# TabulateArea capitalizes the zone for some annoying reason and ArcGIS is case-insensitive to field names
# so we have this work-around:
zone_field_t = '{}_t'.format(zone_field)
DM.AddField(temp_entire_table,
zone_field_t,
'TEXT',
field_length=20)
expr = '!{}!'.format(zone_field.upper())
DM.CalculateField(temp_entire_table, zone_field_t, expr, 'PYTHON')
DM.DeleteField(temp_entire_table, zone_field.upper())
DM.AlterField(temp_entire_table,
zone_field_t,
zone_field,
clear_field_alias=True)
# replaces join to Zonal Stats in previous versions of tool
# no joining, just calculate the area/count from what's produced by TabulateArea
arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE')
cursor_fields = ['AREA', 'COUNT']
value_fields = [
f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*')
]
cursor_fields.extend(value_fields)
with arcpy.da.UpdateCursor(temp_entire_table,
cursor_fields) as uCursor:
for uRow in uCursor:
area, count, value_fields = uRow[0], uRow[1], uRow[2:]
area = sum(value_fields)
count = round(
area / (int(env.cellSize) * int(env.cellSize)), 0)
new_row = [area, count] + value_fields
uCursor.updateRow(new_row)
arcpy.AddMessage("Refining output table...")
arcpy.AddField_management(temp_entire_table, 'datacoveragepct',
'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'ORIGINAL_COUNT', 'LONG')
# calculate datacoveragepct by comparing to original areas in zone raster
# alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count
zone_raster_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(zone_raster,
[zone_field, 'Count'])
}
temp_entire_table_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(temp_entire_table,
[zone_field, 'COUNT'])
}
sum_cell_area = float(env.cellSize) * float(env.cellSize)
orig_cell_area = zone_size * zone_size
with arcpy.da.UpdateCursor(
temp_entire_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT']) as cursor:
for uRow in cursor:
key_value, data_pct, count_orig = uRow
count_orig = zone_raster_dict[key_value]
if key_value in temp_entire_table_dict:
count_summarized = temp_entire_table_dict[key_value]
data_pct = 100 * float((count_summarized * sum_cell_area) /
(count_orig * orig_cell_area))
else:
data_pct = None
cursor.updateRow((key_value, data_pct, count_orig))
# Refine the output
refine_zonal_output(temp_entire_table)
# in order to add vector capabilities back, need to do something with this
# right now we just can't fill in polygon zones that didn't convert to raster in our system
stats_result = cu.one_in_one_out(temp_entire_table, zone_fc,
zone_field, out_table)
# Convert "datacoveragepct" and "ORIGINAL_COUNT" values to 0 for zones with no metrics calculated
with arcpy.da.UpdateCursor(
out_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT', 'CELL_COUNT'
]) as u_cursor:
for row in u_cursor:
# data_coverage pct to 0
if row[1] is None:
row[1] = 0
# original count filled in if a) zone outside raster bounds or b) zone too small to be rasterized
if row[2] is None:
if row[0] in zone_raster_dict:
row[2] = zone_raster_dict[row[0]]
else:
row[2] = 0
# cell count set to 0
if row[3] is None:
row[3] = 0
u_cursor.updateRow(row)
# count whether all zones got an output record or not)
out_count = int(
arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
count_diff = in_count - out_count
# cleanup
if not debug_mode:
for item in [
'temp_zonal_table', temp_entire_table, 'convertraster'
]: # don't add zone_raster, orig
arcpy.Delete_management(item)
arcpy.ResetEnvironments()
env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch
arcpy.CheckInExtension("Spatial")
return [stats_result, count_diff]
def fast_join(fc_target, fc_target_keyfield, fc_join, fc_join_keyfield,
fields_to_join):
start_time = perf_counter()
# make field dict for join fc fields {fname: [dtype, len]}
jfields_names = [f.name for f in ListFields(fc_join)]
jfields_dtypes = [f.type for f in ListFields(fc_join)]
jfields_len = [f.length for f in ListFields(fc_join)]
dts_lens = [[type, len] for type, len in zip(jfields_dtypes, jfields_len)]
jfields_dict = dict(zip(jfields_names, dts_lens))
# field names in the target fc
target_start_fields = [f.name for f in ListFields(fc_target)]
# as needed, add field(s) to target FC if it doesn't already exist.
print(f"Adding fields {fields_to_join} to target table {fc_target}...")
import pdb
pdb.set_trace()
for jfield in fields_to_join:
if jfield not in target_start_fields:
ftype = jfields_dict[jfield][0]
flen = jfields_dict[jfield][1]
management.AddField(in_table=fc_target,
field_name=jfield,
field_type=ftype,
field_length=flen)
else:
print(
f"\t{jfield} already in {fc_target}'s fields. Will be OVERWRITTEN with joined data..."
)
cur_fields = [fc_target_keyfield] + fields_to_join
join_dict = {}
print("reading data from join table...")
with SearchCursor(fc_join, cur_fields) as scur:
for row in scur:
jkey = row[cur_fields.index(fc_join_keyfield)]
vals_to_join = [
row[cur_fields.index(fname)] for fname in fields_to_join
]
join_dict[jkey] = vals_to_join
print("writing join data to target table...")
with UpdateCursor(fc_target, cur_fields) as ucur:
for row in ucur:
jkey = row[cur_fields.index(fc_join_keyfield)]
# if a join id value is in the target table but not the join table,
# skip the join. The values in the resulting joined column will be null for these cases.
if join_dict.get(jkey):
vals_to_join = join_dict[jkey]
else:
continue
row_out = [jkey] + vals_to_join
row = row_out
ucur.updateRow(row)
elapsed_sec = round(perf_counter() - start_time, 1)
print(f"Successfully joined fields {fields_to_join} from {fc_join} onto {fc_target}" \
f" in {elapsed_sec} seconds!")
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field,
other_keep_fields, clip_hu8, lagosne_name):
# dissolve fields by the field that zone_id is based on (the field that identifies a unique zone)
dissolve_fields = [
f for f in "{}, {}, {}".format(zone_id_field, zone_name_field,
other_keep_fields).split(', ')
if f != ''
]
print("Dissolving...")
dissolve1 = DM.Dissolve(zone_fc, 'dissolve1', dissolve_fields)
# update name field to match our standard
DM.AlterField(dissolve1, zone_name_field, 'name')
# original area
DM.AddField(dissolve1, 'originalarea', 'DOUBLE')
DM.CalculateField(dissolve1, 'originalarea', '!shape.area@hectares!',
'PYTHON')
#clip
print("Clipping...")
clip = AN.Clip(dissolve1, MASTER_CLIPPING_POLY, 'clip')
if clip_hu8 == 'Y':
final_clip = AN.Clip(clip, HU8_OUTPUT, 'final_clip')
else:
final_clip = clip
print("Selecting...")
# calc new area, orig area pct, compactness
DM.AddField(final_clip, 'area_ha', 'DOUBLE')
DM.AddField(final_clip, 'originalarea_pct', 'DOUBLE')
DM.AddField(final_clip, 'compactness', 'DOUBLE')
DM.JoinField(final_clip, zone_id_field, dissolve1, zone_id_field,
'originalarea_pct')
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'compactness',
'SHAPE@AREA', 'SHAPE@LENGTH'
]
with arcpy.da.UpdateCursor(final_clip, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, comp, shape_area, shape_length = row
area = shape_area / 10000 # convert from m2 to hectares
orig_area_pct = round(100 * area / orig_area, 2)
comp = 4 * 3.14159 * shape_area / (shape_length**2)
row = (area, orig_area_pct, orig_area, comp, shape_area,
shape_length)
uCursor.updateRow(row)
# if zones are present with <5% of original area and a compactness measure of <.2 (ranges from 0-1)
# AND ALSO they are no bigger than 500 sq. km. (saves Chippewa County and a WWF), filter out
# save eliminated polygons to temp database as a separate layer for inspection
# Different processing for HU4 and HU8, so that they match the extent of HU8 more closely but still throw out tiny slivers
# County also only eliminated if a tiny, tiny, tiny sliver (so: none should be eliminated)
if zone_name not in ('hu4', 'hu12', 'county'):
selected = AN.Select(
final_clip, 'selected',
"originalarea_pct >= 5 OR compactness >= .2 OR area_ha > 50000")
not_selected = AN.Select(
final_clip, '{}_not_selected'.format(output),
"originalarea_pct < 5 AND compactness < .2 AND area_ha < 50000")
else:
selected = final_clip
# eliminate small slivers, re-calc area fields, add perimeter and multipart flag
# leaves the occasional errant sliver but some areas over 25 hectares are more valid so this is
# CONSERVATIVE
print("Trimming...")
trimmed = DM.EliminatePolygonPart(selected,
'trimmed',
'AREA',
'25 Hectares',
part_option='ANY')
# gather up a few calculations into one cursor because this is taking too long over the HU12 layer
DM.AddField(trimmed, 'perimeter_m', 'DOUBLE')
DM.AddField(trimmed, 'multipart', 'TEXT', field_length=1)
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'perimeter_m',
'multipart', 'SHAPE@'
]
with arcpy.da.UpdateCursor(trimmed, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, perim, multipart, shape = row
area = shape.area / 10000 # convert to hectares from m2
orig_area_pct = round(100 * area / orig_area, 2)
perim = shape.length
# multipart flag calc
if shape.isMultipart:
multipart = 'Y'
else:
multipart = 'N'
row = (area, orig_area_pct, orig_area, perim, multipart, shape)
uCursor.updateRow(row)
# delete intermediate fields
DM.DeleteField(trimmed, 'compactness')
DM.DeleteField(trimmed, 'originalarea')
print("Zone IDs....")
# link to LAGOS-NE zone IDs
DM.AddField(trimmed, 'zoneid', 'TEXT', field_length=40)
trimmed_lyr = DM.MakeFeatureLayer(trimmed, 'trimmed_lyr')
if lagosne_name:
# join to the old master GDB path on the same master field and copy in the ids
old_fc = os.path.join(LAGOSNE_GDB, lagosne_name)
old_fc_lyr = DM.MakeFeatureLayer(old_fc, 'old_fc_lyr')
if lagosne_name == 'STATE' or lagosne_name == 'COUNTY':
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr, 'FIPS')
else:
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr,
zone_id_field) # usually works because same source data
# copy
DM.CalculateField(trimmed_lyr, 'zoneid',
'!{}.ZoneID!.lower()'.format(lagosne_name), 'PYTHON')
DM.RemoveJoin(trimmed_lyr)
# generate new zone ids
old_ids = [row[0] for row in arcpy.da.SearchCursor(trimmed, 'zoneid')]
with arcpy.da.UpdateCursor(trimmed, 'zoneid') as cursor:
counter = 1
for row in cursor:
if not row[
0]: # if no existing ID borrowed from LAGOS-NE, assign a new one
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
# ensures new ids don't re-use old numbers but fills in all positive numbers eventually
while new_id in old_ids:
counter += 1
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
row[0] = new_id
cursor.updateRow(row)
counter += 1
print("Edge flags...")
# add flag fields
DM.AddField(trimmed, 'onlandborder', 'TEXT', field_length=2)
DM.AddField(trimmed, 'oncoast', 'TEXT', field_length=2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(trimmed_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'onlandborder', "'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(trimmed_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'oncoast', "'N'", 'PYTHON')
print("State assignment...")
# State?
DM.AddField(trimmed, "state", 'text', field_length='2')
state_center = arcpy.SpatialJoin_analysis(
trimmed,
STATE_FC,
'state_center',
join_type='KEEP_COMMON',
match_option='HAVE_THEIR_CENTER_IN')
state_intersect = arcpy.SpatialJoin_analysis(trimmed,
STATE_FC,
'state_intersect',
match_option='INTERSECT')
state_center_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_center, ['ZoneID', 'STUSPS'])
}
state_intersect_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_intersect, ['ZoneID', 'STUSPS'])
}
with arcpy.da.UpdateCursor(trimmed, ['ZoneID', 'state']) as cursor:
for updateRow in cursor:
keyValue = updateRow[0]
if keyValue in state_center_dict:
updateRow[1] = state_center_dict[keyValue]
else:
updateRow[1] = state_intersect_dict[keyValue]
cursor.updateRow(updateRow)
# glaciation status?
# TODO as version 0.6
# preface the names with the zones
DM.DeleteField(trimmed, 'ORIG_FID')
fields = [
f.name for f in arcpy.ListFields(trimmed, '*')
if f.type not in ('OID',
'Geometry') and not f.name.startswith('Shape_')
]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig=f).lower()
try:
DM.AlterField(trimmed, f, new_fname, clear_field_alias='TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
DM.CopyFeatures(trimmed, output)
# cleanup
lyr_objects = [
lyr_object for var_name, lyr_object in locals().items()
if var_name.endswith('lyr')
]
temp_fcs = arcpy.ListFeatureClasses('*')
for l in lyr_objects + temp_fcs:
DM.Delete(l)
# # make a dict from combining the INTERSECT and CLOSEST results
# zone_dict = {r[0]:r[1] for r in arcpy.da.SearchCursor(join_fc, ['lagoslakeid', zoneid1])}
# for k, v in zone_dict.items():
# if not v:
# zone_dict[k] = update_vals[k]
#
# with arcpy.da.UpdateCursor(lakes_fc, ['lagoslakeid', zoneid]) as u_cursor:
# for row in u_cursor:
# row[1] = zone_dict[row[0]]
# u_cursor.updateRow(row)
#
# DM.Delete('in_memory')
# update the main lakes layer
lakes_poly = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Lakes\LAGOS_US_All_Lakes_1ha'
lakes_point = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Lakes\LAGOS_US_All_Lakes_1ha_points'
#zones = ['hu12', 'hu8', 'hu4', 'county', 'state', 'epanutr4', 'wwf', 'mlra', 'bailey', 'neon']
zones = ['omernik3', 'epanutr']
zoneids = ['{}_zoneid'.format(z) for z in zones]
point_dict = {r[0]:r[1:] for r in arcpy.da.SearchCursor(lakes_point, ['lagoslakeid'] + zoneids)}
for z in zoneids:
if not arcpy.ListFields(lakes_poly, z):
DM.AddField(lakes_poly, z, 'TEXT', field_length=20)
with arcpy.da.UpdateCursor(lakes_poly, ['lagoslakeid'] + zoneids) as u_cursor:
for row in u_cursor:
row[1:] = point_dict[row[0]]
u_cursor.updateRow(row)
# cycle through each zone raster
for i in range(len(hypZ)):
print("working on " + hypZ[i])
# don't assume i is the class level -- extract class here
classLevel = hypZ[i][-1:]
curZo = wrk + "/zon_C" + classLevel
# cycle through each edm
for j in range(len(rasL)):
if j == 0:
inRas = inPath + "/" + rasL[j] + "_c.tif"
curZoT_out = wrk + "/zonTab_C" + str(i) + "_" + str(j)
print(".. zoning " + rasL[j])
curZoT = ZonalStatisticsAsTable(hypZ[i], "Value", inRas,
curZoT_out, "DATA", "MAXIMUM")
man.CopyRaster(hypZ[i], curZo)
man.AddField(curZo, "spp0", "TEXT", "", "", 251)
man.JoinField(curZo, "Value", curZoT, "VALUE", ["MAX"])
expr = "str( !MAX! )"
man.CalculateField(curZo, "spp0", expr, "PYTHON")
man.DeleteField(curZo, "MAX")
man.Delete(curZoT_out)
else:
#jminus = j-1
inRas = inPath + "/" + rasL[j] + "_c.tif"
print(".. zoning " + rasL[j])
curZoT_out = wrk + "/zonTab_C" + str(i) + "_" + str(j)
curZoT = ZonalStatisticsAsTable(hypZ[i], "Value", inRas,
curZoT_out, "DATA", "MAXIMUM")
man.JoinField(curZo, "Value", curZoT, "VALUE", ["MAX"])
expr = "str(!spp0!) + str(!MAX!)"
man.CalculateField(curZo, "spp0", expr, "PYTHON")
