def find_states(fc, state_fc):
"""Populate *_states field. States fc must have field 'states' with length 255 and state abbreviations within."""
states_field = '{}_states'.format(os.path.basename(fc))
if arcpy.ListFields(fc, states_field):
DM.DeleteField(fc, states_field)
# reverse buffer the states slightly to avoid "D", "I", "J" situations in "INTERSECT" illustration
# from graphic examples of ArcGIS join types "Select polygon using polygon" section in Help
# make a field mapping that gathers all the intersecting states into one new value
field_list = [f.name for f in arcpy.ListFields(fc) if f.type <> 'OID' and f.type <> 'Geometry']
field_mapping = arcpy.FieldMappings()
for f in field_list:
map = arcpy.FieldMap()
map.addInputField(fc, f)
field_mapping.addFieldMap(map)
map_states = arcpy.FieldMap()
map_states.addInputField(state_fc, 'states')
map_states.mergeRule = 'Join'
map_states.joinDelimiter = ' '
field_mapping.addFieldMap(map_states)
# perform join and use output to replace original fc
spjoin = AN.SpatialJoin(fc, state_fc, 'in_memory/spjoin_intersect', 'JOIN_ONE_TO_ONE',
field_mapping=field_mapping, match_option='INTERSECT')
DM.AlterField(spjoin, 'states', new_field_name=states_field, clear_field_alias=True)
DM.Delete(fc)
DM.CopyFeatures(spjoin, fc)
DM.Delete(spjoin)
def process_ws(ws_fc, zone_name):
# generate new zone ids
DM.AddField(ws_fc, 'zoneid', 'TEXT', field_length=10)
DM.CalculateField(ws_fc, 'zoneid', '!lagoslakeid!', 'PYTHON')
ws_fc_lyr = DM.MakeFeatureLayer(ws_fc)
# multipart
DM.AddField(ws_fc, 'ismultipart', 'TEXT', field_length=2)
with arcpy.da.UpdateCursor(ws_fc, ['ismultipart', 'SHAPE@']) as u_cursor:
for row in u_cursor:
if row[1].isMultipart:
row[0] = 'Y'
else:
row[0] = 'N'
u_cursor.updateRow(row)
print("Edge flags...")
# add flag fields
DM.AddField(ws_fc, 'onlandborder', 'TEXT', field_length = 2)
DM.AddField(ws_fc, 'oncoast', 'TEXT', field_length = 2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(ws_fc_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'onlandborder' ,"'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(ws_fc_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'oncoast' ,"'N'", 'PYTHON')
print("State assignment...")
# States
state_geo = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Spatial_Classifications\state'
find_states(ws_fc, STATES_GEO)
# glaciation status?
calc_glaciation(ws_fc, 'zoneid')
# preface the names with the zones
DM.DeleteField(ws_fc, 'ORIG_FID')
fields = [f.name for f in arcpy.ListFields(ws_fc, '*') if f.type not in ('OID', 'Geometry') and not f.name.startswith('Shape_')]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig = f).lower()
try:
DM.AlterField(ws_fc, f, new_fname, clear_field_alias = 'TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
# cleanup
lyr_objects = [lyr_object for var_name, lyr_object in locals().items() if var_name.endswith('lyr')]
for l in lyr_objects:
DM.Delete(l)
def rename_to_standard(table):
arcpy.AddMessage("Renaming.")
# datacoverage just gets tag
new_datacov_name = '{}_datacoveragepct'.format(rename_tag)
cu.rename_field(table,
'datacoveragepct',
new_datacov_name,
deleteOld=True)
# DM.AlterField(out_table, 'datacoveragepct', new_datacov_name, clear_field_alias=True)
if not is_thematic:
new_mean_name = '{}_{}'.format(rename_tag, units).rstrip(
'_') # if no units, just rename_tag
cu.rename_field(table, 'MEAN', new_mean_name, deleteOld=True)
# DM.AlterField(out_table, 'MEAN', new_mean_name, clear_field_alias=True)
else:
# look up the values based on the rename tag
geo_file = os.path.abspath('../geo_metric_provenance.csv')
with open(geo_file) as csv_file:
reader = csv.DictReader(csv_file)
mapping = {
row['subgroup_original_code']: row['subgroup']
for row in reader if row['main_feature']
and row['main_feature'] in rename_tag
}
print(mapping)
# update them
for old, new in mapping.items():
old_fname = 'VALUE_{}_pct'.format(old)
new_fname = '{}_{}_pct'.format(rename_tag, new)
if arcpy.ListFields(table, old_fname):
try:
# same problem with AlterField limit of 31 characters here.
DM.AlterField(table,
old_fname,
new_fname,
clear_field_alias=True)
except:
cu.rename_field(table,
old_fname,
new_fname,
deleteOld=True)
return table
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field,
other_keep_fields, clip_hu8, lagosne_name):
# dissolve fields by the field that zone_id is based on (the field that identifies a unique zone)
dissolve_fields = [
f for f in "{}, {}, {}".format(zone_id_field, zone_name_field,
other_keep_fields).split(', ')
if f != ''
]
print("Dissolving...")
dissolve1 = DM.Dissolve(zone_fc, 'dissolve1', dissolve_fields)
# update name field to match our standard
DM.AlterField(dissolve1, zone_name_field, 'name')
# original area
DM.AddField(dissolve1, 'originalarea', 'DOUBLE')
DM.CalculateField(dissolve1, 'originalarea', '!shape.area@hectares!',
'PYTHON')
#clip
print("Clipping...")
clip = AN.Clip(dissolve1, MASTER_CLIPPING_POLY, 'clip')
if clip_hu8 == 'Y':
final_clip = AN.Clip(clip, HU8_OUTPUT, 'final_clip')
else:
final_clip = clip
print("Selecting...")
# calc new area, orig area pct, compactness
DM.AddField(final_clip, 'area_ha', 'DOUBLE')
DM.AddField(final_clip, 'originalarea_pct', 'DOUBLE')
DM.AddField(final_clip, 'compactness', 'DOUBLE')
DM.JoinField(final_clip, zone_id_field, dissolve1, zone_id_field,
'originalarea_pct')
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'compactness',
'SHAPE@AREA', 'SHAPE@LENGTH'
]
with arcpy.da.UpdateCursor(final_clip, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, comp, shape_area, shape_length = row
area = shape_area / 10000 # convert from m2 to hectares
orig_area_pct = round(100 * area / orig_area, 2)
comp = 4 * 3.14159 * shape_area / (shape_length**2)
row = (area, orig_area_pct, orig_area, comp, shape_area,
shape_length)
uCursor.updateRow(row)
# if zones are present with <5% of original area and a compactness measure of <.2 (ranges from 0-1)
# AND ALSO they are no bigger than 500 sq. km. (saves Chippewa County and a WWF), filter out
# save eliminated polygons to temp database as a separate layer for inspection
# Different processing for HU4 and HU8, so that they match the extent of HU8 more closely but still throw out tiny slivers
# County also only eliminated if a tiny, tiny, tiny sliver (so: none should be eliminated)
if zone_name not in ('hu4', 'hu12', 'county'):
selected = AN.Select(
final_clip, 'selected',
"originalarea_pct >= 5 OR compactness >= .2 OR area_ha > 50000")
not_selected = AN.Select(
final_clip, '{}_not_selected'.format(output),
"originalarea_pct < 5 AND compactness < .2 AND area_ha < 50000")
else:
selected = final_clip
# eliminate small slivers, re-calc area fields, add perimeter and multipart flag
# leaves the occasional errant sliver but some areas over 25 hectares are more valid so this is
# CONSERVATIVE
print("Trimming...")
trimmed = DM.EliminatePolygonPart(selected,
'trimmed',
'AREA',
'25 Hectares',
part_option='ANY')
# gather up a few calculations into one cursor because this is taking too long over the HU12 layer
DM.AddField(trimmed, 'perimeter_m', 'DOUBLE')
DM.AddField(trimmed, 'multipart', 'TEXT', field_length=1)
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'perimeter_m',
'multipart', 'SHAPE@'
]
with arcpy.da.UpdateCursor(trimmed, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, perim, multipart, shape = row
area = shape.area / 10000 # convert to hectares from m2
orig_area_pct = round(100 * area / orig_area, 2)
perim = shape.length
# multipart flag calc
if shape.isMultipart:
multipart = 'Y'
else:
multipart = 'N'
row = (area, orig_area_pct, orig_area, perim, multipart, shape)
uCursor.updateRow(row)
# delete intermediate fields
DM.DeleteField(trimmed, 'compactness')
DM.DeleteField(trimmed, 'originalarea')
print("Zone IDs....")
# link to LAGOS-NE zone IDs
DM.AddField(trimmed, 'zoneid', 'TEXT', field_length=40)
trimmed_lyr = DM.MakeFeatureLayer(trimmed, 'trimmed_lyr')
if lagosne_name:
# join to the old master GDB path on the same master field and copy in the ids
old_fc = os.path.join(LAGOSNE_GDB, lagosne_name)
old_fc_lyr = DM.MakeFeatureLayer(old_fc, 'old_fc_lyr')
if lagosne_name == 'STATE' or lagosne_name == 'COUNTY':
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr, 'FIPS')
else:
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr,
zone_id_field) # usually works because same source data
# copy
DM.CalculateField(trimmed_lyr, 'zoneid',
'!{}.ZoneID!.lower()'.format(lagosne_name), 'PYTHON')
DM.RemoveJoin(trimmed_lyr)
# generate new zone ids
old_ids = [row[0] for row in arcpy.da.SearchCursor(trimmed, 'zoneid')]
with arcpy.da.UpdateCursor(trimmed, 'zoneid') as cursor:
counter = 1
for row in cursor:
if not row[
0]: # if no existing ID borrowed from LAGOS-NE, assign a new one
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
# ensures new ids don't re-use old numbers but fills in all positive numbers eventually
while new_id in old_ids:
counter += 1
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
row[0] = new_id
cursor.updateRow(row)
counter += 1
print("Edge flags...")
# add flag fields
DM.AddField(trimmed, 'onlandborder', 'TEXT', field_length=2)
DM.AddField(trimmed, 'oncoast', 'TEXT', field_length=2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(trimmed_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'onlandborder', "'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(trimmed_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'oncoast', "'N'", 'PYTHON')
print("State assignment...")
# State?
DM.AddField(trimmed, "state", 'text', field_length='2')
state_center = arcpy.SpatialJoin_analysis(
trimmed,
STATE_FC,
'state_center',
join_type='KEEP_COMMON',
match_option='HAVE_THEIR_CENTER_IN')
state_intersect = arcpy.SpatialJoin_analysis(trimmed,
STATE_FC,
'state_intersect',
match_option='INTERSECT')
state_center_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_center, ['ZoneID', 'STUSPS'])
}
state_intersect_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_intersect, ['ZoneID', 'STUSPS'])
}
with arcpy.da.UpdateCursor(trimmed, ['ZoneID', 'state']) as cursor:
for updateRow in cursor:
keyValue = updateRow[0]
if keyValue in state_center_dict:
updateRow[1] = state_center_dict[keyValue]
else:
updateRow[1] = state_intersect_dict[keyValue]
cursor.updateRow(updateRow)
# glaciation status?
# TODO as version 0.6
# preface the names with the zones
DM.DeleteField(trimmed, 'ORIG_FID')
fields = [
f.name for f in arcpy.ListFields(trimmed, '*')
if f.type not in ('OID',
'Geometry') and not f.name.startswith('Shape_')
]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig=f).lower()
try:
DM.AlterField(trimmed, f, new_fname, clear_field_alias='TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
DM.CopyFeatures(trimmed, output)
# cleanup
lyr_objects = [
lyr_object for var_name, lyr_object in locals().items()
if var_name.endswith('lyr')
]
temp_fcs = arcpy.ListFeatureClasses('*')
for l in lyr_objects + temp_fcs:
DM.Delete(l)
def stats_area_table(zone_fc=zone_fc,
zone_field=zone_field,
in_value_raster=in_value_raster,
out_table=out_table,
is_thematic=is_thematic):
def refine_zonal_output(t):
"""Makes a nicer output for this tool. Rename some fields, drop unwanted
ones, calculate percentages using raster AREA before deleting that
field."""
if is_thematic:
value_fields = arcpy.ListFields(t, "VALUE*")
pct_fields = [
'{}_pct'.format(f.name) for f in value_fields
] # VALUE_41_pct, etc. Field can't start with number.
# add all the new fields needed
for f, pct_field in zip(value_fields, pct_fields):
arcpy.AddField_management(t, pct_field, f.type)
# calculate the percents
cursor_fields = ['AREA'] + [f.name
for f in value_fields] + pct_fields
uCursor = arcpy.da.UpdateCursor(t, cursor_fields)
for uRow in uCursor:
# unpacks area + 3 tuples of the right fields for each, no matter how many there are
vf_i_end = len(value_fields) + 1
pf_i_end = vf_i_end + len(pct_fields)
# pct_values and ha_values are both null at this point but unpack for clarity
area, value_values, pct_values = uRow[0], uRow[
1:vf_i_end], uRow[vf_i_end:pf_i_end]
new_pct_values = [100 * vv / area for vv in value_values]
new_row = [area] + value_values + new_pct_values
uCursor.updateRow(new_row)
for vf in value_fields:
arcpy.DeleteField_management(t, vf.name)
arcpy.AlterField_management(t, 'COUNT', 'CELL_COUNT')
drop_fields = ['ZONE_CODE', 'COUNT', 'AREA']
if not debug_mode:
for df in drop_fields:
try:
arcpy.DeleteField_management(t, df)
except:
continue
# Set up environments for alignment between zone raster and theme raster
if isinstance(zone_fc, arcpy.Result):
zone_fc = zone_fc.getOutput(0)
this_files_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(this_files_dir)
common_grid = os.path.abspath('../common_grid.tif')
env.snapRaster = common_grid
env.cellSize = common_grid
env.extent = zone_fc
zone_desc = arcpy.Describe(zone_fc)
zone_raster = 'convertraster'
if zone_desc.dataType not in ['RasterDataset', 'RasterLayer']:
zone_raster = arcpy.PolygonToRaster_conversion(
zone_fc,
zone_field,
zone_raster,
'CELL_CENTER',
cellsize=env.cellSize)
print('cell size is {}'.format(env.cellSize))
zone_size = int(env.cellSize)
else:
zone_raster = zone_fc
zone_size = min(
arcpy.Describe(zone_raster).meanCellHeight,
arcpy.Describe(zone_raster).meanCellWidth)
raster_size = min(
arcpy.Describe(in_value_raster).meanCellHeight,
arcpy.Describe(in_value_raster).meanCellWidth)
env.cellSize = min([zone_size, raster_size])
print('cell size is {}'.format(env.cellSize))
# I tested and there is no need to resample the raster being summarized. It will be resampled correctly
# internally in the following tool given that the necessary environments are set above (cell size, snap).
# # in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE)
if not is_thematic:
arcpy.AddMessage("Calculating Zonal Statistics...")
temp_entire_table = arcpy.sa.ZonalStatisticsAsTable(
zone_raster, zone_field, in_value_raster, 'temp_zonal_table',
'DATA', 'MEAN')
if is_thematic:
# for some reason env.cellSize doesn't work
# calculate/doit
arcpy.AddMessage("Tabulating areas...")
temp_entire_table = arcpy.sa.TabulateArea(
zone_raster,
zone_field,
in_value_raster,
'Value',
'temp_area_table',
processing_cell_size=env.cellSize)
# TabulateArea capitalizes the zone for some annoying reason and ArcGIS is case-insensitive to field names
# so we have this work-around:
zone_field_t = '{}_t'.format(zone_field)
DM.AddField(temp_entire_table,
zone_field_t,
'TEXT',
field_length=20)
expr = '!{}!'.format(zone_field.upper())
DM.CalculateField(temp_entire_table, zone_field_t, expr, 'PYTHON')
DM.DeleteField(temp_entire_table, zone_field.upper())
DM.AlterField(temp_entire_table,
zone_field_t,
zone_field,
clear_field_alias=True)
# replaces join to Zonal Stats in previous versions of tool
# no joining, just calculate the area/count from what's produced by TabulateArea
arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE')
cursor_fields = ['AREA', 'COUNT']
value_fields = [
f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*')
]
cursor_fields.extend(value_fields)
with arcpy.da.UpdateCursor(temp_entire_table,
cursor_fields) as uCursor:
for uRow in uCursor:
area, count, value_fields = uRow[0], uRow[1], uRow[2:]
area = sum(value_fields)
count = round(
area / (int(env.cellSize) * int(env.cellSize)), 0)
new_row = [area, count] + value_fields
uCursor.updateRow(new_row)
arcpy.AddMessage("Refining output table...")
arcpy.AddField_management(temp_entire_table, 'datacoveragepct',
'DOUBLE')
arcpy.AddField_management(temp_entire_table, 'ORIGINAL_COUNT', 'LONG')
# calculate datacoveragepct by comparing to original areas in zone raster
# alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count
zone_raster_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(zone_raster,
[zone_field, 'Count'])
}
temp_entire_table_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(temp_entire_table,
[zone_field, 'COUNT'])
}
sum_cell_area = float(env.cellSize) * float(env.cellSize)
orig_cell_area = zone_size * zone_size
with arcpy.da.UpdateCursor(
temp_entire_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT']) as cursor:
for uRow in cursor:
key_value, data_pct, count_orig = uRow
count_orig = zone_raster_dict[key_value]
if key_value in temp_entire_table_dict:
count_summarized = temp_entire_table_dict[key_value]
data_pct = 100 * float((count_summarized * sum_cell_area) /
(count_orig * orig_cell_area))
else:
data_pct = None
cursor.updateRow((key_value, data_pct, count_orig))
# Refine the output
refine_zonal_output(temp_entire_table)
# in order to add vector capabilities back, need to do something with this
# right now we just can't fill in polygon zones that didn't convert to raster in our system
stats_result = cu.one_in_one_out(temp_entire_table, zone_fc,
zone_field, out_table)
# Convert "datacoveragepct" and "ORIGINAL_COUNT" values to 0 for zones with no metrics calculated
with arcpy.da.UpdateCursor(
out_table,
[zone_field, 'datacoveragepct', 'ORIGINAL_COUNT', 'CELL_COUNT'
]) as u_cursor:
for row in u_cursor:
# data_coverage pct to 0
if row[1] is None:
row[1] = 0
# original count filled in if a) zone outside raster bounds or b) zone too small to be rasterized
if row[2] is None:
if row[0] in zone_raster_dict:
row[2] = zone_raster_dict[row[0]]
else:
row[2] = 0
# cell count set to 0
if row[3] is None:
row[3] = 0
u_cursor.updateRow(row)
# count whether all zones got an output record or not)
out_count = int(
arcpy.GetCount_management(temp_entire_table).getOutput(0))
in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0))
count_diff = in_count - out_count
# cleanup
if not debug_mode:
for item in [
'temp_zonal_table', temp_entire_table, 'convertraster'
]: # don't add zone_raster, orig
arcpy.Delete_management(item)
arcpy.ResetEnvironments()
env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch
arcpy.CheckInExtension("Spatial")
return [stats_result, count_diff]