def find_states(fc, state_fc):
"""Populate *_states field. States fc must have field 'states' with length 255 and state abbreviations within."""
states_field = '{}_states'.format(os.path.basename(fc))
if arcpy.ListFields(fc, states_field):
DM.DeleteField(fc, states_field)
# reverse buffer the states slightly to avoid "D", "I", "J" situations in "INTERSECT" illustration
# from graphic examples of ArcGIS join types "Select polygon using polygon" section in Help
# make a field mapping that gathers all the intersecting states into one new value
field_list = [f.name for f in arcpy.ListFields(fc) if f.type <> 'OID' and f.type <> 'Geometry']
field_mapping = arcpy.FieldMappings()
for f in field_list:
map = arcpy.FieldMap()
map.addInputField(fc, f)
field_mapping.addFieldMap(map)
map_states = arcpy.FieldMap()
map_states.addInputField(state_fc, 'states')
map_states.mergeRule = 'Join'
map_states.joinDelimiter = ' '
field_mapping.addFieldMap(map_states)
# perform join and use output to replace original fc
spjoin = AN.SpatialJoin(fc, state_fc, 'in_memory/spjoin_intersect', 'JOIN_ONE_TO_ONE',
field_mapping=field_mapping, match_option='INTERSECT')
DM.AlterField(spjoin, 'states', new_field_name=states_field, clear_field_alias=True)
DM.Delete(fc)
DM.CopyFeatures(spjoin, fc)
DM.Delete(spjoin)
def drought_analysis(date_string):
ARCPY.env.overwriteOutput = True
working_dir = r"C:\Data\git\devsummit-14-python"
zip_name = "USDM_" + date_string + "_M.zip"
url = "http://droughtmonitor.unl.edu/data/shapefiles_m/" + zip_name
mxd_path = OS.path.join(working_dir, "MapTemplate.mxd")
lyr_template = OS.path.join(working_dir, "CurrentDroughtConditions.lyr")
zip_name = OS.path.basename(url)
drought_zip_file = URLLIB.URLopener()
dzf = drought_zip_file.retrieve(url, OS.path.join(r"C:\Temp", zip_name))
zf = ZIPFILE.ZipFile(dzf[0], "r")
shp_name = [n for n in zf.namelist() if n.endswith('.shp')][0]
zf.extractall(working_dir)
drought = OS.path.splitext(shp_name)[0]
DM.MakeFeatureLayer(OS.path.join(working_dir, shp_name), drought)
#### Add Winery Data ####
beerWinePath = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWine")
intermediate_output = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWineDrought")
wine = "BeerWine"
wine_drought = "Wine_Drought"
DM.MakeFeatureLayer(beerWinePath, wine)
DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'")
ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE",
"KEEP_ALL")
try:
DM.DeleteField(intermediate_output, "NAME")
except:
pass
final_wine_drought = "Wine_Drought_Summary"
DM.MakeFeatureLayer(intermediate_output, final_wine_drought)
lf = DM.SaveToLayerFile(
final_wine_drought,
OS.path.join(working_dir, '{}.lyr'.format(final_wine_drought)))
DM.ApplySymbologyFromLayer(lf, lyr_template)
pw = "PASSWORDHERE" #GETPASS.getpass("Enter AGOL password:"******"Drought_Wine_Service"
agol = AGOLHandler("USERNAMEHERE", pw, service_name)
publish_service(agol, service_name, mxd_path, lf[0])
TIME.sleep(5)
fs_url = agol.findItemURL('Feature Service')
TIME.sleep(35)
gp_url, jsondata = enrich(agol, fs_url + '/0',
'{}_Enriched'.format(service_name), agol.token)
check_job_status(gp_url, jsondata, agol.token)
DM.Delete(OS.path.join(working_dir, shp_name))
DM.Delete(OS.path.join(working_dir, lf[0]))
def create_points_feature_class(fc, sr=None):
arcpy.env.addOutputsToMap = False
sr = sr or arcpy.env.outputCoordinateSystem
if sr is None:
arcpy.AddError('No spatial reference system.')
return None
scratch_fc = os.path.join(arcpy.env.scratchWorkspace, os.path.basename(fc))
mgmt.CreateFeatureclass(*os.path.split(scratch_fc),
'POINT',
spatial_reference=sr)
mgmt.AddField(scratch_fc, 'ELEVATION', 'DOUBLE')
mgmt.AddField(scratch_fc, 'TIME', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'NAME', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'DESCRIPTION', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'SYMBOL', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'TYPE', 'TEXT', field_length=64)
mgmt.AddField(scratch_fc, 'SAMPLES', 'LONG')
if fc != scratch_fc:
mgmt.Copy(scratch_fc, fc)
mgmt.Delete(scratch_fc)
return fc
def process_ws(ws_fc, zone_name):
# generate new zone ids
DM.AddField(ws_fc, 'zoneid', 'TEXT', field_length=10)
DM.CalculateField(ws_fc, 'zoneid', '!lagoslakeid!', 'PYTHON')
ws_fc_lyr = DM.MakeFeatureLayer(ws_fc)
# multipart
DM.AddField(ws_fc, 'ismultipart', 'TEXT', field_length=2)
with arcpy.da.UpdateCursor(ws_fc, ['ismultipart', 'SHAPE@']) as u_cursor:
for row in u_cursor:
if row[1].isMultipart:
row[0] = 'Y'
else:
row[0] = 'N'
u_cursor.updateRow(row)
print("Edge flags...")
# add flag fields
DM.AddField(ws_fc, 'onlandborder', 'TEXT', field_length = 2)
DM.AddField(ws_fc, 'oncoast', 'TEXT', field_length = 2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(ws_fc_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'onlandborder' ,"'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(ws_fc_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION')
DM.CalculateField(ws_fc_lyr, 'oncoast' ,"'N'", 'PYTHON')
print("State assignment...")
# States
state_geo = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Spatial_Classifications\state'
find_states(ws_fc, STATES_GEO)
# glaciation status?
calc_glaciation(ws_fc, 'zoneid')
# preface the names with the zones
DM.DeleteField(ws_fc, 'ORIG_FID')
fields = [f.name for f in arcpy.ListFields(ws_fc, '*') if f.type not in ('OID', 'Geometry') and not f.name.startswith('Shape_')]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig = f).lower()
try:
DM.AlterField(ws_fc, f, new_fname, clear_field_alias = 'TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
# cleanup
lyr_objects = [lyr_object for var_name, lyr_object in locals().items() if var_name.endswith('lyr')]
for l in lyr_objects:
DM.Delete(l)
def cleanupNetLayer(netLayer):
"""Tries to delete Network OD Cost Layers.
INPUTS:
netLayer (str): network OD cost layer
"""
try:
DM.Delete(netLayer)
except:
pass
def get_percentile(in_file, percentile=0.5, multiplier=100, skip_value=None):
# mult_rast = Times (in_file, multiplier)
int_rast = Int(Times(in_file, multiplier))
# no idea why this is needed, but something needs to tickle int_rast
print(int_rast)
arcmgt.BuildRasterAttributeTable(int_rast)
table_view = "table_view%d" % int(random.random() * 1000)
arcmgt.MakeTableView(int_rast, table_view)
fields = arcpy.ListFields(in_file)
rows = arcpy.SearchCursor(table_view)
cum_sum = 0
for row in rows:
val = row.getValue("Value")
count = row.getValue("Count")
if val == skip_value:
continue
cum_sum = cum_sum + count
row = None
target = cum_sum * percentile
rows2 = arcpy.SearchCursor(table_view)
cum_sum = 0
val = 0
for row2 in rows2:
val = row2.getValue("Value")
count = row2.getValue("Count")
if val == skip_value:
continue
cum_sum = cum_sum + count
if cum_sum >= target:
break
val = float(val) / multiplier
arcmgt.Delete(table_view)
return val
def setupOptHotSpot():
"""Retrieves the parameters from the User Interface and executes the
appropriate commands."""
#### Input Parameters ####
inputFC = ARCPY.GetParameterAsText(0)
outputFC = ARCPY.GetParameterAsText(1)
varName = UTILS.getTextParameter(2, fieldName=True)
aggMethod = UTILS.getTextParameter(3)
if aggMethod:
aggType = aggTypes[aggMethod.upper()]
else:
aggType = 1
boundaryFC = UTILS.getTextParameter(4)
polygonFC = UTILS.getTextParameter(5)
outputRaster = UTILS.getTextParameter(6)
makeFeatureLayerNoExtent = UTILS.clearExtent(DM.MakeFeatureLayer)
selectLocationNoExtent = UTILS.clearExtent(DM.SelectLayerByLocation)
featureLayer = "InputOHSA_FC"
makeFeatureLayerNoExtent(inputFC, featureLayer)
if boundaryFC:
selectLocationNoExtent(featureLayer, "INTERSECT", boundaryFC, "#",
"NEW_SELECTION")
polygonFC = None
if polygonFC:
selectLocationNoExtent(featureLayer, "INTERSECT", polygonFC, "#",
"NEW_SELECTION")
boundaryFC = None
#### Create SSDO ####
ssdo = SSDO.SSDataObject(featureLayer,
templateFC=outputFC,
useChordal=True)
hs = OptHotSpots(ssdo,
outputFC,
varName=varName,
aggType=aggType,
polygonFC=polygonFC,
boundaryFC=boundaryFC,
outputRaster=outputRaster)
DM.Delete(featureLayer)
def calc_glaciation(fc, zone_field):
# tab area
g_field = '{}_glaciatedlatewisc'.format(os.path.basename(fc))
AN.TabulateIntersection(fc, zone_field, GLACIAL_EXTENT, 'in_memory/glacial_tab')
glacial_pct = {r[0]:r[1] for r in arcpy.da.SearchCursor('in_memory/glacial_tab', [zone_field, 'PERCENTAGE'])}
DM.AddField(fc, g_field, 'TEXT', field_length=20)
with arcpy.da.UpdateCursor(fc, [zone_field, g_field]) as u_cursor:
for row in u_cursor:
zoneid, glaciation = row
if zoneid not in glacial_pct:
glaciation = 'Not_Glaciated'
else:
if glacial_pct[zoneid] >=99.99:
glaciation = 'Glaciated'
elif glacial_pct[zoneid] < 0.01:
glaciation = 'Not_Glaciated'
else:
glaciation = 'Partially_Glaciated'
u_cursor.updateRow((zoneid, glaciation))
DM.Delete('in_memory/glacial_tab')
outWeighted = WeightedSum(myWSTable)
k = time.perf_counter() / 60
print(f"Weighted sum complete: {round(k - j, 2)} minutes")
# Save the output
output = script + r"/modeloutput"
outWeighted.save(output)
#Make contour around high habitat probability model regions with highest kernel density
points = script + r"\points.shp"
points2 = script + r"\points2.shp"
CO.RasterToPoint(output, points, "Value")
AN.Select(points, points2, "grid_code > 0.9")
kernel = KernelDensity(points2, "NONE", 100, None, "SQUARE_KILOMETERS",
"DENSITIES", "PLANAR")
contour = script + r"\contour.shp"
DDD.Contour(kernel, contour, 1000, 20, 1, "CONTOUR", None)
l = time.perf_counter() / 60
print(f"Contour complete: {round(l - k, 2)} minutes")
# Clean things up a bit
if arcpy.Exists(path):
DM.Delete(path)
if arcpy.Exists(points):
DM.Delete(points)
if arcpy.Exists(points2):
DM.Delete(points2)
#All done
print(f"Model complete, total time elapsed: {round(l - a, 2)} minutes")
print("Files located at: " + script)
for i in range(len(classes)):
cl_str = str(classes[i])
cl_int = classes[i]
print("extracting class " + cl_str)
selStr = '"Value" = ' + cl_str
outR = ExtractByAttributes(outReclass, selStr)
outName = "hyp_Class_" + cl_str
outR.save(outName)
print(" .. to poly")
outPoly = "hyp_pol_Class_" + cl_str
arcpy.RasterToPolygon_conversion(outR, outPoly, "NO_SIMPLIFY", "VALUE")
print(" .. buffering")
outBuff = "hyp_pol_buff_Class_" + cl_str
arcpy.Buffer_analysis(outPoly, outBuff, 32, "FULL", "ROUND", "ALL")
man.Delete(outPoly)
print(" .. removing singletons")
outMultiPart = "hyp_pol_buff_m_Class_" + cl_str
arcpy.MultipartToSinglepart_management(outBuff, outMultiPart)
man.Delete(outBuff)
tmpSelLyr = "tmpSelectSet"
expr = '"Shape_Area" < 7900'
arcpy.MakeFeatureLayer_management(outMultiPart, tmpSelLyr)
arcpy.SelectLayerByAttribute_management(tmpSelLyr, "NEW_SELECTION", expr)
if int(arcpy.GetCount_management(tmpSelLyr).getOutput(0)) > 0:
arcpy.DeleteFeatures_management(tmpSelLyr)
#add a field to the attribute table
fldName = "Class"
fldVal = cl_int
arcpy.AddField_management(outMultiPart, fldName, "SHORT")
#expr = '"' + cl_int + '"'
man.CalculateField(curHyp, "spp0", expr, "PYTHON")
else:
iminus = i - 1
prevHyp = wrk + "/hyp" + str(iminus)
print("working on " + elem + ", " + str(i) + " of " + str(listLen))
curHyp = Combine([prevHyp, rasName])
curHyp.save(wrk + "/hyp" + str(i))
man.AddField(curHyp, "spp0", "TEXT", "", "", 251)
jval = "hyp" + str(iminus)
man.JoinField(curHyp, jval, prevHyp, "VALUE", ["spp0"])
rasNoDot = rasName[0:rasName.find(".")]
newCol = rasNoDot[0:11].upper()
expr = "str(!spp0_1!) + str(!" + newCol + "!)"
man.CalculateField(curHyp, "spp0", expr, "PYTHON")
#clean up
man.Delete(prevHyp)
# clean up a little more
man.DeleteField(curHyp, [jval.upper(), newCol, "spp0_1"])
#needed to continue below if you comment out the previous loop for any reason
#curHyp = wrk + "/hyp" + str(len(codeL)-1)
# expand information out to one col for each spp.
print("adding columns...")
for i in range(len(codeL)):
newCol = codeL[i].upper()
print("..." + newCol)
man.AddField(curHyp, newCol, "SHORT")
#expr="str(!supp0!)[i:i+1]"
expr = "str(!spp0!)[" + str(i) + ":" + str(i + 1) + "]"
def flatten_overlaps(zone_fc,
zone_field,
output_fc,
output_table,
cluster_tolerance=' 3 Meters'):
orig_env = arcpy.env.workspace
arcpy.env.workspace = 'in_memory'
objectid = [f.name for f in arcpy.ListFields(zone_fc)
if f.type == 'OID'][0]
zone_type = [f.type for f in arcpy.ListFields(zone_fc, zone_field)][0]
fid1 = 'FID_{}'.format(os.path.basename(zone_fc))
flat_zoneid = 'flat{}'.format(zone_field)
flat_zoneid_prefix = 'flat{}_'.format(zone_field.replace('_zoneid', ''))
# Union with FID_Only (A)
arcpy.AddMessage("Splitting overlaps in polygons...")
zoneid_dict = {
r[0]: r[1]
for r in arcpy.da.SearchCursor(zone_fc, [objectid, zone_field])
}
self_union = AN.Union([zone_fc],
'self_union',
'ONLY_FID',
cluster_tolerance=cluster_tolerance)
# #If you don't run this section, Find Identical fails with error 999999. Seems to have to do with small slivers
# #having 3 vertices and/or only circular arcs in the geometry.
arcpy.AddMessage("Repairing self-union geometries...")
# DM.AddGeometryAttributes(self_union, 'POINT_COUNT; AREA')
# union_fix = DM.MakeFeatureLayer(self_union, 'union_fix', where_clause='PNT_COUNT <= 10 OR POLY_AREA < 5000')
# arcpy.Densify_edit(union_fix, 'DISTANCE', distance = '1 Meters', max_deviation='1 Meters') # selection ON, edits self_union disk
DM.RepairGeometry(
self_union, 'DELETE_NULL'
) # eliminate empty geoms. selection ON, edits self_union disk
# for field in ['PNT_COUNT', 'POLY_AREA']:
# DM.DeleteField(self_union, field)
# Find Identical by Shape (B)
if arcpy.Exists('identical_shapes'):
DM.Delete(
'identical_shapes'
) # causes failure in FindIdentical even when overwrite is allowed
identical_shapes = DM.FindIdentical(self_union, 'identical_shapes',
'Shape')
# Join A to B and calc flat[zone]_zoneid = FEAT_SEQ (C)
DM.AddField(self_union, flat_zoneid, 'TEXT', field_length=20)
union_oid = [
f.name for f in arcpy.ListFields(self_union) if f.type == 'OID'
][0]
identical_shapes_dict = {
r[0]: r[1]
for r in arcpy.da.SearchCursor(identical_shapes,
['IN_FID', 'FEAT_SEQ'])
}
with arcpy.da.UpdateCursor(self_union,
[union_oid, flat_zoneid]) as u_cursor:
for row in u_cursor:
row[1] = '{}{}'.format(flat_zoneid_prefix,
identical_shapes_dict[row[0]])
u_cursor.updateRow(row)
# Add the original zone ids and save to table (E)
arcpy.AddMessage("Assigning temporary IDs to split polygons...")
unflat_table = DM.CopyRows(self_union, 'unflat_table')
DM.AddField(unflat_table, zone_field,
zone_type) # default text length of 50 is fine if needed
with arcpy.da.UpdateCursor(unflat_table, [fid1, zone_field]) as u_cursor:
for row in u_cursor:
row[1] = zoneid_dict[row[0]] # assign zone id
u_cursor.updateRow(row)
# Delete Identical (C) (save as flat[zone])
with arcpy.da.UpdateCursor(self_union, 'OID@') as cursor:
visited = []
for row in cursor:
feat_seq = identical_shapes_dict[row[0]]
if feat_seq in visited:
cursor.deleteRow()
visited.append(feat_seq)
DM.DeleteField(self_union, fid1)
DM.DeleteField(unflat_table, fid1)
# save outputs
output_fc = DM.CopyFeatures(self_union, output_fc)
output_table = DM.CopyRows(unflat_table, output_table)
# cleanup
for item in [self_union, identical_shapes, unflat_table]:
DM.Delete(item)
arcpy.env.workspace = orig_env
return output_fc
def unflatten(intermediate_table):
flat_zoneid = zone_field
unflat_zoneid = zone_field.replace('flat', '')
zone_type = [f.type for f in arcpy.ListFields(zone_fc, flat_zoneid)][0]
# Set up the output table (can't do this until the prior tool is run)
# if os.path.dirname(out_table):
# out_path = os.path.dirname(out_table)
# else:
# out_path = orig_env
unflat_result = DM.CreateTable('in_memory',
os.path.basename(out_table))
# get the fields to add to the table
editable_fields = [
f for f in arcpy.ListFields(intermediate_table)
if f.editable and f.name.lower() != flat_zoneid.lower()
]
# populate the new table schema
DM.AddField(unflat_result, unflat_zoneid, zone_type)
for f in editable_fields:
DM.AddField(unflat_result, f.name, f.type, field_length=f.length)
# map original zone ids to new zone ids
original_flat = defaultdict(list)
with arcpy.da.SearchCursor(unflat_table,
[unflat_zoneid, flat_zoneid]) as cursor:
for row in cursor:
if row[1] not in original_flat[row[0]]:
original_flat[row[0]].append(row[1])
# Use CELL_COUNT as weight for means to calculate final values for each zone.
fixed_fields = [
unflat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
]
other_field_names = [
f.name for f in editable_fields if f.name not in fixed_fields
]
i_cursor = arcpy.da.InsertCursor(
unflat_result,
fixed_fields + other_field_names) # open output table cursor
flat_stats = {
r[0]: r[1:]
for r in arcpy.da.SearchCursor(intermediate_table, [
flat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct'
] + other_field_names)
}
count_diff = 0
for zid, unflat_ids in original_flat.items():
valid_unflat_ids = [id for id in unflat_ids if id in flat_stats
] # skip flatpolys not rasterized
area_vec = [flat_stats[id][0] for id in valid_unflat_ids
] # ORIGINAL_COUNT specified in 0 index earlier
cell_vec = [flat_stats[id][1] for id in valid_unflat_ids]
coverage_vec = [flat_stats[id][2] for id in valid_unflat_ids
] # datacoveragepct special handling
stat_vectors_by_id = [
flat_stats[id][3:] for id in valid_unflat_ids
] # "the rest", list of lists
# calc the new summarized values
original_count = sum(
filter(None, area_vec)
) # None area is functionally equivalent to 0, all Nones = 0 too
cell_count = sum(filter(None, cell_vec))
if cell_count > 0:
weighted_coverage = sum(
[a * b
for a, b in zip(area_vec, coverage_vec)]) / original_count
# this calculation accounts for fractional missing values, both kinds (whole zone is no data, or zone
# was missing some data and had data coverage % < 100). This is done by converting None to 0
# and by using the cell_count (count of cells with data present)
# instead of the full zone original_count. You have to do both or the mean will be distorted.
# hand-verification that this works as intended using test GIS data on was completed 2019-11-01 by NJS
crossprods = []
for i in range(0, len(valid_unflat_ids)):
crossprods.append([
cell_vec[i] * float(s or 0)
for s in stat_vectors_by_id[i]
])
weighted_stat_means = []
for i in range(0, len(other_field_names)):
weighted_stat_means.append(
sum(zip(*crossprods)[i]) / cell_count)
else:
weighted_coverage = 0
weighted_stat_means = [None] * len(other_field_names)
count_diff += 1
new_row = [zid, original_count, cell_count, weighted_coverage
] + weighted_stat_means
i_cursor.insertRow(new_row)
del i_cursor
DM.Delete(intermediate_table)
return [unflat_result, count_diff]
if len(target_fld) == 0 or target_fld == "#":
target_fld = "New_WP"
arcpy.env.overwriteOutput = True
arcpy.env.workspace = workspace
if (arcpy.env.workspace is None):
arcpy.env.workspace = os.getcwd()
#arcpy.env.extent = "MINOF"
arcpy.env.snapRaster = cost_rast
scratch = arcpy.CreateScratchName('xx', '.shp')
arcpy.Buffer_analysis(in_file, scratch, "2000 meters")
desc = arcpy.Describe(scratch)
arcpy.env.extent = desc.extent
arcmgt.Delete(scratch)
print "Extent is %s" % arcpy.env.extent
add_msg_and_print('Currently in directory: %s\n' % os.getcwd())
add_msg_and_print('Workspace is: %s' % arcpy.env.workspace)
#add_msg_and_print ('Scratch table is: %s' % out_table)
table_view = "table_view"
arcmgt.MakeTableView(in_file, table_view)
fields = arcpy.ListFields(in_file)
layer = "feat_layer"
arcmgt.MakeFeatureLayer(in_file, layer)
desc = arcpy.Describe(layer)
fld_names = []
def doFishnet(self):
#### Initial Data Assessment ####
printOHSSection(84428, prependNewLine=True)
printOHSSubject(84431, addNewLine=False)
#### Find Unique Locations ####
msg = ARCPY.GetIDMessage(84441)
ARCPY.SetProgressor("default", msg)
initCount = UTILS.getCount(self.ssdo.inputFC)
self.checkIncidents(initCount)
collectedPointFC = UTILS.returnScratchName("Collect_InitTempFC")
collInfo = EVENTS.collectEvents(self.ssdo, collectedPointFC)
self.cleanUpList.append(collectedPointFC)
collSSDO = SSDO.SSDataObject(collectedPointFC,
explicitSpatialRef=self.ssdo.spatialRef,
useChordal=True)
collSSDO.obtainDataGA(collSSDO.oidName)
#################################
if self.boundaryFC:
#### Assure Boundary FC Has Area and Obtain Chars ####
self.checkBoundary()
#### Location Outliers ####
lo = UTILS.LocationInfo(collSSDO,
concept="EUCLIDEAN",
silentThreshold=True,
stdDeviations=3)
printOHSLocationalOutliers(lo, aggType=self.aggType)
#### Agg Header ####
printOHSSection(84444)
if self.boundaryFC:
extent = self.boundExtent
forMercExtent = self.boundExtent
countMSGNumber = 84453
else:
countMSGNumber = 84452
extent = None
forMercExtent = collSSDO.extent
if collSSDO.useChordal:
extentFC_GCS = UTILS.returnScratchName("TempGCS_Extent")
extentFC_Merc = UTILS.returnScratchName("TempMercator_Extent")
points = NUM.array([[forMercExtent.XMin, forMercExtent.YMax],
[forMercExtent.XMax, forMercExtent.YMin]])
UTILS.createPointFC(extentFC_GCS,
points,
spatialRef=collSSDO.spatialRef)
DM.Project(extentFC_GCS, extentFC_Merc, mercatorProjection)
d = ARCPY.Describe(extentFC_Merc)
extent = d.extent
fishOutputCoords = mercatorProjection
else:
fishOutputCoords = self.ssdo.spatialRef
#### Fish Subject ####
printOHSSubject(84449, addNewLine=False)
dist = scaleDecision(lo.nonZeroAvgDist, lo.nonZeroMedDist)
area = 0.0
#### Construct Fishnet ####
fish = UTILS.FishnetInfo(collSSDO, area, extent, explicitCellSize=dist)
dist = fish.quadLength
snap = self.ssdo.distanceInfo.linearUnitString(dist)
#### Cell Size Answer ####
snapStr = self.ssdo.distanceInfo.printDistance(dist)
msg = ARCPY.GetIDMessage(84450).format(snapStr)
printOHSAnswer(msg)
self.fish = fish
#### Fishnet Count Subject ####
printOHSSubject(84451, addNewLine=False)
#### Create Temp Fishnet Grid ####
gridFC = UTILS.returnScratchName("Fishnet_TempFC")
self.cleanUpList.append(gridFC)
#### Apply Output Coords to Create Fishnet ####
oldSpatRef = ARCPY.env.outputCoordinateSystem
ARCPY.env.outputCoordinateSystem = fishOutputCoords
#### Fish No Extent ####
oldExtent = ARCPY.env.extent
ARCPY.env.extent = ""
#### Apply Max XY Tolerance ####
fishWithXY = UTILS.funWithXYTolerance(DM.CreateFishnet,
self.ssdo.distanceInfo)
#### Execute Fishnet ####
fishWithXY(gridFC, self.fish.origin, self.fish.rotate,
self.fish.quadLength, self.fish.quadLength,
self.fish.numRows, self.fish.numCols, self.fish.corner,
"NO_LABELS", self.fish.extent, "POLYGON")
#### Project Back to GCS if Use Chordal ####
if collSSDO.useChordal:
gridFC_ProjBack = UTILS.returnScratchName("TempFC_Proj")
DM.Project(gridFC, gridFC_ProjBack, collSSDO.spatialRef)
UTILS.passiveDelete(gridFC)
gridFC = gridFC_ProjBack
#### Set Env Output Coords Back ####
ARCPY.env.outputCoordinateSystem = oldSpatRef
#### Create Empty Field Mappings to Ignore Atts ####
fieldMap = ARCPY.FieldMappings()
fieldMap.addTable(self.ssdo.inputFC)
fieldMap.removeAll()
#### Fishnet Count Answer ####
printOHSAnswer(ARCPY.GetIDMessage(countMSGNumber))
#### Create Weighted Fishnet Grid ####
tempFC = UTILS.returnScratchName("Optimized_TempFC")
self.cleanUpList.append(tempFC)
joinWithXY = UTILS.funWithXYTolerance(ANA.SpatialJoin,
self.ssdo.distanceInfo)
joinWithXY(gridFC, self.ssdo.inputFC, tempFC, "JOIN_ONE_TO_ONE",
"KEEP_ALL", "EMPTY")
#### Clean Up Temp FCs ####
UTILS.passiveDelete(gridFC)
#### Remove Locations Outside Boundary FC ####
featureLayer = "ClippedPointFC"
DM.MakeFeatureLayer(tempFC, featureLayer)
if self.boundaryFC:
msg = ARCPY.GetIDMessage(84454)
ARCPY.SetProgressor("default", msg)
DM.SelectLayerByLocation(featureLayer, "INTERSECT",
self.boundaryFC, "#", "NEW_SELECTION")
DM.SelectLayerByLocation(featureLayer, "INTERSECT", "#", "#",
"SWITCH_SELECTION")
DM.DeleteFeatures(featureLayer)
else:
if additionalZeroDistScale == "ALL":
msg = ARCPY.GetIDMessage(84455)
ARCPY.SetProgressor("default", msg)
DM.SelectLayerByAttribute(featureLayer, "NEW_SELECTION",
'"Join_Count" = 0')
DM.DeleteFeatures(featureLayer)
else:
distance = additionalZeroDistScale * fish.quadLength
distanceStr = self.ssdo.distanceInfo.linearUnitString(
distance, convert=True)
nativeStr = self.ssdo.distanceInfo.printDistance(distance)
msg = "Removing cells further than %s from input pointsd...."
ARCPY.AddMessage(msg % nativeStr)
DM.SelectLayerByLocation(featureLayer, "INTERSECT",
self.ssdo.inputFC, distanceStr,
"NEW_SELECTION")
DM.SelectLayerByLocation(featureLayer, "INTERSECT", "#", "#",
"SWITCH_SELECTION")
DM.DeleteFeatures(featureLayer)
DM.Delete(featureLayer)
del collSSDO
ARCPY.env.extent = oldExtent
self.createAnalysisSSDO(tempFC, "JOIN_COUNT")
curZo = wrk + "/zon_C" + classLevel
# cycle through each edm
for j in range(len(rasL)):
if j == 0:
inRas = inPath + "/" + rasL[j] + "_c.tif"
curZoT_out = wrk + "/zonTab_C" + str(i) + "_" + str(j)
print(".. zoning " + rasL[j])
curZoT = ZonalStatisticsAsTable(hypZ[i], "Value", inRas,
curZoT_out, "DATA", "MAXIMUM")
man.CopyRaster(hypZ[i], curZo)
man.AddField(curZo, "spp0", "TEXT", "", "", 251)
man.JoinField(curZo, "Value", curZoT, "VALUE", ["MAX"])
expr = "str( !MAX! )"
man.CalculateField(curZo, "spp0", expr, "PYTHON")
man.DeleteField(curZo, "MAX")
man.Delete(curZoT_out)
else:
#jminus = j-1
inRas = inPath + "/" + rasL[j] + "_c.tif"
print(".. zoning " + rasL[j])
curZoT_out = wrk + "/zonTab_C" + str(i) + "_" + str(j)
curZoT = ZonalStatisticsAsTable(hypZ[i], "Value", inRas,
curZoT_out, "DATA", "MAXIMUM")
man.JoinField(curZo, "Value", curZoT, "VALUE", ["MAX"])
expr = "str(!spp0!) + str(!MAX!)"
man.CalculateField(curZo, "spp0", expr, "PYTHON")
man.DeleteField(curZo, "MAX")
man.Delete(curZoT_out)
# expand information out to one col for each spp.
print("adding columns...")
for i in range(len(rasL)):
def classify_lakes(nhd,
out_feature_class,
exclude_intermit_flowlines=False,
debug_mode=False):
if debug_mode:
arcpy.env.overwriteOutput = True
temp_gdb = cu.create_temp_GDB('classify_lake_connectivity')
arcpy.env.workspace = temp_gdb
arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb))
else:
arcpy.env.workspace = 'in_memory'
if arcpy.Exists("temp_fc"):
print("There is a problem here.")
raise Exception
# Tool temporary feature classes
temp_fc = "temp_fc"
csiwaterbody_10ha = "csiwaterbody_10ha"
nhdflowline_filtered = "nhdflowline_filtered"
dangles = "dangles"
start = "start"
end = "end"
startdangles = "startdangles"
enddangles = "enddangles"
non_artificial_end = "non_artificial_end"
flags_10ha_lake_junctions = "flags_10ha_lake_junctions"
midvertices = "midvertices"
non10vertices = "non10vertices"
non10junctions = "non10junctions"
all_non_flag_points = "all_non_flag_points"
barriers = "barriers"
trace1_junctions = "trace1_junctions"
trace1_flowline = "trace1_flowline"
trace2_junctions = "trace2junctions"
trace2_flowline = "trace2_flowline"
# Clean up workspace in case of bad exit from prior run in same session.
this_tool_layers = [
"dangles_lyr", "nhdflowline_lyr", "junction_lyr", "midvertices_lyr",
"all_non_flag_points_lyr", "non10vertices_lyr", "out_fc_lyr", "trace1",
"trace2"
]
this_tool_temp = [
temp_fc, csiwaterbody_10ha, nhdflowline_filtered, dangles, start, end,
startdangles, enddangles, non_artificial_end,
flags_10ha_lake_junctions, midvertices, non10vertices, non10junctions,
all_non_flag_points, barriers, trace1_junctions, trace1_flowline,
trace2_junctions, trace2_flowline
]
for item in this_tool_layers + this_tool_temp:
try:
DM.Delete(item)
except:
pass
# Local variables:
nhdflowline = os.path.join(nhd, "Hydrography", "NHDFLowline")
nhdjunction = os.path.join(nhd, "Hydrography", "HYDRO_NET_Junctions")
nhdwaterbody = os.path.join(nhd, "Hydrography", "NHDWaterbody")
network = os.path.join(nhd, "Hydrography", "HYDRO_NET")
# Get lakes, ponds and reservoirs over a hectare.
#csi_population_filter = '''"AreaSqKm" >=0.01 AND\
#"FCode" IN (39000,39004,39009,39010,39011,39012,43600,43613,43615,43617,43618,43619,43621)'''
all_lakes_reservoirs_filter = '''"FType" IN (390, 436)'''
# Can't see why we shouldn't just attribute all lakes and reservoirs
# arcpy.Select_analysis(nhdwaterbody, "csiwaterbody", lake_population_filter)
arcpy.AddMessage("Initializing output.")
if exclude_intermit_flowlines:
DM.CopyFeatures(out_feature_class, temp_fc)
DM.Delete(out_feature_class)
else:
arcpy.Select_analysis(nhdwaterbody, temp_fc,
all_lakes_reservoirs_filter)
# Get lakes, ponds and reservoirs over 10 hectares.
lakes_10ha_filter = '''"AreaSqKm" >= 0.1 AND "FType" IN (390, 436)'''
arcpy.Select_analysis(nhdwaterbody, csiwaterbody_10ha, lakes_10ha_filter)
# Exclude intermittent flowlines, if requested
if exclude_intermit_flowlines:
flowline_where_clause = '''"FCode" NOT IN (46003,46007)'''
nhdflowline = arcpy.Select_analysis(nhdflowline, nhdflowline_filtered,
flowline_where_clause)
# Make dangle points at end of nhdflowline
DM.FeatureVerticesToPoints(nhdflowline, dangles, "DANGLE")
DM.MakeFeatureLayer(dangles, "dangles_lyr")
# Isolate start dangles from end dangles.
DM.FeatureVerticesToPoints(nhdflowline, start, "START")
DM.FeatureVerticesToPoints(nhdflowline, end, "END")
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", start)
DM.CopyFeatures("dangles_lyr", startdangles)
DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", end)
DM.CopyFeatures("dangles_lyr", enddangles)
# Special handling for lakes that have some intermittent flow in and some permanent
if exclude_intermit_flowlines:
DM.MakeFeatureLayer(nhdflowline, "nhdflowline_lyr")
DM.SelectLayerByAttribute("nhdflowline_lyr", "NEW_SELECTION",
'''"WBArea_Permanent_Identifier" is null''')
DM.FeatureVerticesToPoints("nhdflowline_lyr", non_artificial_end,
"END")
DM.SelectLayerByAttribute("nhdflowline_lyr", "CLEAR_SELECTION")
arcpy.AddMessage("Found source area nodes.")
# Get junctions from lakes >= 10 hectares.
DM.MakeFeatureLayer(nhdjunction, "junction_lyr")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT", csiwaterbody_10ha,
XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("junction_lyr", flags_10ha_lake_junctions)
arcpy.AddMessage("Found lakes >= 10 ha.")
# Make points shapefile and layer at flowline vertices to act as potential flags and/or barriers.
arcpy.AddMessage("Tracing...")
DM.FeatureVerticesToPoints(nhdflowline, midvertices, "MID")
DM.MakeFeatureLayer(midvertices, "midvertices_lyr")
# Get vertices that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("midvertices_lyr", non10vertices)
# Get junctions that are not coincident with 10 hectare lake junctions.
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "NEW_SELECTION")
DM.SelectLayerByLocation("junction_lyr", "INTERSECT",
flags_10ha_lake_junctions, "", "SWITCH_SELECTION")
DM.CopyFeatures("junction_lyr", non10junctions)
# Merge non10vertices with non10junctions
DM.Merge([non10junctions, non10vertices],
all_non_flag_points) # inputs both point fc in_memory
DM.MakeFeatureLayer(all_non_flag_points, "all_non_flag_points_lyr")
# Tests the counts...for some reason I'm not getting stable behavior from the merge.
mid_n = int(DM.GetCount(non10vertices).getOutput(0))
jxn_n = int(DM.GetCount(non10junctions).getOutput(0))
merge_n = int(DM.GetCount(all_non_flag_points).getOutput(0))
if merge_n < mid_n + jxn_n:
arcpy.AddWarning(
"The total number of flags ({0}) is less than the sum of the input junctions ({1}) "
"and input midpoints ({2})".format(merge_n, jxn_n, mid_n))
# For tracing barriers, select all_non_flag_points points that intersect a 10 ha lake.
DM.SelectLayerByLocation("all_non_flag_points_lyr", "INTERSECT",
csiwaterbody_10ha, XY_TOLERANCE, "NEW_SELECTION")
DM.CopyFeatures("all_non_flag_points_lyr", barriers)
# Trace1-Trace downstream to first barrier (junctions+midvertices in 10 ha lake) starting from flags_10ha_lake_junctions flag points.
DM.TraceGeometricNetwork(network, "trace1", flags_10ha_lake_junctions,
"TRACE_DOWNSTREAM", barriers)
# Save trace1 flowlines and junctions to layers on disk.
DM.CopyFeatures("trace1\HYDRO_NET_Junctions",
trace1_junctions) # extra for debugging
DM.CopyFeatures("trace1\NHDFlowline", trace1_flowline)
# Select vertice midpoints that intersect trace1 flowlines selection for new flags for trace2.
DM.MakeFeatureLayer(non10vertices, "non10vertices_lyr")
DM.SelectLayerByLocation("non10vertices_lyr", "INTERSECT", trace1_flowline,
"", "NEW_SELECTION")
# Trace2-Trace downstream from midpoints of flowlines that intersect the selected flowlines from trace1.
DM.TraceGeometricNetwork(network, "trace2", "non10vertices_lyr",
"TRACE_DOWNSTREAM")
# Save trace1 flowlines and junctions to layers and then shapes on disk.
DM.CopyFeatures("trace2\HYDRO_NET_Junctions", trace2_junctions)
DM.CopyFeatures("trace2\NHDFlowline",
trace2_flowline) # extra for debugging
arcpy.AddMessage("Done tracing.")
# Make shapefile for seepage lakes. (Ones that don't intersect flowlines)
if exclude_intermit_flowlines:
class_field_name = "Lake_Connectivity_Permanent"
else:
class_field_name = "Lake_Connectivity_Class"
DM.AddField(temp_fc, class_field_name, "TEXT", field_length=13)
DM.MakeFeatureLayer(temp_fc, "out_fc_lyr")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline, "",
"SWITCH_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# New type of "Isolated" classification, mostly for "permanent" but there were some oddballs in "maximum" too
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", enddangles,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# Get headwater lakes.
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles,
XY_TOLERANCE, "NEW_SELECTION")
DM.SelectLayerByAttribute(
"out_fc_lyr", "REMOVE_FROM_SELECTION",
'''"{}" = 'Isolated' '''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# Select csiwaterbody that intersect trace2junctions
arcpy.AddMessage("Beginning connectivity attribution...")
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", trace2_junctions,
XY_TOLERANCE, "NEW_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'DrainageLk'""",
"PYTHON")
# Get stream drainage lakes. Either unassigned so far or convert "Headwater" if a permanent stream flows into it,
# which is detected with "non_artificial_end"
DM.SelectLayerByAttribute("out_fc_lyr", "NEW_SELECTION",
'''"{}" IS NULL'''.format(class_field_name))
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
if exclude_intermit_flowlines:
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"{}" = 'Headwater' '''.format(class_field_name))
DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", non_artificial_end,
XY_TOLERANCE, "SUBSET_SELECTION")
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
# Prevent 'upgrades' due to very odd flow situations and artifacts of bad digitization. The effects of these
# are varied--to avoid confusion, just keep the class assigned with all flowlines
# 1--Purely hypothetical, not seen in testing
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Isolated' AND "Lake_Connectivity_Permanent" <> 'Isolated' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""",
"PYTHON")
# 2--Headwater to Drainage upgrade seen in testing with odd multi-inlet flow situation
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Headwater' AND "Lake_Connectivity_Permanent" IN ('Drainage', 'DrainageLk') '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""",
"PYTHON")
# 3--Drainage to DrainageLk upgrade seen in testing when intermittent stream segments were used
# erroneously instead of artificial paths
DM.SelectLayerByAttribute(
"out_fc_lyr", "NEW_SELECTION",
'''"Lake_Connectivity_Class" = 'Drainage' AND "Lake_Connectivity_Permanent" = 'DrainageLk' '''
)
DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""",
"PYTHON")
DM.SelectLayerByAttribute("out_fc_lyr", "CLEAR_SELECTION")
# Add change flag for users
DM.AddField(temp_fc,
"Lake_Connectivity_Fluctuates",
"Text",
field_length="1")
flag_codeblock = """def flag_calculate(arg1, arg2):
if arg1 == arg2:
return 'N'
else:
return 'Y'"""
expression = 'flag_calculate(!Lake_Connectivity_Class!, !Lake_Connectivity_Permanent!)'
DM.CalculateField(temp_fc, "Lake_Connectivity_Fluctuates", expression,
"PYTHON", flag_codeblock)
# Project output once done with both. Switching CRS earlier causes trace problems.
if not exclude_intermit_flowlines:
DM.CopyFeatures(temp_fc, out_feature_class)
else:
DM.Project(temp_fc, out_feature_class, arcpy.SpatialReference(102039))
# Clean up
if not debug_mode:
for item in this_tool_layers + this_tool_temp:
if arcpy.Exists(item):
DM.Delete(item)
if not debug_mode:
DM.Delete("trace1")
DM.Delete("trace2")
arcpy.AddMessage("{} classification is complete.".format(class_field_name))
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field,
other_keep_fields, clip_hu8, lagosne_name):
# dissolve fields by the field that zone_id is based on (the field that identifies a unique zone)
dissolve_fields = [
f for f in "{}, {}, {}".format(zone_id_field, zone_name_field,
other_keep_fields).split(', ')
if f != ''
]
print("Dissolving...")
dissolve1 = DM.Dissolve(zone_fc, 'dissolve1', dissolve_fields)
# update name field to match our standard
DM.AlterField(dissolve1, zone_name_field, 'name')
# original area
DM.AddField(dissolve1, 'originalarea', 'DOUBLE')
DM.CalculateField(dissolve1, 'originalarea', '!shape.area@hectares!',
'PYTHON')
#clip
print("Clipping...")
clip = AN.Clip(dissolve1, MASTER_CLIPPING_POLY, 'clip')
if clip_hu8 == 'Y':
final_clip = AN.Clip(clip, HU8_OUTPUT, 'final_clip')
else:
final_clip = clip
print("Selecting...")
# calc new area, orig area pct, compactness
DM.AddField(final_clip, 'area_ha', 'DOUBLE')
DM.AddField(final_clip, 'originalarea_pct', 'DOUBLE')
DM.AddField(final_clip, 'compactness', 'DOUBLE')
DM.JoinField(final_clip, zone_id_field, dissolve1, zone_id_field,
'originalarea_pct')
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'compactness',
'SHAPE@AREA', 'SHAPE@LENGTH'
]
with arcpy.da.UpdateCursor(final_clip, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, comp, shape_area, shape_length = row
area = shape_area / 10000 # convert from m2 to hectares
orig_area_pct = round(100 * area / orig_area, 2)
comp = 4 * 3.14159 * shape_area / (shape_length**2)
row = (area, orig_area_pct, orig_area, comp, shape_area,
shape_length)
uCursor.updateRow(row)
# if zones are present with <5% of original area and a compactness measure of <.2 (ranges from 0-1)
# AND ALSO they are no bigger than 500 sq. km. (saves Chippewa County and a WWF), filter out
# save eliminated polygons to temp database as a separate layer for inspection
# Different processing for HU4 and HU8, so that they match the extent of HU8 more closely but still throw out tiny slivers
# County also only eliminated if a tiny, tiny, tiny sliver (so: none should be eliminated)
if zone_name not in ('hu4', 'hu12', 'county'):
selected = AN.Select(
final_clip, 'selected',
"originalarea_pct >= 5 OR compactness >= .2 OR area_ha > 50000")
not_selected = AN.Select(
final_clip, '{}_not_selected'.format(output),
"originalarea_pct < 5 AND compactness < .2 AND area_ha < 50000")
else:
selected = final_clip
# eliminate small slivers, re-calc area fields, add perimeter and multipart flag
# leaves the occasional errant sliver but some areas over 25 hectares are more valid so this is
# CONSERVATIVE
print("Trimming...")
trimmed = DM.EliminatePolygonPart(selected,
'trimmed',
'AREA',
'25 Hectares',
part_option='ANY')
# gather up a few calculations into one cursor because this is taking too long over the HU12 layer
DM.AddField(trimmed, 'perimeter_m', 'DOUBLE')
DM.AddField(trimmed, 'multipart', 'TEXT', field_length=1)
uCursor_fields = [
'area_ha', 'originalarea_pct', 'originalarea', 'perimeter_m',
'multipart', 'SHAPE@'
]
with arcpy.da.UpdateCursor(trimmed, uCursor_fields) as uCursor:
for row in uCursor:
area, orig_area_pct, orig_area, perim, multipart, shape = row
area = shape.area / 10000 # convert to hectares from m2
orig_area_pct = round(100 * area / orig_area, 2)
perim = shape.length
# multipart flag calc
if shape.isMultipart:
multipart = 'Y'
else:
multipart = 'N'
row = (area, orig_area_pct, orig_area, perim, multipart, shape)
uCursor.updateRow(row)
# delete intermediate fields
DM.DeleteField(trimmed, 'compactness')
DM.DeleteField(trimmed, 'originalarea')
print("Zone IDs....")
# link to LAGOS-NE zone IDs
DM.AddField(trimmed, 'zoneid', 'TEXT', field_length=40)
trimmed_lyr = DM.MakeFeatureLayer(trimmed, 'trimmed_lyr')
if lagosne_name:
# join to the old master GDB path on the same master field and copy in the ids
old_fc = os.path.join(LAGOSNE_GDB, lagosne_name)
old_fc_lyr = DM.MakeFeatureLayer(old_fc, 'old_fc_lyr')
if lagosne_name == 'STATE' or lagosne_name == 'COUNTY':
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr, 'FIPS')
else:
DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr,
zone_id_field) # usually works because same source data
# copy
DM.CalculateField(trimmed_lyr, 'zoneid',
'!{}.ZoneID!.lower()'.format(lagosne_name), 'PYTHON')
DM.RemoveJoin(trimmed_lyr)
# generate new zone ids
old_ids = [row[0] for row in arcpy.da.SearchCursor(trimmed, 'zoneid')]
with arcpy.da.UpdateCursor(trimmed, 'zoneid') as cursor:
counter = 1
for row in cursor:
if not row[
0]: # if no existing ID borrowed from LAGOS-NE, assign a new one
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
# ensures new ids don't re-use old numbers but fills in all positive numbers eventually
while new_id in old_ids:
counter += 1
new_id = '{name}_{num}'.format(name=zone_name, num=counter)
row[0] = new_id
cursor.updateRow(row)
counter += 1
print("Edge flags...")
# add flag fields
DM.AddField(trimmed, 'onlandborder', 'TEXT', field_length=2)
DM.AddField(trimmed, 'oncoast', 'TEXT', field_length=2)
# identify border zones
border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', border_lyr)
DM.CalculateField(trimmed_lyr, 'onlandborder', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'onlandborder', "'N'", 'PYTHON')
# identify coastal zones
coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr')
DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', coastal_lyr)
DM.CalculateField(trimmed_lyr, 'oncoast', "'Y'", 'PYTHON')
DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION')
DM.CalculateField(trimmed_lyr, 'oncoast', "'N'", 'PYTHON')
print("State assignment...")
# State?
DM.AddField(trimmed, "state", 'text', field_length='2')
state_center = arcpy.SpatialJoin_analysis(
trimmed,
STATE_FC,
'state_center',
join_type='KEEP_COMMON',
match_option='HAVE_THEIR_CENTER_IN')
state_intersect = arcpy.SpatialJoin_analysis(trimmed,
STATE_FC,
'state_intersect',
match_option='INTERSECT')
state_center_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_center, ['ZoneID', 'STUSPS'])
}
state_intersect_dict = {
row[0]: row[1]
for row in arcpy.da.SearchCursor(state_intersect, ['ZoneID', 'STUSPS'])
}
with arcpy.da.UpdateCursor(trimmed, ['ZoneID', 'state']) as cursor:
for updateRow in cursor:
keyValue = updateRow[0]
if keyValue in state_center_dict:
updateRow[1] = state_center_dict[keyValue]
else:
updateRow[1] = state_intersect_dict[keyValue]
cursor.updateRow(updateRow)
# glaciation status?
# TODO as version 0.6
# preface the names with the zones
DM.DeleteField(trimmed, 'ORIG_FID')
fields = [
f.name for f in arcpy.ListFields(trimmed, '*')
if f.type not in ('OID',
'Geometry') and not f.name.startswith('Shape_')
]
for f in fields:
new_fname = '{zn}_{orig}'.format(zn=zone_name, orig=f).lower()
try:
DM.AlterField(trimmed, f, new_fname, clear_field_alias='TRUE')
# sick of debugging the required field message-I don't want to change required fields anyway
except:
pass
DM.CopyFeatures(trimmed, output)
# cleanup
lyr_objects = [
lyr_object for var_name, lyr_object in locals().items()
if var_name.endswith('lyr')
]
temp_fcs = arcpy.ListFeatureClasses('*')
for l in lyr_objects + temp_fcs:
DM.Delete(l)
def snap_points_to_mask_raster (in_file, mask, out_file, distance, workspace):
if distance is None or len (distance) == 0:
distance = "100 METERS"
if arcpy.env.outputCoordinateSystem is None:
arcpy.env.outputCoordinateSystem = mask
print arcpy.env.outputCoordinateSystem.name
if len(workspace):
arcpy.env.workspace = workspace
if arcpy.env.workspace is None or len(arcpy.env.workspace) == 0:
arcpy.env.workspace = os.getcwd()
arcpy.AddMessage ("workspace is %s" % arcpy.env.workspace)
try:
suffix = None
wk = arcpy.env.workspace
if not '.gdb' in wk:
suffix = '.shp'
poly_file = arcpy.CreateScratchName(None, suffix, 'POLYGON')
arcpy.RasterToPolygon_conversion (mask, poly_file, 'NO_SIMPLIFY')
except:
raise
arcpy.AddMessage ("poly_file is %s" % poly_file)
# handle layers and datasets
desc = arcpy.Describe(in_file)
in_file = desc.catalogPath
# add .shp extension if needed - clunky, but otherwise system fails below
re_gdb = re.compile ('\.gdb$')
re_shp = re.compile ('\.shp$')
path = os.path.dirname(out_file)
if len (path) == 0:
path = arcpy.env.workspace
if not re_gdb.search (path) and not re_shp.search (out_file):
out_file += '.shp'
arcpy.AddMessage ("Input point file is %s" % in_file)
arcpy.AddMessage ("Output point file is %s" % out_file)
arcmgt.CopyFeatures (in_file, out_file)
try:
snap_layer_name = 'get_layer_for_snapping'
arcmgt.MakeFeatureLayer (out_file, snap_layer_name)
arcmgt.SelectLayerByLocation (snap_layer_name, 'intersect', poly_file, '#', 'NEW_SELECTION')
arcmgt.SelectLayerByAttribute(snap_layer_name, 'SWITCH_SELECTION')
if arcmgt.GetCount(snap_layer_name) > 0:
arcpy.Snap_edit (snap_layer_name, [[poly_file, "EDGE", distance]])
else:
arcpy.AddMessage ('No features selected, no snapping applied')
except Exception as e:
print arcpy.GetMessages()
raise e
arcmgt.Delete (snap_layer_name)
arcmgt.Delete (poly_file)
print arcpy.GetMessages()
print "Completed"
return
def get_path_residence_times (in_file, cost_rast, out_raster, t_diff_fld_name, workspace):
if len (out_raster) == 0:
arcpy.AddError ("Missing argument: out_rast")
raise Exception
if len (t_diff_fld_name) == 0:
t_diff_fld_name = "T_DIFF_HRS"
arcpy.env.overwriteOutput = True # This is underhanded. It should be an argument.
if arcpy.env.outputCoordinateSystem is None:
arcpy.env.outputCoordinateSystem = cost_rast
arcpy.AddMessage ("coordinate system is %s" % arcpy.env.outputCoordinateSystem.name)
if len(workspace):
arcpy.env.workspace = workspace
if arcpy.env.workspace is None or len(arcpy.env.workspace) == 0:
arcpy.env.workspace = os.getcwd()
if '.gdb' in arcpy.env.workspace:
arcpy.AddError (
"Worskpace is a geodatabase. " +
"This brings too much pain for this script to work.\n" +
"%s" % arcpy.env.workspace
)
raise WorkspaceIsGeodatabase
r = Raster(cost_rast)
if r.maximum == 0 and r.minimum == 0:
arcpy.AddMessage ('Cost raster has only zero value. Cannot calculate cost distances.')
raise CostRasterIsZero
size = r.height * r.width * 4
if size > 2 * 1028 ** 3:
import struct
struct_size = struct.calcsize("P") * 8
if struct_size == 32:
size_in_gb = float (size) / (1028 ** 3)
arcpy.AddMessage (
'Cost raster exceeds 2 GiB in size (%s GiB). This is too large for a 32 bit NumPy.' % size_in_gb
)
raise NumPyArrayExceedsSizeLimits
if not check_points_are_in_cost_raster(in_file, cost_rast):
arcpy.AddError ('One or more input points do not intersect the cost raster')
raise PointNotOnRaster
arcpy.env.snapRaster = cost_rast
suffix = None
wk = arcpy.env.workspace
if not '.gdb' in wk:
suffix = '.shp'
ext = arcpy.env.extent
if ext is None:
arcpy.env.extent = r.extent
arcpy.AddMessage ("Extent is %s" % arcpy.env.extent)
arcpy.env.cellSize = r.meanCellWidth
arcpy.AddMessage ("Cell size is %s" % arcpy.env.cellSize)
cellsize_used = float (arcpy.env.cellSize)
extent = arcpy.env.extent
lower_left_coord = extent.lowerLeft
arcpy.AddMessage ('Currently in directory: %s\n' % os.getcwd())
arcpy.AddMessage ('Workspace is: %s' % arcpy.env.workspace)
arcpy.AddMessage ("lower left is %s" % lower_left_coord)
if arcpy.env.mask is None:
arcpy.AddMessage ("Setting mask to %s" % cost_rast)
arcpy.env.mask = cost_rast
# accumulated transits
transit_array_accum = arcpy.RasterToNumPyArray (Raster(cost_rast) * 0)
feat_layer = "feat_layer"
arcmgt.MakeFeatureLayer(in_file, feat_layer)
desc = arcpy.Describe (feat_layer)
oid_fd_name = desc.OIDFieldName
arcpy.AddMessage("oid_fd_name = %s" % oid_fd_name)
# variable name is redundant now??? - should all calls be to oid_fd_name?
target_fld = oid_fd_name
proc_layer = "process_layer"
arcmgt.MakeFeatureLayer(in_file, proc_layer)
rows = arcpy.SearchCursor(proc_layer)
last_target = None
for row_cur in rows:
transit_time = row_cur.getValue (t_diff_fld_name)
if last_target is None or transit_time == 0:
message = 'Skipping %s = %s' % (oid_fd_name, row_cur.getValue(oid_fd_name))
if transit_time == 0:
message = message + " Transit time is zero"
arcpy.AddMessage(message)
last_target = row_cur.getValue(target_fld)
last_oid = row_cur.getValue(oid_fd_name)
continue
arcpy.AddMessage ("Processing %s %i" % (oid_fd_name, row_cur.getValue(oid_fd_name)))
arcmgt.SelectLayerByAttribute(
feat_layer,
"NEW_SELECTION",
'%s = %s' % (target_fld, last_target)
)
backlink_rast = arcpy.CreateScratchName("backlink")
path_dist_rast = PathDistance(feat_layer, cost_rast, out_backlink_raster = backlink_rast)
# extract the distance from the last point
shp = row_cur.shape
centroid = shp.centroid
(x, y) = (centroid.X, centroid.Y)
result = arcmgt.GetCellValue(path_dist_rast, "%s %s" % (x, y), "1")
res_val = result.getOutput(0)
if res_val == "NoData":
this_oid = row_cur.getValue(oid_fd_name)
arcpy.AddMessage ("Got nodata for coordinate (%s, %s)" % (x, y))
arcpy.AddMessage ("Is the path between features %s and %s wholly contained by the cost raster?" % (last_oid, this_oid))
pras_name = "pth_%s_%s.tif" % (last_oid, this_oid)
arcpy.AddMessage ("Attempting to save path raster as %s" % pras_name)
try:
path_dist_rast.save(pras_name)
except Exception as e:
arcpy.AddMessage (e)
raise PathDistanceIsNoData
try:
path_distance = float (res_val)
except:
# kludge around locale/radix issues
if res_val.find(","):
res_val = res_val.replace(",", ".")
path_distance = float (res_val)
else:
raise
arcpy.AddMessage("Path distance is %s\nTransit time is %s" % (path_distance, transit_time))
# get a raster of the path from origin to destination
condition = '%s in (%i, %i)' % (oid_fd_name, last_oid, row_cur.getValue(oid_fd_name))
dest_layer = "dest_layer" + str (last_oid)
arcmgt.MakeFeatureLayer(in_file, dest_layer, where_clause = condition)
count = arcmgt.GetCount(dest_layer)
count = int (count.getOutput(0))
if count == 0:
raise NoFeatures("No features selected. Possible coordinate system issues.\n" + condition)
try:
path_cost_rast = CostPath(dest_layer, path_dist_rast, backlink_rast)
#path_dist_rast.save("xx_pr" + str (last_oid))
except Exception as e:
raise
try:
pcr_mask = 1 - IsNull (path_cost_rast)
#pcr_mask.save ("xx_pcr_mask" + str (last_oid))
dist_masked = path_dist_rast * pcr_mask
path_array = arcpy.RasterToNumPyArray(dist_masked, nodata_to_value = -9999)
path_array_idx = numpy.where(path_array > 0)
transit_array = numpy.zeros_like(path_array) # past experience suggests we might need to use a different approach to guarantee we get zeroes
except:
raise
path_sum = None
arcpy.AddMessage ("processing %i cells of path raster" % (len(path_array_idx[0])))
if path_distance == 0 or not len(path_array_idx[0]):
path_sum = 1 # stayed in the same cell
mask_array = arcpy.RasterToNumPyArray(pcr_mask, nodata_to_value = -9999)
mask_array_idx = numpy.where(mask_array == 1)
i = mask_array_idx[0][0]
j = mask_array_idx[1][0]
transit_array[i][j] = path_sum
else:
row_count = len (path_array)
col_count = len (path_array[0])
for idx in range (len(path_array_idx[0])):
i = path_array_idx[0][idx]
j = path_array_idx[1][idx]
val = path_array[i][j]
nbrs = []
for k in (i-1, i, i+1):
if k < 0 or k >= row_count:
continue
checkrow = path_array[k]
for l in (j-1, j, j+1):
if l < 0 or l >= col_count:
continue
if k == i and j == l:
continue # don't check self
checkval = checkrow[l]
# negs are nodata, and this way we
# don't need to care what that value is
if checkval >= 0:
diff = val - checkval
if diff > 0:
nbrs.append(diff)
#arcpy.AddMessage ("Check and diff vals are %s %s" % (checkval, diff))
diff = min (nbrs)
#arcpy.AddMessage ("Diff val is %s" % diff)
transit_array[i][j] = diff
path_sum = path_array.max() # could use path_distance?
#arcpy.AddMessage ("path_array.max is %s" % path_sum)
# sometimes we get a zero path_sum even when the path_distance is non-zero
if path_sum == 0:
path_sum = 1
# Increment the cumulative transit array by the fraction of the
# transit time spent in each cell.
# Use path_sum because it corrects for cases where we stayed in the same cell.
transit_array_accum = transit_array_accum + ((transit_array / path_sum) * transit_time)
#xx = arcpy.NumPyArrayToRaster (transit_array, lower_left_coord, cellsize_used, cellsize_used, 0)
#tmpname = "xx_t_arr_" + str (last_oid)
#print "Saving transit array to %s" % tmpname
#xx.save (tmpname)
try:
arcmgt.Delete(backlink_rast)
arcmgt.Delete(dest_layer)
except Exception as e:
arcpy.AddMessage (e)
# getting off-by-one errors when using the environment, so use this directly
ext = path_cost_rast.extent
lower_left_coord = ext.lowerLeft
last_target = row_cur.getValue(target_fld)
last_oid = row_cur.getValue(oid_fd_name)
# need to use env settings to get it to be the correct size
try:
arcpy.AddMessage ("lower left is %s" % lower_left_coord)
xx = arcpy.NumPyArrayToRaster (transit_array_accum, lower_left_coord, cellsize_used, cellsize_used, 0)
print "Saving to %s" % out_raster
xx.save (out_raster)
except:
raise
print "Completed"
return ()
def georeference_lakes(
lake_points_fc,
out_fc,
lake_id_field,
lake_name_field,
lake_county_field='',
state='',
master_gdb=r'C:\Users\smithn78\Dropbox\CL_HUB_GEO\Lake_Georeferencing\Masters_for_georef.gdb'
):
"""
Evaluate water quality sampling point locations and either assign the point to a lake polygon or flag the
point for manual review.
:param lake_points_fc:
:param out_fc:
:param lake_id_field:
:param lake_name_field:
:param lake_county_field:
:param state:
:param master_gdb: Location of master geodatabase used for linking
:return:
"""
master_lakes_fc = os.path.join(master_gdb, MASTER_LAKES_FC)
master_lakes_lines = os.path.join(master_gdb, MASTER_LAKES_LINES)
master_streams_fc = os.path.join(master_gdb, MASTER_STREAMS_FC)
master_xwalk = os.path.join(master_gdb, MASTER_XWALK)
# setup
arcpy.AddMessage("Joining...")
state = state.upper()
if state not in STATES:
raise ValueError('Use the 2-letter state code abbreviation')
arcpy.env.workspace = 'in_memory'
out_short = os.path.splitext(os.path.basename(out_fc))[0]
join1 = '{}_1'.format(out_short)
join2 = '{}_2'.format(out_short)
join3 = '{}_3'.format(out_short)
join3_select = join3 + '_select'
join4 = '{}_4'.format(out_short)
join5 = '{}_5'.format(out_short)
joinx = '{}_x'.format(out_short)
county_name_results = arcpy.ListFields(
lake_points_fc, '{}*'.format(lake_county_field))[0].name
if lake_county_field and not lake_county_field in county_name_results:
print('{} field does not exist in dataset.'.format(lake_county_field))
raise Exception
point_fields = [f.name for f in arcpy.ListFields(lake_points_fc)]
# update the lake id to a text field if not already
lake_id_field_type = arcpy.ListFields(lake_points_fc,
lake_id_field)[0].type
if lake_id_field_type != 'String':
temp_id_field = '{}_t'.format(lake_id_field)
arcpy.AddField_management(lake_points_fc, '{}_t'.format(lake_id_field),
'TEXT', '255')
expr = '!{}!'.format(lake_id_field)
arcpy.CalculateField_management(lake_points_fc, temp_id_field, expr,
'PYTHON')
arcpy.DeleteField_management(lake_points_fc, lake_id_field)
arcpy.AlterField_management(lake_points_fc,
temp_id_field,
new_field_name=lake_id_field)
# Try to make some spatial connections and fulfill some logic to assign a link
join1 = AN.SpatialJoin(lake_points_fc,
master_lakes_fc,
join1,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join2 = AN.SpatialJoin(join1,
master_streams_fc,
join2,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join3 = AN.SpatialJoin(join2,
master_lakes_fc,
join3,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='10 meters')
join4 = AN.SpatialJoin(join3,
master_lakes_fc,
join4,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='100 meters')
# setup for editing lake assignment values
DM.AddField(join4, 'Auto_Comment', 'TEXT', field_length=100)
DM.AddField(join4, 'Manual_Review', 'SHORT')
DM.AddField(join4, 'Shared_Words', 'TEXT', field_length=100)
DM.AddField(join4, 'Linked_lagoslakeid', 'LONG')
DM.AddField(join4, 'GEO_Discovered_Name', 'TEXT', field_length=255)
DM.AddField(join4, 'Duplicate_Candidate', 'TEXT', field_length=1)
DM.AddField(join4, 'Is_Legacy_Link', 'TEXT', field_length=1)
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID,
MASTER_GNIS_NAME, # 0m match
'PERMANENT_IDENTIFIER_1',
'GNIS_NAME_1', # stream match
MASTER_LAKE_ID + '_1',
MASTER_GNIS_NAME + '_12', # 10m match
MASTER_LAKE_ID + '_12',
MASTER_GNIS_NAME + '_12_13', # 100m match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid'
]
# use a cursor to go through each point and evaluate its assignment
cursor = arcpy.da.UpdateCursor(join4, update_fields)
arcpy.AddMessage("Calculating link status...")
for row in cursor:
id, name, mid_0, mname_0, stream_id, streamname_0, mid_10, mname_10, mid_100, mname_100, comment, review, words, lagosid = row
if mid_0 is not None: # if the point is directly in a polygon
if name and mname_0:
words = lagosGIS.list_shared_words(name,
mname_0,
exclude_lake_words=False)
comment = 'Exact location link'
lagosid = mid_0
review = -1
elif mid_0 is None and mid_10 is not None: # if the point is only within 10m of a lake
if name and mname_10:
words = lagosGIS.list_shared_words(name,
mname_10,
exclude_lake_words=False)
if words:
comment = 'Linked by common name and location'
lagosid = mid_10
review = -1
else:
comment = 'Linked by common location'
lagosid = mid_10
review = 1
elif mid_0 is None and mid_10 is None:
if stream_id is not None: # if there is a stream match
comment = 'Not linked because represented as river in NHD'
review = 2
else:
if mid_100 is not None: # if the point is only within 100m of lake(s)
if name and mname_100:
words = lagosGIS.list_shared_words(
name, mname_100, exclude_lake_words=True)
# TODO: Frequency check
if words:
comment = 'Linked by common name and location'
lagosid = mid_100
review = 1
else:
comment = 'Linked by common location'
lagosid = mid_100
review = 2
cursor.updateRow(
(id, name, mid_0, mname_0, stream_id, streamname_0, mid_10,
mname_10, mid_100, mname_100, comment, review, words, lagosid))
# # So I haven't been able to get the county logic to work and it hasn't been that important yet, ignore for now
# Select down to a minimum set because we're about to join on county, which will create lots of duplicate matches
# Then join calculated results back to full set
# if lake_county_field:
# join5 = AN.Select(join4, join5, 'Manual_Review IS NULL')
# lakes_state = AN.Select(MASTER_LAKES_FC, 'lakes_state', "{0} = '{1}'".format(MASTER_STATE_NAME, state))
# lakes_state_lyr = DM.MakeFeatureLayer(lakes_state, 'lakes_state_lyr')
# join5_lyr = DM.MakeFeatureLayer(join5, 'join5_lyr')
# DM.AddJoin(join5_lyr, lake_county_field, lakes_state_lyr, MASTER_COUNTY_NAME)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_cty')
# j5 = 'DEDUPED_CA_SWAMP_data_linked_5.'
#
# county_update_fields = [j5 + lake_id_field, j5 + lake_name_field, j5 + lake_county_field,
# 'lakes_state.' + MASTER_LAKE_ID, 'lakes_state.' + MASTER_GNIS_NAME, 'lakes_state.' + MASTER_COUNTY_NAME,
# j5 + 'Auto_Comment', j5 + 'Manual_Review', j5 + 'Shared_Words',
# j5 + 'Linked_lagoslakeid']
# with arcpy.da.UpdateCursor(join5_lyr, county_update_fields) as cursor:
# for row in cursor:
# id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid = row
# if county is not None and mcounty is not None:
# if name and mname_cty:
# words = lagosGIS.list_shared_words(name, mname_cty, exclude_lake_words=True)
# if words:
# comment = 'PRELIMINARY: Linked by common name and location'
# lagosid = mid_cty
# review = 2
# cursor.updateRow((id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid))
# DM.RemoveJoin(join5_lyr)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_county')
#
# # join5 = DM.JoinField(join5, lake_county_field, lakes_state, MASTER_COUNTY_NAME,
# fields = [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME])
#
# # This is a long way to make a join
# join_dict = {}
# with arcpy.da.SearchCursor(lakes_state, [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME]) as cursor:
# for row in cursor:
# join_value, val1, val2 = row
# join_dict[join_value] = [val1, val2]
#
# arcpy.AddField_management(join5, MASTER_LAKE_ID + 'cntyj', 'LONG')
# arcpy.AddField_management(join5, MASTER_GNIS_NAME + 'cntyj', 'TEXT', 255)
#
# with arcpy.da.SearchCursor(join5, [lake_county_field, MASTER_LAKE_ID + 'cntyj', MASTER_GNIS_NAME + 'cntyj']) as cursor:
# for row in cursor:
# key_value = row[0]
# words = lagosGIS.list_shared_words()
# if join_dict.has_key(key_value):
# row[1] = join_dict[key_value][0]
# row[2] = join_dict[key_value][1]
# else:
# row[1] = None
# row[2] = None
# cursor.updateRow(row)
#
#
# county_update_fields = [lake_id_field, lake_name_field, lake_county_field,
# MASTER_LAKE_ID + '_12_13_14', MASTER_GNIS_NAME + '_12_13', MASTER_COUNTY_NAME + '_12_13', # county
# 'Auto_Comment', 'Manual_Review', 'Shared_Words',
# 'Linked_lagoslakeid']
# cursor = arcpy.da.UpdateCursor(join5, county_update_fields)
# for row in cursor:
# id, name, county, lagosid_cty, lagosname_cty, mcounty, comment, mreview, words, linked_lagosid = row
# if mcounty is not None:
# words = lagosGIS.list_shared_words()
# else:
# join5 = join4
#
if state in LAGOSNE_STATES:
DM.JoinField(join4, lake_id_field, master_xwalk, 'lagosne_legacyid',
['lagoslakeid', 'lagos_lakename', 'lagos_state'])
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID + '_12_13',
'lagos_lakename',
'lagos_state', # crosswalk match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid',
'Is_Legacy_Link'
]
with arcpy.da.UpdateCursor(join4, update_fields) as uCursor:
for uRow in uCursor:
id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag = uRow
# fields are populated already from links above. Revise only if legacy links
if mid_x is not None:
if state == state_x:
legacy_flag = 'Y' # set to Y regardless of whether using legacy comment if state matches
if comment != 'Exact location link':
review = 1
if state != state_x:
review = 3 # downgrade if states mismatch--border lakes OK, random common IDs NOT. Check.
legacy_flag = 'Y'
comment = 'LAGOS-NE legacy link' # only comment non-exact location matches
lagosid = mid_x
if name and mname_x:
words = lagosGIS.list_shared_words(
name,
mname_x) # update words only if legacy comment
new_row = id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag
uCursor.updateRow(new_row)
# # Undo the next line if you ever bring this chunk back.
join5 = join4
# then re-code the no matches as a 3 and copy comments to the editable field
# compress the joined lake ids into one field
# having two fields lets us keep track of how many of the auto matches are bad
if arcpy.ListFields(join5, 'Comment'):
comment_field_name = 'Comment_LAGOS'
else:
comment_field_name = 'Comment'
DM.AddField(join5, comment_field_name, 'TEXT', field_length=100)
with arcpy.da.UpdateCursor(
join5, ['Manual_Review', 'Auto_Comment', 'Comment']) as cursor:
for flag, ac, comment in cursor:
if flag is None:
flag = 3
ac = 'Not linked'
comment = ac
cursor.updateRow((flag, ac, comment))
# Re-code points more than 100m into the polygon of the lake as no need to check
DM.MakeFeatureLayer(join5, 'join5_lyr')
DM.MakeFeatureLayer(master_lakes_lines, 'lake_lines_lyr')
DM.SelectLayerByAttribute('join5_lyr', 'NEW_SELECTION',
"Auto_Comment = 'Exact location link'")
DM.SelectLayerByLocation('join5_lyr', 'INTERSECT', 'lake_lines_lyr',
'100 meters', 'SUBSET_SELECTION', 'INVERT')
DM.CalculateField('join5_lyr', 'Manual_Review', '-2', 'PYTHON')
DM.Delete('join5_lyr', 'lake_lines_lyr')
# Then make sure to only keep the fields necessary when you write to an output
copy_fields = point_fields + [
'Linked_lagoslakeid', 'Auto_Comment', 'Manual_Review',
'Is_Legacy_Link', 'Shared_Words', 'Comment', 'Duplicate_Candidate',
'GEO_Discovered_Name'
]
copy_fields.remove('Shape')
copy_fields.remove('OBJECTID')
lagosGIS.select_fields(join5, out_fc, copy_fields)
DM.AssignDomainToField(out_fc, 'Comment', 'Comment')
DM.AddField(out_fc, 'Total_points_in_lake_poly', 'Short')
# Remove any duplicates. (These originate from the join3/join4 transition because a point can be both
# within 10m and 100m of lakes, this code takes the closest lake as true for my current sanity.)
# Or, in other words, this is a hack solution.
out_fc_fields = [
f.name for f in arcpy.ListFields(out_fc) if f.name != 'OBJECTID'
]
DM.DeleteIdentical(out_fc, out_fc_fields)
# Get the join_count for each limno lake ID
# De-dupe anything resulting from limno ID duplicates first before counting
id_pairs = list(
set(
arcpy.da.SearchCursor(out_fc,
[lake_id_field, 'Linked_lagoslakeid'])))
# THEN pull out LAGOS id. Any duplicate now are only due to multiple distinct points within lake
lagos_ids = [ids[1] for ids in id_pairs]
sample_ids = [ids[0] for ids in id_pairs]
lagos_lake_counts = Counter(lagos_ids)
linked_multiple_lake_counts = Counter(sample_ids)
# Get the count of points in the polygon
with arcpy.da.UpdateCursor(
out_fc,
['Linked_lagoslakeid', 'Total_points_in_lake_poly']) as cursor:
for lagos_id, join_count in cursor:
join_count = lagos_lake_counts[lagos_id]
cursor.updateRow((lagos_id, join_count))
# Mark any samples linked to more than one lake so that the analyst can select the correct lake in the
# manual process
with arcpy.da.UpdateCursor(
out_fc, [lake_id_field, 'Duplicate_Candidate']) as cursor:
for sample_id, duplicate_flag in cursor:
duplicate_count = linked_multiple_lake_counts[sample_id]
if duplicate_count > 1:
duplicate_flag = "Y"
else:
duplicate_flag = "N"
cursor.updateRow((sample_id, duplicate_flag))
# clean up
DM.AddField(out_fc, 'Note', 'TEXT', field_length=140)
DM.Delete('in_memory')
arcpy.AddMessage('Completed.')
DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'")
ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE",
"KEEP_ALL")
try:
DM.DeleteField(intermediate_output, "NAME")
except:
pass
final_wine_drought = "Wine_Drought_Summary"
DM.MakeFeatureLayer(intermediate_output, final_wine_drought)
lf = DM.SaveToLayerFile(
final_wine_drought,
OS.path.join(working_dir, '{}.lyr'.format(final_wine_drought)))
DM.ApplySymbologyFromLayer(lf, lyr_template)
pw = "test" #GETPASS.getpass("Enter AGOL password:"******"Drought_and_Wine"
agol = AGOLHandler("analytics", pw, service_name)
publish_service(agol, service_name, mxd_path, lf[0])
TIME.sleep(5)
fs_url = agol.findItemURL('Feature Service')
TIME.sleep(35)
gp_url, jsondata = enrich(fs_url + '/0',
'{}_Enriched'.format(service_name), agol.token)
check_job_status(gp_url, jsondata, agol.token)
DM.Delete(OS.path.join(working_dir, shp_name))
DM.Delete(OS.path.join(working_dir, lf[0]))
def lake_from_to(nhd_subregion_gdb, output_table):
arcpy.env.workspace = 'in_memory'
waterbody0 = os.path.join(nhd_subregion_gdb, 'NHDWaterbody')
network = os.path.join(nhd_subregion_gdb, 'Hydrography', 'HYDRO_NET')
junctions0 = os.path.join(nhd_subregion_gdb, 'HYDRO_NET_Junctions')
# use layers for selections. We will only work with lakes over 1 hectare for this tool.
waterbody = DM.MakeFeatureLayer(waterbody0,
'waterbody',
where_clause=LAGOS_LAKE_FILTER)
num_wbs = int(arcpy.GetCount_management(waterbody).getOutput(0))
junctions = DM.MakeFeatureLayer(junctions0, 'junctions')
DM.SelectLayerByLocation(junctions, 'INTERSECT', waterbody, '1 Meters',
'NEW_SELECTION')
junctions_1ha = DM.MakeFeatureLayer(junctions, 'junctions_1ha')
# insert results into output table
DM.CreateTable(os.path.dirname(output_table),
os.path.basename(output_table))
DM.AddField(output_table, 'FROM_PERMANENT_ID', 'TEXT', field_length=40)
DM.AddField(output_table, 'TO_PERMANENT_ID', 'TEXT', field_length=40)
# create a dictionary to hold results in memory
results = []
counter = 0
progress = .01
arcpy.AddMessage("Starting network tracing...")
with arcpy.da.SearchCursor(waterbody, 'Permanent_Identifier') as cursor:
for row in cursor:
# set up a progress printer
counter += 1
if counter >= float(num_wbs) * progress:
progress += .01
arcpy.AddMessage("{}% complete...".format(
round(progress * 100), 1))
# select this lake
id = row[0]
where_clause = """"{0}" = '{1}'""".format('Permanent_Identifier',
id)
this_waterbody = DM.MakeFeatureLayer(waterbody, 'this_waterbody',
where_clause)
# select junctions overlapping this lake. only the downstream one matters, rest have no effect
DM.SelectLayerByLocation(junctions_1ha, 'INTERSECT',
this_waterbody, '1 Meters')
count_junctions = int(
arcpy.GetCount_management(junctions_1ha).getOutput(0))
if count_junctions == 0:
# add a row with no "TO" lake to the results
results.append({'FROM': id, 'TO': None})
else:
# copy with selection on
this_junctions = DM.MakeFeatureLayer(junctions_1ha,
'this_junctions')
DM.TraceGeometricNetwork(network, 'downstream', this_junctions,
'TRACE_DOWNSTREAM')
# select lakes that intersect the downstream network with a tolerance of 1 meters
DM.SelectLayerByLocation(waterbody, 'INTERSECT',
'downstream/NHDFlowline', '1 Meters',
'NEW_SELECTION')
# remove this lake
DM.SelectLayerByAttribute(waterbody, 'REMOVE_FROM_SELECTION',
where_clause)
# get the count, if it's 0 then there should be no table entry or something?
count_waterbody = int(
arcpy.GetCount_management(waterbody).getOutput(0))
# copy those into the table that you're storing stuff in
if count_waterbody == 0:
# add a row with no "TO" lake to the results
results.append({'FROM': id, 'TO': None})
else:
# for each ID, how am I getting those
to_ids = [
row[0] for row in arcpy.da.SearchCursor(
waterbody, 'Permanent_Identifier')
]
for to_id in to_ids:
result = {'FROM': id, 'TO': to_id}
results.append(result)
# delete all the intermediates
DM.SelectLayerByAttribute(waterbody, 'CLEAR_SELECTION')
for item in [this_waterbody, this_junctions, 'downstream']:
DM.Delete(item)
# insert the results in the table
insert_cursor = arcpy.da.InsertCursor(
output_table, ['FROM_PERMANENT_ID', 'TO_PERMANENT_ID'])
for result in results:
insert_cursor.insertRow([result['FROM'], result['TO']])
# delete everything
for item in [waterbody, junctions, junctions_1ha, 'in_memory']:
DM.Delete(item)
arcpy.AddMessage("Completed.")
def createOutputShapes(self, outputFC):
#### Shorthand Attributes ####
ssdoBase = self.ssdoBase
ssdoCand = self.ssdoCand
#### Validate Output Workspace ####
ARCPY.overwriteOutput = True
ERROR.checkOutputPath(outputFC)
#### Create Output Feature Class ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84003))
outPath, outName = OS.path.split(outputFC)
tempFC = UTILS.returnScratchName("TempSS_FC", fileType = "FEATURECLASS",
scratchWS = outPath)
outTempPath, outTempName = OS.path.split(tempFC)
try:
DM.CreateFeatureclass(outTempPath, outTempName, ssdoBase.shapeType,
"", ssdoBase.mFlag,
ssdoBase.zFlag, ssdoBase.spatialRefString)
except:
ARCPY.AddIDMessage("ERROR", 210, outputFC)
raise SystemExit()
#### Add Null Value Flag ####
outIsShapeFile = UTILS.isShapeFile(outputFC)
setNullable = outIsShapeFile == False
#### Make Feature Layer and Select Result OIDs/Shapes ####
featureCount = ssdoBase.numObs + ssdoCand.numObs
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84003), 0,
featureCount, 1)
#### Add Shape/ID Field Names ####
matchID, candID = outputIDFieldNames
outFieldNames = ["SHAPE@"] + outputIDFieldNames
inFieldNames = ["OID@", "SHAPE@"]
UTILS.addEmptyField(tempFC, matchID, "LONG", nullable = True)
UTILS.addEmptyField(tempFC, candID, "LONG", nullable = True)
#### Add Append Fields ####
lenAppend = len(self.appendFields)
appendIsDate = []
in2OutFieldNames = {}
if lenAppend:
for fieldName in self.appendFields:
fcField = ssdoCand.allFields[fieldName]
fieldType = UTILS.convertType[fcField.type]
fieldOutName = UTILS.validQFieldName(fcField, outPath)
in2OutFieldNames[fieldName] = fieldOutName
if fieldType == "DATE":
appendIsDate.append(fieldName)
UTILS.addEmptyField(tempFC, fieldOutName, fieldType,
alias = fcField.alias)
outFieldNames.append(fieldOutName)
#### Add Analysis Fields ####
for fieldName in self.fieldNames:
fcField = ssdoBase.allFields[fieldName]
fieldType = UTILS.convertType[fcField.type]
fieldOutName = UTILS.validQFieldName(fcField, outPath)
in2OutFieldNames[fieldName] = fieldOutName
UTILS.addEmptyField(tempFC, fieldOutName, fieldType,
alias = fcField.alias)
outFieldNames.append(fieldOutName)
dataFieldNames = matchFieldInfo[self.similarType]
dataFieldInfo = outputFieldInfo[self.matchMethod]
baseValues = []
for fieldName in dataFieldNames:
outAlias, outType, baseValue = dataFieldInfo[fieldName]
UTILS.addEmptyField(tempFC, fieldName, outType,
alias = outAlias,
nullable = setNullable)
outFieldNames.append(fieldName)
baseValues.append(baseValue)
#### Get Insert Cursor ####
baseRows = DA.SearchCursor(ssdoBase.inputFC, inFieldNames)
candRows = DA.SearchCursor(ssdoCand.inputFC, inFieldNames)
rows = DA.InsertCursor(tempFC, outFieldNames)
#### Set Base Data ####
useShapeNull = outIsShapeFile
if useShapeNull:
nullIntValue = UTILS.shpFileNull['LONG']
else:
nullIntValue = None
#### Set Base Null For Append ####
appendNull = {}
for fieldName in self.appendFields:
if fieldName not in ssdoBase.fields:
if useShapeNull:
outType = ssdoCand.fields[fieldName].type
outNullValue = UTILS.shpFileNull[outType]
else:
outNullValue = None
appendNull[fieldName] = outNullValue
#### Add Base Data ####
for masterID, shp in baseRows:
orderID = ssdoBase.master2Order[masterID]
#### Insert Shape, Match_ID and NULL (Cand_ID) ####
rowRes = [shp, masterID, nullIntValue]
#### Add Append Fields ####
for fieldName in self.appendFields:
if fieldName in appendNull:
rowRes.append(appendNull[fieldName])
else:
value = ssdoBase.fields[fieldName].data[orderID]
if fieldName in appendIsDate:
value = TUTILS.iso2DateTime(value)
rowRes.append(value)
#### Add Analysis Fields ####
for fieldName in self.fieldNames:
rowRes.append(ssdoBase.fields[fieldName].data[orderID])
#### Add Null Base Values ####
rowRes += baseValues
rows.insertRow(rowRes)
ARCPY.SetProgressorPosition()
del baseRows
#### First Add Similar Results ####
for masterID, shp in candRows:
orderID = ssdoCand.master2Order[masterID]
indTop = NUM.where(self.topIDs == orderID)[0]
indBot = NUM.where(self.botIDs == orderID)[0]
if self.similarType in ['MOST_SIMILAR', 'BOTH'] and len(indTop):
ind = indTop[0]
#### Insert Shape, NULL (Match_ID) and Cand_ID ####
rowRes = [shp, nullIntValue, masterID]
#### Add Append Fields ####
for fieldName in self.appendFields:
rowRes.append(ssdoCand.fields[fieldName].data[orderID])
#### Add Analysis Fields ####
for fieldName in self.fieldNames:
rowRes.append(ssdoCand.fields[fieldName].data[orderID])
#### Add Results ####
rank = ind + 1
ss = self.totalDist[orderID]
if self.similarType == 'BOTH':
rowRes += [rank, nullIntValue, ss, rank]
else:
rowRes += [rank, ss, rank]
rows.insertRow(rowRes)
if self.similarType in ['LEAST_SIMILAR', 'BOTH'] and len(indBot):
ind = indBot[0]
#### Insert Shape, NULL (Match_ID) and Cand_ID ####
rowRes = [shp, nullIntValue, masterID]
#### Add Append Fields ####
for fieldName in self.appendFields:
rowRes.append(ssdoCand.fields[fieldName].data[orderID])
#### Add Analysis Fields ####
for fieldName in self.fieldNames:
rowRes.append(ssdoCand.fields[fieldName].data[orderID])
#### Add Results ####
rank = ind + 1
labRank = rank * -1
ss = self.totalDist[orderID]
if self.similarType == 'BOTH':
rowRes += [nullIntValue, rank, ss, labRank]
else:
rowRes += [rank, ss, labRank]
rows.insertRow(rowRes)
ARCPY.SetProgressorPosition()
del candRows
del rows
#### Do Final Sort ####
if self.matchMethod == 'ATTRIBUTE_PROFILES':
if self.similarType == 'MOST_SIMILAR':
sortString = "SIMINDEX DESCENDING;SIMRANK DESCENDING"
else:
sortString = "SIMINDEX DESCENDING"
else:
if self.similarType == 'MOST_SIMILAR':
sortString = "SIMINDEX ASCENDING;SIMRANK ASCENDING"
else:
sortString = "SIMINDEX ASCENDING"
DM.Sort(tempFC, outputFC, sortString, "UR")
#### Clean Up ####
DM.Delete(tempFC)
#### Symbology ####
params = ARCPY.gp.GetParameterInfo()
try:
renderType = UTILS.renderType[self.ssdoBase.shapeType.upper()]
renderKey = (self.similarType, renderType)
renderLayerFile = outputRenderInfo[renderKey]
templateDir = OS.path.dirname(OS.path.dirname(SYS.argv[0]))
fullRLF = OS.path.join(templateDir, "Templates",
"Layers", renderLayerFile)
params[2].Symbology = fullRLF
except:
ARCPY.AddIDMessage("WARNING", 973)
