def find_states(fc, state_fc):
"""Populate *_states field. States fc must have field 'states' with length 255 and state abbreviations within."""
states_field = '{}_states'.format(os.path.basename(fc))
if arcpy.ListFields(fc, states_field):
DM.DeleteField(fc, states_field)
# reverse buffer the states slightly to avoid "D", "I", "J" situations in "INTERSECT" illustration
# from graphic examples of ArcGIS join types "Select polygon using polygon" section in Help
# make a field mapping that gathers all the intersecting states into one new value
field_list = [f.name for f in arcpy.ListFields(fc) if f.type <> 'OID' and f.type <> 'Geometry']
field_mapping = arcpy.FieldMappings()
for f in field_list:
map = arcpy.FieldMap()
map.addInputField(fc, f)
field_mapping.addFieldMap(map)
map_states = arcpy.FieldMap()
map_states.addInputField(state_fc, 'states')
map_states.mergeRule = 'Join'
map_states.joinDelimiter = ' '
field_mapping.addFieldMap(map_states)
# perform join and use output to replace original fc
spjoin = AN.SpatialJoin(fc, state_fc, 'in_memory/spjoin_intersect', 'JOIN_ONE_TO_ONE',
field_mapping=field_mapping, match_option='INTERSECT')
DM.AlterField(spjoin, 'states', new_field_name=states_field, clear_field_alias=True)
DM.Delete(fc)
DM.CopyFeatures(spjoin, fc)
DM.Delete(spjoin)
def drought_analysis(date_string):
ARCPY.env.overwriteOutput = True
working_dir = r"C:\Data\git\devsummit-14-python"
zip_name = "USDM_" + date_string + "_M.zip"
url = "http://droughtmonitor.unl.edu/data/shapefiles_m/" + zip_name
mxd_path = OS.path.join(working_dir, "MapTemplate.mxd")
lyr_template = OS.path.join(working_dir, "CurrentDroughtConditions.lyr")
zip_name = OS.path.basename(url)
drought_zip_file = URLLIB.URLopener()
dzf = drought_zip_file.retrieve(url, OS.path.join(r"C:\Temp", zip_name))
zf = ZIPFILE.ZipFile(dzf[0], "r")
shp_name = [n for n in zf.namelist() if n.endswith('.shp')][0]
zf.extractall(working_dir)
drought = OS.path.splitext(shp_name)[0]
DM.MakeFeatureLayer(OS.path.join(working_dir, shp_name), drought)
#### Add Winery Data ####
beerWinePath = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWine")
intermediate_output = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWineDrought")
wine = "BeerWine"
wine_drought = "Wine_Drought"
DM.MakeFeatureLayer(beerWinePath, wine)
DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'")
ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE",
"KEEP_ALL")
try:
DM.DeleteField(intermediate_output, "NAME")
except:
pass
final_wine_drought = "Wine_Drought_Summary"
DM.MakeFeatureLayer(intermediate_output, final_wine_drought)
lf = DM.SaveToLayerFile(
final_wine_drought,
OS.path.join(working_dir, '{}.lyr'.format(final_wine_drought)))
DM.ApplySymbologyFromLayer(lf, lyr_template)
pw = "PASSWORDHERE" #GETPASS.getpass("Enter AGOL password:"******"Drought_Wine_Service"
agol = AGOLHandler("USERNAMEHERE", pw, service_name)
publish_service(agol, service_name, mxd_path, lf[0])
TIME.sleep(5)
fs_url = agol.findItemURL('Feature Service')
TIME.sleep(35)
gp_url, jsondata = enrich(agol, fs_url + '/0',
'{}_Enriched'.format(service_name), agol.token)
check_job_status(gp_url, jsondata, agol.token)
DM.Delete(OS.path.join(working_dir, shp_name))
DM.Delete(OS.path.join(working_dir, lf[0]))
def georeference_lakes(
lake_points_fc,
out_fc,
lake_id_field,
lake_name_field,
lake_county_field='',
state='',
master_gdb=r'C:\Users\smithn78\Dropbox\CL_HUB_GEO\Lake_Georeferencing\Masters_for_georef.gdb'
):
"""
Evaluate water quality sampling point locations and either assign the point to a lake polygon or flag the
point for manual review.
:param lake_points_fc:
:param out_fc:
:param lake_id_field:
:param lake_name_field:
:param lake_county_field:
:param state:
:param master_gdb: Location of master geodatabase used for linking
:return:
"""
master_lakes_fc = os.path.join(master_gdb, MASTER_LAKES_FC)
master_lakes_lines = os.path.join(master_gdb, MASTER_LAKES_LINES)
master_streams_fc = os.path.join(master_gdb, MASTER_STREAMS_FC)
master_xwalk = os.path.join(master_gdb, MASTER_XWALK)
# setup
arcpy.AddMessage("Joining...")
state = state.upper()
if state not in STATES:
raise ValueError('Use the 2-letter state code abbreviation')
arcpy.env.workspace = 'in_memory'
out_short = os.path.splitext(os.path.basename(out_fc))[0]
join1 = '{}_1'.format(out_short)
join2 = '{}_2'.format(out_short)
join3 = '{}_3'.format(out_short)
join3_select = join3 + '_select'
join4 = '{}_4'.format(out_short)
join5 = '{}_5'.format(out_short)
joinx = '{}_x'.format(out_short)
county_name_results = arcpy.ListFields(
lake_points_fc, '{}*'.format(lake_county_field))[0].name
if lake_county_field and not lake_county_field in county_name_results:
print('{} field does not exist in dataset.'.format(lake_county_field))
raise Exception
point_fields = [f.name for f in arcpy.ListFields(lake_points_fc)]
# update the lake id to a text field if not already
lake_id_field_type = arcpy.ListFields(lake_points_fc,
lake_id_field)[0].type
if lake_id_field_type != 'String':
temp_id_field = '{}_t'.format(lake_id_field)
arcpy.AddField_management(lake_points_fc, '{}_t'.format(lake_id_field),
'TEXT', '255')
expr = '!{}!'.format(lake_id_field)
arcpy.CalculateField_management(lake_points_fc, temp_id_field, expr,
'PYTHON')
arcpy.DeleteField_management(lake_points_fc, lake_id_field)
arcpy.AlterField_management(lake_points_fc,
temp_id_field,
new_field_name=lake_id_field)
# Try to make some spatial connections and fulfill some logic to assign a link
join1 = AN.SpatialJoin(lake_points_fc,
master_lakes_fc,
join1,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join2 = AN.SpatialJoin(join1,
master_streams_fc,
join2,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT')
join3 = AN.SpatialJoin(join2,
master_lakes_fc,
join3,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='10 meters')
join4 = AN.SpatialJoin(join3,
master_lakes_fc,
join4,
'JOIN_ONE_TO_MANY',
'KEEP_ALL',
match_option='INTERSECT',
search_radius='100 meters')
# setup for editing lake assignment values
DM.AddField(join4, 'Auto_Comment', 'TEXT', field_length=100)
DM.AddField(join4, 'Manual_Review', 'SHORT')
DM.AddField(join4, 'Shared_Words', 'TEXT', field_length=100)
DM.AddField(join4, 'Linked_lagoslakeid', 'LONG')
DM.AddField(join4, 'GEO_Discovered_Name', 'TEXT', field_length=255)
DM.AddField(join4, 'Duplicate_Candidate', 'TEXT', field_length=1)
DM.AddField(join4, 'Is_Legacy_Link', 'TEXT', field_length=1)
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID,
MASTER_GNIS_NAME, # 0m match
'PERMANENT_IDENTIFIER_1',
'GNIS_NAME_1', # stream match
MASTER_LAKE_ID + '_1',
MASTER_GNIS_NAME + '_12', # 10m match
MASTER_LAKE_ID + '_12',
MASTER_GNIS_NAME + '_12_13', # 100m match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid'
]
# use a cursor to go through each point and evaluate its assignment
cursor = arcpy.da.UpdateCursor(join4, update_fields)
arcpy.AddMessage("Calculating link status...")
for row in cursor:
id, name, mid_0, mname_0, stream_id, streamname_0, mid_10, mname_10, mid_100, mname_100, comment, review, words, lagosid = row
if mid_0 is not None: # if the point is directly in a polygon
if name and mname_0:
words = lagosGIS.list_shared_words(name,
mname_0,
exclude_lake_words=False)
comment = 'Exact location link'
lagosid = mid_0
review = -1
elif mid_0 is None and mid_10 is not None: # if the point is only within 10m of a lake
if name and mname_10:
words = lagosGIS.list_shared_words(name,
mname_10,
exclude_lake_words=False)
if words:
comment = 'Linked by common name and location'
lagosid = mid_10
review = -1
else:
comment = 'Linked by common location'
lagosid = mid_10
review = 1
elif mid_0 is None and mid_10 is None:
if stream_id is not None: # if there is a stream match
comment = 'Not linked because represented as river in NHD'
review = 2
else:
if mid_100 is not None: # if the point is only within 100m of lake(s)
if name and mname_100:
words = lagosGIS.list_shared_words(
name, mname_100, exclude_lake_words=True)
# TODO: Frequency check
if words:
comment = 'Linked by common name and location'
lagosid = mid_100
review = 1
else:
comment = 'Linked by common location'
lagosid = mid_100
review = 2
cursor.updateRow(
(id, name, mid_0, mname_0, stream_id, streamname_0, mid_10,
mname_10, mid_100, mname_100, comment, review, words, lagosid))
# # So I haven't been able to get the county logic to work and it hasn't been that important yet, ignore for now
# Select down to a minimum set because we're about to join on county, which will create lots of duplicate matches
# Then join calculated results back to full set
# if lake_county_field:
# join5 = AN.Select(join4, join5, 'Manual_Review IS NULL')
# lakes_state = AN.Select(MASTER_LAKES_FC, 'lakes_state', "{0} = '{1}'".format(MASTER_STATE_NAME, state))
# lakes_state_lyr = DM.MakeFeatureLayer(lakes_state, 'lakes_state_lyr')
# join5_lyr = DM.MakeFeatureLayer(join5, 'join5_lyr')
# DM.AddJoin(join5_lyr, lake_county_field, lakes_state_lyr, MASTER_COUNTY_NAME)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_cty')
# j5 = 'DEDUPED_CA_SWAMP_data_linked_5.'
#
# county_update_fields = [j5 + lake_id_field, j5 + lake_name_field, j5 + lake_county_field,
# 'lakes_state.' + MASTER_LAKE_ID, 'lakes_state.' + MASTER_GNIS_NAME, 'lakes_state.' + MASTER_COUNTY_NAME,
# j5 + 'Auto_Comment', j5 + 'Manual_Review', j5 + 'Shared_Words',
# j5 + 'Linked_lagoslakeid']
# with arcpy.da.UpdateCursor(join5_lyr, county_update_fields) as cursor:
# for row in cursor:
# id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid = row
# if county is not None and mcounty is not None:
# if name and mname_cty:
# words = lagosGIS.list_shared_words(name, mname_cty, exclude_lake_words=True)
# if words:
# comment = 'PRELIMINARY: Linked by common name and location'
# lagosid = mid_cty
# review = 2
# cursor.updateRow((id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid))
# DM.RemoveJoin(join5_lyr)
# join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_county')
#
# # join5 = DM.JoinField(join5, lake_county_field, lakes_state, MASTER_COUNTY_NAME,
# fields = [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME])
#
# # This is a long way to make a join
# join_dict = {}
# with arcpy.da.SearchCursor(lakes_state, [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME]) as cursor:
# for row in cursor:
# join_value, val1, val2 = row
# join_dict[join_value] = [val1, val2]
#
# arcpy.AddField_management(join5, MASTER_LAKE_ID + 'cntyj', 'LONG')
# arcpy.AddField_management(join5, MASTER_GNIS_NAME + 'cntyj', 'TEXT', 255)
#
# with arcpy.da.SearchCursor(join5, [lake_county_field, MASTER_LAKE_ID + 'cntyj', MASTER_GNIS_NAME + 'cntyj']) as cursor:
# for row in cursor:
# key_value = row[0]
# words = lagosGIS.list_shared_words()
# if join_dict.has_key(key_value):
# row[1] = join_dict[key_value][0]
# row[2] = join_dict[key_value][1]
# else:
# row[1] = None
# row[2] = None
# cursor.updateRow(row)
#
#
# county_update_fields = [lake_id_field, lake_name_field, lake_county_field,
# MASTER_LAKE_ID + '_12_13_14', MASTER_GNIS_NAME + '_12_13', MASTER_COUNTY_NAME + '_12_13', # county
# 'Auto_Comment', 'Manual_Review', 'Shared_Words',
# 'Linked_lagoslakeid']
# cursor = arcpy.da.UpdateCursor(join5, county_update_fields)
# for row in cursor:
# id, name, county, lagosid_cty, lagosname_cty, mcounty, comment, mreview, words, linked_lagosid = row
# if mcounty is not None:
# words = lagosGIS.list_shared_words()
# else:
# join5 = join4
#
if state in LAGOSNE_STATES:
DM.JoinField(join4, lake_id_field, master_xwalk, 'lagosne_legacyid',
['lagoslakeid', 'lagos_lakename', 'lagos_state'])
update_fields = [
lake_id_field,
lake_name_field,
MASTER_LAKE_ID + '_12_13',
'lagos_lakename',
'lagos_state', # crosswalk match
'Auto_Comment',
'Manual_Review',
'Shared_Words',
'Linked_lagoslakeid',
'Is_Legacy_Link'
]
with arcpy.da.UpdateCursor(join4, update_fields) as uCursor:
for uRow in uCursor:
id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag = uRow
# fields are populated already from links above. Revise only if legacy links
if mid_x is not None:
if state == state_x:
legacy_flag = 'Y' # set to Y regardless of whether using legacy comment if state matches
if comment != 'Exact location link':
review = 1
if state != state_x:
review = 3 # downgrade if states mismatch--border lakes OK, random common IDs NOT. Check.
legacy_flag = 'Y'
comment = 'LAGOS-NE legacy link' # only comment non-exact location matches
lagosid = mid_x
if name and mname_x:
words = lagosGIS.list_shared_words(
name,
mname_x) # update words only if legacy comment
new_row = id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag
uCursor.updateRow(new_row)
# # Undo the next line if you ever bring this chunk back.
join5 = join4
# then re-code the no matches as a 3 and copy comments to the editable field
# compress the joined lake ids into one field
# having two fields lets us keep track of how many of the auto matches are bad
if arcpy.ListFields(join5, 'Comment'):
comment_field_name = 'Comment_LAGOS'
else:
comment_field_name = 'Comment'
DM.AddField(join5, comment_field_name, 'TEXT', field_length=100)
with arcpy.da.UpdateCursor(
join5, ['Manual_Review', 'Auto_Comment', 'Comment']) as cursor:
for flag, ac, comment in cursor:
if flag is None:
flag = 3
ac = 'Not linked'
comment = ac
cursor.updateRow((flag, ac, comment))
# Re-code points more than 100m into the polygon of the lake as no need to check
DM.MakeFeatureLayer(join5, 'join5_lyr')
DM.MakeFeatureLayer(master_lakes_lines, 'lake_lines_lyr')
DM.SelectLayerByAttribute('join5_lyr', 'NEW_SELECTION',
"Auto_Comment = 'Exact location link'")
DM.SelectLayerByLocation('join5_lyr', 'INTERSECT', 'lake_lines_lyr',
'100 meters', 'SUBSET_SELECTION', 'INVERT')
DM.CalculateField('join5_lyr', 'Manual_Review', '-2', 'PYTHON')
DM.Delete('join5_lyr', 'lake_lines_lyr')
# Then make sure to only keep the fields necessary when you write to an output
copy_fields = point_fields + [
'Linked_lagoslakeid', 'Auto_Comment', 'Manual_Review',
'Is_Legacy_Link', 'Shared_Words', 'Comment', 'Duplicate_Candidate',
'GEO_Discovered_Name'
]
copy_fields.remove('Shape')
copy_fields.remove('OBJECTID')
lagosGIS.select_fields(join5, out_fc, copy_fields)
DM.AssignDomainToField(out_fc, 'Comment', 'Comment')
DM.AddField(out_fc, 'Total_points_in_lake_poly', 'Short')
# Remove any duplicates. (These originate from the join3/join4 transition because a point can be both
# within 10m and 100m of lakes, this code takes the closest lake as true for my current sanity.)
# Or, in other words, this is a hack solution.
out_fc_fields = [
f.name for f in arcpy.ListFields(out_fc) if f.name != 'OBJECTID'
]
DM.DeleteIdentical(out_fc, out_fc_fields)
# Get the join_count for each limno lake ID
# De-dupe anything resulting from limno ID duplicates first before counting
id_pairs = list(
set(
arcpy.da.SearchCursor(out_fc,
[lake_id_field, 'Linked_lagoslakeid'])))
# THEN pull out LAGOS id. Any duplicate now are only due to multiple distinct points within lake
lagos_ids = [ids[1] for ids in id_pairs]
sample_ids = [ids[0] for ids in id_pairs]
lagos_lake_counts = Counter(lagos_ids)
linked_multiple_lake_counts = Counter(sample_ids)
# Get the count of points in the polygon
with arcpy.da.UpdateCursor(
out_fc,
['Linked_lagoslakeid', 'Total_points_in_lake_poly']) as cursor:
for lagos_id, join_count in cursor:
join_count = lagos_lake_counts[lagos_id]
cursor.updateRow((lagos_id, join_count))
# Mark any samples linked to more than one lake so that the analyst can select the correct lake in the
# manual process
with arcpy.da.UpdateCursor(
out_fc, [lake_id_field, 'Duplicate_Candidate']) as cursor:
for sample_id, duplicate_flag in cursor:
duplicate_count = linked_multiple_lake_counts[sample_id]
if duplicate_count > 1:
duplicate_flag = "Y"
else:
duplicate_flag = "N"
cursor.updateRow((sample_id, duplicate_flag))
# clean up
DM.AddField(out_fc, 'Note', 'TEXT', field_length=140)
DM.Delete('in_memory')
arcpy.AddMessage('Completed.')
shp_name = [n for n in zf.namelist() if n.endswith('.shp')][0]
zf.extractall(working_dir)
drought = OS.path.splitext(shp_name)[0]
DM.MakeFeatureLayer(OS.path.join(working_dir, shp_name), drought)
#### Add Winery Data ####
beerWinePath = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWine")
intermediate_output = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb",
"BeerWineDrought")
wine = "BeerWine"
wine_drought = "Wine_Drought"
DM.MakeFeatureLayer(beerWinePath, wine)
DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'")
ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE",
"KEEP_ALL")
try:
DM.DeleteField(intermediate_output, "NAME")
except:
pass
final_wine_drought = "Wine_Drought_Summary"
DM.MakeFeatureLayer(intermediate_output, final_wine_drought)
lf = DM.SaveToLayerFile(
final_wine_drought,
OS.path.join(working_dir, '{}.lyr'.format(final_wine_drought)))
DM.ApplySymbologyFromLayer(lf, lyr_template)
pw = "test" #GETPASS.getpass("Enter AGOL password:"******"Drought_and_Wine"