def createPMPfc(): arcpy.AddMessage( "\nCreating feature class: 'PMP_Points' in Scratch.gdb...") dm.MakeFeatureLayer( home + "\\Input\Non_Storm_Data.gdb\Vector_Grid", "vgLayer") # make a feature layer of vector grid cells dm.SelectLayerByLocation( "vgLayer", "INTERSECT", aoiBasin ) # select the vector grid cells that intersect the aoiBasin polygon dm.MakeFeatureLayer(home + "\\Input\Non_Storm_Data.gdb\Grid_Points", "gpLayer") # make a feature layer of grid points dm.SelectLayerByLocation( "gpLayer", "HAVE_THEIR_CENTER_IN", "vgLayer" ) # select the grid points within the vector grid selection con.FeatureClassToFeatureClass( "gpLayer", env.scratchGDB, "PMP_Points") # save feature layer as "PMP_Points" feature class arcpy.AddMessage("(" + str(dm.GetCount("gpLayer")) + " grid points will be analyzed)\n") # Add PMP Fields for dur in durList: arcpy.AddMessage("\t...adding field: PMP_" + str(dur)) dm.AddField(env.scratchGDB + "\\PMP_Points", "PMP_" + dur, "DOUBLE") # Add STORM Fields (this string values identifies the driving storm by SPAS ID number) for dur in durList: arcpy.AddMessage("\t...adding field: STORM_" + str(dur)) dm.AddField(env.scratchGDB + "\\PMP_Points", "STORM_" + dur, "TEXT", "", "", 16) return
def process_ws(ws_fc, zone_name): # generate new zone ids DM.AddField(ws_fc, 'zoneid', 'TEXT', field_length=10) DM.CalculateField(ws_fc, 'zoneid', '!lagoslakeid!', 'PYTHON') ws_fc_lyr = DM.MakeFeatureLayer(ws_fc) # multipart DM.AddField(ws_fc, 'ismultipart', 'TEXT', field_length=2) with arcpy.da.UpdateCursor(ws_fc, ['ismultipart', 'SHAPE@']) as u_cursor: for row in u_cursor: if row[1].isMultipart: row[0] = 'Y' else: row[0] = 'N' u_cursor.updateRow(row) print("Edge flags...") # add flag fields DM.AddField(ws_fc, 'onlandborder', 'TEXT', field_length = 2) DM.AddField(ws_fc, 'oncoast', 'TEXT', field_length = 2) # identify border zones border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr') DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', border_lyr) DM.CalculateField(ws_fc_lyr, 'onlandborder', "'Y'", 'PYTHON') DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION') DM.CalculateField(ws_fc_lyr, 'onlandborder' ,"'N'", 'PYTHON') # identify coastal zones coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr') DM.SelectLayerByLocation(ws_fc_lyr, 'INTERSECT', coastal_lyr) DM.CalculateField(ws_fc_lyr, 'oncoast', "'Y'", 'PYTHON') DM.SelectLayerByAttribute(ws_fc_lyr, 'SWITCH_SELECTION') DM.CalculateField(ws_fc_lyr, 'oncoast' ,"'N'", 'PYTHON') print("State assignment...") # States state_geo = r'D:\Continental_Limnology\Data_Working\LAGOS_US_GIS_Data_v0.6.gdb\Spatial_Classifications\state' find_states(ws_fc, STATES_GEO) # glaciation status? calc_glaciation(ws_fc, 'zoneid') # preface the names with the zones DM.DeleteField(ws_fc, 'ORIG_FID') fields = [f.name for f in arcpy.ListFields(ws_fc, '*') if f.type not in ('OID', 'Geometry') and not f.name.startswith('Shape_')] for f in fields: new_fname = '{zn}_{orig}'.format(zn=zone_name, orig = f).lower() try: DM.AlterField(ws_fc, f, new_fname, clear_field_alias = 'TRUE') # sick of debugging the required field message-I don't want to change required fields anyway except: pass # cleanup lyr_objects = [lyr_object for var_name, lyr_object in locals().items() if var_name.endswith('lyr')] for l in lyr_objects: DM.Delete(l)
def drought_analysis(date_string): ARCPY.env.overwriteOutput = True working_dir = r"C:\Data\git\devsummit-14-python" zip_name = "USDM_" + date_string + "_M.zip" url = "http://droughtmonitor.unl.edu/data/shapefiles_m/" + zip_name mxd_path = OS.path.join(working_dir, "MapTemplate.mxd") lyr_template = OS.path.join(working_dir, "CurrentDroughtConditions.lyr") zip_name = OS.path.basename(url) drought_zip_file = URLLIB.URLopener() dzf = drought_zip_file.retrieve(url, OS.path.join(r"C:\Temp", zip_name)) zf = ZIPFILE.ZipFile(dzf[0], "r") shp_name = [n for n in zf.namelist() if n.endswith('.shp')][0] zf.extractall(working_dir) drought = OS.path.splitext(shp_name)[0] DM.MakeFeatureLayer(OS.path.join(working_dir, shp_name), drought) #### Add Winery Data #### beerWinePath = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb", "BeerWine") intermediate_output = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb", "BeerWineDrought") wine = "BeerWine" wine_drought = "Wine_Drought" DM.MakeFeatureLayer(beerWinePath, wine) DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'") ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE", "KEEP_ALL") try: DM.DeleteField(intermediate_output, "NAME") except: pass final_wine_drought = "Wine_Drought_Summary" DM.MakeFeatureLayer(intermediate_output, final_wine_drought) lf = DM.SaveToLayerFile( final_wine_drought, OS.path.join(working_dir, '{}.lyr'.format(final_wine_drought))) DM.ApplySymbologyFromLayer(lf, lyr_template) pw = "PASSWORDHERE" #GETPASS.getpass("Enter AGOL password:"******"Drought_Wine_Service" agol = AGOLHandler("USERNAMEHERE", pw, service_name) publish_service(agol, service_name, mxd_path, lf[0]) TIME.sleep(5) fs_url = agol.findItemURL('Feature Service') TIME.sleep(35) gp_url, jsondata = enrich(agol, fs_url + '/0', '{}_Enriched'.format(service_name), agol.token) check_job_status(gp_url, jsondata, agol.token) DM.Delete(OS.path.join(working_dir, shp_name)) DM.Delete(OS.path.join(working_dir, lf[0]))
def erase_and_clean(tract_id, block_id): tract_clause = '"STATEFP" = ' + "'" + tract_id + "'" # First, generate the state-specific layer state_tracts = mg.SelectLayerByAttribute(tract_layer, "NEW_SELECTION", tract_clause) # Next, read in the appropriate block filter blocks = "data/blocks/nopop/" + block_id + "_nopop.shp" block_layer = mg.MakeFeatureLayer(blocks) # Run the Erase tool out_erase = "data/erase/" + block_id + "_erase.shp" arcpy.analysis.Erase(state_tracts, block_layer, out_feature_class = out_erase) # Check for slivers and remove them - let's set a threshold of 5000 sq meters out_singlepart = "data/singlepart/" + block_id + "_single.shp" mg.MultipartToSinglepart(out_erase, out_singlepart) single_layer = mg.MakeFeatureLayer(out_singlepart) mg.AddGeometryAttributes(single_layer, Geometry_Properties = "AREA", Area_Unit = "SQUARE_METERS") # Now, specify the where clause and dissolve the remainder area_clause = '"POLY_AREA" >= 5000' mg.SelectLayerByAttribute(single_layer, "NEW_SELECTION", area_clause) out_dissolve = "data/dissolve/" + block_id + "_dissolve.shp" mg.Dissolve(single_layer, out_dissolve, dissolve_field = "GISJOIN")
def createOutput(self, outputFC): """Creates an Output Feature Class with the Directional Mean Results. INPUTS: outputFC (str): path to the output feature class """ #### Validate Output Workspace #### ERROR.checkOutputPath(outputFC) #### Shorthand Attributes #### ssdo = self.ssdo caseField = self.caseField #### Create Output Feature Class #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84003)) tempCFLayer = "tmpCFLayer" try: DM.MakeFeatureLayer(ssdo.inputFC, tempCFLayer) first = True for key, value in self.cf.iteritems(): oids = value[0] for oid in oids: sqlString = ssdo.oidName + '=' + str(oid) if first: DM.SelectLayerByAttribute(tempCFLayer, "NEW_SELECTION", sqlString) first = False else: DM.SelectLayerByAttribute(tempCFLayer, "ADD_TO_SELECTION", sqlString) UTILS.clearExtent(DM.CopyFeatures(tempCFLayer, outputFC)) except: ARCPY.AddIDMessage("ERROR", 210, outputFC) raise SystemExit() #### Set Attribute #### self.outputFC = outputFC
def simplifyParks(): """Simplify parks so that they can effectively be displayed at a scale of 1:100,000 on the principal system map""" # limit parks to sites that are at least 100 acres and that are named parks_lyr = 'parks_layer' park_type = 'Park and/or Natural Area' park_size = 100 # acres where_clause = """"TYPE" = '{0}' AND "ACREAGE" > {1} AND "SITENAME" <> ' '""" where_populated = where_clause.format(park_type, park_size) management.MakeFeatureLayer(orca_sites, parks_lyr, where_populated) # use union tool to get rid of any holes in park features gaps_setting = 'NO_GAPS' parks_union = os.path.join('in_memory', 'parks_union') analysis.Union(parks_lyr, parks_union, gaps=gaps_setting) # the holes have been filled in by the union tool, but parks_dissolve = os.path.join('in_memory', 'parks_dissolve') management.Dissolve(parks_union, parks_dissolve) # split mulitpart features to single part single_part_parks = os.path.join('in_memory', 'single_part_parks') management.MultipartToSinglepart(parks_dissolve, single_part_parks) # delete any park fragments parks_fields = ['OID@', 'SHAPE@AREA'] with da.UpdateCursor(single_part_parks, parks_fields) as cursor: for oid, area in cursor: if area < 1000000: # square feet cursor.deleteRow() # simplify the parks by smoothing out their edges algorithm = 'PAEK' tolerance = 5000 # feet endpoint_option = 'NO_FIXED' cartography.SmoothPolygon(single_part_parks, simplified_parks, algorithm, tolerance, endpoint_option)
def check_points_are_in_cost_raster(in_file, raster): proc_layer = "checker" arcmgt.MakeFeatureLayer(in_file, proc_layer) rows = arcpy.SearchCursor(proc_layer) for row_cur in rows: shp = row_cur.shape try: centroid = shp.centroid (x, y) = (centroid.X, centroid.Y) except: arcpy.AddError ( 'One or more input points have no geometry (X or Y coords are not defined). ' + 'This can be caused by imported spreadsheets having excess rows.' ) raise PointHasNoGeometry result = arcmgt.GetCellValue(raster, "%s %s" % (x, y), "1") value = result.getOutput(0) if value == 'NoData': return 0 #print value return 1
def georeference_lakes( lake_points_fc, out_fc, lake_id_field, lake_name_field, lake_county_field='', state='', master_gdb=r'C:\Users\smithn78\Dropbox\CL_HUB_GEO\Lake_Georeferencing\Masters_for_georef.gdb' ): """ Evaluate water quality sampling point locations and either assign the point to a lake polygon or flag the point for manual review. :param lake_points_fc: :param out_fc: :param lake_id_field: :param lake_name_field: :param lake_county_field: :param state: :param master_gdb: Location of master geodatabase used for linking :return: """ master_lakes_fc = os.path.join(master_gdb, MASTER_LAKES_FC) master_lakes_lines = os.path.join(master_gdb, MASTER_LAKES_LINES) master_streams_fc = os.path.join(master_gdb, MASTER_STREAMS_FC) master_xwalk = os.path.join(master_gdb, MASTER_XWALK) # setup arcpy.AddMessage("Joining...") state = state.upper() if state not in STATES: raise ValueError('Use the 2-letter state code abbreviation') arcpy.env.workspace = 'in_memory' out_short = os.path.splitext(os.path.basename(out_fc))[0] join1 = '{}_1'.format(out_short) join2 = '{}_2'.format(out_short) join3 = '{}_3'.format(out_short) join3_select = join3 + '_select' join4 = '{}_4'.format(out_short) join5 = '{}_5'.format(out_short) joinx = '{}_x'.format(out_short) county_name_results = arcpy.ListFields( lake_points_fc, '{}*'.format(lake_county_field))[0].name if lake_county_field and not lake_county_field in county_name_results: print('{} field does not exist in dataset.'.format(lake_county_field)) raise Exception point_fields = [f.name for f in arcpy.ListFields(lake_points_fc)] # update the lake id to a text field if not already lake_id_field_type = arcpy.ListFields(lake_points_fc, lake_id_field)[0].type if lake_id_field_type != 'String': temp_id_field = '{}_t'.format(lake_id_field) arcpy.AddField_management(lake_points_fc, '{}_t'.format(lake_id_field), 'TEXT', '255') expr = '!{}!'.format(lake_id_field) arcpy.CalculateField_management(lake_points_fc, temp_id_field, expr, 'PYTHON') arcpy.DeleteField_management(lake_points_fc, lake_id_field) arcpy.AlterField_management(lake_points_fc, temp_id_field, new_field_name=lake_id_field) # Try to make some spatial connections and fulfill some logic to assign a link join1 = AN.SpatialJoin(lake_points_fc, master_lakes_fc, join1, 'JOIN_ONE_TO_MANY', 'KEEP_ALL', match_option='INTERSECT') join2 = AN.SpatialJoin(join1, master_streams_fc, join2, 'JOIN_ONE_TO_MANY', 'KEEP_ALL', match_option='INTERSECT') join3 = AN.SpatialJoin(join2, master_lakes_fc, join3, 'JOIN_ONE_TO_MANY', 'KEEP_ALL', match_option='INTERSECT', search_radius='10 meters') join4 = AN.SpatialJoin(join3, master_lakes_fc, join4, 'JOIN_ONE_TO_MANY', 'KEEP_ALL', match_option='INTERSECT', search_radius='100 meters') # setup for editing lake assignment values DM.AddField(join4, 'Auto_Comment', 'TEXT', field_length=100) DM.AddField(join4, 'Manual_Review', 'SHORT') DM.AddField(join4, 'Shared_Words', 'TEXT', field_length=100) DM.AddField(join4, 'Linked_lagoslakeid', 'LONG') DM.AddField(join4, 'GEO_Discovered_Name', 'TEXT', field_length=255) DM.AddField(join4, 'Duplicate_Candidate', 'TEXT', field_length=1) DM.AddField(join4, 'Is_Legacy_Link', 'TEXT', field_length=1) update_fields = [ lake_id_field, lake_name_field, MASTER_LAKE_ID, MASTER_GNIS_NAME, # 0m match 'PERMANENT_IDENTIFIER_1', 'GNIS_NAME_1', # stream match MASTER_LAKE_ID + '_1', MASTER_GNIS_NAME + '_12', # 10m match MASTER_LAKE_ID + '_12', MASTER_GNIS_NAME + '_12_13', # 100m match 'Auto_Comment', 'Manual_Review', 'Shared_Words', 'Linked_lagoslakeid' ] # use a cursor to go through each point and evaluate its assignment cursor = arcpy.da.UpdateCursor(join4, update_fields) arcpy.AddMessage("Calculating link status...") for row in cursor: id, name, mid_0, mname_0, stream_id, streamname_0, mid_10, mname_10, mid_100, mname_100, comment, review, words, lagosid = row if mid_0 is not None: # if the point is directly in a polygon if name and mname_0: words = lagosGIS.list_shared_words(name, mname_0, exclude_lake_words=False) comment = 'Exact location link' lagosid = mid_0 review = -1 elif mid_0 is None and mid_10 is not None: # if the point is only within 10m of a lake if name and mname_10: words = lagosGIS.list_shared_words(name, mname_10, exclude_lake_words=False) if words: comment = 'Linked by common name and location' lagosid = mid_10 review = -1 else: comment = 'Linked by common location' lagosid = mid_10 review = 1 elif mid_0 is None and mid_10 is None: if stream_id is not None: # if there is a stream match comment = 'Not linked because represented as river in NHD' review = 2 else: if mid_100 is not None: # if the point is only within 100m of lake(s) if name and mname_100: words = lagosGIS.list_shared_words( name, mname_100, exclude_lake_words=True) # TODO: Frequency check if words: comment = 'Linked by common name and location' lagosid = mid_100 review = 1 else: comment = 'Linked by common location' lagosid = mid_100 review = 2 cursor.updateRow( (id, name, mid_0, mname_0, stream_id, streamname_0, mid_10, mname_10, mid_100, mname_100, comment, review, words, lagosid)) # # So I haven't been able to get the county logic to work and it hasn't been that important yet, ignore for now # Select down to a minimum set because we're about to join on county, which will create lots of duplicate matches # Then join calculated results back to full set # if lake_county_field: # join5 = AN.Select(join4, join5, 'Manual_Review IS NULL') # lakes_state = AN.Select(MASTER_LAKES_FC, 'lakes_state', "{0} = '{1}'".format(MASTER_STATE_NAME, state)) # lakes_state_lyr = DM.MakeFeatureLayer(lakes_state, 'lakes_state_lyr') # join5_lyr = DM.MakeFeatureLayer(join5, 'join5_lyr') # DM.AddJoin(join5_lyr, lake_county_field, lakes_state_lyr, MASTER_COUNTY_NAME) # join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_cty') # j5 = 'DEDUPED_CA_SWAMP_data_linked_5.' # # county_update_fields = [j5 + lake_id_field, j5 + lake_name_field, j5 + lake_county_field, # 'lakes_state.' + MASTER_LAKE_ID, 'lakes_state.' + MASTER_GNIS_NAME, 'lakes_state.' + MASTER_COUNTY_NAME, # j5 + 'Auto_Comment', j5 + 'Manual_Review', j5 + 'Shared_Words', # j5 + 'Linked_lagoslakeid'] # with arcpy.da.UpdateCursor(join5_lyr, county_update_fields) as cursor: # for row in cursor: # id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid = row # if county is not None and mcounty is not None: # if name and mname_cty: # words = lagosGIS.list_shared_words(name, mname_cty, exclude_lake_words=True) # if words: # comment = 'PRELIMINARY: Linked by common name and location' # lagosid = mid_cty # review = 2 # cursor.updateRow((id, name, county, mid_cty, mname_cty, mcounty, comment, review, words, lagosid)) # DM.RemoveJoin(join5_lyr) # join5_with_county = DM.CopyFeatures(join5_lyr, 'join5_with_county') # # # join5 = DM.JoinField(join5, lake_county_field, lakes_state, MASTER_COUNTY_NAME, # fields = [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME]) # # # This is a long way to make a join # join_dict = {} # with arcpy.da.SearchCursor(lakes_state, [MASTER_COUNTY_NAME, MASTER_LAKE_ID, MASTER_GNIS_NAME]) as cursor: # for row in cursor: # join_value, val1, val2 = row # join_dict[join_value] = [val1, val2] # # arcpy.AddField_management(join5, MASTER_LAKE_ID + 'cntyj', 'LONG') # arcpy.AddField_management(join5, MASTER_GNIS_NAME + 'cntyj', 'TEXT', 255) # # with arcpy.da.SearchCursor(join5, [lake_county_field, MASTER_LAKE_ID + 'cntyj', MASTER_GNIS_NAME + 'cntyj']) as cursor: # for row in cursor: # key_value = row[0] # words = lagosGIS.list_shared_words() # if join_dict.has_key(key_value): # row[1] = join_dict[key_value][0] # row[2] = join_dict[key_value][1] # else: # row[1] = None # row[2] = None # cursor.updateRow(row) # # # county_update_fields = [lake_id_field, lake_name_field, lake_county_field, # MASTER_LAKE_ID + '_12_13_14', MASTER_GNIS_NAME + '_12_13', MASTER_COUNTY_NAME + '_12_13', # county # 'Auto_Comment', 'Manual_Review', 'Shared_Words', # 'Linked_lagoslakeid'] # cursor = arcpy.da.UpdateCursor(join5, county_update_fields) # for row in cursor: # id, name, county, lagosid_cty, lagosname_cty, mcounty, comment, mreview, words, linked_lagosid = row # if mcounty is not None: # words = lagosGIS.list_shared_words() # else: # join5 = join4 # if state in LAGOSNE_STATES: DM.JoinField(join4, lake_id_field, master_xwalk, 'lagosne_legacyid', ['lagoslakeid', 'lagos_lakename', 'lagos_state']) update_fields = [ lake_id_field, lake_name_field, MASTER_LAKE_ID + '_12_13', 'lagos_lakename', 'lagos_state', # crosswalk match 'Auto_Comment', 'Manual_Review', 'Shared_Words', 'Linked_lagoslakeid', 'Is_Legacy_Link' ] with arcpy.da.UpdateCursor(join4, update_fields) as uCursor: for uRow in uCursor: id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag = uRow # fields are populated already from links above. Revise only if legacy links if mid_x is not None: if state == state_x: legacy_flag = 'Y' # set to Y regardless of whether using legacy comment if state matches if comment != 'Exact location link': review = 1 if state != state_x: review = 3 # downgrade if states mismatch--border lakes OK, random common IDs NOT. Check. legacy_flag = 'Y' comment = 'LAGOS-NE legacy link' # only comment non-exact location matches lagosid = mid_x if name and mname_x: words = lagosGIS.list_shared_words( name, mname_x) # update words only if legacy comment new_row = id, name, mid_x, mname_x, state_x, comment, review, words, lagosid, legacy_flag uCursor.updateRow(new_row) # # Undo the next line if you ever bring this chunk back. join5 = join4 # then re-code the no matches as a 3 and copy comments to the editable field # compress the joined lake ids into one field # having two fields lets us keep track of how many of the auto matches are bad if arcpy.ListFields(join5, 'Comment'): comment_field_name = 'Comment_LAGOS' else: comment_field_name = 'Comment' DM.AddField(join5, comment_field_name, 'TEXT', field_length=100) with arcpy.da.UpdateCursor( join5, ['Manual_Review', 'Auto_Comment', 'Comment']) as cursor: for flag, ac, comment in cursor: if flag is None: flag = 3 ac = 'Not linked' comment = ac cursor.updateRow((flag, ac, comment)) # Re-code points more than 100m into the polygon of the lake as no need to check DM.MakeFeatureLayer(join5, 'join5_lyr') DM.MakeFeatureLayer(master_lakes_lines, 'lake_lines_lyr') DM.SelectLayerByAttribute('join5_lyr', 'NEW_SELECTION', "Auto_Comment = 'Exact location link'") DM.SelectLayerByLocation('join5_lyr', 'INTERSECT', 'lake_lines_lyr', '100 meters', 'SUBSET_SELECTION', 'INVERT') DM.CalculateField('join5_lyr', 'Manual_Review', '-2', 'PYTHON') DM.Delete('join5_lyr', 'lake_lines_lyr') # Then make sure to only keep the fields necessary when you write to an output copy_fields = point_fields + [ 'Linked_lagoslakeid', 'Auto_Comment', 'Manual_Review', 'Is_Legacy_Link', 'Shared_Words', 'Comment', 'Duplicate_Candidate', 'GEO_Discovered_Name' ] copy_fields.remove('Shape') copy_fields.remove('OBJECTID') lagosGIS.select_fields(join5, out_fc, copy_fields) DM.AssignDomainToField(out_fc, 'Comment', 'Comment') DM.AddField(out_fc, 'Total_points_in_lake_poly', 'Short') # Remove any duplicates. (These originate from the join3/join4 transition because a point can be both # within 10m and 100m of lakes, this code takes the closest lake as true for my current sanity.) # Or, in other words, this is a hack solution. out_fc_fields = [ f.name for f in arcpy.ListFields(out_fc) if f.name != 'OBJECTID' ] DM.DeleteIdentical(out_fc, out_fc_fields) # Get the join_count for each limno lake ID # De-dupe anything resulting from limno ID duplicates first before counting id_pairs = list( set( arcpy.da.SearchCursor(out_fc, [lake_id_field, 'Linked_lagoslakeid']))) # THEN pull out LAGOS id. Any duplicate now are only due to multiple distinct points within lake lagos_ids = [ids[1] for ids in id_pairs] sample_ids = [ids[0] for ids in id_pairs] lagos_lake_counts = Counter(lagos_ids) linked_multiple_lake_counts = Counter(sample_ids) # Get the count of points in the polygon with arcpy.da.UpdateCursor( out_fc, ['Linked_lagoslakeid', 'Total_points_in_lake_poly']) as cursor: for lagos_id, join_count in cursor: join_count = lagos_lake_counts[lagos_id] cursor.updateRow((lagos_id, join_count)) # Mark any samples linked to more than one lake so that the analyst can select the correct lake in the # manual process with arcpy.da.UpdateCursor( out_fc, [lake_id_field, 'Duplicate_Candidate']) as cursor: for sample_id, duplicate_flag in cursor: duplicate_count = linked_multiple_lake_counts[sample_id] if duplicate_count > 1: duplicate_flag = "Y" else: duplicate_flag = "N" cursor.updateRow((sample_id, duplicate_flag)) # clean up DM.AddField(out_fc, 'Note', 'TEXT', field_length=140) DM.Delete('in_memory') arcpy.AddMessage('Completed.')
def __init__(self, inputFC, templateFC=None, explicitSpatialRef=None, silentWarnings=False, useChordal=True): #### Validate Input Feature Class #### ERROR.checkFC(inputFC) try: self.inPath, self.inName = OS.path.split(inputFC) except: self.inPath = None self.inName = inputFC #### Validate Template FC #### if templateFC != None: if ARCPY.Exists(templateFC) == False: templateFC = None #### ShapeFile Boolean #### self.shapeFileBool = False if self.inPath: self.shapeFileBool = UTILS.isShapeFile(inputFC) #### Create Feature Layer if LYR File #### path, ext = OS.path.splitext(inputFC) if ext.upper() == ".LYR": tempFC = "SSDO_FeatureLayer" DM.MakeFeatureLayer(inputFC, tempFC) inputFC = tempFC #### Describe Input #### self.info = ARCPY.Describe(inputFC) #### Assure Input are Features with OIDs #### if not self.info.hasOID: ARCPY.AddIDMessage("ERROR", 339, self.inName) raise SystemExit() #### Assign Describe Objects to Class Attributes #### self.inputFC = inputFC self.catPath = self.info.CatalogPath self.shapeType = self.info.ShapeType self.oidName = self.info.oidFieldName self.dataType = self.info.DataType self.shapeField = self.info.ShapeFieldName self.templateFC = templateFC self.hasM = self.info.HasM self.hasZ = self.info.HasZ self.silentWarnings = silentWarnings #### Set Initial Extent Depending on DataType #### if self.dataType in ["FeatureLayer", "Layer"]: try: tempInfo = ARCPY.Describe(self.catPath) extent = tempInfo.extent except: #### in_memory, SDE, NetCDF etc... #### extent = self.info.extent self.fidSet = self.info.FIDSet if self.fidSet == "": self.selectionSet = False else: self.selectionSet = True else: extent = self.info.extent self.fidSet = "" self.selectionSet = False self.extent = extent #### Set Spatial Reference #### inputSpatRef = self.info.SpatialReference inputSpatRefName = inputSpatRef.name if explicitSpatialRef: #### Explicitely Override Spatial Reference #### self.templateFC = None self.spatialRef = explicitSpatialRef else: #### 1. Feature Dataset, 2. Env Setting, 3. Input Hierarchy #### self.spatialRef = UTILS.returnOutputSpatialRef(inputSpatRef, outputFC=templateFC) self.spatialRefString = UTILS.returnOutputSpatialString( self.spatialRef) self.spatialRefName = self.spatialRef.name self.spatialRefType = self.spatialRef.type #### Warn if Spatial Reference Changed #### if not silentWarnings: UTILS.compareSpatialRefNames(inputSpatRefName, self.spatialRefName) #### Check for Projection #### if self.spatialRefType.upper() != "PROJECTED": if self.spatialRefType.upper() == "GEOGRAPHIC": self.useChordal = useChordal if not explicitSpatialRef: if self.useChordal: ARCPY.AddIDMessage("WARNING", 1605) else: ARCPY.AddIDMessage("WARNING", 916) else: self.useChordal = False if not explicitSpatialRef: ARCPY.AddIDMessage("WARNING", 916) else: self.useChordal = False #### Angular/Linear Unit Info #### self.distanceInfo = UTILS.DistanceInfo( self.spatialRef, useChordalDistances=self.useChordal) #### Create Composition and Accounting Structure #### self.fields = {} self.master2Order = {} self.order2Master = {} #### Obtain a Full List of Field Names/Type #### self.allFields = {} listFields = self.info.fields for field in listFields: name = field.name.upper() self.allFields[name] = FCField(field) #### Set Z and M Flags and Defaults #### zmInfo = UTILS.setZMFlagInfo(self.hasM, self.hasZ, self.spatialRef) self.zFlag, self.mFlag, self.defaultZ = zmInfo self.zBool = self.zFlag == "ENABLED" #### Render Type #### self.renderType = UTILS.renderType[self.shapeType.upper()]
desc = arcpy.Describe(scratch) arcpy.env.extent = desc.extent arcmgt.Delete(scratch) print "Extent is %s" % arcpy.env.extent add_msg_and_print('Currently in directory: %s\n' % os.getcwd()) add_msg_and_print('Workspace is: %s' % arcpy.env.workspace) #add_msg_and_print ('Scratch table is: %s' % out_table) table_view = "table_view" arcmgt.MakeTableView(in_file, table_view) fields = arcpy.ListFields(in_file) layer = "feat_layer" arcmgt.MakeFeatureLayer(in_file, layer) desc = arcpy.Describe(layer) fld_names = [] for fld in desc.fields: fld_names.append(fld.name) try: fields = ["PATH_FROM", "PATH_TO", "PATH_DIST"] #arcmgt.DeleteField(layer, fields) #arcmgt.DeleteField(layer, "FROM_") for fld in fields: if not fld in fld_names: arcmgt.AddField(table_view, fld, "DOUBLE") # SHOULD GET TYPE FROM target_fld except Exception as e:
def network2SWM(inputFC, masterField, swmFile, inputNetwork, impedance, cutoff = "#", numberOfNeighs = "#", inputBarrier = "#", uturnPolicy = "ALLOW_UTURNS", restrictions = "#", hierarchy = 'NO_HIERARCHY', searchTolerance = "#", fixed = 0, exponent = 1.0, rowStandard = True): """Creates spatial weights in SWM format from a combination of network data and feature classes. INPUTS: inputFC (str): path to the input feature class masterField (str): field in table that serves as the mapping swmFile (str): path to the SWM file inputNetwork (str): path to the network dataset (*.nd) impedance (str): attribute from network dataset (1) cutoff {float, "#"}: impedance threshold numberOfNeighs {int, "#"}: number of neighbors to return inputBarrier {str, "#"}: path to the input barrier feature class uturnPolicy {str, ALLOW_UTURNS}: uturn policy (2) restrictions {str, "#"}: attribute from network dataset (3) hierarchy {str, NO_HIERARCHY}: NO_HIERARCHY or USE_HIERARCHY searchTolerance {linear measure, "#"}: snap tolerance for network (4) fixed {int, 0}: Invert impedance as weight or return a weight = 1? exponent {float, 1.0}: distance decay rowStandard {bool, True}: row standardize weights? NOTES: (1) E.g. MINUTES and METERS (2) E.g. ALLOW_UTURNS or NO_UTURNS (3) E.g. ONEWAY (4) E.g. 5000 METERS """ #### Check out Network Analyst #### try: ARCPY.CheckOutExtension("Network") except: ARCPY.AddIDMessage("ERROR", 849) raise SystemExit() #### OD Matrix and Layers #### ODCostMatrix = "ODMatrix" BarriersLayerNames = {"POINT": 'Barriers', "POLYLINE" : 'PolylineBarriers', "LINE" : 'PolylineBarriers', "POLYGON" : 'PolygonBarriers'} lines = ODCostMatrix + "\\Lines" destFCLayer = "NetSWM_Dest" ##### Delete Layers If They Exist #### cleanupNetLayer(ODCostMatrix) cleanupNetLayer(destFCLayer) cleanupNetLayer(lines) #### Get Master Field From inputFC #### ssdo = SSDO.SSDataObject(inputFC, useChordal = False) ssdo.obtainDataGA(masterField, minNumObs = 2) master2Order = ssdo.master2Order masterFieldObj = ssdo.allFields[masterField.upper()] allMaster = master2Order.keys() numObs = ssdo.numObs numPossNeighs = numObs - 1 #### Get Spatial Ref From Net Data Set #### netDesc = ARCPY.Describe(inputNetwork) netSpatialRef = netDesc.SpatialReference netSpatName = netSpatialRef.Name #### Set Maximum Neighbor Argument #### if numberOfNeighs == "#": numberOfNeighs = min( [numPossNeighs, 30] ) ARCPY.AddIDMessage("WARNING", 1012, numberOfNeighs) if numberOfNeighs >= numObs: numberOfNeighs = numPossNeighs ARCPY.AddIDMessage("WARNING", 1013, numberOfNeighs) if numberOfNeighs == 0: numberOfNeighs = numPossNeighs #### All Features are Related. Force Inverse Impedance #### if (numObs - numberOfNeighs) <= 1: if fixed: ARCPY.AddIDMessage("WARNING", 974) fixed = 0 #### Add Self Neighbor For OD Solve #### numberOfNeighsOD = numberOfNeighs + 1 #### Make OD Cost Matrix Layer #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84132)) odCostMatrixLayer = NET.MakeODCostMatrixLayer(inputNetwork, ODCostMatrix, impedance, cutoff, numberOfNeighsOD, "#", uturnPolicy, restrictions, hierarchy, "#", "NO_LINES").getOutput(0) #### OD Matrix and Layers #### naClassNames = NET.GetNAClassNames(odCostMatrixLayer) destinationLayer = ODCostMatrix + OS.sep + naClassNames["Destinations"] originLayer = ODCostMatrix + OS.sep + naClassNames["Origins"] lines = ODCostMatrix + OS.sep + naClassNames["ODLines"] #### Add Barriers #### if inputBarrier != "" and inputBarrier != "#": ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84147)) barDesc = ARCPY.Describe(inputBarrier) barShapeType = barDesc.ShapeType.upper() if barShapeType in BarriersLayerNames: barString = naClassNames[BarriersLayerNames[barShapeType]] NET.AddLocations(ODCostMatrix, barString, inputBarrier, "", searchTolerance) #### Add Master Field to OD for Selection #### masterType = UTILS.convertType[masterFieldObj.type] NET.AddFieldToAnalysisLayer(ODCostMatrix, naClassNames["Destinations"], masterField, masterType) #### Add Destinations #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84133)) masterToken = "Name " + masterField + " #;" masterToken += masterField + " " + masterField + " #" NET.AddLocations(ODCostMatrix, naClassNames["Destinations"], inputFC, masterToken, searchTolerance, exclude_restricted_elements = "EXCLUDE") #### Initialize Spatial Weights Matrix File #### hierarchyBool = hierarchy == 'USE_HIERARCHY' addConcept = WU.wTypeDispatch[fixed].split("_")[0] forceFixed = (fixed == True) swmWriter = WU.SWMWriter(swmFile, masterField, netSpatName, numObs, rowStandard, inputFC = inputFC, wType = 10, inputNet = inputNetwork, impedanceField = impedance, barrierFC = inputBarrier, uturnPolicy = uturnPolicy, restrictions = restrictions, useHierarchy = hierarchyBool, searchTolerance = searchTolerance, addConcept = addConcept, exponent = exponent, forceFixed = forceFixed) #### Create FieldList for Subset Searching #### totalImpedance = "Total_" + impedance fieldList = ";".join( ["NAME", totalImpedance] ) #### Get Chunks if Necessary #### numOrigins = int(10000000. / numObs) allMaster.sort() chunkedIDs = UTILS.chunk(allMaster, numOrigins) sqlStrings = UTILS.sqlChunkStrings(inputFC, masterField, chunkedIDs) numChunks = len(sqlStrings) #### Create Field Map for Origins #### masterToken = "Name " + masterField + " #" orgFieldMap = [masterToken, 'CurbApproach CurbApproach 0', 'SourceID SourceID #', 'SourceOID SourceOID #', 'PosAlong PosAlong #', 'SideOfEdge SideOfEdge #'] orgFieldMap = ";".join(orgFieldMap) #### Keep Track of Features That Snap to Network #### snappedFeatures = set([]) for chunkNum in xrange(numChunks): progMsg = ARCPY.GetIDMessage(84145).format(chunkNum + 1, numChunks) ARCPY.SetProgressor("default", progMsg) #### Make Origins from Chunk of Destinations #### sqlValue = sqlStrings[chunkNum] DM.MakeFeatureLayer(destinationLayer, destFCLayer, sqlValue) NET.AddLocations(ODCostMatrix, naClassNames["Origins"], destFCLayer, orgFieldMap, "#", "#", "#", "#", "CLEAR") #### Solve OD Matrix and Select Data #### NET.Solve(ODCostMatrix, "SKIP") #### Count the Number of NonZero Spatial Linkages #### numLinks = UTILS.getCount(lines) #### Create Search Cursor for OD Line Info #### rows = ARCPY.SearchCursor(lines, "", None, fieldList) row = rows.next() #### Set Tool Progressor and Process Information #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, numLinks, 1) #### Process First Record #### ODInfo = row.getValue("NAME") lastID, neighID = [ int(i) for i in ODInfo.split(" - ") ] impValue = row.getValue(totalImpedance) weight = WU.distance2Weight(impValue, wType = fixed, exponent = exponent) neighs = [] weights = [] if lastID != neighID: neighs.append(neighID) weights.append(weight) #### Process Remaining Records #### progMsg = ARCPY.GetIDMessage(84146).format(chunkNum + 1, numChunks) ARCPY.SetProgressor("step", progMsg, 0, numLinks, 1) while row: #### Get Origin and Destination Unique IDs #### ODInfo = row.getValue("NAME") masterID, neighID = [ int(i) for i in ODInfo.split(" - ") ] #### Obtain Impedance and Create Weight #### impValue = row.getValue(totalImpedance) weight = WU.distance2Weight(impValue, wType = fixed, exponent = exponent) #### Check Whether it is the Same ID #### if masterID == lastID: if masterID != neighID: neighs.append(neighID) weights.append(weight) else: #### New ID, Add Last ID Result to SWM File #### swmWriter.swm.writeEntry(lastID, neighs, weights) snappedFeatures.add(lastID) #### Reset and Initialize Containers #### neighs = [] weights = [] if masterID != neighID: neighs.append(neighID) weights.append(weight) lastID = masterID ARCPY.SetProgressorPosition() row = rows.next() #### Write Last ID Result #### swmWriter.swm.writeEntry(lastID, neighs, weights) snappedFeatures.add(lastID) #### Clean Up #### del rows ##### Delete Layers If They Exist #### cleanupNetLayer(ODCostMatrix) cleanupNetLayer(destFCLayer) cleanupNetLayer(lines) #### Add Empty SWM Entries for Features Not Snapped to Network #### notSnapped = snappedFeatures.symmetric_difference(allMaster) for masterID in notSnapped: swmWriter.swm.writeEntry(masterID, [], []) #### Report Warning/Max Neighbors #### swmWriter.reportNeighInfo() #### Clean Up #### swmWriter.close() #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM()
def snap_points_to_mask_raster (in_file, mask, out_file, distance, workspace): if distance is None or len (distance) == 0: distance = "100 METERS" if arcpy.env.outputCoordinateSystem is None: arcpy.env.outputCoordinateSystem = mask print arcpy.env.outputCoordinateSystem.name if len(workspace): arcpy.env.workspace = workspace if arcpy.env.workspace is None or len(arcpy.env.workspace) == 0: arcpy.env.workspace = os.getcwd() arcpy.AddMessage ("workspace is %s" % arcpy.env.workspace) try: suffix = None wk = arcpy.env.workspace if not '.gdb' in wk: suffix = '.shp' poly_file = arcpy.CreateScratchName(None, suffix, 'POLYGON') arcpy.RasterToPolygon_conversion (mask, poly_file, 'NO_SIMPLIFY') except: raise arcpy.AddMessage ("poly_file is %s" % poly_file) # handle layers and datasets desc = arcpy.Describe(in_file) in_file = desc.catalogPath # add .shp extension if needed - clunky, but otherwise system fails below re_gdb = re.compile ('\.gdb$') re_shp = re.compile ('\.shp$') path = os.path.dirname(out_file) if len (path) == 0: path = arcpy.env.workspace if not re_gdb.search (path) and not re_shp.search (out_file): out_file += '.shp' arcpy.AddMessage ("Input point file is %s" % in_file) arcpy.AddMessage ("Output point file is %s" % out_file) arcmgt.CopyFeatures (in_file, out_file) try: snap_layer_name = 'get_layer_for_snapping' arcmgt.MakeFeatureLayer (out_file, snap_layer_name) arcmgt.SelectLayerByLocation (snap_layer_name, 'intersect', poly_file, '#', 'NEW_SELECTION') arcmgt.SelectLayerByAttribute(snap_layer_name, 'SWITCH_SELECTION') if arcmgt.GetCount(snap_layer_name) > 0: arcpy.Snap_edit (snap_layer_name, [[poly_file, "EDGE", distance]]) else: arcpy.AddMessage ('No features selected, no snapping applied') except Exception as e: print arcpy.GetMessages() raise e arcmgt.Delete (snap_layer_name) arcmgt.Delete (poly_file) print arcpy.GetMessages() print "Completed" return
#desc = arcpy.Describe(polygon_file) #sr = desc.spatialReference arcpy.env.outputCoordinateSystem = default_coord_sys arcpy.env.geographicTransformations = "GDA_1994_To_WGS_1984" arcpy.env.XYResolution = "0.0000000001 Meters" arcpy.env.XYTolerance = "0.0000000001 Meters" temp_xy = arcpy.CreateScratchName("xx", ".shp") add_msg_and_print("temp_xy is %s" % temp_xy) try: arcmgt.XYToLine(in_table, temp_xy, "Longitude_dd", "Lattitude_dd", "Longitude_dd_2", "Lattitude_dd_2", "GEODESIC", "New_WP") except: add_msg_and_print("Unable to create XY to line feature class") raise layer = "feat_layer" arcmgt.MakeFeatureLayer(temp_xy, layer) arcmgt.SelectLayerByLocation(layer, "COMPLETELY_WITHIN", polygon_file) arcmgt.SelectLayerByAttribute(layer, "SWITCH_SELECTION") temp_overlap = arcpy.CreateScratchName("xx_overlap_", ".shp") arcpy.CopyFeatures_management(layer, temp_overlap) # now we need to iterate over those overlapping vertices and integrate them with the boundary polygon print "Completed"
def lake_from_to(nhd_subregion_gdb, output_table): arcpy.env.workspace = 'in_memory' waterbody0 = os.path.join(nhd_subregion_gdb, 'NHDWaterbody') network = os.path.join(nhd_subregion_gdb, 'Hydrography', 'HYDRO_NET') junctions0 = os.path.join(nhd_subregion_gdb, 'HYDRO_NET_Junctions') # use layers for selections. We will only work with lakes over 1 hectare for this tool. waterbody = DM.MakeFeatureLayer(waterbody0, 'waterbody', where_clause=LAGOS_LAKE_FILTER) num_wbs = int(arcpy.GetCount_management(waterbody).getOutput(0)) junctions = DM.MakeFeatureLayer(junctions0, 'junctions') DM.SelectLayerByLocation(junctions, 'INTERSECT', waterbody, '1 Meters', 'NEW_SELECTION') junctions_1ha = DM.MakeFeatureLayer(junctions, 'junctions_1ha') # insert results into output table DM.CreateTable(os.path.dirname(output_table), os.path.basename(output_table)) DM.AddField(output_table, 'FROM_PERMANENT_ID', 'TEXT', field_length=40) DM.AddField(output_table, 'TO_PERMANENT_ID', 'TEXT', field_length=40) # create a dictionary to hold results in memory results = [] counter = 0 progress = .01 arcpy.AddMessage("Starting network tracing...") with arcpy.da.SearchCursor(waterbody, 'Permanent_Identifier') as cursor: for row in cursor: # set up a progress printer counter += 1 if counter >= float(num_wbs) * progress: progress += .01 arcpy.AddMessage("{}% complete...".format( round(progress * 100), 1)) # select this lake id = row[0] where_clause = """"{0}" = '{1}'""".format('Permanent_Identifier', id) this_waterbody = DM.MakeFeatureLayer(waterbody, 'this_waterbody', where_clause) # select junctions overlapping this lake. only the downstream one matters, rest have no effect DM.SelectLayerByLocation(junctions_1ha, 'INTERSECT', this_waterbody, '1 Meters') count_junctions = int( arcpy.GetCount_management(junctions_1ha).getOutput(0)) if count_junctions == 0: # add a row with no "TO" lake to the results results.append({'FROM': id, 'TO': None}) else: # copy with selection on this_junctions = DM.MakeFeatureLayer(junctions_1ha, 'this_junctions') DM.TraceGeometricNetwork(network, 'downstream', this_junctions, 'TRACE_DOWNSTREAM') # select lakes that intersect the downstream network with a tolerance of 1 meters DM.SelectLayerByLocation(waterbody, 'INTERSECT', 'downstream/NHDFlowline', '1 Meters', 'NEW_SELECTION') # remove this lake DM.SelectLayerByAttribute(waterbody, 'REMOVE_FROM_SELECTION', where_clause) # get the count, if it's 0 then there should be no table entry or something? count_waterbody = int( arcpy.GetCount_management(waterbody).getOutput(0)) # copy those into the table that you're storing stuff in if count_waterbody == 0: # add a row with no "TO" lake to the results results.append({'FROM': id, 'TO': None}) else: # for each ID, how am I getting those to_ids = [ row[0] for row in arcpy.da.SearchCursor( waterbody, 'Permanent_Identifier') ] for to_id in to_ids: result = {'FROM': id, 'TO': to_id} results.append(result) # delete all the intermediates DM.SelectLayerByAttribute(waterbody, 'CLEAR_SELECTION') for item in [this_waterbody, this_junctions, 'downstream']: DM.Delete(item) # insert the results in the table insert_cursor = arcpy.da.InsertCursor( output_table, ['FROM_PERMANENT_ID', 'TO_PERMANENT_ID']) for result in results: insert_cursor.insertRow([result['FROM'], result['TO']]) # delete everything for item in [waterbody, junctions, junctions_1ha, 'in_memory']: DM.Delete(item) arcpy.AddMessage("Completed.")
newCol = "Richness" man.AddField(curZo, newCol, "SHORT") expr = "sum(int(i) for i in !spp0!)" man.CalculateField(curZo, newCol, expr, "PYTHON") # cycle through each zone raster for i in range(len(hypZ)): print("linking zones up to polys") print("working on " + hypZ[i]) # don't assume i is the class level -- extract class here classLevel = hypZ[i][-1:] curZo = wrk + "/zon_C" + classLevel polyZo = wrk + "/hyp_backOut_dissolve_" + classLevel polyZoLyr = "polyZoLayer" # join the table from the raster to the poly zone layer man.MakeFeatureLayer(polyZo, polyZoLyr) man.AddJoin(polyZoLyr, "OBJECTID", curZo, "OBJECTID", "KEEP_ALL") # find any polys with Richness below zone level # each dict entry is [zone: min richness] dictMinRich = {1: 1, 2: 2, 3: 5} targMinRich = dictMinRich[int(classLevel)] expr = "Richness >= " + str(targMinRich) man.SelectLayerByAttribute(polyZoLyr, "NEW_SELECTION", expr) # write out the selected set outFeat = wrk + "/zon_Joined_C" + classLevel man.CopyFeatures(polyZoLyr, outFeat) # if rows were dropped AND we are above level 1, then need # to add dropped polys to one level down. numRowsSelSet = int(man.GetCount(polyZoLyr).getOutput(0)) numRowsLyr = int(man.GetCount(polyZo).getOutput(0)) if numRowsSelSet < numRowsLyr & int(classLevel) > 1:
state_codes = { 'WA': '53', 'DE': '10', 'DC': '11', 'WI': '55', 'WV': '54', 'HI': '15', 'FL': '12', 'WY': '56', 'NJ': '34', 'NM': '35', 'TX': '48', 'LA': '22', 'NC': '37', 'ND': '38', 'NE': '31', 'TN': '47', 'NY': '36', 'PA': '42', 'AK': '02', 'NV': '32', 'NH': '33', 'VA': '51', 'CO': '08', 'CA': '06', 'AL': '01', 'AR': '05', 'VT': '50', 'IL': '17', 'GA': '13', 'IN': '18', 'IA': '19', 'MA': '25', 'AZ': '04', 'ID': '16', 'CT': '09', 'ME': '23', 'MD': '24', 'OK': '40', 'OH': '39', 'UT': '49', 'MO': '29', 'MN': '27', 'MI': '26', 'RI': '44', 'KS': '20', 'MT': '30', 'MS': '28', 'SC': '45', 'KY': '21', 'OR': '41', 'SD': '46' } tracts = "data-raw/nhgis0077_shape/us_tract_2015.shp" tract_layer = mg.MakeFeatureLayer(tracts) def erase_and_clean(tract_id, block_id): tract_clause = '"STATEFP" = ' + "'" + tract_id + "'" # First, generate the state-specific layer state_tracts = mg.SelectLayerByAttribute(tract_layer, "NEW_SELECTION", tract_clause) # Next, read in the appropriate block filter blocks = "data/blocks/nopop/" + block_id + "_nopop.shp" block_layer = mg.MakeFeatureLayer(blocks) # Run the Erase tool out_erase = "data/erase/" + block_id + "_erase.shp" arcpy.analysis.Erase(state_tracts, block_layer, out_feature_class = out_erase) # Check for slivers and remove them - let's set a threshold of 5000 sq meters out_singlepart = "data/singlepart/" + block_id + "_single.shp" mg.MultipartToSinglepart(out_erase, out_singlepart) single_layer = mg.MakeFeatureLayer(out_singlepart)
ARCPY.env.overwriteOutput = True working_dir = r"C:\Data\DevSummit14" zip_name = "USDM_" + date_string + "_M.zip" url = "http://droughtmonitor.unl.edu/data/shapefiles_m/" + zip_name mxd_path = OS.path.join(working_dir, "MapTemplate.mxd") lyr_template = OS.path.join(working_dir, "CurrentDroughtConditions.lyr") zip_name = OS.path.basename(url) drought_zip_file = URLLIB.URLopener() dzf = drought_zip_file.retrieve(url, OS.path.join(r"C:\Temp", zip_name)) zf = ZIPFILE.ZipFile(dzf[0], "r") shp_name = [n for n in zf.namelist() if n.endswith('.shp')][0] zf.extractall(working_dir) drought = OS.path.splitext(shp_name)[0] DM.MakeFeatureLayer(OS.path.join(working_dir, shp_name), drought) #### Add Winery Data #### beerWinePath = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb", "BeerWine") intermediate_output = OS.path.join(working_dir, "BeerWine", "BeerWine.gdb", "BeerWineDrought") wine = "BeerWine" wine_drought = "Wine_Drought" DM.MakeFeatureLayer(beerWinePath, wine) DM.SelectLayerByAttribute(wine, "NEW_SELECTION", "Type = 'Winery'") ANALYSIS.SpatialJoin(drought, wine, intermediate_output, "JOIN_ONE_TO_ONE", "KEEP_ALL") try: DM.DeleteField(intermediate_output, "NAME") except:
def doIntegrate(self): #### Initial Data Assessment #### printOHSSection(84428, prependNewLine=True) printOHSSubject(84431, addNewLine=False) #### Find Unique Locations #### msg = ARCPY.GetIDMessage(84441) ARCPY.SetProgressor("default", msg) initCount = UTILS.getCount(self.ssdo.inputFC) self.checkIncidents(initCount) collectedPointFC = UTILS.returnScratchName("Collect_InitTempFC") collInfo = EVENTS.collectEvents(self.ssdo, collectedPointFC) self.cleanUpList.append(collectedPointFC) collSSDO = SSDO.SSDataObject(collectedPointFC, explicitSpatialRef=self.ssdo.spatialRef, useChordal=True) collSSDO.obtainDataGA(collSSDO.oidName) ################################# #### Locational Outliers #### lo = UTILS.LocationInfo(collSSDO, concept="EUCLIDEAN", silentThreshold=True, stdDeviations=3) printOHSLocationalOutliers(lo, aggType=self.aggType) #### Raster Boundary #### if self.outputRaster: self.validateRaster(collSSDO.xyCoords) #### Agg Header #### printOHSSection(84444) #### Copy Features for Integrate #### msg = ARCPY.GetIDMessage(84443) ARCPY.SetProgressor("default", msg) intFC = UTILS.returnScratchName("Integrated_TempFC") self.cleanUpList.append(intFC) DM.CopyFeatures(self.ssdo.inputFC, intFC) #### Make Feature Layer To Avoid Integrate Bug with Spaces #### mfc = "Integrate_MFC_2" DM.MakeFeatureLayer(intFC, mfc) self.cleanUpList.append(mfc) #### Snap Subject #### printOHSSubject(84442, addNewLine=False) nScale = (collSSDO.numObs * 1.0) / self.cnt if lo.nonZeroAvgDist < lo.nonZeroMedDist: useDist = lo.nonZeroAvgDist * nScale useType = "average" else: useDist = lo.nonZeroMedDist * nScale useType = "median" distance2Integrate = lo.distances[lo.distances < useDist] distance2Integrate = NUM.sort(distance2Integrate) numDists = len(distance2Integrate) #### Max Snap Answer #### msg = ARCPY.GetIDMessage(84445) useDistStr = self.ssdo.distanceInfo.printDistance(useDist) msg = msg.format(useDistStr) printOHSAnswer(msg) percs = [10, 25, 100] indices = [int(numDists * (i * .01)) for i in percs] if indices[-1] >= numDists: indices[-1] = -1 ARCPY.SetProgressor("default", msg) for pInd, dInd in enumerate(indices): dist = distance2Integrate[dInd] snap = self.ssdo.distanceInfo.linearUnitString(dist, convert=True) DM.Integrate(mfc, snap) del collSSDO #### Run Collect Events #### collectedFC = UTILS.returnScratchName("Collect_TempFC") self.cleanUpList.append(collectedFC) intSSDO = SSDO.SSDataObject(intFC, explicitSpatialRef=self.ssdo.spatialRef, silentWarnings=True, useChordal=True) intSSDO.obtainDataGA(intSSDO.oidName) EVENTS.collectEvents(intSSDO, collectedFC) descTemp = ARCPY.Describe(collectedFC) oidName = descTemp.oidFieldName #### Delete Integrated FC #### del intSSDO #### Set VarName, MasterField, AnalysisSSDO #### self.createAnalysisSSDO(collectedFC, "ICOUNT")
def pmpAnalysis(aoiBasin, stormType, durList): ########################################################################### ## Create PMP Point Feature Class from points within AOI basin and add fields def createPMPfc(): arcpy.AddMessage( "\nCreating feature class: 'PMP_Points' in Scratch.gdb...") dm.MakeFeatureLayer( home + "\\Input\Non_Storm_Data.gdb\Vector_Grid", "vgLayer") # make a feature layer of vector grid cells dm.SelectLayerByLocation( "vgLayer", "INTERSECT", aoiBasin ) # select the vector grid cells that intersect the aoiBasin polygon dm.MakeFeatureLayer(home + "\\Input\Non_Storm_Data.gdb\Grid_Points", "gpLayer") # make a feature layer of grid points dm.SelectLayerByLocation( "gpLayer", "HAVE_THEIR_CENTER_IN", "vgLayer" ) # select the grid points within the vector grid selection con.FeatureClassToFeatureClass( "gpLayer", env.scratchGDB, "PMP_Points") # save feature layer as "PMP_Points" feature class arcpy.AddMessage("(" + str(dm.GetCount("gpLayer")) + " grid points will be analyzed)\n") # Add PMP Fields for dur in durList: arcpy.AddMessage("\t...adding field: PMP_" + str(dur)) dm.AddField(env.scratchGDB + "\\PMP_Points", "PMP_" + dur, "DOUBLE") # Add STORM Fields (this string values identifies the driving storm by SPAS ID number) for dur in durList: arcpy.AddMessage("\t...adding field: STORM_" + str(dur)) dm.AddField(env.scratchGDB + "\\PMP_Points", "STORM_" + dur, "TEXT", "", "", 16) return ########################################################################### ## Define getAOIarea() function: ## getAOIarea() calculates the area of AOI (basin outline) input shapefile/ ## featureclass. The basin outline shapefile must be projected. The area ## is sqaure miles, converted from the basin layers projected units (feet ## or meters). The aoiBasin feature class should only have a single feature ## (the basin outline). If there are multiple features, the area will be stored ## for the final feature only. def getAOIarea(): sr = arcpy.Describe( aoiBasin ).SpatialReference # Determine aoiBasin spatial reference system srname = sr.name srtype = sr.type srunitname = sr.linearUnitName # Units arcpy.AddMessage("\nAOI basin spatial reference: " + srname + "\nUnit type: " + srunitname + "\nSpatial reference type: " + srtype) aoiArea = 0.0 rows = arcpy.SearchCursor(aoiBasin) for row in rows: feat = row.getValue("Shape") aoiArea += feat.area if srtype == 'Geographic': # Must have a surface projection. If one doesn't exist it projects a temporary file and uses that. arcpy.AddMessage( "\n***The basin shapefile's spatial reference 'Geographic' is not supported. Projecting temporary shapefile for AOI.***" ) arcpy.Project_management( aoiBasin, env.scratchGDB + "\\TempBasin", 102039 ) #Projects AOI Basin (102039 = USA_Contiguous_Albers_Equal_Area_Conic_USGS_version) TempBasin = env.scratchGDB + "\\TempBasin" # Path to temporary basin created in scratch geodatabase sr = arcpy.Describe( TempBasin ).SpatialReference # Determine Spatial Reference of temporary basin aoiArea = 0.0 rows = arcpy.SearchCursor( TempBasin) # Assign area size in square meters for row in rows: feat = row.getValue("Shape") aoiArea += feat.area aoiArea = aoiArea * 0.000000386102 # Converts square meters to square miles elif srtype == 'Projected': # If a projection exists, it re-projects a temporary file and uses that for data consistency. arcpy.AddMessage( "\n***The basin shapefile's spatial reference will be reprojected to USA_Contiguous_Albers_Equal_Area_Conic_USGS_version for data consistency. Projecting temporary shapefile for AOI.***" ) arcpy.Project_management( aoiBasin, env.scratchGDB + "\\TempBasin", 102039 ) #Projects AOI Basin (102039 = USA_Contiguous_Albers_Equal_Area_Conic_USGS_version) TempBasin = env.scratchGDB + "\\TempBasin" # Path to temporary basin created in scratch geodatabase sr = arcpy.Describe( TempBasin ).SpatialReference # Determine Spatial Reference of temporary basin aoiArea = 0.0 rows = arcpy.SearchCursor( TempBasin) # Assign area size in square meters for row in rows: feat = row.getValue("Shape") aoiArea += feat.area aoiArea = aoiArea * 0.000000386102 # Converts square meters to square miles aoiArea = round(aoiArea, 3) arcpy.AddMessage("\nArea of interest: " + str(aoiArea) + " square miles.") if arcpy.GetParameter(5) == False: aoiArea = arcpy.GetParameter(6) # Enable a constant area size aoiArea = round(aoiArea, 1) arcpy.AddMessage("\n***Area used for PMP analysis: " + str(aoiArea) + " sqmi***") return aoiArea ########################################################################### ## Define dadLookup() function: ## The dadLookup() function determines the DAD value for the current storm ## and duration according to the basin area size. The DAD depth is interpolated ## linearly between the two nearest areal values within the DAD table. def dadLookup( stormLayer, duration, area ): # dadLookup() accepts the current storm layer name (string), the current duration (string), and AOI area size (float) #arcpy.AddMessage("\t\tfunction dadLookup() called.") durField = "H_" + duration # defines the name of the duration field (eg., "H_06" for 6-hour) dadTable = dadGDB + "\\" + stormLayer rows = arcpy.SearchCursor(dadTable) try: row = rows.next( ) # Sets DAD area x1 to the value in the first row of the DAD table. x1 = row.AREASQMI y1 = row.getValue(durField) xFlag = "FALSE" # xFlag will remain false for basins that are larger than the largest DAD area. except RuntimeError: # return if duration does not exist in DAD table return row = rows.next() i = 0 while row: # iterates through the DAD table - assiging the bounding values directly above and below the basin area size i += 1 if row.AREASQMI < area: x1 = row.AREASQMI y1 = row.getValue(durField) else: xFlag = "TRUE" # xFlag is switched to "TRUE" indicating area is within DAD range x2 = row.AREASQMI y2 = row.getValue(durField) break row = rows.next() del row, rows, i if xFlag == "FALSE": x2 = area # If x2 is equal to the basin area, this means that the largest DAD area is smaller than the basin and the resulting DAD value must be extrapolated. arcpy.AddMessage( "\t\tThe basin area size: " + str(area) + " sqmi is greater than the largest DAD area: " + str(x1) + " sqmi.\n\t\tDAD value is estimated by extrapolation.") y = x1 / x2 * y1 # y (the DAD depth) is estimated by extrapolating the DAD area to the basin area size. return y # The extrapolated DAD depth (in inches) is returned. # arcpy.AddMessage("\nArea = " + str(area) + "\nx1 = " + str(x1) + "\nx2 = " + str(x2) + "\ny1 = " + str(y1) + "\ny2 = " + str(y2)) x = area # If the basin area size is within the DAD table area range, the DAD depth is interpolated deltax = x2 - x1 # to determine the DAD value (y) at area (x) based on next lower (x1) and next higher (x2) areas. deltay = y2 - y1 diffx = x - x1 y = y1 + diffx * deltay / deltax if x < x1: arcpy.AddMessage( "\t\tThe basin area size: " + str(area) + " sqmi is less than the smallest DAD table area: " + str(x1) + " sqmi.\n\t\tDAD value is estimated by extrapolation.") return y # The interpolated DAD depth (in inches) is returned. ########################################################################### ## Define updatePMP() function: ## This function updates the 'PMP_XX_' and 'STORM_XX' fields of the PMP_Points ## feature class with the largest value from all analyzed storms stored in the ## pmpValues list. def updatePMP( pmpValues, stormID, duration ): # Accepts four arguments: pmpValues - largest adjusted rainfall for current duration (float list); stormID - driver storm ID for each PMP value (text list); and duration (string) pmpfield = "PMP_" + duration stormfield = "STORM_" + duration gridRows = arcpy.UpdateCursor( env.scratchGDB + "\\PMP_Points") # iterates through PMP_Points rows i = 0 for row in gridRows: row.setValue( pmpfield, pmpValues[i] ) # Sets the PMP field value equal to the Max Adj. Rainfall value (if larger than existing value). row.setValue( stormfield, stormID[i] ) # Sets the storm ID field to indicate the driving storm event gridRows.updateRow(row) i += 1 del row, gridRows, pmpfield, stormfield arcpy.AddMessage("\n\t" + duration + "-hour PMP values update complete. \n") return ########################################################################### ## The outputPMP() function produces raster GRID files for each of the PMP durations. ## Aslo, a space-delimited PMP_Distribition.txt file is created in the 'Text_Output' folder. def outputPMP(type, area, outPath): desc = arcpy.Describe(basin) basinName = desc.baseName pmpPoints = env.scratchGDB + "\\PMP_Points" # Location of 'PMP_Points' feature class which will provide data for output outType = type[:1] outArea = str(int(round(area, 0))) + "sqmi" outFC = outType + "_" + outArea #I don't think I need this..... arcpy.AddMessage("\nCopying PMP_Points feature class to " + outFC + "...") #outFC might be replaced with outpath... dm.Merge( pmpPoints, outPath ) # merge the scratch feature layer(s) of vector grid cells into the outputs arcpy.AddMessage("\nCreating Basin Summary Table...") tableName = type + "_PMP_Basin_Average" + "_" + outArea tablePath = env.scratchGDB + "\\" + tableName dm.CreateTable(env.scratchGDB, tableName) # Create blank table cursor = arcpy.da.InsertCursor( tablePath, "*") # Create Insert cursor and add a blank row to the table cursor.insertRow([0]) del cursor dm.AddField(tablePath, "STORM_TYPE", "TEXT", "", "", 10, "Storm Type") # Create "Storm Type" field dm.CalculateField(tablePath, "STORM_TYPE", "'" + type + "'", "PYTHON_9.3") # populate storm type field i = 0 for field in arcpy.ListFields( pmpPoints, "PMP_*" ): # Add fields for each PMP duration and calculate the basin average fieldName = field.name fieldAve = basinAve( basin, fieldName ) # Calls the basinAve() function - returns the average (weighted or not) dm.AddField(tablePath, fieldName, "DOUBLE", "", 2) # Add duration field dm.CalculateField(tablePath, fieldName, fieldAve, "PYTHON_9.3") # Assigns the basin average i += 1 arcpy.AddMessage("\nSummary table complete.") basAveTables.append(tablePath) return ########################################################################### ## The basin() returns the basin average PMP value for a given duration field. ## If the option for a weighted average is checked in the tool parameter the script ## will weight the grid point values based on proportion of area inside the basin. def basinAve(aoiBasin, pmpField): pmpPoints = env.scratchGDB + "\\PMP_Points" # Path of 'PMP_Points' scratch feature class if weightedAve: arcpy.AddMessage("\tCalculating basin average for " + pmpField + "(weighted)...") vectorGridClip = env.scratchGDB + "\\VectorGridClip" # Path of 'PMP_Points' scratch feature class sumstats = env.scratchGDB + "\\SummaryStats" dm.MakeFeatureLayer( home + "\\Input\Non_Storm_Data.gdb\\Vector_Grid", "vgLayer") # make a feature layer of vector grid cells dm.SelectLayerByLocation( "vgLayer", "INTERSECT", aoiBasin ) # select the vector grid cells that intersect the aoiBasin polygon an.Clip("vgLayer", aoiBasin, vectorGridClip) # clips aoi vector grid to basin dm.AddField( pmpPoints, "WEIGHT", "DOUBLE" ) # adds 'WEIGHT' field to PMP_Points scratch feature class dm.MakeFeatureLayer( vectorGridClip, "vgClipLayer" ) # make a feature layer of basin clipped vector grid cells dm.MakeFeatureLayer( pmpPoints, "pmpPointsLayer" ) # make a feature layer of PMP_Points feature class dm.AddJoin("pmpPointsLayer", "ID", "vgClipLayer", "ID") # joins PMP_Points and vectorGridBasin tables dm.CalculateField( "pmpPointsLayer", "WEIGHT", "!vectorGridClip.Shape_Area!", "PYTHON_9.3" ) # Calculates basin area proportion to use as weight for each grid cell. dm.RemoveJoin("pmpPointsLayer", "vectorGridClip") an.Statistics(pmpPoints, sumstats, [["WEIGHT", "SUM"]], "") stats = arcpy.SearchCursor(sumstats) pmpWgtAve = pmpField + "_WgtAve" for row in stats: calc = row.getValue("SUM_WEIGHT") express = "(!WEIGHT!/{})* !{}!".format(calc, pmpField) i = 0 for field in arcpy.ListFields(pmpPoints, pmpField): dm.AddField(pmpPoints, pmpWgtAve, "DOUBLE", 2) dm.CalculateField(pmpPoints, pmpWgtAve, express, "PYTHON_9.3") i += 1 del stats, row an.Statistics(pmpPoints, sumstats, [[pmpWgtAve, "SUM"]], "") sumwgtave = "SUM_" + pmpWgtAve with arcpy.da.SearchCursor(sumstats, sumwgtave) as stats: for row in stats: wgtAve = row[0] return round(wgtAve, 2) ## na = arcpy.da.TableToNumPyArray(pmpPoints,(pmpField, 'WEIGHT')) # Assign pmpPoints values and weights to Numpy array (na) ## wgtAve = numpy.average(na[pmpField], weights=na['WEIGHT']) # Calculate weighted average with Numpy average ## del na ## return round(wgtAve, 2) else: arcpy.AddMessage("\tCalculating basin average for " + pmpField + "(not weighted)...") sumstats = env.scratchGDB + "\\SummaryStats" an.Statistics(pmpPoints, sumstats, [[pmpField, "MEAN"]], "") mean = "MEAN_" + pmpField with arcpy.da.SearchCursor(sumstats, mean) as stats: for row in stats: fieldAve = row[0] return round(fieldAve, 2) ## na = arcpy.da.TableToNumPyArray(pmpPoints, pmpField) # Assign pmpPoints values to Numpy array (na) ## fieldAve = numpy.average(na[pmpField]) # Calculates aritmetic mean ## del na ## return round(fieldAve, 2) ########################################################################### ## This portion of the code iterates through each storm feature class in the ## 'Storm_Adj_Factors' geodatabase (evaluating the feature class only within ## the Local, Tropical, or general feature dataset). For each duration, ## at each grid point within the aoi basin, the transpositionality is ## confirmed. Then the DAD precip depth is retrieved and applied to the ## total adjustement factor to yield the total adjusted rainfall. This ## value is then sent to the updatePMP() function to update the 'PMP_Points' ## feature class. ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~## desc = arcpy.Describe( basin) # Check to ensure AOI input shape is a Polygon. If not - exit. basinShape = desc.shapeType if desc.shapeType == "Polygon": arcpy.AddMessage("\nBasin shape type: " + desc.shapeType) else: arcpy.AddMessage("\nBasin shape type: " + desc.shapeType) arcpy.AddMessage("\nError: Input shapefile must be a polygon!\n") sys.exit() createPMPfc( ) # Call the createPMPfc() function to create the PMP_Points feature class. env.workspace = adjFactGDB # the workspace environment is set to the 'Storm_Adj_Factors' file geodatabase aoiSQMI = round( getAOIarea(), 2 ) # Calls the getAOIarea() function to assign area of AOI shapefile to 'aoiSQMI' for dur in durList: stormList = arcpy.ListFeatureClasses( "", "Point", stormType ) # List all the total adjustment factor feature classes within the storm type feature dataset. arcpy.AddMessage( "\n*************************************************************\nEvaluating " + dur + "-hour duration...") pmpList = [] driverList = [] gridRows = arcpy.SearchCursor(env.scratchGDB + "\\PMP_Points") try: for row in gridRows: pmpList.append( 0.0 ) # creates pmpList of empty float values for each grid point to store final PMP values driverList.append( "STORM" ) # creates driverList of empty text values for each grid point to store final Driver Storm IDs del row, gridRows except UnboundLocalError: arcpy.AddMessage( "\n***Error: No data present within basin/AOI area.***\n") sys.exit() for storm in stormList: arcpy.AddMessage("\n\tEvaluating storm: " + storm + "...") dm.MakeFeatureLayer( storm, "stormLayer") # creates a feature layer for the current storm dm.SelectLayerByLocation( "stormLayer", "HAVE_THEIR_CENTER_IN", "vgLayer" ) # examines only the grid points that lie within the AOI gridRows = arcpy.SearchCursor("stormLayer") pmpField = "PMP_" + dur i = 0 try: dadPrecip = round(dadLookup(storm, dur, aoiSQMI), 3) arcpy.AddMessage("\t\t" + dur + "-hour DAD value: " + str(dadPrecip) + chr(34)) except TypeError: # In no duration exists in the DAD table - move to the next storm arcpy.AddMessage("\t***Duration '" + str(dur) + "-hour' is not present for " + str(storm) + ".***\n") continue arcpy.AddMessage( "\t\tComparing " + storm + " adjusted rainfall values against current driver values...\n") for row in gridRows: if row.TRANS == 1: # Only continue if grid point is transpositionable ('1' is transpostionable, '0' is not). try: # get total adj. factor if duration exists adjRain = round(dadPrecip * row.TAF, 1) if adjRain > pmpList[i]: pmpList[i] = adjRain driverList[i] = storm except RuntimeError: arcpy.AddMessage( "\t\t *Warning* Total Adjusted Raifnall value falied to set for row " + str(row.CNT)) break del adjRain i += 1 del row del storm, stormList, gridRows, dadPrecip updatePMP(pmpList, driverList, dur) # calls function to update "PMP Points" feature class del dur, pmpList arcpy.AddMessage( "\n'PMP_Points' Feature Class 'PMP_XX' fields update complete for all '" + stormType + "' storms.") outputPMP(stormType, aoiSQMI, outPath) # calls outputPMP() function del aoiSQMI return
def basinAve(aoiBasin, pmpField): pmpPoints = env.scratchGDB + "\\PMP_Points" # Path of 'PMP_Points' scratch feature class if weightedAve: arcpy.AddMessage("\tCalculating basin average for " + pmpField + "(weighted)...") vectorGridClip = env.scratchGDB + "\\VectorGridClip" # Path of 'PMP_Points' scratch feature class sumstats = env.scratchGDB + "\\SummaryStats" dm.MakeFeatureLayer( home + "\\Input\Non_Storm_Data.gdb\\Vector_Grid", "vgLayer") # make a feature layer of vector grid cells dm.SelectLayerByLocation( "vgLayer", "INTERSECT", aoiBasin ) # select the vector grid cells that intersect the aoiBasin polygon an.Clip("vgLayer", aoiBasin, vectorGridClip) # clips aoi vector grid to basin dm.AddField( pmpPoints, "WEIGHT", "DOUBLE" ) # adds 'WEIGHT' field to PMP_Points scratch feature class dm.MakeFeatureLayer( vectorGridClip, "vgClipLayer" ) # make a feature layer of basin clipped vector grid cells dm.MakeFeatureLayer( pmpPoints, "pmpPointsLayer" ) # make a feature layer of PMP_Points feature class dm.AddJoin("pmpPointsLayer", "ID", "vgClipLayer", "ID") # joins PMP_Points and vectorGridBasin tables dm.CalculateField( "pmpPointsLayer", "WEIGHT", "!vectorGridClip.Shape_Area!", "PYTHON_9.3" ) # Calculates basin area proportion to use as weight for each grid cell. dm.RemoveJoin("pmpPointsLayer", "vectorGridClip") an.Statistics(pmpPoints, sumstats, [["WEIGHT", "SUM"]], "") stats = arcpy.SearchCursor(sumstats) pmpWgtAve = pmpField + "_WgtAve" for row in stats: calc = row.getValue("SUM_WEIGHT") express = "(!WEIGHT!/{})* !{}!".format(calc, pmpField) i = 0 for field in arcpy.ListFields(pmpPoints, pmpField): dm.AddField(pmpPoints, pmpWgtAve, "DOUBLE", 2) dm.CalculateField(pmpPoints, pmpWgtAve, express, "PYTHON_9.3") i += 1 del stats, row an.Statistics(pmpPoints, sumstats, [[pmpWgtAve, "SUM"]], "") sumwgtave = "SUM_" + pmpWgtAve with arcpy.da.SearchCursor(sumstats, sumwgtave) as stats: for row in stats: wgtAve = row[0] return round(wgtAve, 2) ## na = arcpy.da.TableToNumPyArray(pmpPoints,(pmpField, 'WEIGHT')) # Assign pmpPoints values and weights to Numpy array (na) ## wgtAve = numpy.average(na[pmpField], weights=na['WEIGHT']) # Calculate weighted average with Numpy average ## del na ## return round(wgtAve, 2) else: arcpy.AddMessage("\tCalculating basin average for " + pmpField + "(not weighted)...") sumstats = env.scratchGDB + "\\SummaryStats" an.Statistics(pmpPoints, sumstats, [[pmpField, "MEAN"]], "") mean = "MEAN_" + pmpField with arcpy.da.SearchCursor(sumstats, mean) as stats: for row in stats: fieldAve = row[0] return round(fieldAve, 2)
def get_path_residence_times (in_file, cost_rast, out_raster, t_diff_fld_name, workspace): if len (out_raster) == 0: arcpy.AddError ("Missing argument: out_rast") raise Exception if len (t_diff_fld_name) == 0: t_diff_fld_name = "T_DIFF_HRS" arcpy.env.overwriteOutput = True # This is underhanded. It should be an argument. if arcpy.env.outputCoordinateSystem is None: arcpy.env.outputCoordinateSystem = cost_rast arcpy.AddMessage ("coordinate system is %s" % arcpy.env.outputCoordinateSystem.name) if len(workspace): arcpy.env.workspace = workspace if arcpy.env.workspace is None or len(arcpy.env.workspace) == 0: arcpy.env.workspace = os.getcwd() if '.gdb' in arcpy.env.workspace: arcpy.AddError ( "Worskpace is a geodatabase. " + "This brings too much pain for this script to work.\n" + "%s" % arcpy.env.workspace ) raise WorkspaceIsGeodatabase r = Raster(cost_rast) if r.maximum == 0 and r.minimum == 0: arcpy.AddMessage ('Cost raster has only zero value. Cannot calculate cost distances.') raise CostRasterIsZero size = r.height * r.width * 4 if size > 2 * 1028 ** 3: import struct struct_size = struct.calcsize("P") * 8 if struct_size == 32: size_in_gb = float (size) / (1028 ** 3) arcpy.AddMessage ( 'Cost raster exceeds 2 GiB in size (%s GiB). This is too large for a 32 bit NumPy.' % size_in_gb ) raise NumPyArrayExceedsSizeLimits if not check_points_are_in_cost_raster(in_file, cost_rast): arcpy.AddError ('One or more input points do not intersect the cost raster') raise PointNotOnRaster arcpy.env.snapRaster = cost_rast suffix = None wk = arcpy.env.workspace if not '.gdb' in wk: suffix = '.shp' ext = arcpy.env.extent if ext is None: arcpy.env.extent = r.extent arcpy.AddMessage ("Extent is %s" % arcpy.env.extent) arcpy.env.cellSize = r.meanCellWidth arcpy.AddMessage ("Cell size is %s" % arcpy.env.cellSize) cellsize_used = float (arcpy.env.cellSize) extent = arcpy.env.extent lower_left_coord = extent.lowerLeft arcpy.AddMessage ('Currently in directory: %s\n' % os.getcwd()) arcpy.AddMessage ('Workspace is: %s' % arcpy.env.workspace) arcpy.AddMessage ("lower left is %s" % lower_left_coord) if arcpy.env.mask is None: arcpy.AddMessage ("Setting mask to %s" % cost_rast) arcpy.env.mask = cost_rast # accumulated transits transit_array_accum = arcpy.RasterToNumPyArray (Raster(cost_rast) * 0) feat_layer = "feat_layer" arcmgt.MakeFeatureLayer(in_file, feat_layer) desc = arcpy.Describe (feat_layer) oid_fd_name = desc.OIDFieldName arcpy.AddMessage("oid_fd_name = %s" % oid_fd_name) # variable name is redundant now??? - should all calls be to oid_fd_name? target_fld = oid_fd_name proc_layer = "process_layer" arcmgt.MakeFeatureLayer(in_file, proc_layer) rows = arcpy.SearchCursor(proc_layer) last_target = None for row_cur in rows: transit_time = row_cur.getValue (t_diff_fld_name) if last_target is None or transit_time == 0: message = 'Skipping %s = %s' % (oid_fd_name, row_cur.getValue(oid_fd_name)) if transit_time == 0: message = message + " Transit time is zero" arcpy.AddMessage(message) last_target = row_cur.getValue(target_fld) last_oid = row_cur.getValue(oid_fd_name) continue arcpy.AddMessage ("Processing %s %i" % (oid_fd_name, row_cur.getValue(oid_fd_name))) arcmgt.SelectLayerByAttribute( feat_layer, "NEW_SELECTION", '%s = %s' % (target_fld, last_target) ) backlink_rast = arcpy.CreateScratchName("backlink") path_dist_rast = PathDistance(feat_layer, cost_rast, out_backlink_raster = backlink_rast) # extract the distance from the last point shp = row_cur.shape centroid = shp.centroid (x, y) = (centroid.X, centroid.Y) result = arcmgt.GetCellValue(path_dist_rast, "%s %s" % (x, y), "1") res_val = result.getOutput(0) if res_val == "NoData": this_oid = row_cur.getValue(oid_fd_name) arcpy.AddMessage ("Got nodata for coordinate (%s, %s)" % (x, y)) arcpy.AddMessage ("Is the path between features %s and %s wholly contained by the cost raster?" % (last_oid, this_oid)) pras_name = "pth_%s_%s.tif" % (last_oid, this_oid) arcpy.AddMessage ("Attempting to save path raster as %s" % pras_name) try: path_dist_rast.save(pras_name) except Exception as e: arcpy.AddMessage (e) raise PathDistanceIsNoData try: path_distance = float (res_val) except: # kludge around locale/radix issues if res_val.find(","): res_val = res_val.replace(",", ".") path_distance = float (res_val) else: raise arcpy.AddMessage("Path distance is %s\nTransit time is %s" % (path_distance, transit_time)) # get a raster of the path from origin to destination condition = '%s in (%i, %i)' % (oid_fd_name, last_oid, row_cur.getValue(oid_fd_name)) dest_layer = "dest_layer" + str (last_oid) arcmgt.MakeFeatureLayer(in_file, dest_layer, where_clause = condition) count = arcmgt.GetCount(dest_layer) count = int (count.getOutput(0)) if count == 0: raise NoFeatures("No features selected. Possible coordinate system issues.\n" + condition) try: path_cost_rast = CostPath(dest_layer, path_dist_rast, backlink_rast) #path_dist_rast.save("xx_pr" + str (last_oid)) except Exception as e: raise try: pcr_mask = 1 - IsNull (path_cost_rast) #pcr_mask.save ("xx_pcr_mask" + str (last_oid)) dist_masked = path_dist_rast * pcr_mask path_array = arcpy.RasterToNumPyArray(dist_masked, nodata_to_value = -9999) path_array_idx = numpy.where(path_array > 0) transit_array = numpy.zeros_like(path_array) # past experience suggests we might need to use a different approach to guarantee we get zeroes except: raise path_sum = None arcpy.AddMessage ("processing %i cells of path raster" % (len(path_array_idx[0]))) if path_distance == 0 or not len(path_array_idx[0]): path_sum = 1 # stayed in the same cell mask_array = arcpy.RasterToNumPyArray(pcr_mask, nodata_to_value = -9999) mask_array_idx = numpy.where(mask_array == 1) i = mask_array_idx[0][0] j = mask_array_idx[1][0] transit_array[i][j] = path_sum else: row_count = len (path_array) col_count = len (path_array[0]) for idx in range (len(path_array_idx[0])): i = path_array_idx[0][idx] j = path_array_idx[1][idx] val = path_array[i][j] nbrs = [] for k in (i-1, i, i+1): if k < 0 or k >= row_count: continue checkrow = path_array[k] for l in (j-1, j, j+1): if l < 0 or l >= col_count: continue if k == i and j == l: continue # don't check self checkval = checkrow[l] # negs are nodata, and this way we # don't need to care what that value is if checkval >= 0: diff = val - checkval if diff > 0: nbrs.append(diff) #arcpy.AddMessage ("Check and diff vals are %s %s" % (checkval, diff)) diff = min (nbrs) #arcpy.AddMessage ("Diff val is %s" % diff) transit_array[i][j] = diff path_sum = path_array.max() # could use path_distance? #arcpy.AddMessage ("path_array.max is %s" % path_sum) # sometimes we get a zero path_sum even when the path_distance is non-zero if path_sum == 0: path_sum = 1 # Increment the cumulative transit array by the fraction of the # transit time spent in each cell. # Use path_sum because it corrects for cases where we stayed in the same cell. transit_array_accum = transit_array_accum + ((transit_array / path_sum) * transit_time) #xx = arcpy.NumPyArrayToRaster (transit_array, lower_left_coord, cellsize_used, cellsize_used, 0) #tmpname = "xx_t_arr_" + str (last_oid) #print "Saving transit array to %s" % tmpname #xx.save (tmpname) try: arcmgt.Delete(backlink_rast) arcmgt.Delete(dest_layer) except Exception as e: arcpy.AddMessage (e) # getting off-by-one errors when using the environment, so use this directly ext = path_cost_rast.extent lower_left_coord = ext.lowerLeft last_target = row_cur.getValue(target_fld) last_oid = row_cur.getValue(oid_fd_name) # need to use env settings to get it to be the correct size try: arcpy.AddMessage ("lower left is %s" % lower_left_coord) xx = arcpy.NumPyArrayToRaster (transit_array_accum, lower_left_coord, cellsize_used, cellsize_used, 0) print "Saving to %s" % out_raster xx.save (out_raster) except: raise print "Completed" return ()
def classify_lakes(nhd, out_feature_class, exclude_intermit_flowlines=False, debug_mode=False): if debug_mode: arcpy.env.overwriteOutput = True temp_gdb = cu.create_temp_GDB('classify_lake_connectivity') arcpy.env.workspace = temp_gdb arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb)) else: arcpy.env.workspace = 'in_memory' if arcpy.Exists("temp_fc"): print("There is a problem here.") raise Exception # Tool temporary feature classes temp_fc = "temp_fc" csiwaterbody_10ha = "csiwaterbody_10ha" nhdflowline_filtered = "nhdflowline_filtered" dangles = "dangles" start = "start" end = "end" startdangles = "startdangles" enddangles = "enddangles" non_artificial_end = "non_artificial_end" flags_10ha_lake_junctions = "flags_10ha_lake_junctions" midvertices = "midvertices" non10vertices = "non10vertices" non10junctions = "non10junctions" all_non_flag_points = "all_non_flag_points" barriers = "barriers" trace1_junctions = "trace1_junctions" trace1_flowline = "trace1_flowline" trace2_junctions = "trace2junctions" trace2_flowline = "trace2_flowline" # Clean up workspace in case of bad exit from prior run in same session. this_tool_layers = [ "dangles_lyr", "nhdflowline_lyr", "junction_lyr", "midvertices_lyr", "all_non_flag_points_lyr", "non10vertices_lyr", "out_fc_lyr", "trace1", "trace2" ] this_tool_temp = [ temp_fc, csiwaterbody_10ha, nhdflowline_filtered, dangles, start, end, startdangles, enddangles, non_artificial_end, flags_10ha_lake_junctions, midvertices, non10vertices, non10junctions, all_non_flag_points, barriers, trace1_junctions, trace1_flowline, trace2_junctions, trace2_flowline ] for item in this_tool_layers + this_tool_temp: try: DM.Delete(item) except: pass # Local variables: nhdflowline = os.path.join(nhd, "Hydrography", "NHDFLowline") nhdjunction = os.path.join(nhd, "Hydrography", "HYDRO_NET_Junctions") nhdwaterbody = os.path.join(nhd, "Hydrography", "NHDWaterbody") network = os.path.join(nhd, "Hydrography", "HYDRO_NET") # Get lakes, ponds and reservoirs over a hectare. #csi_population_filter = '''"AreaSqKm" >=0.01 AND\ #"FCode" IN (39000,39004,39009,39010,39011,39012,43600,43613,43615,43617,43618,43619,43621)''' all_lakes_reservoirs_filter = '''"FType" IN (390, 436)''' # Can't see why we shouldn't just attribute all lakes and reservoirs # arcpy.Select_analysis(nhdwaterbody, "csiwaterbody", lake_population_filter) arcpy.AddMessage("Initializing output.") if exclude_intermit_flowlines: DM.CopyFeatures(out_feature_class, temp_fc) DM.Delete(out_feature_class) else: arcpy.Select_analysis(nhdwaterbody, temp_fc, all_lakes_reservoirs_filter) # Get lakes, ponds and reservoirs over 10 hectares. lakes_10ha_filter = '''"AreaSqKm" >= 0.1 AND "FType" IN (390, 436)''' arcpy.Select_analysis(nhdwaterbody, csiwaterbody_10ha, lakes_10ha_filter) # Exclude intermittent flowlines, if requested if exclude_intermit_flowlines: flowline_where_clause = '''"FCode" NOT IN (46003,46007)''' nhdflowline = arcpy.Select_analysis(nhdflowline, nhdflowline_filtered, flowline_where_clause) # Make dangle points at end of nhdflowline DM.FeatureVerticesToPoints(nhdflowline, dangles, "DANGLE") DM.MakeFeatureLayer(dangles, "dangles_lyr") # Isolate start dangles from end dangles. DM.FeatureVerticesToPoints(nhdflowline, start, "START") DM.FeatureVerticesToPoints(nhdflowline, end, "END") DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", start) DM.CopyFeatures("dangles_lyr", startdangles) DM.SelectLayerByLocation("dangles_lyr", "ARE_IDENTICAL_TO", end) DM.CopyFeatures("dangles_lyr", enddangles) # Special handling for lakes that have some intermittent flow in and some permanent if exclude_intermit_flowlines: DM.MakeFeatureLayer(nhdflowline, "nhdflowline_lyr") DM.SelectLayerByAttribute("nhdflowline_lyr", "NEW_SELECTION", '''"WBArea_Permanent_Identifier" is null''') DM.FeatureVerticesToPoints("nhdflowline_lyr", non_artificial_end, "END") DM.SelectLayerByAttribute("nhdflowline_lyr", "CLEAR_SELECTION") arcpy.AddMessage("Found source area nodes.") # Get junctions from lakes >= 10 hectares. DM.MakeFeatureLayer(nhdjunction, "junction_lyr") DM.SelectLayerByLocation("junction_lyr", "INTERSECT", csiwaterbody_10ha, XY_TOLERANCE, "NEW_SELECTION") DM.CopyFeatures("junction_lyr", flags_10ha_lake_junctions) arcpy.AddMessage("Found lakes >= 10 ha.") # Make points shapefile and layer at flowline vertices to act as potential flags and/or barriers. arcpy.AddMessage("Tracing...") DM.FeatureVerticesToPoints(nhdflowline, midvertices, "MID") DM.MakeFeatureLayer(midvertices, "midvertices_lyr") # Get vertices that are not coincident with 10 hectare lake junctions. DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT", flags_10ha_lake_junctions, "", "NEW_SELECTION") DM.SelectLayerByLocation("midvertices_lyr", "INTERSECT", flags_10ha_lake_junctions, "", "SWITCH_SELECTION") DM.CopyFeatures("midvertices_lyr", non10vertices) # Get junctions that are not coincident with 10 hectare lake junctions. DM.SelectLayerByLocation("junction_lyr", "INTERSECT", flags_10ha_lake_junctions, "", "NEW_SELECTION") DM.SelectLayerByLocation("junction_lyr", "INTERSECT", flags_10ha_lake_junctions, "", "SWITCH_SELECTION") DM.CopyFeatures("junction_lyr", non10junctions) # Merge non10vertices with non10junctions DM.Merge([non10junctions, non10vertices], all_non_flag_points) # inputs both point fc in_memory DM.MakeFeatureLayer(all_non_flag_points, "all_non_flag_points_lyr") # Tests the counts...for some reason I'm not getting stable behavior from the merge. mid_n = int(DM.GetCount(non10vertices).getOutput(0)) jxn_n = int(DM.GetCount(non10junctions).getOutput(0)) merge_n = int(DM.GetCount(all_non_flag_points).getOutput(0)) if merge_n < mid_n + jxn_n: arcpy.AddWarning( "The total number of flags ({0}) is less than the sum of the input junctions ({1}) " "and input midpoints ({2})".format(merge_n, jxn_n, mid_n)) # For tracing barriers, select all_non_flag_points points that intersect a 10 ha lake. DM.SelectLayerByLocation("all_non_flag_points_lyr", "INTERSECT", csiwaterbody_10ha, XY_TOLERANCE, "NEW_SELECTION") DM.CopyFeatures("all_non_flag_points_lyr", barriers) # Trace1-Trace downstream to first barrier (junctions+midvertices in 10 ha lake) starting from flags_10ha_lake_junctions flag points. DM.TraceGeometricNetwork(network, "trace1", flags_10ha_lake_junctions, "TRACE_DOWNSTREAM", barriers) # Save trace1 flowlines and junctions to layers on disk. DM.CopyFeatures("trace1\HYDRO_NET_Junctions", trace1_junctions) # extra for debugging DM.CopyFeatures("trace1\NHDFlowline", trace1_flowline) # Select vertice midpoints that intersect trace1 flowlines selection for new flags for trace2. DM.MakeFeatureLayer(non10vertices, "non10vertices_lyr") DM.SelectLayerByLocation("non10vertices_lyr", "INTERSECT", trace1_flowline, "", "NEW_SELECTION") # Trace2-Trace downstream from midpoints of flowlines that intersect the selected flowlines from trace1. DM.TraceGeometricNetwork(network, "trace2", "non10vertices_lyr", "TRACE_DOWNSTREAM") # Save trace1 flowlines and junctions to layers and then shapes on disk. DM.CopyFeatures("trace2\HYDRO_NET_Junctions", trace2_junctions) DM.CopyFeatures("trace2\NHDFlowline", trace2_flowline) # extra for debugging arcpy.AddMessage("Done tracing.") # Make shapefile for seepage lakes. (Ones that don't intersect flowlines) if exclude_intermit_flowlines: class_field_name = "Lake_Connectivity_Permanent" else: class_field_name = "Lake_Connectivity_Class" DM.AddField(temp_fc, class_field_name, "TEXT", field_length=13) DM.MakeFeatureLayer(temp_fc, "out_fc_lyr") DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline, XY_TOLERANCE, "NEW_SELECTION") DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", nhdflowline, "", "SWITCH_SELECTION") DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""", "PYTHON") # New type of "Isolated" classification, mostly for "permanent" but there were some oddballs in "maximum" too DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles, XY_TOLERANCE, "NEW_SELECTION") DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", enddangles, XY_TOLERANCE, "SUBSET_SELECTION") DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""", "PYTHON") # Get headwater lakes. DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", startdangles, XY_TOLERANCE, "NEW_SELECTION") DM.SelectLayerByAttribute( "out_fc_lyr", "REMOVE_FROM_SELECTION", '''"{}" = 'Isolated' '''.format(class_field_name)) DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""", "PYTHON") # Select csiwaterbody that intersect trace2junctions arcpy.AddMessage("Beginning connectivity attribution...") DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", trace2_junctions, XY_TOLERANCE, "NEW_SELECTION") DM.CalculateField("out_fc_lyr", class_field_name, """'DrainageLk'""", "PYTHON") # Get stream drainage lakes. Either unassigned so far or convert "Headwater" if a permanent stream flows into it, # which is detected with "non_artificial_end" DM.SelectLayerByAttribute("out_fc_lyr", "NEW_SELECTION", '''"{}" IS NULL'''.format(class_field_name)) DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""", "PYTHON") if exclude_intermit_flowlines: DM.SelectLayerByAttribute( "out_fc_lyr", "NEW_SELECTION", '''"{}" = 'Headwater' '''.format(class_field_name)) DM.SelectLayerByLocation("out_fc_lyr", "INTERSECT", non_artificial_end, XY_TOLERANCE, "SUBSET_SELECTION") DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""", "PYTHON") # Prevent 'upgrades' due to very odd flow situations and artifacts of bad digitization. The effects of these # are varied--to avoid confusion, just keep the class assigned with all flowlines # 1--Purely hypothetical, not seen in testing DM.SelectLayerByAttribute( "out_fc_lyr", "NEW_SELECTION", '''"Lake_Connectivity_Class" = 'Isolated' AND "Lake_Connectivity_Permanent" <> 'Isolated' ''' ) DM.CalculateField("out_fc_lyr", class_field_name, """'Isolated'""", "PYTHON") # 2--Headwater to Drainage upgrade seen in testing with odd multi-inlet flow situation DM.SelectLayerByAttribute( "out_fc_lyr", "NEW_SELECTION", '''"Lake_Connectivity_Class" = 'Headwater' AND "Lake_Connectivity_Permanent" IN ('Drainage', 'DrainageLk') ''' ) DM.CalculateField("out_fc_lyr", class_field_name, """'Headwater'""", "PYTHON") # 3--Drainage to DrainageLk upgrade seen in testing when intermittent stream segments were used # erroneously instead of artificial paths DM.SelectLayerByAttribute( "out_fc_lyr", "NEW_SELECTION", '''"Lake_Connectivity_Class" = 'Drainage' AND "Lake_Connectivity_Permanent" = 'DrainageLk' ''' ) DM.CalculateField("out_fc_lyr", class_field_name, """'Drainage'""", "PYTHON") DM.SelectLayerByAttribute("out_fc_lyr", "CLEAR_SELECTION") # Add change flag for users DM.AddField(temp_fc, "Lake_Connectivity_Fluctuates", "Text", field_length="1") flag_codeblock = """def flag_calculate(arg1, arg2): if arg1 == arg2: return 'N' else: return 'Y'""" expression = 'flag_calculate(!Lake_Connectivity_Class!, !Lake_Connectivity_Permanent!)' DM.CalculateField(temp_fc, "Lake_Connectivity_Fluctuates", expression, "PYTHON", flag_codeblock) # Project output once done with both. Switching CRS earlier causes trace problems. if not exclude_intermit_flowlines: DM.CopyFeatures(temp_fc, out_feature_class) else: DM.Project(temp_fc, out_feature_class, arcpy.SpatialReference(102039)) # Clean up if not debug_mode: for item in this_tool_layers + this_tool_temp: if arcpy.Exists(item): DM.Delete(item) if not debug_mode: DM.Delete("trace1") DM.Delete("trace2") arcpy.AddMessage("{} classification is complete.".format(class_field_name))
def mergeDualCarriageways(): """Collapse dual carriageways and turning circles in single, striagt-line roadways, the tools that achieve these effects are run on each route separately then the routes are added back to a single feature class as this yields better results""" generateMatchCode() # create at feature class to store all of the outputs geom_type = 'POLYLINE' template = distinct_routes_src oregon_spn = arcpy.SpatialReference(2913) management.CreateFeatureclass(os.path.dirname(collapsed_routes), os.path.basename(collapsed_routes), geom_type, template, spatial_reference=oregon_spn) # make a feature layer of the source routes so that selections can be made on it distinct_rte_lyr = 'distinct_transit_routes' management.MakeFeatureLayer(distinct_routes, distinct_rte_lyr) route_service_list = getRouteServicePairs() temp_merge = os.path.join(temp_shp_dir, 'temp_merge.shp') temp_collapse = os.path.join(temp_shp_dir, 'temp_collapse.shp') route_fields = ['Shape@', 'route_id', 'serv_level', 'route_type'] i_cursor = da.InsertCursor(collapsed_routes, route_fields) for route, service in route_service_list: select_type = 'NEW_SELECTION' where_clause = """"route_id" = {0} AND "serv_level" = '{1}'""".format( route, service) management.SelectLayerByAttribute(distinct_rte_lyr, select_type, where_clause) # merge dual carriageways merge_field = 'merge_id' # '0' in this field means won't be merged merge_distance = 100 # feet cartography.MergeDividedRoads(distinct_rte_lyr, merge_field, merge_distance, temp_merge) # collapse turing circles collapse_distance = 550 cartography.CollapseRoadDetail(temp_merge, collapse_distance, temp_collapse) with da.SearchCursor(temp_collapse, route_fields) as s_cursor: for row in s_cursor: i_cursor.insertRow(row) del i_cursor # now merge contiguous line segments with common attributes, now that dual carriage- # ways have been collapsed the data can be reduced to fewer segments dissolve_fields = ['route_id', 'serv_level', 'route_type'] geom_class = 'SINGLE_PART' line_handling = 'UNSPLIT_LINES' management.Dissolve(collapsed_routes, dissolved_routes, dissolve_fields, multi_part=geom_class, unsplit_lines=line_handling)
temp = r"memory" env.workspace = temp #Load layers gis = GIS("https://sfsu.maps.arcgis.com/sharing") dsm = gis.content.get("b690f8d64d0145f1baf74d73690da154") dem = gis.content.get("1adaa794cbf2477da2660f190c785fa7") veg = gis.content.get("6341228ec82a4bfbaf52d977a14e99ce") b = time.perf_counter() / 60 print(f"Layers loaded: {round(b - a, 2)} minutes") #Load vegetation layer veg2 = veg.layers[0] veg3 = veg2.url veg4 = r"memory\veg" DM.MakeFeatureLayer(veg3, veg4) c = time.perf_counter() / 60 print(f"Veg feature generated: {round(c - b, 2)} minutes") #Select vegetation type selection = 'Forest & Woodland' #input habitat type here where = "LIFEFORM = '" + selection + "'" DM.SelectLayerByAttribute(veg4, "NEW_SELECTION", where, None) d = time.perf_counter() / 60 print(f"Veg type selected: {round(d - c, 2)} minutes") #Make conditional vegetation layer raster vegCon = r"memory\veg2" path = script + r"\veg.shp" CO.FeatureClassToShapefile(veg4, script) CO.FeatureToRaster(path, "LIFEFORM", vegCon, 30)
def process_zone(zone_fc, output, zone_name, zone_id_field, zone_name_field, other_keep_fields, clip_hu8, lagosne_name): # dissolve fields by the field that zone_id is based on (the field that identifies a unique zone) dissolve_fields = [ f for f in "{}, {}, {}".format(zone_id_field, zone_name_field, other_keep_fields).split(', ') if f != '' ] print("Dissolving...") dissolve1 = DM.Dissolve(zone_fc, 'dissolve1', dissolve_fields) # update name field to match our standard DM.AlterField(dissolve1, zone_name_field, 'name') # original area DM.AddField(dissolve1, 'originalarea', 'DOUBLE') DM.CalculateField(dissolve1, 'originalarea', '!shape.area@hectares!', 'PYTHON') #clip print("Clipping...") clip = AN.Clip(dissolve1, MASTER_CLIPPING_POLY, 'clip') if clip_hu8 == 'Y': final_clip = AN.Clip(clip, HU8_OUTPUT, 'final_clip') else: final_clip = clip print("Selecting...") # calc new area, orig area pct, compactness DM.AddField(final_clip, 'area_ha', 'DOUBLE') DM.AddField(final_clip, 'originalarea_pct', 'DOUBLE') DM.AddField(final_clip, 'compactness', 'DOUBLE') DM.JoinField(final_clip, zone_id_field, dissolve1, zone_id_field, 'originalarea_pct') uCursor_fields = [ 'area_ha', 'originalarea_pct', 'originalarea', 'compactness', 'SHAPE@AREA', 'SHAPE@LENGTH' ] with arcpy.da.UpdateCursor(final_clip, uCursor_fields) as uCursor: for row in uCursor: area, orig_area_pct, orig_area, comp, shape_area, shape_length = row area = shape_area / 10000 # convert from m2 to hectares orig_area_pct = round(100 * area / orig_area, 2) comp = 4 * 3.14159 * shape_area / (shape_length**2) row = (area, orig_area_pct, orig_area, comp, shape_area, shape_length) uCursor.updateRow(row) # if zones are present with <5% of original area and a compactness measure of <.2 (ranges from 0-1) # AND ALSO they are no bigger than 500 sq. km. (saves Chippewa County and a WWF), filter out # save eliminated polygons to temp database as a separate layer for inspection # Different processing for HU4 and HU8, so that they match the extent of HU8 more closely but still throw out tiny slivers # County also only eliminated if a tiny, tiny, tiny sliver (so: none should be eliminated) if zone_name not in ('hu4', 'hu12', 'county'): selected = AN.Select( final_clip, 'selected', "originalarea_pct >= 5 OR compactness >= .2 OR area_ha > 50000") not_selected = AN.Select( final_clip, '{}_not_selected'.format(output), "originalarea_pct < 5 AND compactness < .2 AND area_ha < 50000") else: selected = final_clip # eliminate small slivers, re-calc area fields, add perimeter and multipart flag # leaves the occasional errant sliver but some areas over 25 hectares are more valid so this is # CONSERVATIVE print("Trimming...") trimmed = DM.EliminatePolygonPart(selected, 'trimmed', 'AREA', '25 Hectares', part_option='ANY') # gather up a few calculations into one cursor because this is taking too long over the HU12 layer DM.AddField(trimmed, 'perimeter_m', 'DOUBLE') DM.AddField(trimmed, 'multipart', 'TEXT', field_length=1) uCursor_fields = [ 'area_ha', 'originalarea_pct', 'originalarea', 'perimeter_m', 'multipart', 'SHAPE@' ] with arcpy.da.UpdateCursor(trimmed, uCursor_fields) as uCursor: for row in uCursor: area, orig_area_pct, orig_area, perim, multipart, shape = row area = shape.area / 10000 # convert to hectares from m2 orig_area_pct = round(100 * area / orig_area, 2) perim = shape.length # multipart flag calc if shape.isMultipart: multipart = 'Y' else: multipart = 'N' row = (area, orig_area_pct, orig_area, perim, multipart, shape) uCursor.updateRow(row) # delete intermediate fields DM.DeleteField(trimmed, 'compactness') DM.DeleteField(trimmed, 'originalarea') print("Zone IDs....") # link to LAGOS-NE zone IDs DM.AddField(trimmed, 'zoneid', 'TEXT', field_length=40) trimmed_lyr = DM.MakeFeatureLayer(trimmed, 'trimmed_lyr') if lagosne_name: # join to the old master GDB path on the same master field and copy in the ids old_fc = os.path.join(LAGOSNE_GDB, lagosne_name) old_fc_lyr = DM.MakeFeatureLayer(old_fc, 'old_fc_lyr') if lagosne_name == 'STATE' or lagosne_name == 'COUNTY': DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr, 'FIPS') else: DM.AddJoin(trimmed_lyr, zone_id_field, old_fc_lyr, zone_id_field) # usually works because same source data # copy DM.CalculateField(trimmed_lyr, 'zoneid', '!{}.ZoneID!.lower()'.format(lagosne_name), 'PYTHON') DM.RemoveJoin(trimmed_lyr) # generate new zone ids old_ids = [row[0] for row in arcpy.da.SearchCursor(trimmed, 'zoneid')] with arcpy.da.UpdateCursor(trimmed, 'zoneid') as cursor: counter = 1 for row in cursor: if not row[ 0]: # if no existing ID borrowed from LAGOS-NE, assign a new one new_id = '{name}_{num}'.format(name=zone_name, num=counter) # ensures new ids don't re-use old numbers but fills in all positive numbers eventually while new_id in old_ids: counter += 1 new_id = '{name}_{num}'.format(name=zone_name, num=counter) row[0] = new_id cursor.updateRow(row) counter += 1 print("Edge flags...") # add flag fields DM.AddField(trimmed, 'onlandborder', 'TEXT', field_length=2) DM.AddField(trimmed, 'oncoast', 'TEXT', field_length=2) # identify border zones border_lyr = DM.MakeFeatureLayer(LAND_BORDER, 'border_lyr') DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', border_lyr) DM.CalculateField(trimmed_lyr, 'onlandborder', "'Y'", 'PYTHON') DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION') DM.CalculateField(trimmed_lyr, 'onlandborder', "'N'", 'PYTHON') # identify coastal zones coastal_lyr = DM.MakeFeatureLayer(COASTLINE, 'coastal_lyr') DM.SelectLayerByLocation(trimmed_lyr, 'INTERSECT', coastal_lyr) DM.CalculateField(trimmed_lyr, 'oncoast', "'Y'", 'PYTHON') DM.SelectLayerByAttribute(trimmed_lyr, 'SWITCH_SELECTION') DM.CalculateField(trimmed_lyr, 'oncoast', "'N'", 'PYTHON') print("State assignment...") # State? DM.AddField(trimmed, "state", 'text', field_length='2') state_center = arcpy.SpatialJoin_analysis( trimmed, STATE_FC, 'state_center', join_type='KEEP_COMMON', match_option='HAVE_THEIR_CENTER_IN') state_intersect = arcpy.SpatialJoin_analysis(trimmed, STATE_FC, 'state_intersect', match_option='INTERSECT') state_center_dict = { row[0]: row[1] for row in arcpy.da.SearchCursor(state_center, ['ZoneID', 'STUSPS']) } state_intersect_dict = { row[0]: row[1] for row in arcpy.da.SearchCursor(state_intersect, ['ZoneID', 'STUSPS']) } with arcpy.da.UpdateCursor(trimmed, ['ZoneID', 'state']) as cursor: for updateRow in cursor: keyValue = updateRow[0] if keyValue in state_center_dict: updateRow[1] = state_center_dict[keyValue] else: updateRow[1] = state_intersect_dict[keyValue] cursor.updateRow(updateRow) # glaciation status? # TODO as version 0.6 # preface the names with the zones DM.DeleteField(trimmed, 'ORIG_FID') fields = [ f.name for f in arcpy.ListFields(trimmed, '*') if f.type not in ('OID', 'Geometry') and not f.name.startswith('Shape_') ] for f in fields: new_fname = '{zn}_{orig}'.format(zn=zone_name, orig=f).lower() try: DM.AlterField(trimmed, f, new_fname, clear_field_alias='TRUE') # sick of debugging the required field message-I don't want to change required fields anyway except: pass DM.CopyFeatures(trimmed, output) # cleanup lyr_objects = [ lyr_object for var_name, lyr_object in locals().items() if var_name.endswith('lyr') ] temp_fcs = arcpy.ListFeatureClasses('*') for l in lyr_objects + temp_fcs: DM.Delete(l)
def doFishnet(self): #### Initial Data Assessment #### printOHSSection(84428, prependNewLine=True) printOHSSubject(84431, addNewLine=False) #### Find Unique Locations #### msg = ARCPY.GetIDMessage(84441) ARCPY.SetProgressor("default", msg) initCount = UTILS.getCount(self.ssdo.inputFC) self.checkIncidents(initCount) collectedPointFC = UTILS.returnScratchName("Collect_InitTempFC") collInfo = EVENTS.collectEvents(self.ssdo, collectedPointFC) self.cleanUpList.append(collectedPointFC) collSSDO = SSDO.SSDataObject(collectedPointFC, explicitSpatialRef=self.ssdo.spatialRef, useChordal=True) collSSDO.obtainDataGA(collSSDO.oidName) ################################# if self.boundaryFC: #### Assure Boundary FC Has Area and Obtain Chars #### self.checkBoundary() #### Location Outliers #### lo = UTILS.LocationInfo(collSSDO, concept="EUCLIDEAN", silentThreshold=True, stdDeviations=3) printOHSLocationalOutliers(lo, aggType=self.aggType) #### Agg Header #### printOHSSection(84444) if self.boundaryFC: extent = self.boundExtent forMercExtent = self.boundExtent countMSGNumber = 84453 else: countMSGNumber = 84452 extent = None forMercExtent = collSSDO.extent if collSSDO.useChordal: extentFC_GCS = UTILS.returnScratchName("TempGCS_Extent") extentFC_Merc = UTILS.returnScratchName("TempMercator_Extent") points = NUM.array([[forMercExtent.XMin, forMercExtent.YMax], [forMercExtent.XMax, forMercExtent.YMin]]) UTILS.createPointFC(extentFC_GCS, points, spatialRef=collSSDO.spatialRef) DM.Project(extentFC_GCS, extentFC_Merc, mercatorProjection) d = ARCPY.Describe(extentFC_Merc) extent = d.extent fishOutputCoords = mercatorProjection else: fishOutputCoords = self.ssdo.spatialRef #### Fish Subject #### printOHSSubject(84449, addNewLine=False) dist = scaleDecision(lo.nonZeroAvgDist, lo.nonZeroMedDist) area = 0.0 #### Construct Fishnet #### fish = UTILS.FishnetInfo(collSSDO, area, extent, explicitCellSize=dist) dist = fish.quadLength snap = self.ssdo.distanceInfo.linearUnitString(dist) #### Cell Size Answer #### snapStr = self.ssdo.distanceInfo.printDistance(dist) msg = ARCPY.GetIDMessage(84450).format(snapStr) printOHSAnswer(msg) self.fish = fish #### Fishnet Count Subject #### printOHSSubject(84451, addNewLine=False) #### Create Temp Fishnet Grid #### gridFC = UTILS.returnScratchName("Fishnet_TempFC") self.cleanUpList.append(gridFC) #### Apply Output Coords to Create Fishnet #### oldSpatRef = ARCPY.env.outputCoordinateSystem ARCPY.env.outputCoordinateSystem = fishOutputCoords #### Fish No Extent #### oldExtent = ARCPY.env.extent ARCPY.env.extent = "" #### Apply Max XY Tolerance #### fishWithXY = UTILS.funWithXYTolerance(DM.CreateFishnet, self.ssdo.distanceInfo) #### Execute Fishnet #### fishWithXY(gridFC, self.fish.origin, self.fish.rotate, self.fish.quadLength, self.fish.quadLength, self.fish.numRows, self.fish.numCols, self.fish.corner, "NO_LABELS", self.fish.extent, "POLYGON") #### Project Back to GCS if Use Chordal #### if collSSDO.useChordal: gridFC_ProjBack = UTILS.returnScratchName("TempFC_Proj") DM.Project(gridFC, gridFC_ProjBack, collSSDO.spatialRef) UTILS.passiveDelete(gridFC) gridFC = gridFC_ProjBack #### Set Env Output Coords Back #### ARCPY.env.outputCoordinateSystem = oldSpatRef #### Create Empty Field Mappings to Ignore Atts #### fieldMap = ARCPY.FieldMappings() fieldMap.addTable(self.ssdo.inputFC) fieldMap.removeAll() #### Fishnet Count Answer #### printOHSAnswer(ARCPY.GetIDMessage(countMSGNumber)) #### Create Weighted Fishnet Grid #### tempFC = UTILS.returnScratchName("Optimized_TempFC") self.cleanUpList.append(tempFC) joinWithXY = UTILS.funWithXYTolerance(ANA.SpatialJoin, self.ssdo.distanceInfo) joinWithXY(gridFC, self.ssdo.inputFC, tempFC, "JOIN_ONE_TO_ONE", "KEEP_ALL", "EMPTY") #### Clean Up Temp FCs #### UTILS.passiveDelete(gridFC) #### Remove Locations Outside Boundary FC #### featureLayer = "ClippedPointFC" DM.MakeFeatureLayer(tempFC, featureLayer) if self.boundaryFC: msg = ARCPY.GetIDMessage(84454) ARCPY.SetProgressor("default", msg) DM.SelectLayerByLocation(featureLayer, "INTERSECT", self.boundaryFC, "#", "NEW_SELECTION") DM.SelectLayerByLocation(featureLayer, "INTERSECT", "#", "#", "SWITCH_SELECTION") DM.DeleteFeatures(featureLayer) else: if additionalZeroDistScale == "ALL": msg = ARCPY.GetIDMessage(84455) ARCPY.SetProgressor("default", msg) DM.SelectLayerByAttribute(featureLayer, "NEW_SELECTION", '"Join_Count" = 0') DM.DeleteFeatures(featureLayer) else: distance = additionalZeroDistScale * fish.quadLength distanceStr = self.ssdo.distanceInfo.linearUnitString( distance, convert=True) nativeStr = self.ssdo.distanceInfo.printDistance(distance) msg = "Removing cells further than %s from input pointsd...." ARCPY.AddMessage(msg % nativeStr) DM.SelectLayerByLocation(featureLayer, "INTERSECT", self.ssdo.inputFC, distanceStr, "NEW_SELECTION") DM.SelectLayerByLocation(featureLayer, "INTERSECT", "#", "#", "SWITCH_SELECTION") DM.DeleteFeatures(featureLayer) DM.Delete(featureLayer) del collSSDO ARCPY.env.extent = oldExtent self.createAnalysisSSDO(tempFC, "JOIN_COUNT")