class SearchButtonClass4(object): global emexditems global featclasses global selattrib global ProjectName featclasses = emexditems arcpy.SetLogHistory(False) def __init__(self): self.enabled = True self.checked = False def onClick(self): featclasses = emexditems for fc in featclasses: SAS_Stormwater_SDE_swManhole = fc try: arcpy.SelectLayerByAttribute_management( SAS_Stormwater_SDE_swManhole, "NEW_SELECTION", selattrib + " LIKE '%" + ProjectName + "%'") except: print " Skip " + fc del SAS_Stormwater_SDE_swManhole arcpy.SetLogHistory(True)
def field_reorder(self): _field_lists = [('building_id', '建筑ID'), ('type', '类型'), ('floor', '楼层'), ('floor_z', '层高'), ('status', '状态'), ('fire_area', '防火分区'), ('function_type', '功能类型'), ('door_aspect', '门_方向'), ('area_type', '分区类型'), ('name', '建筑名称'), ('z', '显示高度'), ('floor_d', '层差'), ('city', '市'), ('province', '省')] _field_order_lists = OrderedDict(_field_lists) # 按照输入的顺序,输出字典的key和value # Field Copy for _key in _field_order_lists.keys(): # arcpy.AddMessage(_key) for fld in arcpy.ListFields(self._feature): if fld.name == _key: _field_new = _key + '_1' arcpy.AddField_management(self._feature, _field_new, fld.type, fld.precision, fld.scale, fld.length) expression = '!{}!'.format(_key) # 字符串无法直接变成变量名,需要加一个转换 # arcpy.AddMessage(expression) arcpy.CalculateField_management(self._feature, _field_new, expression, "PYTHON_9.3") arcpy.DeleteField_management(self._feature, _key) # arcpy.AddMessage("The {0} was deleted.".format(_key)) # Field Reorder for _key in _field_order_lists.keys(): for fld in arcpy.ListFields(self._feature): if fld.name == _key + '_1': arcpy.AddField_management(self._feature, _key, fld.type, fld.precision, fld.scale, fld.length, _field_order_lists[_key]) expression = '!{}!'.format(fld.name) # 字符串无法直接变成变量名,需要加一个转换 # arcpy.AddMessage(expression) arcpy.CalculateField_management(self._feature, _key, expression, "PYTHON_9.3") arcpy.DeleteField_management(self._feature, fld.name) # arcpy.AddMessage("The {0} was deleted.".format(fld.name)) arcpy.SetLogHistory(True)
def setup_arcpy_environment(): ''' Process: Define some misc. arcpy environment variables''' arcpy.env.overwriteOutput = True arcpy.SetLogHistory(False) arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "None" arcpy.env.XYResolution = "0.0005 METERS" arcpy.env.XYTolerance = "0.001 METERS" arcpy.env.outputCoordinateSystem = 2927 # WASPSNAD83HARNFEET
def __init__(self): """Define the toolbox (the name of the toolbox is the name of the .pyt file).""" self.label = "Toolbox" self.alias = "" # List of tool classes associated with this toolbox self.tools = [CutTool, OverlapReport, UpdateOverlapReportConfig] arcpy.SetLogHistory(False) arcpy.env.overwriteOutput = True
def getTrainLocations(gdbws, fcdatasetname, trainTableName): """ 公共交通APIの列車ロケーションから変換済みのCSVをもとに 列車ロケーション情報を取得しDelay_Railroad_Metro, Delay_Station_Metro を最新化 第1引数:challenge_db.gdb へのフルパス 第2引数:フィーチャデータセット名 第3引数:現在の列車遅延csvファイル """ logging.debug('--arcgis processing:getTrainLocations') # Network Analystのライセンスをチェックアウト arcpy.CheckOutExtension("Network") # Tips! バッチ処理など繰り返しするだんだん速度低下するのでログをオフに arcpy.SetLogHistory(False) train_ws = os.path.dirname(gdbws) arcpy.env.workspace = train_ws # 抽出する路線名 operator_name = 'odpt.Operator:TokyoMetro' # 抽出する対象の遅れ時間 delayTime = 0 # 動作確認用 #q_wh ="odpt_delay >= 0 AND odpt_operator = 'odpt.Operator:TokyoMetro' AND odpt_tostation IS NOT NULL AND odpt_fromstation IS NOT NULL" #q_wh = "odpt_delay >= {0} AND odpt_operator = '{1}' AND odpt_tostation IS NOT NULL AND odpt_fromstation IS NOT NULL".format(delayTime, operator_name) # 本番用(FromStation と ToStationが入っている場合) q_wh = "odpt_delay > {0} AND odpt_operator = '{1}' AND odpt_tostation IS NOT NULL AND odpt_fromstation IS NOT NULL".format(delayTime, operator_name) #print q_wh fields = ['dc_date','odpt_delay','odpt_railway','odpt_fromstation','odpt_tostation','odpt_operator','odpt_trainnumber'] try: with arcpy.da.SearchCursor(trainTableName, fields, q_wh) as cursor: # バッチ処理高速化-ネットワーク解析を行なう関数側にRecordSetを渡すようにした trainLocationAnalBatch(gdbws, fcdatasetname, cursor) except arcpy.ExecuteError as e: # GP エラー logging.debug(str(e).decode('utf-8')) #print str(e).decode('utf-8') except Exception as e: # Python エラー logging.debug(str(e).decode('utf-8')) #print str(e).decode('utf-8') finally: # ファイルの後始末 arcpy.Delete_management(trainTableName)
class SearchComboBoxClass3(object): global emexditems global featclasses global selattrib global ProjectName featclasses = emexditems arcpy.SetLogHistory(False) def __init__(self): self.items = [] self.editable = True self.enabled = True self.dropdownWidth = 'WWWWWWW' self.width = 'WWWWWWW' def onSelChange(self, selection): pass def onEditChange(self, text): global ProjectName ProjectName = text def onFocus(self, focused): pass def onEnter(self): featclasses = emexditems for fc in featclasses: SAS_Stormwater_SDE_swManhole = fc try: arcpy.SelectLayerByAttribute_management( SAS_Stormwater_SDE_swManhole, "NEW_SELECTION", selattrib + " LIKE '%" + ProjectName + "%'") except: print " Skip " + fc del SAS_Stormwater_SDE_swManhole def refresh(self): pass arcpy.SetLogHistory(True)
def __init__(self, inputfc, outputfc): arcpy.SetLogHistory(False) arcpy.env.overwriteOutput = True self.inputfc = inputfc self.inputtype = arcpy.Describe(inputfc).dataType self.geomtype = arcpy.Describe(inputfc).shapeType self.outputfc = outputfc self.sr = arcpy.SpatialReference(4612) arcpy.CreateFeatureclass_management(os.path.dirname(self.outputfc), os.path.basename(self.outputfc), self.geomtype, self.inputfc, spatial_reference=self.sr)
def setup_arcpy_environment(): ''' Process: Define some misc. arcpy environment variables''' try: import arcpy arcpy.env.overwriteOutput = True arcpy.SetLogHistory(False) arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "None" arcpy.env.XYResolution = "0.0005 METERS" arcpy.env.XYTolerance = "0.001 METERS" arcpy.env.outputCoordinateSystem = 2927 # WASPSNAD83HARNFEET except ImportError: print("The module arcpy does not seem to be available." "Are you working in a virtual environment?")
def generate_data(values): arcpy.env.overwriteOutput = True arcpy.SetLogHistory(False) queue_item, lock_item = values results = [] try: while True: item = queue_item.get_nowait() scenario = item.split("\\")[0] print_message(lock_item, "Starting {}".format(scenario)) except: return results
def __init__(self, ToolName, Params, licenceLevel="arceditor"): """Class initialization procedure Args: self: The reserved object 'self' ToolName: Name of the tool (used for output) Params: parameter object licenceLevel: arcinfo, arceditor, arcview, arcserver, arcenginegeodb, or arcengine """ self.ExecutePatch = os.path.dirname(os.path.realpath(__file__)) if licenceLevel == "arcinfo": print(u'...Using licence level : ' + licenceLevel) import arcinfo elif licenceLevel == "arceditor": print(u'...Using licence level : ' + licenceLevel) import arceditor elif licenceLevel == "arcview": print(u'...Using licence level : ' + licenceLevel) import arcview elif licenceLevel == "arcserver": print(u'...Using licence level : ' + licenceLevel) import arcserver elif licenceLevel == "arcenginegeodb": print(u'...Using licence level : ' + licenceLevel) import arcenginegeodb elif licenceLevel == "arcengine": print(u'...Using licence level : ' + licenceLevel) import arcengine else: print(u'...Incorect licence suplied - using : arceditor') import arceditor import arcpy self.Pr = Params LogDir = self.Pr.ErrorLogDir LogDirArh = self.Pr.ErrorLogDirArh OutDir = self.Pr.OutDir OutDirArh = self.Pr.OutDirArh self.StartTime = datetime.datetime.now() self.gp = arcpy if hasattr(Params, 'SetLogHistory'): arcpy.SetLogHistory(Params.SetLogHistory) else: arcpy.SetLogHistory(False) self.GDBHelper = GDBHelper.GDBHelper(self.gp) self.ToolName = ToolName self.SR = MySR() self.OutputStr = u'' self.OutputErrStr = u'' self.State = "Started" try: self.AchiveFiles(LogDir, LogDirArh, ToolName, False) except Exception, err: print(u'Error archiving errlog files') if hasattr(err, 'strerror'): if hasattr(err, 'strerror'): print(err.strerror) else: print('{}'.format(err)) else: print(err.message)
sr_mol = arcpy.SpatialReference( 'Mollweide (world)') # projected Mollweide (54009) sr_gcs = arcpy.SpatialReference( 'WGS 1984') # geographic coordinate system WGS84 (4326) # environment os.chdir(wd) if not os.path.exists('tmp'): os.makedirs('tmp') if not os.path.exists(dir_tmp): os.makedirs(dir_tmp) if not arcpy.Exists(gdb): print 'creating gdb' arcpy.CreateFileGDB_management(os.path.dirname(gdb), os.path.basename(gdb)) arcpy.env.overwriteOutput = True arcpy.env.workspace = gdb arcpy.SetLogHistory( True ) # C:\Users\visitor\AppData\Roaming\ESRI\Desktop10.2\ArcToolbox\History # copy shapefiles to local gdb arcpy.env.outputCoordinateSystem = sr_gcs shps_rgn = ['rgn_gcs'] + ['rgn_%s_gcs' % b for b in buffers] for fc_rgn in [fc_gadm] + ['%s\\%s.shp' % (dir_rgn, x) for x in shps_rgn]: fc_gdb = os.path.splitext(os.path.basename(fc_rgn))[0] if not arcpy.Exists(fc_gdb): print 'copying', fc_gdb arcpy.CopyFeatures_management(fc_rgn, fc_gdb) # convert gcs to mol fc_mol = fc_gdb.replace('_gcs', '_mol') if not arcpy.Exists(fc_mol): print 'projecting', fc_mol
# * go through slivers of FAO and holes from earth box erased by the rest and manually associate with legit region # * convert EEZ of Black Sea to eez-inland # * merge all products and peform checks for overlap and geometry repair # # Built using: # ArcGIS 10.2.1 # Python Data Analysis Library (pandas) installed with easy_install # # Changes since OHI 2013 # * New EEZ splits: # - 140 Guadeloupe and Martinique # - 116 Puerto Rico and Virgin Islands of the United States import arcpy, os, re, numpy as np, socket, pandas, time from collections import Counter arcpy.SetLogHistory(True) # %USERPROFILE%\AppData\Roaming\ESRI\Desktop10.2\ArcToolbox\History # configuration based on machine name conf = { 'Amphitrite': {'dir_git' :'G:/ohiprep', 'dir_neptune':'N:', 'dir_tmp' :'C:/tmp', }}[socket.gethostname()] # paths nm = 'NCEAS-Regions_v2014' # name of data product td = '{0}/Global/{1}'.format(conf['dir_tmp'], nm) # temp directory on local filesystem gdb = '{0}/geodb.gdb'.format(td) # file geodatabase ad = '{0}/git-annex/Global/{1}'.format(conf['dir_neptune'], nm) # git annex directory on neptune gd = '{0}/Global/{1}'.format(conf['dir_git'], nm) # git directory on local filesystem
def polygons_in_zones(zone_fc, zone_field, polygons_of_interest, output_table, interest_selection_expr): old_workspace = arcpy.env.workspace arcpy.env.workspace = 'in_memory' arcpy.SetLogHistory(False) arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(102039) selected_polys = 'selected_polys' # fixes some stupid ArcGIS thing with the interactive Python window if arcpy.Exists(selected_polys): arcpy.env.overwriteOutput = True arcpy.AddMessage('Copying/selecting polygon features...') if interest_selection_expr: arcpy.Select_analysis(polygons_of_interest, selected_polys, interest_selection_expr) else: arcpy.CopyFeatures_management(polygons_of_interest, selected_polys) # use tabulate intersection for the areas overlapping arcpy.AddMessage('Tabulating intersection between zones and polygons...') tab_table = arcpy.TabulateIntersection_analysis( zone_fc, zone_field, selected_polys, 'tabulate_intersection_table') # area was calculated in map units which was m2 so convert to hectares arcpy.AddField_management(tab_table, 'Poly_Ha', 'DOUBLE') arcpy.CalculateField_management(tab_table, 'Poly_Ha', '!AREA!/10000', 'PYTHON') # just change the name of the percent field arcpy.AlterField_management(tab_table, 'PERCENTAGE', 'Poly_Pct') # Now just get the count as there is no other area metric anymore spjoin_fc = arcpy.SpatialJoin_analysis(zone_fc, selected_polys, 'spatial_join_output') arcpy.AlterField_management(spjoin_fc, 'Join_Count', 'Poly_n') # Add the density arcpy.AddField_management(spjoin_fc, 'Poly_nperha', 'DOUBLE') arcpy.CalculateField_management(spjoin_fc, 'Poly_nperha', '!Poly_n!/!shape.area@hectares!', 'PYTHON') arcpy.AddMessage('Refining output...') arcpy.JoinField_management(tab_table, zone_field, spjoin_fc, zone_field, ["Poly_n", 'Poly_nperha']) final_fields = ['Poly_Ha', 'Poly_Pct', 'Poly_n', 'Poly_nperha'] # make output nice arcpy.env.overwriteOutput = False cu.one_in_one_out(tab_table, final_fields, zone_fc, zone_field, output_table) cu.redefine_nulls(output_table, final_fields, [0, 0, 0, 0]) # clean up # can't delete all of in_memory because this function is meant to be called from another one that uses in_memory for item in [selected_polys, tab_table, spjoin_fc]: arcpy.Delete_management(item) arcpy.env.workspace = old_workspace arcpy.AddMessage('Polygons in zones tool complete.') arcpy.SetLogHistory(True)
def Process_TRMM(Package,Basin,Year,zoneField): # assembling the list of file addresses TRMM_path= Package + '/Datos/NASA_Datos/TRMM/' + Year inZoneData = Package + '/Datos/Cuencas/' + Basin + '/Parametros/Shapefile/HighShape.tif' OutputTRMM = Package + '/Datos/Cuencas/' +Basin + '/Datos_Intermedia/TRMM/TRMM_Precip' + Year +'.dbf' OutputTRMMpath = Package + '/Datos/Cuencas/' +Basin + '/Datos_Intermedia/TRMM' #import modules and manage temporary folders import arcpy, arcgisscripting, sys, os, csv, string, shutil from arcpy import env from arcpy.sa import * try: from dbfpy import dbf except: print 'You do not have "dbfpy" installed, Process_TRMM cannot run!' return arcpy.env.overwriteOutput = True arcpy.SetLogHistory(True) arcpy.CheckOutExtension("Spatial") IntermediateOutput = TRMM_path + '/temp' # make sure the TRMM output path is there if not os.path.exists(OutputTRMMpath): os.makedirs(OutputTRMMpath) # If a temp data folder already exists, delete its contents and recreate it. if os.path.exists(IntermediateOutput): shutil.rmtree(IntermediateOutput) os.makedirs(IntermediateOutput) else: ###### If no temporary folder exists, NetCDFs are converted to tiffs for the first time. os.makedirs(IntermediateOutput) arcpy.env.workspace = TRMM_path arcpy.env.overwriteOutput = True NCfiles = arcpy.ListFiles("*.nc") for filename in NCfiles: print 'Process_TRMM: Converting netCDF file ' + filename + ' to Raster' inNCfiles = arcpy.env.workspace + "/" + filename fileroot = filename[0:(len(filename)-3)] outRasterLayer = TRMM_path + "/" + fileroot arcpy.MakeNetCDFRasterLayer_md(inNCfiles, "r", "longitude", "latitude", "r", "", "", "BY_VALUE") arcpy.CopyRaster_management("r", outRasterLayer + ".tif", "", "", "", "NONE", "NONE", "") print 'Process_TRMM: Finished creating TIFs!' ####### # Execute zonal statistics functions on selected basin and store output as a dbf. arcpy.env.workspace = TRMM_path TIFfiles = arcpy.ListFiles("*.tif") try: for filename in TIFfiles: fileroot = filename[0:(len(filename)-4)] print "Process_TRMM: Calculating zonal statistics on " + filename inValueRaster = TRMM_path + '/'+ filename arcpy.CheckOutExtension("Spatial") tempout=IntermediateOutput + '/tempdbf.dbf' outZstat = ZonalStatisticsAsTable(inZoneData, zoneField, inValueRaster, tempout, "DATA", "ALL") # arcmap is a terribly programmed software package, so we have # save the output dbf as a static filename then rename it appropriately here. # this took forever to figure out (and we think it has something to do with "." in filename) outTable = IntermediateOutput + "/" + fileroot + '.dbf' os.rename(tempout,outTable) print 'Process_TRMM: Finished calculating zonal statistics!' except: print 'Error: Process_TRMM: Error encountered while calculating zonal statistics!' # Create csvs from DBFs arcpy.env.workspace = IntermediateOutput DBFfiles = arcpy.ListFiles("*.dbf") try: for filename in DBFfiles: fileroot = filename[0:(len(filename)-4)] # replace the '.' with '_', because the merge function is apparently very finicky with inputs. csv_fn = IntermediateOutput + '/' + string.replace(fileroot, '.', '_') + '.csv' inDBFfiles = arcpy.env.workspace + '/' + filename with open(csv_fn,'wb') as csvfile: in_db = dbf.Dbf(inDBFfiles) out_csv = csv.writer(csvfile) names = [] for field in in_db.header.fields: names.append(field.name) out_csv.writerow(names) for rec in in_db: out_csv.writerow(rec.fieldData) in_db.close() print 'Process_TRMM: Finished conversion to CSVs!' except: print 'Error: Process_TRMM: Error encountered while creating CSVs from dbfs!' # Merge CSVs together and print a bunch of progress items. arcpy.env.workspace = IntermediateOutput CSVfiles = arcpy.ListFiles("*.csv") print 'Process_TRMM: Creating Output file at ' + OutputTRMM print 'Process_TRMM: This may take up to 15 minutes' arcpy.env.workspace = IntermediateOutput CSVfiles = arcpy.ListFiles("*.csv") arcpy.Merge_management(CSVfiles, OutputTRMM) print 'Process_TRMM: Output file created for year ' + Year
def makeServiceAreas(outGDB, accFeat, costRastLoc, costRastHwy, rampPts, rampPtsID, grpFld=None, maxCost=None, attFld=None): # Checks on attFld if attFld: if not maxCost: print( 'Must specify a `maxCost` value if using `attFld`, exiting...') return if isinstance(attFld, str) and not [ attFld in [a.name for a in arcpy.ListFields(accFeat)] ]: print('Field ' + attFld + ' not found in access features, exiting...') return arcpy.env.snapRaster = costRastLoc arcpy.env.cellSize = costRastLoc arcpy.env.extent = costRastLoc arcpy.env.outputCoordinateSystem = costRastLoc make_gdb(outGDB) arcpy.env.workspace = outGDB arcpy.SetLogHistory(False) # copy access points to gdb accFeat = arcpy.CopyFeatures_management(accFeat, 'accFeat_orig') if not grpFld: # add a field to assign all rows to one group. grpFld = 'serviceArea_group' arcpy.CalculateField_management(accFeat, grpFld, "1", field_type="SHORT") grps = unique_values(accFeat, grpFld) # assign max costs if maxCost: if isinstance(maxCost, str): arcpy.CalculateField_management(accFeat, 'minutes_SA', '!' + maxCost + '!', field_type="FLOAT") else: arcpy.CalculateField_management(accFeat, 'minutes_SA', maxCost, 'PYTHON', field_type="FLOAT") # dictionary: grps: minutes grp_min = { a[0]: a[1] for a in arcpy.da.SearchCursor(accFeat, [grpFld, 'minutes_SA']) } for i in grps: n = grps.index(i) + 1 if isinstance(i, str): rastout = "grp_" + i + "_servArea" cdpts = "grp_" + i + "_inputFeat" i_q = "'" + i + "'" else: rastout = "grp_" + str(int(i)) + "_servArea" cdpts = "grp_" + str(int(i)) + "_inputFeat" i_q = i if arcpy.Exists(rastout): # skip already existing continue print("working on group " + str(i) + " (" + str(n) + " of " + str(len(grps)) + ")...") arcpy.env.extent = costRastLoc # reset extent prior to every run t0 = time.time() c = 1 # counter arcpy.Select_analysis(accFeat, cdpts, grpFld + " = " + str(i_q)) print('Number of access pts: ' + arcpy.GetCount_management(cdpts)[0]) # get service area in minutes if maxCost is not None: grpMaxCost = grp_min[i] # Make buffer to set a smaller extent, to reduce processing time. buffd = str( int(grpMaxCost * 1609) ) + ' METERS' # buffer set to straightline distance at ~60 mph (1 mile per minute) print('Cost in minutes: ' + str(grpMaxCost)) arcpy.Buffer_analysis(cdpts, "buffext", buffd) arcpy.env.extent = "buffext" else: grpMaxCost = None # local CD cd1 = arcpy.sa.CostDistance(cdpts, costRastLoc, grpMaxCost) nm = "cd" + str(c) cd1.save(nm) cds = [nm] # values to ramps rp1 = arcpy.sa.ExtractValuesToPoints(rampPts, cd1, "rp1", "NONE", "VALUE_ONLY") rp1s = arcpy.MakeFeatureLayer_management( rp1, where_clause="RASTERVALU IS NOT NULL") if int(arcpy.GetCount_management(rp1s)[0]) == 0: # No ramps reached: just output local roads only service area if attFld is not None: if isinstance(attFld, str): areaval = unique_values(cdpts, attFld)[0] area = arcpy.sa.Con("cd1", areaval, "", "Value <= " + str(grpMaxCost)) area.save(rastout) elif isinstance(attFld, int): area = arcpy.sa.Con("cd1", attFld, "", "Value <= " + str(grpMaxCost)) area.save(rastout) else: cd1.save(rastout) else: # Some ramps reached: Run highways/local loop until there is no improvement in travel time. notin = [1] while len(notin) != 0: print('Limited-access cost distance run # ' + str(int((c + 1) / 2)) + '...') arcpy.CopyFeatures_management(rp1s, "rp1s") # highway CD cd2 = arcpy.sa.CostDistance("rp1s", costRastHwy, grpMaxCost, source_start_cost="RASTERVALU") c += 1 nm = "cd" + str(c) cd2.save(nm) cds = cds + [nm] rp2 = arcpy.sa.ExtractValuesToPoints(rampPts, cd2, "rp2", "NONE", "VALUE_ONLY") # change name to avoid confusion with local ramp points arcpy.AlterField_management(rp2, "RASTERVALU", "costLAH", clear_field_alias=True) rp2s = arcpy.MakeFeatureLayer_management( rp2, where_clause="costLAH IS NOT NULL") # Check for new ramps or ramps reached at least one minute faster after latest run (LAH) notin = [] lahr = { a[0]: a[1] for a in arcpy.da.SearchCursor(rp2s, [rampPtsID, 'costLAH']) } locr = { a[0]: a[1] for a in arcpy.da.SearchCursor('rp1s', [rampPtsID, 'RASTERVALU']) } for a in lahr: if a not in locr: notin.append(a) else: if lahr[a] - locr[a] < -1: notin.append(a) if len(notin) == 0: print('No new ramps reached after LAH, moving on...') break # back to local arcpy.CopyFeatures_management(rp2s, "rp2s") cd3 = arcpy.sa.CostDistance("rp2s", costRastLoc, grpMaxCost, source_start_cost="costLAH") c += 1 nm = "cd" + str(c) cd3.save(nm) cds = cds + [nm] rp1 = arcpy.sa.ExtractValuesToPoints(rampPts, cd3, "rp1", "NONE", "VALUE_ONLY") rp1s = arcpy.MakeFeatureLayer_management( rp1, where_clause="RASTERVALU IS NOT NULL") # Check for new ramps or ramps reached at least one minute faster after latest run (Local) # Similar to process earlier, but with names reversed notin = [] locr = { a[0]: a[1] for a in arcpy.da.SearchCursor(rp1s, [rampPtsID, 'RASTERVALU']) } lahr = { a[0]: a[1] for a in arcpy.da.SearchCursor('rp2s', [rampPtsID, 'costLAH']) } for a in locr: if a not in lahr: notin.append(a) else: if locr[a] - lahr[a] < -1: notin.append(a) # end while loop if attFld is not None: if isinstance(attFld, str): # cell statistics areaval = round(unique_values(cdpts, attFld)[0], 3) area = arcpy.sa.Con( arcpy.sa.CellStatistics(cds, "MINIMUM", "DATA"), areaval, "", "Value <= " + str(grpMaxCost)) area.save(rastout) elif isinstance(attFld, int): area = arcpy.sa.Con( arcpy.sa.CellStatistics(cds, "MINIMUM", "DATA"), attFld, "", "Value <= " + str(grpMaxCost)) area.save(rastout) else: arcpy.sa.CellStatistics(cds, "MINIMUM", "DATA").save(rastout) print("Done with group: " + str(i)) t1 = time.time() print('That took ' + str(int(t1 - t0)) + ' seconds.') # garbage pickup every 10 runs, last run if n == round(n, -1) or n == len(grps): print("Deleting files...") r = arcpy.ListRasters("cd*") fc = arcpy.ListFeatureClasses("rp*") fc.append("buffext") garbagePickup(r) garbagePickup(fc) # reset extent arcpy.env.extent = costRastLoc arcpy.BuildPyramids_management(rastout) return rastout # General usage # # Environment settings # arcpy.env.parallelProcessingFactor = "100%" # Adjust to some percent (e.g. 100%) for large extent analyses (e.g. maxCost = None) # arcpy.env.mask = r'L:\David\projects\RCL_processing\RCL_processing.gdb\VA_Buff50mi_wgs84' # arcpy.env.overwriteOutput = True # # # Cost surface variables # costRastLoc = r'E:\RCL_cost_surfaces\Tiger_2019\cost_surfaces.gdb\costSurf_no_lah' # costRastHwy = r'E:\RCL_cost_surfaces\Tiger_2019\cost_surfaces.gdb\costSurf_only_lah' # rampPts = r'E:\RCL_cost_surfaces\Tiger_2019\cost_surfaces.gdb\rmpt_final' # rampPtsID = 'UniqueID' # unique ramp segment ID attribute field, since some ramps have multiple points # # # Facilities features and settings # accFeat = r'accessFeatures' # outGDB = r'serviceAreas.gdb' # # Attributes # grpFld = 'facil_code' # maxCost = 30 # attFld = None # makeServiceAreas(outGDB, accFeat, costRastLoc, costRastHwy, rampPts, rampPtsID, grpFld, maxCost, attFld)
# try: # break # except exception as e: # logger.exception('Unhandled Exception') start = time.time() Start = datetime.datetime.now() msgFolder = "C:/logs/" sender = "*****@*****.**" recipient = "*****@*****.**" fc = "C:\MYLATesting.gdb\MYLA311" if arcpy.Exists(fc): arcpy.Delete_management(fc) arcpy.SetLogHistory(True) f2 = open('C:\Users\Administrator\Desktop\DetailView.json', 'r') data2 = jsonpickle.encode(jsonpickle.decode(f2.read())) url2 = "https://myla311.lacity.org/myla311router/mylasrbe/1/QuerySR" headers2 = {'Content-type': 'text/plain', 'Accept': '/'} r2 = requests.post(url2, data=data2, headers=headers2) decoded2 = json.loads(r2.text) Start = datetime.datetime.now() # print Start
import arcpy from arcproc import dataset from arcproc import features from arcproc.helpers import elapsed, log_entity_states, slugify, unique_path from arcproc.metadata import Field, SpatialReferenceSourceItem LOG: logging.Logger = logging.getLogger(__name__) """Module-level logger.""" # Py3.7: Can replace usage with `typing.Self` in Py3.11. TProcedure = TypeVar("TProcedure", bound="Procedure") """Type variable to enable method return of self on Procedure.""" arcpy.SetLogHistory(False) class Procedure(ContextDecorator): """Manager for a single Arc-style procedure.""" keep_transforms: bool = False """Preserve transformation datasets if True.""" name: str = "Unnamed Procedure" """Procedure name.""" time_started: _datetime """Timestamp for when procedure started.""" transform_path: Path = None """Path to current transformation dataset.""" workspace_path: Path = "memory" """Path to workspace for transformation datasets."""
def handle_overlaps(zone_fc, zone_field, in_value_raster, out_table, is_thematic, unflat_table='', rename_tag='', units='', debug_mode=False): orig_env = env.workspace if debug_mode: env.overwriteOutput = True temp_gdb = cu.create_temp_GDB('zonal_tabarea') env.workspace = temp_gdb arcpy.AddMessage('Debugging workspace located at {}'.format(temp_gdb)) else: env.workspace = 'in_memory' arcpy.SetLogHistory(False) arcpy.CheckOutExtension("Spatial") def stats_area_table(zone_fc=zone_fc, zone_field=zone_field, in_value_raster=in_value_raster, out_table=out_table, is_thematic=is_thematic): def refine_zonal_output(t): """Makes a nicer output for this tool. Rename some fields, drop unwanted ones, calculate percentages using raster AREA before deleting that field.""" if is_thematic: value_fields = arcpy.ListFields(t, "VALUE*") pct_fields = [ '{}_pct'.format(f.name) for f in value_fields ] # VALUE_41_pct, etc. Field can't start with number. # add all the new fields needed for f, pct_field in zip(value_fields, pct_fields): arcpy.AddField_management(t, pct_field, f.type) # calculate the percents cursor_fields = ['AREA'] + [f.name for f in value_fields] + pct_fields uCursor = arcpy.da.UpdateCursor(t, cursor_fields) for uRow in uCursor: # unpacks area + 3 tuples of the right fields for each, no matter how many there are vf_i_end = len(value_fields) + 1 pf_i_end = vf_i_end + len(pct_fields) # pct_values and ha_values are both null at this point but unpack for clarity area, value_values, pct_values = uRow[0], uRow[ 1:vf_i_end], uRow[vf_i_end:pf_i_end] new_pct_values = [100 * vv / area for vv in value_values] new_row = [area] + value_values + new_pct_values uCursor.updateRow(new_row) for vf in value_fields: arcpy.DeleteField_management(t, vf.name) arcpy.AlterField_management(t, 'COUNT', 'CELL_COUNT') drop_fields = ['ZONE_CODE', 'COUNT', 'AREA'] if not debug_mode: for df in drop_fields: try: arcpy.DeleteField_management(t, df) except: continue # Set up environments for alignment between zone raster and theme raster if isinstance(zone_fc, arcpy.Result): zone_fc = zone_fc.getOutput(0) this_files_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(this_files_dir) common_grid = os.path.abspath('../common_grid.tif') env.snapRaster = common_grid env.cellSize = common_grid env.extent = zone_fc zone_desc = arcpy.Describe(zone_fc) zone_raster = 'convertraster' if zone_desc.dataType not in ['RasterDataset', 'RasterLayer']: zone_raster = arcpy.PolygonToRaster_conversion( zone_fc, zone_field, zone_raster, 'CELL_CENTER', cellsize=env.cellSize) print('cell size is {}'.format(env.cellSize)) zone_size = int(env.cellSize) else: zone_raster = zone_fc zone_size = min( arcpy.Describe(zone_raster).meanCellHeight, arcpy.Describe(zone_raster).meanCellWidth) raster_size = min( arcpy.Describe(in_value_raster).meanCellHeight, arcpy.Describe(in_value_raster).meanCellWidth) env.cellSize = min([zone_size, raster_size]) print('cell size is {}'.format(env.cellSize)) # I tested and there is no need to resample the raster being summarized. It will be resampled correctly # internally in the following tool given that the necessary environments are set above (cell size, snap). # # in_value_raster = arcpy.Resample_management(in_value_raster, 'in_value_raster_resampled', CELL_SIZE) if not is_thematic: arcpy.AddMessage("Calculating Zonal Statistics...") temp_entire_table = arcpy.sa.ZonalStatisticsAsTable( zone_raster, zone_field, in_value_raster, 'temp_zonal_table', 'DATA', 'MEAN') if is_thematic: # for some reason env.cellSize doesn't work # calculate/doit arcpy.AddMessage("Tabulating areas...") temp_entire_table = arcpy.sa.TabulateArea( zone_raster, zone_field, in_value_raster, 'Value', 'temp_area_table', processing_cell_size=env.cellSize) # TabulateArea capitalizes the zone for some annoying reason and ArcGIS is case-insensitive to field names # so we have this work-around: zone_field_t = '{}_t'.format(zone_field) DM.AddField(temp_entire_table, zone_field_t, 'TEXT', field_length=20) expr = '!{}!'.format(zone_field.upper()) DM.CalculateField(temp_entire_table, zone_field_t, expr, 'PYTHON') DM.DeleteField(temp_entire_table, zone_field.upper()) DM.AlterField(temp_entire_table, zone_field_t, zone_field, clear_field_alias=True) # replaces join to Zonal Stats in previous versions of tool # no joining, just calculate the area/count from what's produced by TabulateArea arcpy.AddField_management(temp_entire_table, 'AREA', 'DOUBLE') arcpy.AddField_management(temp_entire_table, 'COUNT', 'DOUBLE') cursor_fields = ['AREA', 'COUNT'] value_fields = [ f.name for f in arcpy.ListFields(temp_entire_table, 'VALUE*') ] cursor_fields.extend(value_fields) with arcpy.da.UpdateCursor(temp_entire_table, cursor_fields) as uCursor: for uRow in uCursor: area, count, value_fields = uRow[0], uRow[1], uRow[2:] area = sum(value_fields) count = round( area / (int(env.cellSize) * int(env.cellSize)), 0) new_row = [area, count] + value_fields uCursor.updateRow(new_row) arcpy.AddMessage("Refining output table...") arcpy.AddField_management(temp_entire_table, 'datacoveragepct', 'DOUBLE') arcpy.AddField_management(temp_entire_table, 'ORIGINAL_COUNT', 'LONG') # calculate datacoveragepct by comparing to original areas in zone raster # alternative to using JoinField, which is prohibitively slow if zones exceed hu12 count zone_raster_dict = { row[0]: row[1] for row in arcpy.da.SearchCursor(zone_raster, [zone_field, 'Count']) } temp_entire_table_dict = { row[0]: row[1] for row in arcpy.da.SearchCursor(temp_entire_table, [zone_field, 'COUNT']) } sum_cell_area = float(env.cellSize) * float(env.cellSize) orig_cell_area = zone_size * zone_size with arcpy.da.UpdateCursor( temp_entire_table, [zone_field, 'datacoveragepct', 'ORIGINAL_COUNT']) as cursor: for uRow in cursor: key_value, data_pct, count_orig = uRow count_orig = zone_raster_dict[key_value] if key_value in temp_entire_table_dict: count_summarized = temp_entire_table_dict[key_value] data_pct = 100 * float((count_summarized * sum_cell_area) / (count_orig * orig_cell_area)) else: data_pct = None cursor.updateRow((key_value, data_pct, count_orig)) # Refine the output refine_zonal_output(temp_entire_table) # in order to add vector capabilities back, need to do something with this # right now we just can't fill in polygon zones that didn't convert to raster in our system stats_result = cu.one_in_one_out(temp_entire_table, zone_fc, zone_field, out_table) # Convert "datacoveragepct" and "ORIGINAL_COUNT" values to 0 for zones with no metrics calculated with arcpy.da.UpdateCursor( out_table, [zone_field, 'datacoveragepct', 'ORIGINAL_COUNT', 'CELL_COUNT' ]) as u_cursor: for row in u_cursor: # data_coverage pct to 0 if row[1] is None: row[1] = 0 # original count filled in if a) zone outside raster bounds or b) zone too small to be rasterized if row[2] is None: if row[0] in zone_raster_dict: row[2] = zone_raster_dict[row[0]] else: row[2] = 0 # cell count set to 0 if row[3] is None: row[3] = 0 u_cursor.updateRow(row) # count whether all zones got an output record or not) out_count = int( arcpy.GetCount_management(temp_entire_table).getOutput(0)) in_count = int(arcpy.GetCount_management(zone_fc).getOutput(0)) count_diff = in_count - out_count # cleanup if not debug_mode: for item in [ 'temp_zonal_table', temp_entire_table, 'convertraster' ]: # don't add zone_raster, orig arcpy.Delete_management(item) arcpy.ResetEnvironments() env.workspace = orig_env # hope this prevents problems using list of FCs from workspace as batch arcpy.CheckInExtension("Spatial") return [stats_result, count_diff] def unflatten(intermediate_table): flat_zoneid = zone_field unflat_zoneid = zone_field.replace('flat', '') zone_type = [f.type for f in arcpy.ListFields(zone_fc, flat_zoneid)][0] # Set up the output table (can't do this until the prior tool is run) # if os.path.dirname(out_table): # out_path = os.path.dirname(out_table) # else: # out_path = orig_env unflat_result = DM.CreateTable('in_memory', os.path.basename(out_table)) # get the fields to add to the table editable_fields = [ f for f in arcpy.ListFields(intermediate_table) if f.editable and f.name.lower() != flat_zoneid.lower() ] # populate the new table schema DM.AddField(unflat_result, unflat_zoneid, zone_type) for f in editable_fields: DM.AddField(unflat_result, f.name, f.type, field_length=f.length) # map original zone ids to new zone ids original_flat = defaultdict(list) with arcpy.da.SearchCursor(unflat_table, [unflat_zoneid, flat_zoneid]) as cursor: for row in cursor: if row[1] not in original_flat[row[0]]: original_flat[row[0]].append(row[1]) # Use CELL_COUNT as weight for means to calculate final values for each zone. fixed_fields = [ unflat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct' ] other_field_names = [ f.name for f in editable_fields if f.name not in fixed_fields ] i_cursor = arcpy.da.InsertCursor( unflat_result, fixed_fields + other_field_names) # open output table cursor flat_stats = { r[0]: r[1:] for r in arcpy.da.SearchCursor(intermediate_table, [ flat_zoneid, 'ORIGINAL_COUNT', 'CELL_COUNT', 'datacoveragepct' ] + other_field_names) } count_diff = 0 for zid, unflat_ids in original_flat.items(): valid_unflat_ids = [id for id in unflat_ids if id in flat_stats ] # skip flatpolys not rasterized area_vec = [flat_stats[id][0] for id in valid_unflat_ids ] # ORIGINAL_COUNT specified in 0 index earlier cell_vec = [flat_stats[id][1] for id in valid_unflat_ids] coverage_vec = [flat_stats[id][2] for id in valid_unflat_ids ] # datacoveragepct special handling stat_vectors_by_id = [ flat_stats[id][3:] for id in valid_unflat_ids ] # "the rest", list of lists # calc the new summarized values original_count = sum( filter(None, area_vec) ) # None area is functionally equivalent to 0, all Nones = 0 too cell_count = sum(filter(None, cell_vec)) if cell_count > 0: weighted_coverage = sum( [a * b for a, b in zip(area_vec, coverage_vec)]) / original_count # this calculation accounts for fractional missing values, both kinds (whole zone is no data, or zone # was missing some data and had data coverage % < 100). This is done by converting None to 0 # and by using the cell_count (count of cells with data present) # instead of the full zone original_count. You have to do both or the mean will be distorted. # hand-verification that this works as intended using test GIS data on was completed 2019-11-01 by NJS crossprods = [] for i in range(0, len(valid_unflat_ids)): crossprods.append([ cell_vec[i] * float(s or 0) for s in stat_vectors_by_id[i] ]) weighted_stat_means = [] for i in range(0, len(other_field_names)): weighted_stat_means.append( sum(zip(*crossprods)[i]) / cell_count) else: weighted_coverage = 0 weighted_stat_means = [None] * len(other_field_names) count_diff += 1 new_row = [zid, original_count, cell_count, weighted_coverage ] + weighted_stat_means i_cursor.insertRow(new_row) del i_cursor DM.Delete(intermediate_table) return [unflat_result, count_diff] def rename_to_standard(table): arcpy.AddMessage("Renaming.") # datacoverage just gets tag new_datacov_name = '{}_datacoveragepct'.format(rename_tag) cu.rename_field(table, 'datacoveragepct', new_datacov_name, deleteOld=True) # DM.AlterField(out_table, 'datacoveragepct', new_datacov_name, clear_field_alias=True) if not is_thematic: new_mean_name = '{}_{}'.format(rename_tag, units).rstrip( '_') # if no units, just rename_tag cu.rename_field(table, 'MEAN', new_mean_name, deleteOld=True) # DM.AlterField(out_table, 'MEAN', new_mean_name, clear_field_alias=True) else: # look up the values based on the rename tag geo_file = os.path.abspath('../geo_metric_provenance.csv') with open(geo_file) as csv_file: reader = csv.DictReader(csv_file) mapping = { row['subgroup_original_code']: row['subgroup'] for row in reader if row['main_feature'] and row['main_feature'] in rename_tag } print(mapping) # update them for old, new in mapping.items(): old_fname = 'VALUE_{}_pct'.format(old) new_fname = '{}_{}_pct'.format(rename_tag, new) if arcpy.ListFields(table, old_fname): try: # same problem with AlterField limit of 31 characters here. DM.AlterField(table, old_fname, new_fname, clear_field_alias=True) except: cu.rename_field(table, old_fname, new_fname, deleteOld=True) return table if unflat_table: if not arcpy.Exists(unflat_table): raise Exception('Unflat_table must exist.') intermediate_stats = stats_area_table(out_table='intermediate_stats') named_as_original = unflatten(intermediate_stats[0]) else: named_as_original = stats_area_table(out_table='named_as_original') if rename_tag: named_as_standard = rename_to_standard(named_as_original[0]) out_table = DM.CopyRows(named_as_standard, out_table) else: out_table = DM.CopyRows(named_as_original[0], out_table) total_count_diff = named_as_original[1] if total_count_diff > 0: warn_msg = ( "WARNING: {0} zones have null zonal statistics. There are 2 possible reasons:\n" "1) Presence of zones that are fully outside the extent of the raster summarized.\n" "2) Zones are too small relative to the raster resolution.".format( total_count_diff)) arcpy.AddWarning(warn_msg) arcpy.SetLogHistory(True) return out_table