def getBasin(self, comID, isCatchmentLevel=False, xpoint=None, ypoint=None, crs=4326): try: if isCatchmentLevel == True: resource = Config()['queryParams']['nldiWFS'].format( crs, xpoint, ypoint) # else: resource = Config()['queryParams']['nldiQuery'].format(comID) try: results = self.Execute(resource) return results #Converted json.load(results) to this implimentation except: tb = traceback.format_exc() self._sm( "Exception raised for " + os.path.basename(resource) + ". Moving to next ComID.", "ERROR") except: tb = traceback.format_exc() self._sm("NLDIService getBasin Error " + tb, "ERROR")
def __init__(self, workspacePath, workspaceID): super(StreamStatsNationalOps, self).__init__(workspacePath) self.WorkspaceID = workspaceID self.mask = os.path.join( os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb', "Layers"), Config()["catchment"]["downstream"]) self.mask2 = os.path.join( os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb', "Layers"), Config()["catchment"]["global"]) self.mask3 = os.path.join( os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb', "Layers"), Config()["catchment"]["upstream"]) if not arcpy.Exists(self.mask): raise Exception("Mask does not exist: " + self.mask) self._sm("initialized StreamStatsNationalOps") arcpy.ResetEnvironments()
def MergeCatchment(self, inbasin): dstemp = None try: catchments = Config()["catchment"] env.workspace = self._TempLocation resultworkspace = os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb', "Layers") downstreamcatchment = os.path.join(resultworkspace, catchments["downstream"]) dsDesc = Describe(downstreamcatchment) if not Exists(downstreamcatchment): raise Exception("downstream catchment doesn't exist") sr = dsDesc.spatialReference wrkingbasin = self.ProjectFeature(inbasin, sr) #strip downstream catchment from mask dstemp = Erase_analysis(wrkingbasin, downstreamcatchment, "dstemp") #remove any weird verticies associated with Erase globalbasin = os.path.join(resultworkspace, catchments["global"]) FeatureToPolygon_management(dstemp, globalbasin, cluster_tolerance="50 Meters", attributes="ATTRIBUTES", label_features="") #Delete multipolys created by Featuretopolygon maxOid = max({ key: value for (key, value) in arcpy.da.SearchCursor( globalbasin, ['OID@', 'Shape_Area']) }.iteritems(), key=operator.itemgetter(1))[0] with arcpy.da.UpdateCursor(globalbasin, 'OID@') as cursor: for row in cursor: if row[0] != maxOid: cursor.deleteRow() #https://gis.stackexchange.com/questions/152481/how-to-delete-selected-rows-using-arcpy if not Exists(globalbasin): raise Exception("Failed to create basin " + GetMessages()) return True except: tb = traceback.format_exc() self._sm("Merge basin Error " + tb, "ERROR") return False finally: #cleanup for fs in arcpy.ListFeatureClasses(): arcpy.Delete_management(fs) for rs in arcpy.ListRasters(): arcpy.Delete_management(rs)
def getBasinCharacteristics(self, comID): results = {} try: resource = Config()['queryParams']['nldiChars'].format(comID) results = self.Execute(resource) #Removed json.load from results for x in results['characteristics']: results[str( x['characteristic_id'])] = x['characteristic_value'] return results except: tb = traceback.format_exc() self._sm("NLDIService getBasinCharacteristics Error " + tb, "ERROR")
def getDisturbanceIndex(self,comID): results={} key = comID+'.0' try: resource = Config()['queryParams']['damindex'] url = self.BaseUrl + resource results = json.load(open(url)) if(key in results): return results[key] else: return 0 except: tb = traceback.format_exc() self._sm("NLDIService getBasinCharacteristics Error "+tb, "ERROR")
def test_globalBasin(self): result = None try: Config({"NLDIService": "https://cida.usgs.gov", "queryParams": { "nldiQuery": "https://labs.waterdata.usgs.gov/api/nldi/linked-data/comid/{0}/basin", "nldiWFS": "/nwc/geoserver/nhdplus/ows?service=wfs&version=1.0.0&request=GetFeature&typeName=nhdplus:catchmentsp&srsName=EPSG:{0}&outputFormat=json&filter=<Filter xmlns=\"http://www.opengis.net/ogc\" xmlns:gml=\"http://www.opengis.net/gml\"><Contains><PropertyName>the_geom</PropertyName><gml:Point srsName=\"EPSG:{0}\"><gml:coordinates>{1},{2}</gml:coordinates></gml:Point></Contains></Filter>" }}) with NLDIServiceAgent() as sa: basin = sa.getBasin(5335813, False) result = json.dumps(basin) self.assertFalse(result==None or result == '') except: tb = traceback.format_exc() self.fail(tb)
def get_config(): return Config( json.load( open(os.path.join(os.path.dirname(__file__), '../config.json'))))
import json import threading from WIMLib.WiMLogging import WiMLogging from WIMLib import Shared from WIMLib.Config import Config from ServiceAgents.StreamStatsServiceAgent import StreamStatsServiceAgent from threading import Thread import random import queue #Initial number of thread calls will be (N+1) simulThreads = 0 queue_list = queue.Queue() #Open config file and define workspace config = Config( json.load(open(os.path.join(os.path.dirname(__file__), 'config.json')))) workingDir = Shared.GetWorkspaceDirectory( config["workingdirectory"]) #initialize and create logging folder w file #Create Summary.txt in the root folder of streamstats (Need to change directory) sumPath = os.path.join(workingDir, 'Summary.txt') fSummary = open(sumPath, 'w+') fSummary.write('Starting Summary' + '\n') fSummary.write('Total all runs: 0' + '\n') fSummary.write('Total bchar runs: 0' + '\n') fSummary.write('Total bdel runs: 0' + '\n') fSummary.write('Total bcharNoteq runs: 0' + '\n') fSummary.write('Total bdelNoteq runs: 0' + '\n') fSummary.write('Total bcharfail runs: 0' + '\n') fSummary.write('Total bdelfail runs: 0' + '\n') fSummary.write('Total bcharNew runs: 0' + '\n')
def __init__(self): ServiceAgentBase.ServiceAgentBase.__init__(self, Config()["NLDIService"]) self._sm("initialized NLDIServiceAgent")
def __init__(self): ServiceAgentBase.ServiceAgentBase.__init__(self, Config()["NLDIServiceFiles"]) self._sm("initialized DamIndexServiceAgent")
def _delineate(self, gage, workspace): try: ppoint = arcpy.CreateFeatureclass_management( "in_memory", "ppFC" + gage.comid, "POINT", spatial_reference=gage.sr) pnt = { "type": "Feature", "geometry": { "type": "Point", "coordinates": [gage.long, gage.lat] } } if (pnt["type"].lower() == "feature"): GeoJsonHandler.read_feature(pnt, ppoint, gage.sr) else: GeoJsonHandler.read_feature_collection(pnt, ppoint, gage.sr) if Config()["UseNLDIServices"] == False: #Toggler sa = NLDIFileServiceAgent() else: sa = NLDIServiceAgent() maskjson = sa.getBasin(gage.comid, True, gage.long, gage.lat, gage.sr.factoryCode) if (not maskjson): return None mask = arcpy.CreateFeatureclass_management( "in_memory", "maskFC" + gage.comid, "POLYGON", spatial_reference=gage.sr) if (maskjson["type"].lower() == "feature"): if not maskjson["geometry"]["type"].lower() in [ "polygon", "multipolygon" ]: raise Exception( 'Mask Geometry is not polygon output will be erroneous!' ) GeoJsonHandler.read_feature(maskjson, mask, gage.sr) else: for feature in maskjson["features"]: if not feature["geometry"]["type"].lower() in [ "polygon", "multipolygon" ]: raise Exception( 'Mask Geometry within the Feature Collection is not polygon output will be erroneous!' ) GeoJsonHandler.read_feature_collection(maskjson, mask, gage.sr) # fileSA = NLDIFileServiceAgent() basinjson = sa.getBasin(gage.comid, False) if (not basinjson): return None self.globalCatchment = arcpy.CreateFeatureclass_management( "in_memory", "globalBasin" + gage.comid, "POLYGON", spatial_reference=gage.sr) if (basinjson["type"].lower() == "feature"): if not basinjson["geometry"]["type"].lower() in [ "polygon", "multipolygon" ]: raise Exception( 'Basin Geometry is not polygon output will be erroneous!' ) GeoJsonHandler.read_feature(basinjson, self.globalCatchment, gage.sr) else: for feature in basinjson["features"]: if not feature["geometry"]["type"].lower() in [ "polygon", "multipolygon" ]: raise Exception( 'Basin Geometry within the Feature Collection is not polygon output will be erroneous!' ) GeoJsonHandler.read_feature_collection(basinjson, self.globalCatchment, gage.sr) ssdel = HydroOps(workspace, gage.id) ssdel.Delineate(ppoint, mask) ssdel.MergeCatchment(self.globalCatchment) return ssdel.WorkspaceID except: tb = traceback.format_exc() self._sm("Error delineating basin " + tb) return None
def _run(projectID, in_file, outwkid, parameters, today_date, arr, start_idx, end_idx): config = Config( json.load(open(os.path.join(os.path.dirname(__file__), 'config.json')))) if projectID == '#' or not projectID: raise Exception('Input Study Area required') workingDir = os.path.join(config["workingdirectory"], "temp", projectID + "_" + today_date) if not os.path.exists(workingDir): os.makedirs(workingDir) WiMLogging.init(os.path.join(workingDir, "temp"), "gage.log") WiMLogging.sm("Starting routine") params = parameters.split(";") if ( parameters) else config["characteristics"].keys() gage_file = Shared.readCSVFile(in_file) headers = gage_file[0] if "Gage_no" in headers: idindex = headers.index("Gage_no") if "Gage_name" in headers: nmindex = headers.index("Gage_name") if "COMID" in headers: comIDindex = headers.index("COMID") if "Lat_snap" in headers: latindex = headers.index("Lat_snap") if "Long_snap" in headers: longindex = headers.index("Long_snap") if "State" in headers: stateindex = headers.index("State") # strip the header line gage_file.pop(0) header = [] header.append("-+-+-+-+-+-+-+-+-+ NEW RUN -+-+-+-+-+-+-+-+-+") header.append("Execute Date: " + str(datetime.date.today())) header.append("-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+") header.append(",".join([ 'GAGEID', 'COMID', 'WorkspaceID', 'Description', 'LAT', 'LONG', 'STATE' ] + _formatRow(params))) Shared.writeToFile(os.path.join(workingDir, config["outputFile"]), header) ##not working yet, need to refresh rerun file #if not projectID == 'FH-10': # rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s.csv" % today_date #else: # rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s_Final.csv" % today_date rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s.csv" % today_date if not arcpy.Exists(rerunFile): rerunFileHeader = [] rerunFileHeader.append(",".join(headers)) Shared.writeToFile(rerunFile, rerunFileHeader) gagelist = gage_file[start_idx:end_idx] if not arcpy.Exists(r"D:\Applications\output\gage_iii\temp"): os.mkdir(r"D:\Applications\output\gage_iii\temp") for station in gagelist: #For use in temp directory name (see line #86) idx = start_idx # Create temp directory so ARC does not run out of internal memory newTempDir = r"D:\Applications\output\gage_iii\temp\gptmpenvr_" + time.strftime( '%Y%m%d%H%M%S') + '2018' + str(idx) os.mkdir(newTempDir) os.environ["TEMP"] = newTempDir os.environ["TMP"] = newTempDir with FederalHighwayWrapper(workingDir, projectID) as fh: results = {'Values': [{}]} g = gage.gage(station[idindex], station[comIDindex], station[latindex], station[longindex], outwkid, station[nmindex].replace(",", " "), '', station[stateindex]) results = fh.Run(g, params, arr) if results is None: results = {'Values': [{}]} Shared.appendLineToFile( os.path.join(workingDir, config["outputFile"]), ",".join( str(v) for v in [ g.id, g.comid, fh.workspaceID, results.Description, g.lat, g.long, g.state ] + _formatRow(results.Values, params))) formatresults = _formatRow(results.Values, params) if (fh.workspaceID is None or "not reported. see log file" in formatresults) and not g.comid == "-9999": f = open(rerunFile, "a") f.writelines('\n' + ",".join(station)) f.close() gc.collect() idx += 1 #The with statement should automatically take care of gc operations #but just in case fh = None gc.collect()
def Delineate(self, PourPoint, inmask=None): #http://pro.arcgis.com/en/pro-app/tool-reference/spatial-analyst/watershed.htm fdr = None sr = None mask = None datasetPath = None featurePath = None outWatershedRaster = None upCatch = None dstemp = None downCatch = None try: catchments = Config()["catchment"] arcpy.env.workspace = self._TempLocation self._sm("Delineating catchment") fdr = MapLayer(MapLayerDef("fdr"), "", PourPoint) if not fdr.Activated: raise Exception("Flow direction could not be activated.") fac = MapLayer(MapLayerDef("fac"), fdr.TileID) if not fac.Activated: raise Exception("Flow accumulation could not be activated.") sr = fdr.spatialreference if inmask is not None: mask = self.ProjectFeature(inmask, sr) datasetPath = arcpy.CreateFileGDB_management( self._WorkspaceDirectory, self.WorkspaceID + '.gdb')[0] featurePath = arcpy.CreateFeatureDataset_management( datasetPath, 'Layers', sr)[0] self._sm("creating workspace environment. " + datasetPath) arcpy.CheckOutExtension("Spatial") self._sm("Starting Delineation") arcpy.env.extent = arcpy.Describe(mask).extent #Build snap pour point then use it for watershed. # Pour point search distance of 60 m is equal to two cells. outSnapPour = SnapPourPoint(PourPoint, fac.Dataset, 60) outWatershedRaster = Watershed(fdr.Dataset, outSnapPour) upCatch = os.path.join(featurePath, catchments["upstream"]) arcpy.RasterToPolygon_conversion(outWatershedRaster, upCatch, "NO_SIMPLIFY") #strip downstream catchment from mask dstemp = arcpy.Erase_analysis(mask, upCatch, "dstemp") downCatch = self.__removePolygonHoles(dstemp, featurePath, catchments["downstream"]) self._sm(arcpy.GetMessages(), 'AHMSG') self._sm("Finished") except: tb = traceback.format_exc() self._sm("Delineation Error " + tb, "ERROR") finally: arcpy.CheckInExtension("Spatial") #Local cleanup if fdr is not None: del fdr if sr is not None: del sr if mask is not None: arcpy.Delete_management(mask) if dstemp is not None: arcpy.Delete_management(dstemp) dstemp = None for raster in arcpy.ListRasters("*", "GRID"): arcpy.Delete_management(raster) del raster raster = None if datasetPath is not None: del datasetPath if featurePath is not None: del featurePath if upCatch is not None: del upCatch upCatch = None if downCatch is not None: del downCatch downCatch = None arcpy.env.extent = ""
def __init__(self): try: parser = argparse.ArgumentParser() parser.add_argument("-projectID", help="specifies the projectID", type=str, default="FH_short") parser.add_argument( "-file", help= "specifies csv file location including gage lat/long and comid's to estimate", type=str, default=r'D:\Applications\input\CATCHMENT_gageloc_v1_short.csv' ) parser.add_argument( "-outwkid", help="specifies the esri well known id of pourpoint ", type=int, default='4326') parser.add_argument("-parameters", help="specifies the ';' separated list of parameters to be computed", type=str, default = "TOT_BASIN_AREA;" \ +"TOT_IMPV11;" \ +"TOT_IMPV11_NODATA;"\ +"TOT_MIRAD_2012;"\ +"TOT_MIRAD_2012_NODATA;") args = parser.parse_args() projectID = args.projectID if projectID == '#' or not projectID: raise Exception('Input Study Area required') config = Config( json.load( open(os.path.join(os.path.dirname(__file__), 'config.json')))) workingDir = Shared.GetWorkspaceDirectory( config["workingdirectory"], projectID) WiMLogging.init(os.path.join(workingDir, "Temp"), "gage.log") WiMLogging.sm("Starting routine") params = args.parameters.split(";") if ( args.parameters) else config["characteristics"].keys() file = Shared.readCSVFile(args.file) headers = file[0] if "Gage_no" in headers: idindex = headers.index("Gage_no") if "Gage_name" in headers: nmindex = headers.index("Gage_name") if "COMID" in headers: comIDindex = headers.index("COMID") if "Lat_snap" in headers: latindex = headers.index("Lat_snap") if "Long_snap" in headers: longindex = headers.index("Long_snap") #strip the header line file.pop(0) header = [] header.append("-+-+-+-+-+-+-+-+-+ NEW RUN -+-+-+-+-+-+-+-+-+") header.append("Execute Date: " + str(datetime.date.today())) header.append("-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+") header.append(",".join( ['COMID', 'WorkspaceID', 'Description', 'LAT', 'LONG'] + self._formatRow(params))) Shared.writeToFile(os.path.join(workingDir, config["outputFile"]), header) with FederalHighwayWrapper(workingDir, projectID) as fh: for station in file: results = {'Values': [{}]} try: g = gage.gage(station[idindex], station[comIDindex], station[latindex], station[longindex], args.outwkid, station[nmindex].replace(",", " ")) results = fh.Run(g, params) if results is None: results = {'Values': [{}]} Shared.appendLineToFile( os.path.join(workingDir, config["outputFile"]), ",".join( str(v) for v in [ g.comid, fh.workspaceID, results.Description, g.lat, g.long ] + self._formatRow(results.Values, params))) except: tb = traceback.format_exc() WiMLogging.sm("error computing gage " + g.id + " " + tb) continue finally: #ensure gc has collected before next gage gc.collect() #next station #endwith except: tb = traceback.format_exc() WiMLogging.sm("error running " + tb)
def __init__(self, chardefname): try: CharObj = Config()["characteristics"][chardefname] if (not CharObj): return None self.ID = CharObj["ID"] if ("ID" in CharObj) else 0 self.Name = chardefname self.MapLayers = CharObj["MapLayers"] if ("MapLayers" in CharObj) else None self.Method = CharObj["Method"] if ("Method" in CharObj) else None self.UnitID = CharObj["UnitID"] if ("UnitID" in CharObj) else "" self.Procedure = CharObj["Procedure"] self.Description = CharObj["Description"] if ("Description" in CharObj) else "" self.QueryField = CharObj["QueryField"] if ("QueryField" in CharObj) else None self.ClassCodes = CharObj["ClassCodes"] if ("ClassCodes" in CharObj) else None self.Count = CharObj["Count"] if ("Count" in CharObj) else None self.Data = os.path.join(Config()["parentdirectory"], CharObj["Data"]) if ("Data" in CharObj) else None self.MethField = CharObj["methodField"] if ("methodField" in CharObj) else None self.Field = CharObj["selectorField"] if ("selectorField" in CharObj) else None self.Operator = CharObj["Operator"] if ("Operator" in CharObj) else None self.Keyword = CharObj["Keyword"] if ("Keyword" in CharObj) else None self.Variables = CharObj["Variables"] if ("Variables" in CharObj) else None self.Equation = CharObj["Equation"] if ("Equation" in CharObj) else None self.EquationVariables = CharObj["EquationVariables"] if ( "EquationVariables" in CharObj) else None self.SubProcedure = CharObj["SubProcedure"] if ( "SubProcedure" in CharObj) else None self.WhereClause = CharObj["WhereClause"] if ("WhereClause" in CharObj) else "" self.MultiplicationFactor = CharObj["MultiplicationFactor"] if ( "MultiplicationFactor" in CharObj) else 1 # Added by JWX self.TimeRange = CharObj["TimeRange"] if ("TimeRange" in CharObj) else "" self.TimeMethod = CharObj["TimeMethod"] if ("TimeMethod" in CharObj) else "" self.AggregationMethod = CharObj["AggregationMethod"] if ( "AggregationMethod" in CharObj) else "weighteddifference" #seeWIMLib.ExpressionOps self.IDX = CharObj["IDX"] if ("IDX" in CharObj) else "" self.TOT_IDX = CharObj["TOT_IDX"] if ("TOT_IDX" in CharObj) else "", self.JoinTables = CharObj["JoinTables"] if ("JoinTables" in CharObj) else None, self.JoinField = CharObj["JoinField"] if ("JoinField" in CharObj) else None except: WiMLogging.sm( chardefname + " not available to compute. Returning none value.", "ERROR") return None
def __init__(self): ServiceAgentBase.__init__(self, Config()["WIM"]["baseurl"]) self.resources = Config()["WIM"]["resources"] self._sm("initialized WIMServiceAgent")
def __init__(self): self.BaseUrl = Config()["StreamStats"]["baseurl"] self.resources = Config()["StreamStats"]["resources"]