def getBasin(self,
comID,
isCatchmentLevel=False,
xpoint=None,
ypoint=None,
crs=4326):
try:
if isCatchmentLevel == True:
resource = Config()['queryParams']['nldiWFS'].format(
crs, xpoint, ypoint)
#
else:
resource = Config()['queryParams']['nldiQuery'].format(comID)
try:
results = self.Execute(resource)
return results #Converted json.load(results) to this implimentation
except:
tb = traceback.format_exc()
self._sm(
"Exception raised for " + os.path.basename(resource) +
". Moving to next ComID.", "ERROR")
except:
tb = traceback.format_exc()
self._sm("NLDIService getBasin Error " + tb, "ERROR")
def __init__(self, workspacePath, workspaceID):
super(StreamStatsNationalOps, self).__init__(workspacePath)
self.WorkspaceID = workspaceID
self.mask = os.path.join(
os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb',
"Layers"),
Config()["catchment"]["downstream"])
self.mask2 = os.path.join(
os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb',
"Layers"),
Config()["catchment"]["global"])
self.mask3 = os.path.join(
os.path.join(self._WorkspaceDirectory, self.WorkspaceID + '.gdb',
"Layers"),
Config()["catchment"]["upstream"])
if not arcpy.Exists(self.mask):
raise Exception("Mask does not exist: " + self.mask)
self._sm("initialized StreamStatsNationalOps")
arcpy.ResetEnvironments()
def MergeCatchment(self, inbasin):
dstemp = None
try:
catchments = Config()["catchment"]
env.workspace = self._TempLocation
resultworkspace = os.path.join(self._WorkspaceDirectory,
self.WorkspaceID + '.gdb', "Layers")
downstreamcatchment = os.path.join(resultworkspace,
catchments["downstream"])
dsDesc = Describe(downstreamcatchment)
if not Exists(downstreamcatchment):
raise Exception("downstream catchment doesn't exist")
sr = dsDesc.spatialReference
wrkingbasin = self.ProjectFeature(inbasin, sr)
#strip downstream catchment from mask
dstemp = Erase_analysis(wrkingbasin, downstreamcatchment, "dstemp")
#remove any weird verticies associated with Erase
globalbasin = os.path.join(resultworkspace, catchments["global"])
FeatureToPolygon_management(dstemp,
globalbasin,
cluster_tolerance="50 Meters",
attributes="ATTRIBUTES",
label_features="")
#Delete multipolys created by Featuretopolygon
maxOid = max({
key: value
for (key, value) in arcpy.da.SearchCursor(
globalbasin, ['OID@', 'Shape_Area'])
}.iteritems(),
key=operator.itemgetter(1))[0]
with arcpy.da.UpdateCursor(globalbasin, 'OID@') as cursor:
for row in cursor:
if row[0] != maxOid:
cursor.deleteRow()
#https://gis.stackexchange.com/questions/152481/how-to-delete-selected-rows-using-arcpy
if not Exists(globalbasin):
raise Exception("Failed to create basin " + GetMessages())
return True
except:
tb = traceback.format_exc()
self._sm("Merge basin Error " + tb, "ERROR")
return False
finally:
#cleanup
for fs in arcpy.ListFeatureClasses():
arcpy.Delete_management(fs)
for rs in arcpy.ListRasters():
arcpy.Delete_management(rs)
def getBasinCharacteristics(self, comID):
results = {}
try:
resource = Config()['queryParams']['nldiChars'].format(comID)
results = self.Execute(resource) #Removed json.load from results
for x in results['characteristics']:
results[str(
x['characteristic_id'])] = x['characteristic_value']
return results
except:
tb = traceback.format_exc()
self._sm("NLDIService getBasinCharacteristics Error " + tb,
"ERROR")
def getDisturbanceIndex(self,comID):
results={}
key = comID+'.0'
try:
resource = Config()['queryParams']['damindex']
url = self.BaseUrl + resource
results = json.load(open(url))
if(key in results):
return results[key]
else:
return 0
except:
tb = traceback.format_exc()
self._sm("NLDIService getBasinCharacteristics Error "+tb, "ERROR")
def test_globalBasin(self):
result = None
try:
Config({"NLDIService": "https://cida.usgs.gov",
"queryParams": {
"nldiQuery": "https://labs.waterdata.usgs.gov/api/nldi/linked-data/comid/{0}/basin",
"nldiWFS": "/nwc/geoserver/nhdplus/ows?service=wfs&version=1.0.0&request=GetFeature&typeName=nhdplus:catchmentsp&srsName=EPSG:{0}&outputFormat=json&filter=<Filter xmlns=\"http://www.opengis.net/ogc\" xmlns:gml=\"http://www.opengis.net/gml\"><Contains><PropertyName>the_geom</PropertyName><gml:Point srsName=\"EPSG:{0}\"><gml:coordinates>{1},{2}</gml:coordinates></gml:Point></Contains></Filter>"
}})
with NLDIServiceAgent() as sa:
basin = sa.getBasin(5335813, False)
result = json.dumps(basin)
self.assertFalse(result==None or result == '')
except:
tb = traceback.format_exc()
self.fail(tb)
def get_config():
return Config(
json.load(
open(os.path.join(os.path.dirname(__file__), '../config.json'))))
import json
import threading
from WIMLib.WiMLogging import WiMLogging
from WIMLib import Shared
from WIMLib.Config import Config
from ServiceAgents.StreamStatsServiceAgent import StreamStatsServiceAgent
from threading import Thread
import random
import queue
#Initial number of thread calls will be (N+1)
simulThreads = 0
queue_list = queue.Queue()
#Open config file and define workspace
config = Config(
json.load(open(os.path.join(os.path.dirname(__file__), 'config.json'))))
workingDir = Shared.GetWorkspaceDirectory(
config["workingdirectory"]) #initialize and create logging folder w file
#Create Summary.txt in the root folder of streamstats (Need to change directory)
sumPath = os.path.join(workingDir, 'Summary.txt')
fSummary = open(sumPath, 'w+')
fSummary.write('Starting Summary' + '\n')
fSummary.write('Total all runs: 0' + '\n')
fSummary.write('Total bchar runs: 0' + '\n')
fSummary.write('Total bdel runs: 0' + '\n')
fSummary.write('Total bcharNoteq runs: 0' + '\n')
fSummary.write('Total bdelNoteq runs: 0' + '\n')
fSummary.write('Total bcharfail runs: 0' + '\n')
fSummary.write('Total bdelfail runs: 0' + '\n')
fSummary.write('Total bcharNew runs: 0' + '\n')
def __init__(self):
ServiceAgentBase.ServiceAgentBase.__init__(self,
Config()["NLDIService"])
self._sm("initialized NLDIServiceAgent")
def __init__(self):
ServiceAgentBase.ServiceAgentBase.__init__(self, Config()["NLDIServiceFiles"])
self._sm("initialized DamIndexServiceAgent")
def _delineate(self, gage, workspace):
try:
ppoint = arcpy.CreateFeatureclass_management(
"in_memory",
"ppFC" + gage.comid,
"POINT",
spatial_reference=gage.sr)
pnt = {
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [gage.long, gage.lat]
}
}
if (pnt["type"].lower() == "feature"):
GeoJsonHandler.read_feature(pnt, ppoint, gage.sr)
else:
GeoJsonHandler.read_feature_collection(pnt, ppoint, gage.sr)
if Config()["UseNLDIServices"] == False: #Toggler
sa = NLDIFileServiceAgent()
else:
sa = NLDIServiceAgent()
maskjson = sa.getBasin(gage.comid, True, gage.long, gage.lat,
gage.sr.factoryCode)
if (not maskjson): return None
mask = arcpy.CreateFeatureclass_management(
"in_memory",
"maskFC" + gage.comid,
"POLYGON",
spatial_reference=gage.sr)
if (maskjson["type"].lower() == "feature"):
if not maskjson["geometry"]["type"].lower() in [
"polygon", "multipolygon"
]:
raise Exception(
'Mask Geometry is not polygon output will be erroneous!'
)
GeoJsonHandler.read_feature(maskjson, mask, gage.sr)
else:
for feature in maskjson["features"]:
if not feature["geometry"]["type"].lower() in [
"polygon", "multipolygon"
]:
raise Exception(
'Mask Geometry within the Feature Collection is not polygon output will be erroneous!'
)
GeoJsonHandler.read_feature_collection(maskjson, mask, gage.sr)
# fileSA = NLDIFileServiceAgent()
basinjson = sa.getBasin(gage.comid, False)
if (not basinjson): return None
self.globalCatchment = arcpy.CreateFeatureclass_management(
"in_memory",
"globalBasin" + gage.comid,
"POLYGON",
spatial_reference=gage.sr)
if (basinjson["type"].lower() == "feature"):
if not basinjson["geometry"]["type"].lower() in [
"polygon", "multipolygon"
]:
raise Exception(
'Basin Geometry is not polygon output will be erroneous!'
)
GeoJsonHandler.read_feature(basinjson, self.globalCatchment,
gage.sr)
else:
for feature in basinjson["features"]:
if not feature["geometry"]["type"].lower() in [
"polygon", "multipolygon"
]:
raise Exception(
'Basin Geometry within the Feature Collection is not polygon output will be erroneous!'
)
GeoJsonHandler.read_feature_collection(basinjson,
self.globalCatchment,
gage.sr)
ssdel = HydroOps(workspace, gage.id)
ssdel.Delineate(ppoint, mask)
ssdel.MergeCatchment(self.globalCatchment)
return ssdel.WorkspaceID
except:
tb = traceback.format_exc()
self._sm("Error delineating basin " + tb)
return None
def _run(projectID, in_file, outwkid, parameters, today_date, arr, start_idx,
end_idx):
config = Config(
json.load(open(os.path.join(os.path.dirname(__file__),
'config.json'))))
if projectID == '#' or not projectID:
raise Exception('Input Study Area required')
workingDir = os.path.join(config["workingdirectory"], "temp",
projectID + "_" + today_date)
if not os.path.exists(workingDir):
os.makedirs(workingDir)
WiMLogging.init(os.path.join(workingDir, "temp"), "gage.log")
WiMLogging.sm("Starting routine")
params = parameters.split(";") if (
parameters) else config["characteristics"].keys()
gage_file = Shared.readCSVFile(in_file)
headers = gage_file[0]
if "Gage_no" in headers: idindex = headers.index("Gage_no")
if "Gage_name" in headers: nmindex = headers.index("Gage_name")
if "COMID" in headers: comIDindex = headers.index("COMID")
if "Lat_snap" in headers: latindex = headers.index("Lat_snap")
if "Long_snap" in headers: longindex = headers.index("Long_snap")
if "State" in headers: stateindex = headers.index("State")
# strip the header line
gage_file.pop(0)
header = []
header.append("-+-+-+-+-+-+-+-+-+ NEW RUN -+-+-+-+-+-+-+-+-+")
header.append("Execute Date: " + str(datetime.date.today()))
header.append("-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+")
header.append(",".join([
'GAGEID', 'COMID', 'WorkspaceID', 'Description', 'LAT', 'LONG', 'STATE'
] + _formatRow(params)))
Shared.writeToFile(os.path.join(workingDir, config["outputFile"]), header)
##not working yet, need to refresh rerun file
#if not projectID == 'FH-10':
# rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s.csv" % today_date
#else:
# rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s_Final.csv" % today_date
rerunFile = "D:\Applications\output\gage_iii\RerunGageFiles\%s.csv" % today_date
if not arcpy.Exists(rerunFile):
rerunFileHeader = []
rerunFileHeader.append(",".join(headers))
Shared.writeToFile(rerunFile, rerunFileHeader)
gagelist = gage_file[start_idx:end_idx]
if not arcpy.Exists(r"D:\Applications\output\gage_iii\temp"):
os.mkdir(r"D:\Applications\output\gage_iii\temp")
for station in gagelist:
#For use in temp directory name (see line #86)
idx = start_idx
# Create temp directory so ARC does not run out of internal memory
newTempDir = r"D:\Applications\output\gage_iii\temp\gptmpenvr_" + time.strftime(
'%Y%m%d%H%M%S') + '2018' + str(idx)
os.mkdir(newTempDir)
os.environ["TEMP"] = newTempDir
os.environ["TMP"] = newTempDir
with FederalHighwayWrapper(workingDir, projectID) as fh:
results = {'Values': [{}]}
g = gage.gage(station[idindex], station[comIDindex],
station[latindex], station[longindex], outwkid,
station[nmindex].replace(",", " "), '',
station[stateindex])
results = fh.Run(g, params, arr)
if results is None: results = {'Values': [{}]}
Shared.appendLineToFile(
os.path.join(workingDir, config["outputFile"]), ",".join(
str(v) for v in [
g.id, g.comid, fh.workspaceID, results.Description,
g.lat, g.long, g.state
] + _formatRow(results.Values, params)))
formatresults = _formatRow(results.Values, params)
if (fh.workspaceID is None or "not reported. see log file"
in formatresults) and not g.comid == "-9999":
f = open(rerunFile, "a")
f.writelines('\n' + ",".join(station))
f.close()
gc.collect()
idx += 1
#The with statement should automatically take care of gc operations
#but just in case
fh = None
gc.collect()
def Delineate(self, PourPoint, inmask=None):
#http://pro.arcgis.com/en/pro-app/tool-reference/spatial-analyst/watershed.htm
fdr = None
sr = None
mask = None
datasetPath = None
featurePath = None
outWatershedRaster = None
upCatch = None
dstemp = None
downCatch = None
try:
catchments = Config()["catchment"]
arcpy.env.workspace = self._TempLocation
self._sm("Delineating catchment")
fdr = MapLayer(MapLayerDef("fdr"), "", PourPoint)
if not fdr.Activated:
raise Exception("Flow direction could not be activated.")
fac = MapLayer(MapLayerDef("fac"), fdr.TileID)
if not fac.Activated:
raise Exception("Flow accumulation could not be activated.")
sr = fdr.spatialreference
if inmask is not None:
mask = self.ProjectFeature(inmask, sr)
datasetPath = arcpy.CreateFileGDB_management(
self._WorkspaceDirectory, self.WorkspaceID + '.gdb')[0]
featurePath = arcpy.CreateFeatureDataset_management(
datasetPath, 'Layers', sr)[0]
self._sm("creating workspace environment. " + datasetPath)
arcpy.CheckOutExtension("Spatial")
self._sm("Starting Delineation")
arcpy.env.extent = arcpy.Describe(mask).extent
#Build snap pour point then use it for watershed.
# Pour point search distance of 60 m is equal to two cells.
outSnapPour = SnapPourPoint(PourPoint, fac.Dataset, 60)
outWatershedRaster = Watershed(fdr.Dataset, outSnapPour)
upCatch = os.path.join(featurePath, catchments["upstream"])
arcpy.RasterToPolygon_conversion(outWatershedRaster, upCatch,
"NO_SIMPLIFY")
#strip downstream catchment from mask
dstemp = arcpy.Erase_analysis(mask, upCatch, "dstemp")
downCatch = self.__removePolygonHoles(dstemp, featurePath,
catchments["downstream"])
self._sm(arcpy.GetMessages(), 'AHMSG')
self._sm("Finished")
except:
tb = traceback.format_exc()
self._sm("Delineation Error " + tb, "ERROR")
finally:
arcpy.CheckInExtension("Spatial")
#Local cleanup
if fdr is not None: del fdr
if sr is not None: del sr
if mask is not None: arcpy.Delete_management(mask)
if dstemp is not None:
arcpy.Delete_management(dstemp)
dstemp = None
for raster in arcpy.ListRasters("*", "GRID"):
arcpy.Delete_management(raster)
del raster
raster = None
if datasetPath is not None: del datasetPath
if featurePath is not None: del featurePath
if upCatch is not None:
del upCatch
upCatch = None
if downCatch is not None:
del downCatch
downCatch = None
arcpy.env.extent = ""
def __init__(self):
try:
parser = argparse.ArgumentParser()
parser.add_argument("-projectID",
help="specifies the projectID",
type=str,
default="FH_short")
parser.add_argument(
"-file",
help=
"specifies csv file location including gage lat/long and comid's to estimate",
type=str,
default=r'D:\Applications\input\CATCHMENT_gageloc_v1_short.csv'
)
parser.add_argument(
"-outwkid",
help="specifies the esri well known id of pourpoint ",
type=int,
default='4326')
parser.add_argument("-parameters", help="specifies the ';' separated list of parameters to be computed", type=str,
default = "TOT_BASIN_AREA;" \
+"TOT_IMPV11;" \
+"TOT_IMPV11_NODATA;"\
+"TOT_MIRAD_2012;"\
+"TOT_MIRAD_2012_NODATA;")
args = parser.parse_args()
projectID = args.projectID
if projectID == '#' or not projectID:
raise Exception('Input Study Area required')
config = Config(
json.load(
open(os.path.join(os.path.dirname(__file__),
'config.json'))))
workingDir = Shared.GetWorkspaceDirectory(
config["workingdirectory"], projectID)
WiMLogging.init(os.path.join(workingDir, "Temp"), "gage.log")
WiMLogging.sm("Starting routine")
params = args.parameters.split(";") if (
args.parameters) else config["characteristics"].keys()
file = Shared.readCSVFile(args.file)
headers = file[0]
if "Gage_no" in headers: idindex = headers.index("Gage_no")
if "Gage_name" in headers: nmindex = headers.index("Gage_name")
if "COMID" in headers: comIDindex = headers.index("COMID")
if "Lat_snap" in headers: latindex = headers.index("Lat_snap")
if "Long_snap" in headers: longindex = headers.index("Long_snap")
#strip the header line
file.pop(0)
header = []
header.append("-+-+-+-+-+-+-+-+-+ NEW RUN -+-+-+-+-+-+-+-+-+")
header.append("Execute Date: " + str(datetime.date.today()))
header.append("-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+")
header.append(",".join(
['COMID', 'WorkspaceID', 'Description', 'LAT', 'LONG'] +
self._formatRow(params)))
Shared.writeToFile(os.path.join(workingDir, config["outputFile"]),
header)
with FederalHighwayWrapper(workingDir, projectID) as fh:
for station in file:
results = {'Values': [{}]}
try:
g = gage.gage(station[idindex], station[comIDindex],
station[latindex], station[longindex],
args.outwkid,
station[nmindex].replace(",", " "))
results = fh.Run(g, params)
if results is None: results = {'Values': [{}]}
Shared.appendLineToFile(
os.path.join(workingDir, config["outputFile"]),
",".join(
str(v) for v in [
g.comid, fh.workspaceID,
results.Description, g.lat, g.long
] + self._formatRow(results.Values, params)))
except:
tb = traceback.format_exc()
WiMLogging.sm("error computing gage " + g.id + " " +
tb)
continue
finally:
#ensure gc has collected before next gage
gc.collect()
#next station
#endwith
except:
tb = traceback.format_exc()
WiMLogging.sm("error running " + tb)
def __init__(self, chardefname):
try:
CharObj = Config()["characteristics"][chardefname]
if (not CharObj): return None
self.ID = CharObj["ID"] if ("ID" in CharObj) else 0
self.Name = chardefname
self.MapLayers = CharObj["MapLayers"] if ("MapLayers"
in CharObj) else None
self.Method = CharObj["Method"] if ("Method" in CharObj) else None
self.UnitID = CharObj["UnitID"] if ("UnitID" in CharObj) else ""
self.Procedure = CharObj["Procedure"]
self.Description = CharObj["Description"] if ("Description"
in CharObj) else ""
self.QueryField = CharObj["QueryField"] if ("QueryField"
in CharObj) else None
self.ClassCodes = CharObj["ClassCodes"] if ("ClassCodes"
in CharObj) else None
self.Count = CharObj["Count"] if ("Count" in CharObj) else None
self.Data = os.path.join(Config()["parentdirectory"],
CharObj["Data"]) if ("Data"
in CharObj) else None
self.MethField = CharObj["methodField"] if ("methodField"
in CharObj) else None
self.Field = CharObj["selectorField"] if ("selectorField"
in CharObj) else None
self.Operator = CharObj["Operator"] if ("Operator"
in CharObj) else None
self.Keyword = CharObj["Keyword"] if ("Keyword"
in CharObj) else None
self.Variables = CharObj["Variables"] if ("Variables"
in CharObj) else None
self.Equation = CharObj["Equation"] if ("Equation"
in CharObj) else None
self.EquationVariables = CharObj["EquationVariables"] if (
"EquationVariables" in CharObj) else None
self.SubProcedure = CharObj["SubProcedure"] if (
"SubProcedure" in CharObj) else None
self.WhereClause = CharObj["WhereClause"] if ("WhereClause"
in CharObj) else ""
self.MultiplicationFactor = CharObj["MultiplicationFactor"] if (
"MultiplicationFactor" in CharObj) else 1 # Added by JWX
self.TimeRange = CharObj["TimeRange"] if ("TimeRange"
in CharObj) else ""
self.TimeMethod = CharObj["TimeMethod"] if ("TimeMethod"
in CharObj) else ""
self.AggregationMethod = CharObj["AggregationMethod"] if (
"AggregationMethod"
in CharObj) else "weighteddifference" #seeWIMLib.ExpressionOps
self.IDX = CharObj["IDX"] if ("IDX" in CharObj) else ""
self.TOT_IDX = CharObj["TOT_IDX"] if ("TOT_IDX"
in CharObj) else "",
self.JoinTables = CharObj["JoinTables"] if ("JoinTables"
in CharObj) else None,
self.JoinField = CharObj["JoinField"] if ("JoinField"
in CharObj) else None
except:
WiMLogging.sm(
chardefname +
" not available to compute. Returning none value.", "ERROR")
return None
def __init__(self):
ServiceAgentBase.__init__(self, Config()["WIM"]["baseurl"])
self.resources = Config()["WIM"]["resources"]
self._sm("initialized WIMServiceAgent")
def __init__(self):
self.BaseUrl = Config()["StreamStats"]["baseurl"]
self.resources = Config()["StreamStats"]["resources"]