def getEditArea(self, editAreaName, argDict):
# Returns an AFPS.ReferenceData object given an edit area name
# as defined in the GFE
# Apply suffix if appropriate
refID = ReferenceID(editAreaName)
#print("Getting edit area")
definition = argDict["forecastDef"]
if definition.has_key("editAreaSuffix"):
eaSuffix = definition["editAreaSuffix"]
#print("eaSuffix", eaSuffix)
if eaSuffix is not None:
#inv = argDict["ifpClient"].getReferenceInventory()
inv = self.dataMgr.getRefManager().getAvailableSets()
inventory = []
sz = inv.size()
for x in range(sz):
invID = inv.get(x)
inventory.append(str(invID.getName()))
suffName = editAreaName + eaSuffix
if suffName in inventory:
#print(" Setting suffix id", suffName)
refID = ReferenceID(suffName)
#print(" Adding editArea", refID)
from java.util import ArrayList
refList = ArrayList()
refList.add(refID)
tmp = self.dataMgr.getRefManager().getReferenceData(refList).get(0)
#tmp.getGrid()
return tmp
def getCombinations(self, combinations, argDict):
editAreas = []
newCombinations = []
for comboList, areaLabel in combinations:
newComboList = []
for editArea in comboList:
#print("Get edit area TF: get edit area combo", editArea)
#print("TF2: Get Edit Area set up edit areas", editArea)
newArea = self.getEditArea(editArea, argDict)
if comboList.index(editArea) == 0:
comboNumber = self.getComboNumber()
label = "Combo" + repr(comboNumber)
refId = ReferenceID(label)
#global GridLoc
#GridLoc = newArea.getGloc()
area = ReferenceData(newArea)
area.setId(refId)
#GridLoc, refId, newArea.getPolygons("LATLON"), "LATLON")
# randerso: I don't think this is necessary
# area.convertToAWIPS()
newComboList.append(newArea.getId().getName())
area = self.unionAreas(label, area, newArea)
if argDict["fcstType"] == "table":
# Allow user-supplied area labels to be used for simple tables
editAreas.append((area, areaLabel))
else:
editAreas.append((area, label))
newCombinations.append((newComboList, label))
return editAreas, newCombinations
def makeArea(gridLoc, pointList, refname=None):
" Make a Reference Area with a unique ReferenceID"
from com.vividsolutions.jts.geom import GeometryFactory, LinearRing, Coordinate, Polygon
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
CoordinateType = ReferenceData.CoordinateType
geomFactory = GeometryFactory()
import jep
size = len(pointList)
if pointList[0] != pointList[size - 1]: # closing the loop
pointList.append(pointList[0])
pointArray = jep.jarray(len(pointList), Coordinate)
for i in range(len(pointList)):
pointArray[i] = pointList[i]
lr = geomFactory.createLinearRing(pointArray)
poly = geomFactory.createPolygon(lr, jep.jarray(0, LinearRing))
polyArray = jep.jarray(1, Polygon)
polyArray[0] = poly
region = geomFactory.createMultiPolygon(polyArray)
if refname is None:
refname = "Ref" + getTime()
refId = ReferenceID(refname)
refData = ReferenceData(gridLoc, refId, region, CoordinateType.LATLON)
# randerso: I don't think this is necessary
# refData.convertToAWIPS()
return refData
def __init__(self, imageWidth=None, imageHeight=None, expandLeft=25.0, expandRight=25.0, expandTop=25.0, expandBottom=25.0, mask=None, wholeDomain=0, bgColor=None):
# Create empty display and data manager for display
display = GFEMapRenderableDisplay()
self.dataMgr = DataManagerOffscreenFactory.getInstance(display)
self.refId = None
envelope = None
gloc = self.dataMgr.getParmManager().compositeGridLocation()
if mask is not None:
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
CoordinateType = ReferenceData.CoordinateType
self.refId = ReferenceID(mask)
if wholeDomain == 0:
envelope = self.dataMgr.getRefManager().loadRefSet(self.refId).overallDomain(CoordinateType.GRID)
if imageWidth is not None:
imageWidth = Integer(int(imageWidth))
if imageHeight is not None:
imageHeight = Integer(int(imageHeight))
geom = GfeImageUtil.getLocationGeometry(gloc, envelope, imageWidth, imageHeight, expandLeft / 100.0, expandRight / 100.0, expandTop / 100.0, expandBottom / 100.0)
# Create descriptor for display
desc = MapDescriptor(geom)
display.setDescriptor(desc)
VizPainter.VizPainter.__init__(self, display, backgroundColor=bgColor)
gfeSystem = GFESystemResource(self.dataMgr)
self.addVizResource(gfeSystem)
desc.getResourceList().getProperties(gfeSystem).setSystemResource(True)
self.primaryRsc = None
def evaluate(self, expression, timeInfluence=None):
# if no expression, return the empty ref set
if len(expression) == 0:
return self.__dm.getRefManager().emptyRefSet()
if timeInfluence is None:
timeInfluence = self.__dm.getSpatialDisplayManager().getSpatialEditorTime()
if not isinstance(timeInfluence, AbsTime.AbsTime):
timeInfluence = AbsTime.AbsTime(timeInfluence)
self._query.setTime(timeInfluence.unixTime())
grid = self._query.eval(expression)
if type(grid) != type(numpy.array([])) or grid.shape != self._shape:
raise TypeError("query did not eval to a grid of shape: "
+ `self._shape`)
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData, ReferenceID
from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit
if grid.dtype == 'bool':
grid = numpy.array(grid, 'byte')
bits = Grid2DBit.createBitGrid(grid.shape[1], grid.shape[0], grid)
return ReferenceData(self.gloc(),
ReferenceID(expression),
expression, bits)
def unionAreas(self, name, area1, area2):
# OR the areas (ReferenceData objects)
# together and return a ReferenceData object
refData = area1.orEquals(area2)
#refData.convertToLatLon()
refData.setId(ReferenceID(name))
refData.getGrid()
return refData
def intersectAreas(self, name, area1, area2):
# AND the areas (ReferenceData objects)
# together and return a ReferenceData object
cpy = ReferenceData(area1)
refData = cpy.andEquals(area2)
#refData.convertToLatLon()
refData.setId(ReferenceID(name))
refData.getGrid()
return refData
def __init__(self, procName, configFile, startTime, endTime, timeRange,
editArea, mutableModel, varDict):
# import the config file
prefs = loadConfig.loadPreferences(configFile)
LogStream.logEvent("Configuration File: ", configFile)
if mutableModel is None:
mutableModel = prefs.getString('mutableModel')
else:
prefs.setValue('mutableModel', mutableModel)
self.__dataMgr = DataManagerUIFactory.getInstance(None)
currentMutableModel = self.__dataMgr.getParmManager(
).getMutableDatabase()
desiredMutableModel = self.__dataMgr.getParmManager(
).decodeDbString(mutableModel)
if currentMutableModel != desiredMutableModel:
DataManagerUIFactory.dispose(None)
self.__dataMgr = DataManagerUIFactory.getInstance(None)
# Create Time Range
if startTime is not None and endTime is not None:
start = self.decodeTimeStruct(startTime)
end = self.decodeTimeStruct(endTime)
self.__timeRange = TimeRange.TimeRange(start, end)
elif timeRange is not None:
self.__timeRange = TimeRange.TimeRange(
self.__dataMgr.getSelectTimeRangeManager().getRange(
timeRange).toTimeRange())
else:
self.__timeRange = TimeRange.default()
if editArea is not None:
refID = ReferenceID(editArea)
self.__editArea = \
self.__dataMgr.getRefManager().loadRefSet(refID)
else:
self.__editArea = self.__dataMgr.getRefManager().emptyRefSet()
LogStream.logVerbose("varDict=", varDict)
runner = ProcedureRunner(procName)
errors = runner.getImportErrors()
if len(errors) > 0:
msg = "\n\t".join(
["Error importing the following procedures:"] + errors)
LogStream.error(msg)
runner.instantiate(procName, CLASS_NAME,
**{'dbss': self.__dataMgr})
runner.run(self.__dataMgr, procName, CLASS_NAME, METHOD_NAME,
varDict, self.__editArea, self.__timeRange)
def getEditArea(self, editAreaName, argDict):
# Returns an AFPS.ReferenceData object given an edit area name
# as defined in the GFE
refID = ReferenceID(editAreaName)
refList = ArrayList()
refList.add(refID)
tmp = argDict["ifpClient"].getReferenceData(refList).get(0)
if tmp is not None:
tmp.getGrid()
return tmp
def __saveEA(self, name, grid):
#save edit area from a grid
gloc = self.__dataMgr.getParmManager().compositeGridLocation()
id = ReferenceID(name)
# convert grid to polygons
grid2d = Grid2DBit.createBitGrid(int(gloc.getNx()), int(gloc.getNy()),
grid)
refdata = ReferenceData(gloc, id, grid2d)
# save the edit area
self.__dataMgr.getRefManager().saveRefSet(refdata)
def deleteEditAreas(self, editAreas):
# Delete a list of ReferenceData, ReferenceID, or string objects
ifpClient = self.getIFPClient()
ids = ArrayList()
for area in editAreas:
if type(area) is str:
ids.add(ReferenceID(area))
else:
try:
# reference data
ids.add(area.getId())
except:
# reference id
ids.add(area)
ifpClient.deleteReferenceData(ids)
def getStatDict(self, sampler, analysisList, timeRange, area, componentName=None):
# Get Statistic dictionary for given analysisList, timeRange,
# and area
# Area can be ReferenceData or text string
# Convert area to ID if necessary
if area is None:
return None
if isinstance(area, str):
area = ReferenceID(area)
else:
area = area.getId()
return Analysis.Analysis(sampler).createStats(
analysisList, area, timeRange, componentName)
def getSampler(self, argDict, sampleInfo, sampleFromServer=0):
# Get a HistoSampler given
# sampleInfo, which is a list of tuples, or just a single tuple
# of tuples ([elements], [periods], [areas])
# the elements are [(name, method)] -- basically the analysis list
# the periods [(timeRange, label)]
# areas [(name,label)] or [(refData, label)] or [(refID, label)]
ifpClient = argDict["ifpClient"]
databaseID = argDict["databaseID"]
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceID, ReferenceData
from com.raytheon.viz.gfe.sampler import SamplerRequest, HistoSampler
from java.util import ArrayList
# convert input sampleInfo to samplerRequests
samplerRequests = ArrayList()
if type(sampleInfo) == tuple:
sampleInfo = [sampleInfo]
for si in sampleInfo:
elements, periods, areas = si
for e in elements:
parmID = self.getParmID(e[0], databaseID)
for p in periods:
for a in areas:
if type(a[0]) is str:
samplerRequests.add(SamplerRequest( \
parmID, ReferenceID(a[0]), p[0].toJavaObj()))
elif hasattr(a[0], "java_name") and \
(a[0].java_name == ReferenceID.java_name or
a[0].java_name == ReferenceData.java_name):
samplerRequests.add(
SamplerRequest(parmID, a[0], p[0].toJavaObj()))
else:
raise TypeError(
"area is not String, ReferenceID, or ReferenceData"
)
# do sampling
if sampleFromServer:
sampler = ifpClient.sampleRequest(samplerRequests)
else:
sampler = HistoSampler(ifpClient.getJavaClient(), samplerRequests)
if sampler.isValid() != 1:
print "Cannot Sample: Check for invalid Weather Elements, ",\
"Invalid Areas", str(samplerRequests)
return None
#print "sampler ", sampler
return sampler
def __init__(self, procName, host, port, userName, configFile, startTime,
endTime, timeRange, editArea, mutableModel, varDict):
# import the config file
prefs = loadConfig.loadPreferences(configFile)
LogStream.logEvent("Configuration File: ", configFile)
if mutableModel is None:
mutableModel = prefs.getString('mutableModel')
else:
prefs.setValue('mutableModel', mutableModel)
self.__dataMgr = DataManager.getInstance(None)
# Create Time Range
if startTime is not None and endTime is not None:
start = self.getAbsTime(startTime)
end = self.getAbsTime(endTime)
self.__timeRange = TimeRange.TimeRange(start, end)
elif timeRange is not None:
self.__timeRange = TimeRange.TimeRange(
self.__dataMgr.getSelectTimeRangeManager().getRange(
timeRange).toTimeRange())
else:
self.__timeRange = TimeRange.default()
if editArea is not None:
refID = ReferenceID(editArea)
self.__editArea = \
self.__dataMgr.getRefManager().loadRefSet(refID)
else:
self.__editArea = self.__dataMgr.getRefManager().emptyRefSet()
LogStream.logVerbose("varDict=", varDict)
runner = ProcedureRunner(procName)
errors = runner.getImportErrors()
if len(errors) > 0:
print "Error importing the following procedures:\n"
for s in errors:
print s
sys.exit(1)
runner.instantiate(procName, CLASS_NAME, **{'dbss': self.__dataMgr})
runner.run(self.__dataMgr, procName, CLASS_NAME, METHOD_NAME, varDict,
self.__editArea, self.__timeRange)
def getUnion(self, argDict, areaLabels, areaPrefix):
GridLoc = self.getIFPClient().getDBGridLocation()
area = None
for areaLabel in areaLabels:
newArea = self.getEditArea(areaLabel, argDict)
if areaLabels.index(areaLabel) == 0:
comboNumber = self.getComboNumber()
label = areaPrefix + ` int(time.time()) ` + ` comboNumber `
refId = ReferenceID(label)
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
CoordinateType = ReferenceData.CoordinateType
coordType = CoordinateType.valueOf('LATLON')
area = ReferenceData(GridLoc, refId,
newArea.getPolygons(coordType), coordType)
# randerso: I don't think this is necessary
# area.convertToAWIPS()
area = self.unionAreas(label, area, newArea)
return area
def __createArea(self, latLonTuple, argDict):
# Return a ReferenceData created for the given lat,lon and dimension
# If dim is zero, make edit area of the one grid
# point closest to the lat/lon value.
lat, lon, dim = latLonTuple
name = self.__getLatLonAreaName((lat, lon, dim))
#print("\ncreateArea", lat, lon, dim, name)
if dim != 0:
for x in range(100):
points = makeSquare(lat, lon, dim)
pointList = []
for point in points:
pointList.append(makePoint(point))
refData = makeArea(self.ifpClient.getDBGridLocation(),
pointList,
refname=name)
# Make sure we have at least one grid point in
# the edit area
if refData.getGrid().isAnyBitsSet():
#print("returning", dim)
return refData
# Increment dim and try again
#print("iterating", dim)
dim += 0.25
msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n"
self.log.warning(msg)
return None
else:
from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit
# Get grid cell coordinates for lat/lon
gridLoc = self.ifpClient.getDBGridLocation()
nx = int(gridLoc.getNx())
ny = int(gridLoc.getNy())
cc2D = gridLoc.gridCell(float(lat), float(lon))
# convert grid cell to Grid2DBit with single bit set
grid2Dbit = Grid2DBit(nx, ny)
if (nx > cc2D.x >= 0 and ny > cc2D.y >= 0):
grid2Dbit.set(int(cc2D.x), int(cc2D.y))
#refData = GridLoc.convertToReferenceData(grid2Dbit)
refID = ReferenceID(name)
refData = ReferenceData(gridLoc, refID, grid2Dbit)
return refData
def getEditAreas(self, argDict, areas):
# Get the ReferenceArea given
# a name or ReferenceID or ReferenceData(just return)
refDataList = []
for area in areas:
if type(area) is str: # Get area from name
id = ReferenceID(area)
jlist = ArrayList()
jlist.add(id)
tmp = argDict["ifpClient"].getReferenceData(jlist)
refDataList.append(tmp.get(0))
else:
if str(area).find('Id') > -1:
refDataList.append(area)
else:
jlist = ArrayList()
jlist.add(id)
tmp = argDict["ifpClient"].getReferenceData(jlist)
refDataList.append(tmp.get(0))
return refDataList
def createLatLonArea(self, lat, lon, dim):
# Create a square edit area given a latitude, longitude,
# and kilometer dimension for the sides.
# Example:
# area = self.createLatLonArea(40.93, -106.26, 5)
#
# If dim is zero, make edit area of the one grid
# point closest to the lat/lon value.
#
name = self.getLatLonAreaName((lat, lon, dim))
if dim != 0:
for x in range(100):
points = self.makeSquare(lat, lon, dim)
pointList = []
for point in points:
pointList.append(self.makePoint(point))
refData = self.makeArea(pointList, refname=name)
# Make sure we have at least one grid point in
# the edit area
if refData.getGrid().isAnyBitsSet():
return refData
# Increment dim and try again
dim += 0.25
msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n"
self.log.warning(msg)
return None
else:
# Get grid cell coordinates for lat/lon
gridLoc = self.getGridLoc()
cc2D = gridLoc.gridCell(lat, lon)
# convert grid cell to Grid2DBit with single bit set
grid2Dbit = JavaGrid2DBit(gridLoc.gridSize().x,
gridLoc.gridSize().y)
grid2Dbit.set(int(cc2D.x), int(cc2D.y))
#refData = gridLoc.convertToReferenceData(grid2Dbit)
refID = ReferenceID(name)
refData = ReferenceData(gridLoc, refID, grid2Dbit)
#refData.setId(refID)
return refData
def process(self):
import TimeRange
# get list of edit areas that are part of the Zones/FireWx group
from com.raytheon.viz.gfe.smarttool import TextFileUtil, GridCycler
textID = TextFileUtil.getTextFile('Zones', 'editAreaGroups')
zoneList = []
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
zoneList.append(line.rstrip())
textFile.close()
textID = TextFileUtil.getTextFile('FireWxZones', 'editAreaGroups')
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
zoneList.append(line.rstrip())
textFile.close()
refMgr = self.__dataMgr.getRefManager()
# make the basic edit areas that are required, go sequentially through
# the zoneList
requiredEA = [
"west_half", "east_half", "east_one_third", "west_one_third",
"east_two_thirds", "west_two_thirds", "east_one_quarter",
"west_one_quarter", "east_three_quarters", "west_three_quarters",
"Superior"
]
for x in xrange(len(requiredEA)):
refData = refMgr.loadRefSet(ReferenceID(zoneList[x]))
ea = ReferenceData(refData)
ea.setId(ReferenceID(requiredEA[x]))
refMgr.saveRefSet(ea)
#ea = self.__client.getEditAreaPolygons(zoneList[x])
#self.__client.saveEditArea(requiredEA[x], ea)
LogStream.logEvent("Saved ", zoneList[x], "under", requiredEA[x])
# special EAs (source,destination)
special = [("ISC_Send_Area", "FireArea"), ("ISC_Send_Area", "area3")]
for s in special:
refData = refMgr.loadRefSet(ReferenceID(s[0]))
ea = ReferenceData(refData)
ea.setId(ReferenceID(s[1]))
refMgr.saveRefSet(ea)
#ea = self.__client.getEditAreaPolygons(s[0])
#self.__client.saveEditArea(s[1], ea)
LogStream.logEvent("Saved ", s[0], "under", s[1])
# topography simulated based edit areas
# area3 = whole area, AboveElev, BelowElev
LogStream.logEvent("Calculating topo-dependent edit areas...")
topo = self.__dataMgr.getParmManager().getParmInExpr("Topo", True)
topogrid = GridCycler.getInstance().getCorrespondingResult(
topo,
TimeRange.allTimes().toJavaObj(), "TimeWtAverage")
topogrid = topogrid[0].getGridSlice().getNDArray()
iscSend = ReferenceID('ISC_Send_Area')
#wholeGrid = self.__client.getEditArea("ISC_Send_Area")
wholeGrid = refMgr.loadRefSet(iscSend).getGrid().getNDArray()
topoAve = 0
count = 0
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
for x in range(minx, maxx):
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
count = count + 1
topoAve = topoAve + topogrid[y, x]
topoAve = topoAve / count
aboveGrid = wholeGrid * 0
belowGrid = wholeGrid * 0
for x in xrange(topogrid.shape[1]):
for y in xrange(topogrid.shape[0]):
if wholeGrid[y, x] == 1:
if topogrid[y, x] > topoAve:
aboveGrid[y, x] = 1
else:
belowGrid[y, x] = 1
# area1 and area2 need to be "BelowElev", but should be different
# than area3
desiredCount = 2000
count = 0
area1 = wholeGrid * 0
area2 = wholeGrid * 0
for x in xrange(topogrid.shape[1]):
if count < desiredCount:
for y in xrange(topogrid.shape[0]):
if wholeGrid[y, x] == 0 and topogrid[y, x] < topoAve:
area1[y, x] = 1
belowGrid[y, x] = 1
count = count + 1
count = 0
for x in xrange(topogrid.shape[1]):
if count < desiredCount:
for y in xrange(topogrid.shape[0]):
if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve and \
area1[y,x] == 0:
area2[y, x] = 1
belowGrid[y, x] = 1
count = count + 1
# save all topography-dependent edit areas
self.__saveEA("area1", area1)
LogStream.logEvent("Saved area1 based on area2, area3, and topo <",
topoAve)
self.__saveEA("area2", area2)
LogStream.logEvent("Saved area2 based on area1, area3, and topo <",
topoAve)
self.__saveEA("AboveElev", aboveGrid)
LogStream.logEvent("Saved AboveElev based on area3 > ", topoAve)
self.__saveEA("BelowElev", belowGrid)
LogStream.logEvent("Saved BelowElev based on area3 <= ", topoAve)
self.__saveEA("Ridges", aboveGrid)
LogStream.logEvent("Saved Ridges based on area3 > ", topoAve)
self.__saveEA("Valleys", belowGrid)
LogStream.logEvent("Saved Valleys based on area3 < ", topoAve)
self.__saveEA("Inland", aboveGrid)
LogStream.logEvent("Saved Ridges based on area3 > ", topoAve)
self.__saveEA("Coastal", belowGrid)
LogStream.logEvent("Saved Valleys based on area3 < ", topoAve)
#city areas, which are a small part of other edit areas
cityBased = [("area1", ["city1", "city2"]), ("area2", ["city3"]),
("area3", ["city4", "area3_pt"])]
for baseArea, cityAreas in cityBased:
#wholeGrid = self.__client.getEditArea(baseArea)
wholeGrid = refMgr.loadRefSet(
ReferenceID(baseArea)).getGrid().getNDArray()
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
cNumber = 0
print minx, maxx, miny, maxy, wholeGrid.shape
for x in range(minx, maxx):
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
if cNumber >= len(cityAreas):
break
cityGrid = numpy.logical_and(wholeGrid, 0)
cityGrid[y, x] = 1
self.__saveEA(cityAreas[cNumber],
cityGrid.astype('int8'))
LogStream.logEvent("Saved ", cityAreas[cNumber],
"based on:", baseArea)
cNumber = cNumber + 1
# special for ISC areas for CCF database source test
#txt = self.__eagdb["ISC"]
#iscList = cPickle.loads(txt)
textID = TextFileUtil.getTextFile('ISC', 'editAreaGroups')
iscList = []
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
iscList.append(line.rstrip())
textFile.close()
count = 0
while count < 6:
for i in iscList:
if i == "ISC_Send_Area" or i == "ISC_Tool_Area":
continue
wholeGrid = refMgr.loadRefSet(
ReferenceID(i)).getGrid().getNDArray()
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
if minx == -1:
continue
ok = 1
print minx, maxx, miny, maxy, wholeGrid.shape
for x in range(minx, maxx):
if ok:
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
ptGrid = numpy.logical_and(wholeGrid, 0)
ptGrid[y, x] = 1
name = "isc" + ` count `
self.__saveEA(name, ptGrid.astype('int8'))
requiredEA.append(name)
LogStream.logEvent("Saved ", name, "based on ",
i)
ok = 0
break
else:
break
count = count + 1
if count > 6:
break
# store an edit area group with all of the generated edit areas
requiredEA.append("FireArea")
requiredEA.append("AboveElev")
requiredEA.append("BelowElev")
requiredEA.append("Valleys")
requiredEA.append("Ridges")
requiredEA.append("Inland")
requiredEA.append("Coastal")
requiredEA.append("city1")
requiredEA.append("city2")
requiredEA.append("city3")
requiredEA.append("city4")
requiredEA.append("area3")
requiredEA.append("area2")
requiredEA.append("area1")
refMgr.saveGroup("GFETest", JUtil.pylistToJavaStringList(requiredEA))
time.sleep(.5)
def __getRunTimeVariables(self, fcstName, forecastDef, fcstType, module,
argDict):
# Input variables can come from various sources:
# varDict from command line
# command line switch e.g. -l language (will be in argDict)
# definition section of product
# We must check all three in that order.
varDict = argDict["varDict"]
#print("varDict", varDict)
for item, default in [
("language", "english"),
("appendFile", None),
("lineLength", 69), # no command line option
("timePeriod", 3),
]:
try: # Try the varDict
#print("trying varDict", item)
argDict[item] = varDict[item]
#print("got it in varDict")
except:
try: # Try argDict i.e. from command line switch
# If so, argDict is already set
#print("trying argDict", item)
argValue = argDict[item]
#print("got it in argDict", argValue)
except:
argValue = None
# Try
#print("getting from definition", item)
if argValue is None:
argDict[item] = self.__ut.set(forecastDef, item, default)
#print("value from definition", argDict[item])
# These need to be integers
for item in ["lineLength", "timePeriod"]:
if argDict[item] is not None:
argDict[item] = int(argDict[item])
# Edit Areas and Time Ranges
#
# Set up these argDict values:
# editAreas -- list of (refData, label) pairs
# timeRanges -- list of named ranges
# rawRanges -- list of (rawRange, rangeName) pairs
#
# As we eventually loop through the product, these values will be set:
# editArea -- current edit area pair
# timeRange -- current raw time range
# issueTime -- first raw time range
# timeRangeName -- current time range name (if available)
# combinations -- list of (editAreaList, comboLabel) tuples
# where editAreaList is the list of edit areas composing
# the combination
# Set up Edit Areas
# May be from these sources:
# "AreaLabel"
# varDict from command line -- list of names
# command line as named reference areas
# defaultEditAreas
# list of (name, label) pairs
# "LatLon"
# OR list of ((lat,lon,dimension), label)
# "Combinations"
# OR A Combinations file
# We may have had edit areas entered from the command line
# or from the Interfaces "generateForecast" command
# If so, argDict["editAreas"] will be a list of either
# (name, label) or (refData, label) pairs
editAreaType = "AreaLabel"
# We may have had edit areas entered from the command line
# If so, argDict["editAreas"] will be a list of either
# (name, label) or (refData, label) pairs
if len(argDict["editAreas"]) == 0:
dfEditAreas = self.__ut.set(forecastDef, "defaultEditAreas", [])
try: # Try the varDict
chosenAreas = varDict["Choose Edit Areas"]
# Turn the list of chosen areas into (name, label) pairs
# using the defaultEditAreas list
dfEditAreas = self.__pairAreaWithLabel(chosenAreas,
dfEditAreas)
except:
pass
# Set up edit areas as ReferenceData's for AreaLabel and LatLon areas
if type(dfEditAreas) == types.StringType:
editAreaType = "Combinations"
# Get edit areas from file with format:
# Combinations = [
# ([editArea1, editArea2,...],label)
# ...
# ]
# For example:
# Combinations = [
# (["Zones48", "Zones49", "Zones50"],"/48/49/50"),
# (["Zones37","Zones38"], "/37/38"),"/37/38"),
# (["Zones57","Zones58","Zones59"],"57/58/59")
# ]
comboName = dfEditAreas
for retryCount in xrange(MAX_TRIES):
accessor = ModuleAccessor.ModuleAccessor()
dfEditAreas = accessor.variable(comboName, "Combinations")
if dfEditAreas is None:
if sys.modules.has_key(comboName):
comboMod = sys.modules[comboName]
if comboMod.__file__.endswith(".pyo"):
os.remove(comboMod.__file__)
comboMod = None
del sys.modules[comboName]
# if not last try, log and try again
if retryCount < MAX_TRIES - 1:
# log but don't pop up
self.log.error(
"Error loading combinations file: %s, retrying",
comboName)
else:
return "COMBINATION FILE NOT FOUND: " + \
self.__ut.set(forecastDef, "defaultEditAreas", [])
else:
break
elif len(dfEditAreas) > 0:
refDataList = []
tempRefData = []
for area, label in dfEditAreas:
if type(area) is types.TupleType:
# Create a referenceData given lat, lon, dim
refData = self.__createArea(area, argDict)
tempRefData.append(refData)
else: # Get named Reference Data
refId = ReferenceID(area)
refData = self.getEditArea(area, argDict)
if refData is None:
return "EDIT AREA NOT FOUND: " + str(refId)
refDataList.append((refData, label))
argDict["editAreas"] = refDataList
storeReferenceData(self.dataMgr.getRefManager(), tempRefData)
# Set up HazardsTable
# Product must be:
# --Type "smart"
# --Have an "filterMethod" method
if fcstType == "smart":
product = module.TextProduct()
# Test Code: Uncomment to test
#allowedHazards = product.allowedHazards()
#filterMethod = product.filterMethod
#print("allowedHazards", allowedHazards)
try:
allowedHazards = product.allowedHazards()
filterMethod = product.filterMethod
except:
allowedHazards = None
if allowedHazards is not None and allowedHazards != []:
# Set up editAreas as a list of combinations
# Cases:
# lat/lon or (area, label) pairs -- call HazardsTable,
# but the edit areas will not change
# Combinations -- call HazardsTable and check for changed combinations
# Set up edit areas as list of lists
editAreas = []
for area, label in dfEditAreas:
if type(area) is types.ListType:
editAreas.append(area)
elif type(area) is types.TupleType: #LatLon
editAreas.append([self.__getLatLonAreaName(area)])
else:
editAreas.append([area])
# if Definition['separateByTimeZone'] set to "effectiveTZ"
# or "actualTZ", change the set of edit areas to ensure
# that time zones are same in each grouping.
separateByTZ = product.Definition.get('separateByTimeZone',
None)
if separateByTZ is not None:
areaDictName = product.Definition.get(
'areaDictionary', "AreaDictionary")
editAreas = self._separateByTimeZone(
editAreas,
areaDictName,
argDict['creationTime'],
effectiveTZ=separateByTZ)
accurateCities = product.Definition.get('accurateCities', 0)
cityRefData = []
if accurateCities:
cityLocationName = product.Definition.get(
'cityLocation', "CityLocation")
accessor = ModuleAccessor.ModuleAccessor()
citydict = accessor.variable(cityLocationName,
"CityLocation")
cityEAs = []
if citydict is None:
msg = "CityLocation dictionary module was not found for"\
" city location:"
self.log.error(msg + repr(cityLocationName))
else:
for ea in editAreas:
for ean in ea:
if ean not in citydict:
msg = "CityLocation dictionary does not "\
"contain entry for edit area: "
self.log.error(msg + repr(ean))
continue
for city, llrec in citydict[ean].iteritems():
# Create a referenceData given lat, lon, dim
area = (llrec[0], llrec[1], 0)
refData = self.__createArea(area, argDict)
cityEAs.append(refData)
cityRefData.append((refData, city))
# Store temporary reference data in the server
storeReferenceData(self.dataMgr.getRefManager(), cityEAs)
# Get Product ID and other info for HazardsTable
pil = self.__ut.set(forecastDef, "pil", None)
stationID4 = product.Definition['fullStationID']
productCategory = pil[0:3] #part of the pil
sampleThreshold = product.Definition.get(\
"hazardSamplingThreshold", (10, None))
# Process the hazards
import HazardsTable
hazards = HazardsTable.HazardsTable(
argDict["ifpClient"],
editAreas,
productCategory,
filterMethod,
argDict["databaseID"],
stationID4,
argDict["vtecActiveTable"],
argDict["vtecMode"],
sampleThreshold,
creationTime=argDict["creationTime"],
dataMgr=self.dataMgr,
accurateCities=accurateCities,
cityEditAreas=cityRefData)
# Store hazards object for later use
argDict["hazards"] = hazards
# Get revised combinations
if editAreaType == "Combinations":
reorganizeCombos = product.Definition.get(
"reorganizeCombinations", 1)
if reorganizeCombos:
hazardAreas = hazards.getHazardAreaCombinations()
# Add a bogus label
newCombos = []
for combo in hazardAreas:
newCombos.append((combo, ""))
# Re-assign new combinations
dfEditAreas = newCombos
# Set up Combinations as ReferenceDatas
if editAreaType == "Combinations":
argDict["editAreas"], dfEditAreas = self.getCombinations(
dfEditAreas, argDict)
argDict["combinations"] = dfEditAreas
# Set up Time Ranges
# May be from these sources:
# varDict from command line
# defaultTimeRanges
# command line as named time ranges
# command line as start and end times OR
# from argDict already set up by Interfaces::generateProduct
# In these cases "useRawTR" will be set to 1
if len(argDict["timeRanges"]) > 0:
# Use named time ranges from command line
dfRanges = argDict["timeRanges"]
else:
try: # Try the varDict
dfRanges = varDict["Choose Time Ranges"]
except:
dfRanges = self.__ut.set(forecastDef, "defaultRanges", [])
argDict["timeRanges"] = dfRanges
rawRanges = []
argDict["rawRanges"] = rawRanges
if argDict["useRawTR"] == 1:
tr = argDict["timeRange"]
try:
trName = argDict["timeRangeName"]
except:
trName = ""
if tr is not None:
rawRanges.append((tr, trName))
elif len(dfRanges) == 0:
pass
else:
import TimeRangeUtils
forecast = TimeRangeUtils.TimeRangeUtils()
for rangeName in dfRanges:
rawRange = forecast.getTimeRange(rangeName, argDict)
rawRanges.append((rawRange, rangeName))
argDict["rawRanges"] = rawRanges
#print("rawRanges", rawRanges)
# Row Label
areaType = self.__ut.set(forecastDef, "areaType", "")
rowVariable = self.__ut.set(forecastDef, "rowVariable", "EditArea")
if rowVariable == "EditArea":
rowLabel = areaType
elif rowVariable == "WeatherElement":
rowLabel = "Weather Element"
else:
rowLabel = "Time Period"
argDict["heading"] = rowLabel
def executeIscExtract(parmNames, databaseName, startTime, endTime,
irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost,
ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None):
startT = time.time()
parms = parmNames
dbid = databaseName
startTR = startTime
endTR = endTime
xmlDestinations = destinations
ancf = irtTableAddressA
bncf = irtTableAddressB
xmtScript = transmitScript
serverHost = ourServerHost
serverPort = ourServerPort
serverProtocol = ourServerProtocol
mhsid = ourMHSid
siteID = ourSiteID
myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType()
#--------------------------------------------------------------------
# determine the list of destination servers
#--------------------------------------------------------------------
try:
nowT = time.time() #current time
useUntilTime = None #cached use until time
cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract"
cachedXmlDestinations = None
#if xmlDestinations is None: #destinations not on command line
# # check the cache
# try:
# fd = open(cacheFilename, 'rb')
# buf = fd.read()
# fd.close()
# useUntilTime, cachedXmlDestinations = cPickle.loads(buf)
# nowT = time.time() #current time
# if nowT > useUntilTime:
# xmlDestinations = None #cache is too old
# useUntilTime = None
# else:
# logEvent('Using xmlDestinations cache')
# xmlDestinations = cachedXmlDestinations
# except:
# pass
# need to contact IRT to get destinations
irt = IrtAccess.IrtAccess(ancf, bncf)
if xmlDestinations is None:
logEvent('contacting IRT to get destinations')
count = 1
while True:
status, xmlDestinations = irt.getSendAddrs(siteID)
logEvent('IRT getSendAddrs status:', status)
if status:
# if we obtained XML destinations from IRT, then decode
# the useUntilTime field
try:
d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
dE = d.getroot()
for e in dE:
if e.tag == "useuntil":
isoTimeStr = e.text
idx = isoTimeStr.find(".")
if idx != - 1:
isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds
useUntilTime = time.mktime(time.strptime(isoTimeStr,
"%Y-%m-%dT%H:%M:%S"))
logEvent("Use Until: ", isoTimeStr)
except:
logProblem("Malformed XML on getSendAddrs()")
logProblem("XML=", xmlDestinations)
return
if useUntilTime is None:
useUntilTime = time.time() + 180.0 #3 minutes default
logEvent("Using default 180 second useUntilTime")
# write the cache
fd = open(cacheFilename, 'wb')
buf = cPickle.dumps((useUntilTime, xmlDestinations))
fd.write(buf)
fd.close()
break #success from the irt
else:
# try again and again for 10 minutes, then use cache
# if available and alert GFE users
if time.time() - nowT > 600.00:
logProblem("Unable to access IRT for send addrs")
if cachedXmlDestinations is None:
s = "Unable to access IRT for send addrs. Previous" + \
" cache not available."
logProblem(s)
return
# use cached value, even if out of date
else:
xmlDestinations = cachedXmlDestinations
if useUntilTime is not None:
s = time.asctime(time.gmtime(useUntilTime))
else:
s = "Unknown"
logProblem("Using expired cache. Date=", s)
#determine when we issued our last GFE alert
#we alert every 30 minutes.
try:
fd = open(cacheFilename + "-warn", 'rb')
buf = fd.read()
fd.close()
lastAlertTime = cPickle.loads(buf)
except:
lastAlertTime = 0 #for way long ago
if time.time() - lastAlertTime > 1800.0:
logProblem("Sending GFE notification")
msg = """
Contact NCF. ifpServer is unable to contact IRT central server. ISC
traffic routing information is old and possibly incorrect."""
os.system("sendGfeMessage -u -c GFE -m '" + \
msg + "'")
fd = open(cacheFilename + "-warn", 'wb')
fd.write(cPickle.dumps(time.time()))
fd.close()
break
time.sleep(15.0) #sleep awhile and then try again
count = count + 1
logProblem("Retrying to getSendAddrs()", count)
# qc the XML
try:
destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
destE = destTree.getroot()
except:
logProblem("Malformed XML on getSendAddrs() or provided xmlDest")
logProblem("XML=", xmlDestinations)
return
#--------------------------------------------------------------------
# determine how many transmissions are necessary
#--------------------------------------------------------------------
xmt = []
logEvent("XML dest:", xmlDestinations)
if destE.tag != "destinations":
logProblem("Destinations packet missing from web service")
return
# create list of individual transmissions (before attempting to combine
doClip = 1 #0 to send entire domain, 1 to do destination clipping (default)
destStr = "Destination Servers:\n"
for addressE in destE:
if addressE.tag == "doclip":
for name, value in addressE.items():
if name == "clip":
if value == "1":
doClip = 1
elif value == "0":
doClip = 0
logEvent("Clipping State: ", doClip)
for addressE in destE:
if addressE.tag != "address":
continue
# find destination server info and domain information
serverInfo = irt.decodeXMLAddress(addressE)
if doClip == 0:
serverInfo['domain'] = None
keycheckfail = False
for key in ['mhsid', 'host', 'port', 'protocol', 'site']:
if not serverInfo.has_key(key):
logProblem("Fail to decode XML. Skipping serverInfo:",
serverInfo)
keycheckfail = True
continue
if keycheckfail:
continue #skipping this destination due to insufficient info
# get the destination office type
try:
siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site'])
destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex))
except:
logProblem("Unknown site id to get office type. ",
"Skipping serverInfo:", serverInfo)
continue #skipping this destination due to unknown site id
# find weather elements that remote ifpServer wants
# that is available in our server and in the -p parm switches
any = False
for parm in serverInfo['parms']:
p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists
# translation of parm name needed, also if no office type, then
# not wanted from this office.
# example: changes QPFwfo to QPF if we are wfo
# example: discards T if we are wfo and site is non-wfo
if myOfficeType != destOfficeType:
if p1.find(myOfficeType) != - 1:
p1 = string.replace(p1, myOfficeType, "") #remove type
else:
continue #no type, so not intended for our type
# see if parm was listed in the command line switches
if parms.contains(p1):
xmt.append({'serverInfo':[serverInfo], 'parms':[p1],
'domain': serverInfo['domain'], 'area': serverInfo['area']})
if not any:
destStr += irt.printServerInfo(serverInfo) + "\n"
any = True
logEvent(destStr)
# now combine transmissions
# find same domains, same parms, to combine servers/destinations
i = 0
while i < len(xmt):
j = i + 1
while j < len(xmt):
if xmt[i]['domain'] == xmt[j]['domain'] and \
xmt[i]['area'] == xmt[j]['area'] and \
xmt[i]['parms'] == xmt[j]['parms']:
for si in xmt[j]['serverInfo']:
if si not in xmt[i]['serverInfo']:
dests = xmt[i]['serverInfo']
dests.append(si)
xmt[j]['serverInfo'] = dests
del xmt[j] #delete the entry
j = j - 1 #redo this entry index next loop
j = j + 1
i = i + 1
# now try to combine common parm lists (same domain, same servers/destinations)
i = 0
while i < len(xmt):
j = i + 1
while j < len(xmt):
if xmt[i]['domain'] == xmt[j]['domain'] and \
xmt[i]['area'] == xmt[j]['area'] and \
xmt[i]['serverInfo'] == xmt[j]['serverInfo'] :
iparms = xmt[i]['parms']
for p in xmt[j]['parms']:
if p not in iparms:
iparms.append(p)
xmt[i]['parms'] = iparms
del xmt[j] #delete the entry
j = j - 1 #redo this entry index for next loop
j = j + 1
i = i + 1
# if doClip, gather some required information
if doClip:
#get the isc send area and grid domain from the ifpServer
iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID)
sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain()
iscSendAreaGrid.setGloc(sourceDomain)
iscSendAreaGrid = iscSendAreaGrid.getGrid()
#--------------------------------------------------------------------
# prepare output files
#--------------------------------------------------------------------
for dest in xmt:
s = "Processing Xmt Pass:\n"
for sv in dest['serverInfo']:
s += irt.printServerInfo(sv) + '\n'
s += "Domain:" + `dest['domain']` + '\n'
s += "Area:" + `dest['area']` + '\n'
s += "Parms:" + `dest['parms']` + '\n\n'
logEvent(s)
# extract the data using ifpnetCDF
if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False:
os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC")
tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC"
fname = tempfile.mktemp(".isc")
# Determine domain edit area.
if doClip == 1 and dest['domain'] is not None and \
dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID():
#make a GridLocation for our domain
gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy())))
origin = sourceDomain.getOrigin()
extent = sourceDomain.getExtent()
domain = CartDomain2D(origin, extent)
gloc = sourceDomain
#make a GridLocation covering the area for the destination, expanded
#by 1/2 grid cell
dd = dest['domain']
da = dest['area']
cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0)
cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0)
originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0,
float(dd['origy']) - cellsizeY / 2.0)
extentD = Coordinate(float(dd['extx']) + cellsizeX,
float(dd['exty']) + cellsizeY)
domainD = CartDomain2D(originD, extentD)
#check for overlap
if not domainD.overlaps(domain):
logEvent("No intersection of domain box, skipping....")
continue #no bits set in the resulting mask, no intersect
domainD.trim(domain) #trim it to just the overlapping section
gridSize = Point(int(da['xdim']),int(da['ydim']))
destGridLocation = GridLocation("Dest",sourceDomain.getProjection(),
gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT")
# make a Reference Set
refid = ReferenceID("jibberish")
refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON)
# convert destination site's domain to gridpoints
iscMask = refSet.getGrid()
# "and" it with our ISC_Send_Area
iscMask.andEquals(iscSendAreaGrid)
if not iscMask.isAnyBitsSet():
logEvent("No intersection of domain points, skipping....")
continue #no bits set in the resulting mask, no intersect
# store the grid back into the ifpServer
maskName = "iscExtract" + `time.time()`
refSet.setGrid(iscMask)
iscUtil.saveEditAreaGrid(maskName, refSet, siteID)
else: #no clipping, or different projection
maskName = "ISC_Send_Area"
# Run ifpnetCDF for the data
argv = {"outputFilename": fname,
"parmList": dest['parms'],
"databaseID": dbid,
"startTime": startTR,
"endTime": endTR,
"mask": maskName,
"geoInfo": False,
"compressFileFlag": True,
"configFileName": "iscSendSampleDef",
"compressFileFactor": 6,
"trim": True,
"krunch": True,
"userID": "iscExtract",
"logFileName": None,
"siteIdOverride": None}
ifpnetCDF.main(**argv)
fname = fname + '.gz'
size = os.stat(fname)[stat.ST_SIZE]
endT = time.time()
logEvent('File Size: ', size)
logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT),
',cputime:', "%-6.2f" % time.clock())
# create XML destinations file for this output
iscE = Element('isc') #create the XML tree root
sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort,
'protocol': serverProtocol, 'site': siteID}
irt.addSourceXML(iscE, sourceServer)
irt.addDestinationXML(iscE, dest['serverInfo'])
#get the unique list of mhs sites
mhsSites = []
for si in dest['serverInfo']:
if si['mhsid'] not in mhsSites:
mhsSites.append(si['mhsid'])
# create the XML file
fnameXML = tempfile.mktemp(".xml")
fd = open(fnameXML, 'wb')
fd.write(ElementTree.tostring(iscE))
fd.close()
# Transmit files - do string substitution
irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript)
# Delete temporary files
if maskName != "ISC_Send_Area":
iscUtil.deleteEditArea(maskName,siteID)
endT = time.time()
logEvent('After transmission pass, ,wctime:',
"%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock())
except:
logProblem("Failure", traceback.format_exc())
def getEditArea(self, name):
rs = self._refmgr.loadRefSet(ReferenceID(name))
if rs.isQuery():
return rs.getQuery()
return rs.getGrid().getNDArray()