def getStatDict(self, sampler, analysisList, timeRange, area, componentName=None):
# Get Statistic dictionary for given analysisList, timeRange,
# and area
# Area can be ReferenceData or text string
# Convert area to ID if necessary
if area is None:
return None
if isinstance(area, str):
area = ReferenceID(area)
else:
area = area.getId()
return Analysis.Analysis(sampler).createStats(
analysisList, area, timeRange, componentName)
def getUnion(self, argDict, areaLabels, areaPrefix):
GridLoc = self.getIFPClient().getDBGridLocation()
area = None
for areaLabel in areaLabels:
newArea = self.getEditArea(areaLabel, argDict)
if areaLabels.index(areaLabel) == 0:
comboNumber = self.getComboNumber()
label = areaPrefix + ` int(time.time()) ` + ` comboNumber `
refId = ReferenceID(label)
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
CoordinateType = ReferenceData.CoordinateType
coordType = CoordinateType.valueOf('LATLON')
area = ReferenceData(GridLoc, refId,
newArea.getPolygons(coordType), coordType)
# randerso: I don't think this is necessary
# area.convertToAWIPS()
area = self.unionAreas(label, area, newArea)
return area
def getSampler(self, argDict, sampleInfo, sampleFromServer=0):
# Get a HistoSampler given
# sampleInfo, which is a list of tuples, or just a single tuple
# of tuples ([elements], [periods], [areas])
# the elements are [(name, method)] -- basically the analysis list
# the periods [(timeRange, label)]
# areas [(name,label)] or [(refData, label)] or [(refID, label)]
ifpClient = argDict["ifpClient"]
databaseID = argDict["databaseID"]
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceID, ReferenceData
from com.raytheon.viz.gfe.sampler import SamplerRequest, HistoSampler
from java.util import ArrayList
# convert input sampleInfo to samplerRequests
samplerRequests = ArrayList()
if type(sampleInfo) == tuple:
sampleInfo = [sampleInfo]
for si in sampleInfo:
elements, periods, areas = si
for e in elements:
parmID = self.getParmID(e[0], databaseID)
for p in periods:
for a in areas:
if type(a[0]) is str:
samplerRequests.add(SamplerRequest( \
parmID, ReferenceID(a[0]), p[0].toJavaObj()))
elif str(type(a[0])) == "<type 'PyJobject'>":
samplerRequests.add(SamplerRequest( \
parmID, a[0], p[0].toJavaObj()))
else:
raise Exception, "area specification incorrect"
# do sampling
if sampleFromServer:
sampler = ifpClient.sampleRequest(samplerRequests)
else:
sampler = HistoSampler(ifpClient.getJavaClient(), samplerRequests)
if sampler.isValid() != 1:
print "Cannot Sample: Check for invalid Weather Elements, ",\
"Invalid Areas", str(samplerRequests)
return None
#print "sampler ", sampler
return sampler
def __createArea(self, latLonTuple, argDict):
# Return a ReferenceData created for the given lat,lon and dimension
# If dim is zero, make edit area of the one grid
# point closest to the lat/lon value.
lat, lon, dim = latLonTuple
name = self.__getLatLonAreaName((lat, lon, dim))
#print("\ncreateArea", lat, lon, dim, name)
if dim != 0:
for x in range(100):
points = makeSquare(lat, lon, dim)
pointList = []
for point in points:
pointList.append(makePoint(point))
refData = makeArea(self.ifpClient.getDBGridLocation(),
pointList,
refname=name)
# Make sure we have at least one grid point in
# the edit area
if refData.getGrid().isAnyBitsSet():
#print("returning", dim)
return refData
# Increment dim and try again
#print("iterating", dim)
dim += 0.25
msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n"
self.log.warning(msg)
return None
else:
from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit
# Get grid cell coordinates for lat/lon
gridLoc = self.ifpClient.getDBGridLocation()
nx = int(gridLoc.getNx())
ny = int(gridLoc.getNy())
cc2D = gridLoc.gridCell(float(lat), float(lon))
# convert grid cell to Grid2DBit with single bit set
grid2Dbit = Grid2DBit(nx, ny)
if (nx > cc2D.x >= 0 and ny > cc2D.y >= 0):
grid2Dbit.set(int(cc2D.x), int(cc2D.y))
#refData = GridLoc.convertToReferenceData(grid2Dbit)
refID = ReferenceID(name)
refData = ReferenceData(gridLoc, refID, grid2Dbit)
return refData
def getEditAreas(self, argDict, areas):
# Get the ReferenceArea given
# a name or ReferenceID or ReferenceData(just return)
refDataList = []
for area in areas:
if type(area) is str: # Get area from name
id = ReferenceID(area)
jlist = ArrayList()
jlist.add(id)
tmp = argDict["ifpClient"].getReferenceData(jlist)
refDataList.append(tmp.get(0))
else:
if str(area).find('Id') > -1:
refDataList.append(area)
else:
jlist = ArrayList()
jlist.add(id)
tmp = argDict["ifpClient"].getReferenceData(jlist)
refDataList.append(tmp.get(0))
return refDataList
def __init__(self,
imageWidth=None,
imageHeight=None,
expandLeft=25.0,
expandRight=25.0,
expandTop=25.0,
expandBottom=25.0,
mask=None,
wholeDomain=0,
bgColor=None):
# Create empty display and data manager for display
display = GFEMapRenderableDisplay()
self.dataMgr = DataManagerOffscreenFactory.getInstance(display)
self.refId = None
envelope = None
gloc = self.dataMgr.getParmManager().compositeGridLocation()
if mask is not None:
from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData
CoordinateType = ReferenceData.CoordinateType
self.refId = ReferenceID(mask)
if wholeDomain == 0:
envelope = self.dataMgr.getRefManager().loadRefSet(
self.refId).overallDomain(CoordinateType.GRID)
if imageWidth is not None:
imageWidth = Integer(int(imageWidth))
if imageHeight is not None:
imageHeight = Integer(int(imageHeight))
geom = GfeImageUtil.getLocationGeometry(
gloc, envelope, imageWidth, imageHeight, expandLeft / 100.0,
expandRight / 100.0, expandTop / 100.0, expandBottom / 100.0)
# Create descriptor for display
desc = MapDescriptor(geom)
display.setDescriptor(desc)
VizPainter.VizPainter.__init__(self, display, backgroundColor=bgColor)
gfeSystem = GFESystemResource(self.dataMgr)
self.addVizResource(gfeSystem)
desc.getResourceList().getProperties(gfeSystem).setSystemResource(True)
self.primaryRsc = None
def makeArea(gridLoc, pointList, refname=None):
" Make a Reference Area with a unique ReferenceID"
geomFactory = GeometryFactory()
import jep
size = len(pointList)
if pointList[0] != pointList[size - 1]: # closing the loop
pointList.append(pointList[0])
pointArray = jep.jarray(len(pointList), Coordinate)
for i in range(len(pointList)):
pointArray[i] = pointList[i]
lr = geomFactory.createLinearRing(pointArray)
poly = geomFactory.createPolygon(lr, jep.jarray(0, LinearRing))
polyArray = jep.jarray(1, Polygon)
polyArray[0] = poly
region = geomFactory.createMultiPolygon(polyArray)
if refname is None:
refname = "Ref" + getTime()
refId = ReferenceID(refname)
refData = ReferenceData(gridLoc, refId, region, CoordinateType.LATLON)
# randerso: I don't think this is necessary
# refData.convertToAWIPS()
return refData
def createLatLonArea(self, lat, lon, dim):
# Create a square edit area given a latitude, longitude,
# and kilometer dimension for the sides.
# Example:
# area = self.createLatLonArea(40.93, -106.26, 5)
#
# If dim is zero, make edit area of the one grid
# point closest to the lat/lon value.
#
name = self.getLatLonAreaName((lat, lon, dim))
if dim != 0:
for x in range(100):
points = self.makeSquare(lat, lon, dim)
pointList = []
for point in points:
pointList.append(self.makePoint(point))
refData = self.makeArea(pointList, refname=name)
# Make sure we have at least one grid point in
# the edit area
if refData.getGrid().isAnyBitsSet():
return refData
# Increment dim and try again
dim += 0.25
msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n"
self.log.warning(msg)
return None
else:
# Get grid cell coordinates for lat/lon
gridLoc = self.getGridLoc()
cc2D = gridLoc.gridCell(lat, lon)
# convert grid cell to Grid2DBit with single bit set
grid2Dbit = JavaGrid2DBit(gridLoc.gridSize().x,
gridLoc.gridSize().y)
grid2Dbit.set(int(cc2D.x), int(cc2D.y))
#refData = gridLoc.convertToReferenceData(grid2Dbit)
refID = ReferenceID(name)
refData = ReferenceData(gridLoc, refID, grid2Dbit)
#refData.setId(refID)
return refData
def executeIscExtract(parmNames, databaseName, startTime, endTime,
irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost,
ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None):
startT = time.time()
parms = parmNames
dbid = databaseName
startTR = startTime
endTR = endTime
xmlDestinations = destinations
ancf = irtTableAddressA
bncf = irtTableAddressB
xmtScript = transmitScript
serverHost = ourServerHost
serverPort = ourServerPort
serverProtocol = ourServerProtocol
mhsid = ourMHSid
siteID = ourSiteID
myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType()
#--------------------------------------------------------------------
# determine the list of destination servers
#--------------------------------------------------------------------
try:
nowT = time.time() #current time
useUntilTime = None #cached use until time
cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract"
cachedXmlDestinations = None
#if xmlDestinations is None: #destinations not on command line
# # check the cache
# try:
# fd = open(cacheFilename, 'rb')
# buf = fd.read()
# fd.close()
# useUntilTime, cachedXmlDestinations = cPickle.loads(buf)
# nowT = time.time() #current time
# if nowT > useUntilTime:
# xmlDestinations = None #cache is too old
# useUntilTime = None
# else:
# logEvent('Using xmlDestinations cache')
# xmlDestinations = cachedXmlDestinations
# except:
# pass
# need to contact IRT to get destinations
irt = IrtAccess.IrtAccess(ancf, bncf)
if xmlDestinations is None:
logEvent('contacting IRT to get destinations')
count = 1
while True:
status, xmlDestinations = irt.getSendAddrs(siteID)
logEvent('IRT getSendAddrs status:', status)
if status:
# if we obtained XML destinations from IRT, then decode
# the useUntilTime field
try:
d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
dE = d.getroot()
for e in dE:
if e.tag == "useuntil":
isoTimeStr = e.text
idx = isoTimeStr.find(".")
if idx != - 1:
isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds
useUntilTime = time.mktime(time.strptime(isoTimeStr,
"%Y-%m-%dT%H:%M:%S"))
logEvent("Use Until: ", isoTimeStr)
except:
logProblem("Malformed XML on getSendAddrs()")
logProblem("XML=", xmlDestinations)
return
if useUntilTime is None:
useUntilTime = time.time() + 180.0 #3 minutes default
logEvent("Using default 180 second useUntilTime")
# write the cache
fd = open(cacheFilename, 'wb')
buf = cPickle.dumps((useUntilTime, xmlDestinations))
fd.write(buf)
fd.close()
break #success from the irt
else:
# try again and again for 10 minutes, then use cache
# if available and alert GFE users
if time.time() - nowT > 600.00:
logProblem("Unable to access IRT for send addrs")
if cachedXmlDestinations is None:
s = "Unable to access IRT for send addrs. Previous" + \
" cache not available."
logProblem(s)
return
# use cached value, even if out of date
else:
xmlDestinations = cachedXmlDestinations
if useUntilTime is not None:
s = time.asctime(time.gmtime(useUntilTime))
else:
s = "Unknown"
logProblem("Using expired cache. Date=", s)
#determine when we issued our last GFE alert
#we alert every 30 minutes.
try:
fd = open(cacheFilename + "-warn", 'rb')
buf = fd.read()
fd.close()
lastAlertTime = cPickle.loads(buf)
except:
lastAlertTime = 0 #for way long ago
if time.time() - lastAlertTime > 1800.0:
logProblem("Sending GFE notification")
msg = """
Contact NCF. ifpServer is unable to contact IRT central server. ISC
traffic routing information is old and possibly incorrect."""
os.system("sendGfeMessage -u -c GFE -m '" + \
msg + "'")
fd = open(cacheFilename + "-warn", 'wb')
fd.write(cPickle.dumps(time.time()))
fd.close()
break
time.sleep(15.0) #sleep awhile and then try again
count = count + 1
logProblem("Retrying to getSendAddrs()", count)
# qc the XML
try:
destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations))
destE = destTree.getroot()
except:
logProblem("Malformed XML on getSendAddrs() or provided xmlDest")
logProblem("XML=", xmlDestinations)
return
#--------------------------------------------------------------------
# determine how many transmissions are necessary
#--------------------------------------------------------------------
xmt = []
logEvent("XML dest:", xmlDestinations)
if destE.tag != "destinations":
logProblem("Destinations packet missing from web service")
return
# create list of individual transmissions (before attempting to combine
doClip = 1 #0 to send entire domain, 1 to do destination clipping (default)
destStr = "Destination Servers:\n"
for addressE in destE:
if addressE.tag == "doclip":
for name, value in addressE.items():
if name == "clip":
if value == "1":
doClip = 1
elif value == "0":
doClip = 0
logEvent("Clipping State: ", doClip)
for addressE in destE:
if addressE.tag != "address":
continue
# find destination server info and domain information
serverInfo = irt.decodeXMLAddress(addressE)
if doClip == 0:
serverInfo['domain'] = None
keycheckfail = False
for key in ['mhsid', 'host', 'port', 'protocol', 'site']:
if not serverInfo.has_key(key):
logProblem("Fail to decode XML. Skipping serverInfo:",
serverInfo)
keycheckfail = True
continue
if keycheckfail:
continue #skipping this destination due to insufficient info
# get the destination office type
try:
siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site'])
destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex))
except:
logProblem("Unknown site id to get office type. ",
"Skipping serverInfo:", serverInfo)
continue #skipping this destination due to unknown site id
# find weather elements that remote ifpServer wants
# that is available in our server and in the -p parm switches
any = False
for parm in serverInfo['parms']:
p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists
# translation of parm name needed, also if no office type, then
# not wanted from this office.
# example: changes QPFwfo to QPF if we are wfo
# example: discards T if we are wfo and site is non-wfo
if myOfficeType != destOfficeType:
if p1.find(myOfficeType) != - 1:
p1 = string.replace(p1, myOfficeType, "") #remove type
else:
continue #no type, so not intended for our type
# see if parm was listed in the command line switches
if parms.contains(p1):
xmt.append({'serverInfo':[serverInfo], 'parms':[p1],
'domain': serverInfo['domain'], 'area': serverInfo['area']})
if not any:
destStr += irt.printServerInfo(serverInfo) + "\n"
any = True
logEvent(destStr)
# now combine transmissions
# find same domains, same parms, to combine servers/destinations
i = 0
while i < len(xmt):
j = i + 1
while j < len(xmt):
if xmt[i]['domain'] == xmt[j]['domain'] and \
xmt[i]['area'] == xmt[j]['area'] and \
xmt[i]['parms'] == xmt[j]['parms']:
for si in xmt[j]['serverInfo']:
if si not in xmt[i]['serverInfo']:
dests = xmt[i]['serverInfo']
dests.append(si)
xmt[j]['serverInfo'] = dests
del xmt[j] #delete the entry
j = j - 1 #redo this entry index next loop
j = j + 1
i = i + 1
# now try to combine common parm lists (same domain, same servers/destinations)
i = 0
while i < len(xmt):
j = i + 1
while j < len(xmt):
if xmt[i]['domain'] == xmt[j]['domain'] and \
xmt[i]['area'] == xmt[j]['area'] and \
xmt[i]['serverInfo'] == xmt[j]['serverInfo'] :
iparms = xmt[i]['parms']
for p in xmt[j]['parms']:
if p not in iparms:
iparms.append(p)
xmt[i]['parms'] = iparms
del xmt[j] #delete the entry
j = j - 1 #redo this entry index for next loop
j = j + 1
i = i + 1
# if doClip, gather some required information
if doClip:
#get the isc send area and grid domain from the ifpServer
iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID)
sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain()
iscSendAreaGrid.setGloc(sourceDomain)
iscSendAreaGrid = iscSendAreaGrid.getGrid()
#--------------------------------------------------------------------
# prepare output files
#--------------------------------------------------------------------
for dest in xmt:
s = "Processing Xmt Pass:\n"
for sv in dest['serverInfo']:
s += irt.printServerInfo(sv) + '\n'
s += "Domain:" + `dest['domain']` + '\n'
s += "Area:" + `dest['area']` + '\n'
s += "Parms:" + `dest['parms']` + '\n\n'
logEvent(s)
# extract the data using ifpnetCDF
if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False:
os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC")
tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC"
fname = tempfile.mktemp(".isc")
# Determine domain edit area.
if doClip == 1 and dest['domain'] is not None and \
dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID():
#make a GridLocation for our domain
gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy())))
origin = sourceDomain.getOrigin()
extent = sourceDomain.getExtent()
domain = CartDomain2D(origin, extent)
gloc = sourceDomain
#make a GridLocation covering the area for the destination, expanded
#by 1/2 grid cell
dd = dest['domain']
da = dest['area']
cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0)
cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0)
originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0,
float(dd['origy']) - cellsizeY / 2.0)
extentD = Coordinate(float(dd['extx']) + cellsizeX,
float(dd['exty']) + cellsizeY)
domainD = CartDomain2D(originD, extentD)
#check for overlap
if not domainD.overlaps(domain):
logEvent("No intersection of domain box, skipping....")
continue #no bits set in the resulting mask, no intersect
domainD.trim(domain) #trim it to just the overlapping section
gridSize = Point(int(da['xdim']),int(da['ydim']))
destGridLocation = GridLocation("Dest",sourceDomain.getProjection(),
gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT")
# make a Reference Set
refid = ReferenceID("jibberish")
refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON)
# convert destination site's domain to gridpoints
iscMask = refSet.getGrid()
# "and" it with our ISC_Send_Area
iscMask.andEquals(iscSendAreaGrid)
if not iscMask.isAnyBitsSet():
logEvent("No intersection of domain points, skipping....")
continue #no bits set in the resulting mask, no intersect
# store the grid back into the ifpServer
maskName = "iscExtract" + `time.time()`
refSet.setGrid(iscMask)
iscUtil.saveEditAreaGrid(maskName, refSet, siteID)
else: #no clipping, or different projection
maskName = "ISC_Send_Area"
# Run ifpnetCDF for the data
argv = {"outputFilename": fname,
"parmList": dest['parms'],
"databaseID": dbid,
"startTime": startTR,
"endTime": endTR,
"mask": maskName,
"geoInfo": False,
"compressFileFlag": True,
"configFileName": "iscSendSampleDef",
"compressFileFactor": 6,
"trim": True,
"krunch": True,
"userID": "iscExtract",
"logFileName": None,
"siteIdOverride": None}
ifpnetCDF.main(**argv)
fname = fname + '.gz'
size = os.stat(fname)[stat.ST_SIZE]
endT = time.time()
logEvent('File Size: ', size)
logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT),
',cputime:', "%-6.2f" % time.clock())
# create XML destinations file for this output
iscE = Element('isc') #create the XML tree root
sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort,
'protocol': serverProtocol, 'site': siteID}
irt.addSourceXML(iscE, sourceServer)
irt.addDestinationXML(iscE, dest['serverInfo'])
#get the unique list of mhs sites
mhsSites = []
for si in dest['serverInfo']:
if si['mhsid'] not in mhsSites:
mhsSites.append(si['mhsid'])
# create the XML file
fnameXML = tempfile.mktemp(".xml")
fd = open(fnameXML, 'wb')
fd.write(ElementTree.tostring(iscE))
fd.close()
# Transmit files - do string substitution
irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript)
# Delete temporary files
if maskName != "ISC_Send_Area":
iscUtil.deleteEditArea(maskName,siteID)
endT = time.time()
logEvent('After transmission pass, ,wctime:',
"%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock())
except:
logProblem("Failure", traceback.format_exc())
def __getRunTimeVariables(self, fcstName, forecastDef, fcstType, module,
argDict):
# Input variables can come from various sources:
# varDict from command line
# command line switch e.g. -l language (will be in argDict)
# definition section of product
# We must check all three in that order.
varDict = argDict["varDict"]
#print("varDict", varDict)
for item, default in [
("language", "english"),
("appendFile", None),
("lineLength", 69), # no command line option
("timePeriod", 3),
]:
try: # Try the varDict
#print("trying varDict", item)
argDict[item] = varDict[item]
#print("got it in varDict")
except:
try: # Try argDict i.e. from command line switch
# If so, argDict is already set
#print("trying argDict", item)
argValue = argDict[item]
#print("got it in argDict", argValue)
except:
argValue = None
# Try
#print("getting from definition", item)
if argValue is None:
argDict[item] = self.__ut.set(forecastDef, item, default)
#print("value from definition", argDict[item])
# These need to be integers
for item in ["lineLength", "timePeriod"]:
if argDict[item] is not None:
argDict[item] = int(argDict[item])
# Edit Areas and Time Ranges
#
# Set up these argDict values:
# editAreas -- list of (refData, label) pairs
# timeRanges -- list of named ranges
# rawRanges -- list of (rawRange, rangeName) pairs
#
# As we eventually loop through the product, these values will be set:
# editArea -- current edit area pair
# timeRange -- current raw time range
# issueTime -- first raw time range
# timeRangeName -- current time range name (if available)
# combinations -- list of (editAreaList, comboLabel) tuples
# where editAreaList is the list of edit areas composing
# the combination
# Set up Edit Areas
# May be from these sources:
# "AreaLabel"
# varDict from command line -- list of names
# command line as named reference areas
# defaultEditAreas
# list of (name, label) pairs
# "LatLon"
# OR list of ((lat,lon,dimension), label)
# "Combinations"
# OR A Combinations file
# We may have had edit areas entered from the command line
# or from the Interfaces "generateForecast" command
# If so, argDict["editAreas"] will be a list of either
# (name, label) or (refData, label) pairs
editAreaType = "AreaLabel"
# We may have had edit areas entered from the command line
# If so, argDict["editAreas"] will be a list of either
# (name, label) or (refData, label) pairs
if len(argDict["editAreas"]) == 0:
dfEditAreas = self.__ut.set(forecastDef, "defaultEditAreas", [])
try: # Try the varDict
chosenAreas = varDict["Choose Edit Areas"]
# Turn the list of chosen areas into (name, label) pairs
# using the defaultEditAreas list
dfEditAreas = self.__pairAreaWithLabel(chosenAreas,
dfEditAreas)
except:
pass
# Set up edit areas as ReferenceData's for AreaLabel and LatLon areas
if type(dfEditAreas) == types.StringType:
editAreaType = "Combinations"
# Get edit areas from file with format:
# Combinations = [
# ([editArea1, editArea2,...],label)
# ...
# ]
# For example:
# Combinations = [
# (["Zones48", "Zones49", "Zones50"],"/48/49/50"),
# (["Zones37","Zones38"], "/37/38"),"/37/38"),
# (["Zones57","Zones58","Zones59"],"57/58/59")
# ]
comboName = dfEditAreas
for retryCount in xrange(MAX_TRIES):
accessor = ModuleAccessor.ModuleAccessor()
dfEditAreas = accessor.variable(comboName, "Combinations")
if dfEditAreas is None:
if sys.modules.has_key(comboName):
comboMod = sys.modules[comboName]
if comboMod.__file__.endswith(".pyo"):
os.remove(comboMod.__file__)
comboMod = None
del sys.modules[comboName]
# if not last try, log and try again
if retryCount < MAX_TRIES - 1:
# log but don't pop up
self.log.error(
"Error loading combinations file: %s, retrying",
comboName)
else:
return "COMBINATION FILE NOT FOUND: " + \
self.__ut.set(forecastDef, "defaultEditAreas", [])
else:
break
elif len(dfEditAreas) > 0:
refDataList = []
tempRefData = []
for area, label in dfEditAreas:
if type(area) is types.TupleType:
# Create a referenceData given lat, lon, dim
refData = self.__createArea(area, argDict)
tempRefData.append(refData)
else: # Get named Reference Data
refId = ReferenceID(area)
refData = self.getEditArea(area, argDict)
if refData is None:
return "EDIT AREA NOT FOUND: " + str(refId)
refDataList.append((refData, label))
argDict["editAreas"] = refDataList
storeReferenceData(self.dataMgr.getRefManager(), tempRefData)
# Set up HazardsTable
# Product must be:
# --Type "smart"
# --Have an "filterMethod" method
if fcstType == "smart":
product = module.TextProduct()
# Test Code: Uncomment to test
#allowedHazards = product.allowedHazards()
#filterMethod = product.filterMethod
#print("allowedHazards", allowedHazards)
try:
allowedHazards = product.allowedHazards()
filterMethod = product.filterMethod
except:
allowedHazards = None
if allowedHazards is not None and allowedHazards != []:
# Set up editAreas as a list of combinations
# Cases:
# lat/lon or (area, label) pairs -- call HazardsTable,
# but the edit areas will not change
# Combinations -- call HazardsTable and check for changed combinations
# Set up edit areas as list of lists
editAreas = []
for area, label in dfEditAreas:
if type(area) is types.ListType:
editAreas.append(area)
elif type(area) is types.TupleType: #LatLon
editAreas.append([self.__getLatLonAreaName(area)])
else:
editAreas.append([area])
# if Definition['separateByTimeZone'] set to "effectiveTZ"
# or "actualTZ", change the set of edit areas to ensure
# that time zones are same in each grouping.
separateByTZ = product.Definition.get('separateByTimeZone',
None)
if separateByTZ is not None:
areaDictName = product.Definition.get(
'areaDictionary', "AreaDictionary")
editAreas = self._separateByTimeZone(
editAreas,
areaDictName,
argDict['creationTime'],
effectiveTZ=separateByTZ)
accurateCities = product.Definition.get('accurateCities', 0)
cityRefData = []
if accurateCities:
cityLocationName = product.Definition.get(
'cityLocation', "CityLocation")
accessor = ModuleAccessor.ModuleAccessor()
citydict = accessor.variable(cityLocationName,
"CityLocation")
cityEAs = []
if citydict is None:
msg = "CityLocation dictionary module was not found for"\
" city location:"
self.log.error(msg + repr(cityLocationName))
else:
for ea in editAreas:
for ean in ea:
if ean not in citydict:
msg = "CityLocation dictionary does not "\
"contain entry for edit area: "
self.log.error(msg + repr(ean))
continue
for city, llrec in citydict[ean].iteritems():
# Create a referenceData given lat, lon, dim
area = (llrec[0], llrec[1], 0)
refData = self.__createArea(area, argDict)
cityEAs.append(refData)
cityRefData.append((refData, city))
# Store temporary reference data in the server
storeReferenceData(self.dataMgr.getRefManager(), cityEAs)
# Get Product ID and other info for HazardsTable
pil = self.__ut.set(forecastDef, "pil", None)
stationID4 = product.Definition['fullStationID']
productCategory = pil[0:3] #part of the pil
sampleThreshold = product.Definition.get(\
"hazardSamplingThreshold", (10, None))
# Process the hazards
import HazardsTable
hazards = HazardsTable.HazardsTable(
argDict["ifpClient"],
editAreas,
productCategory,
filterMethod,
argDict["databaseID"],
stationID4,
argDict["vtecActiveTable"],
argDict["vtecMode"],
sampleThreshold,
creationTime=argDict["creationTime"],
dataMgr=self.dataMgr,
accurateCities=accurateCities,
cityEditAreas=cityRefData)
# Store hazards object for later use
argDict["hazards"] = hazards
# Get revised combinations
if editAreaType == "Combinations":
reorganizeCombos = product.Definition.get(
"reorganizeCombinations", 1)
if reorganizeCombos:
hazardAreas = hazards.getHazardAreaCombinations()
# Add a bogus label
newCombos = []
for combo in hazardAreas:
newCombos.append((combo, ""))
# Re-assign new combinations
dfEditAreas = newCombos
# Set up Combinations as ReferenceDatas
if editAreaType == "Combinations":
argDict["editAreas"], dfEditAreas = self.getCombinations(
dfEditAreas, argDict)
argDict["combinations"] = dfEditAreas
# Set up Time Ranges
# May be from these sources:
# varDict from command line
# defaultTimeRanges
# command line as named time ranges
# command line as start and end times OR
# from argDict already set up by Interfaces::generateProduct
# In these cases "useRawTR" will be set to 1
if len(argDict["timeRanges"]) > 0:
# Use named time ranges from command line
dfRanges = argDict["timeRanges"]
else:
try: # Try the varDict
dfRanges = varDict["Choose Time Ranges"]
except:
dfRanges = self.__ut.set(forecastDef, "defaultRanges", [])
argDict["timeRanges"] = dfRanges
rawRanges = []
argDict["rawRanges"] = rawRanges
if argDict["useRawTR"] == 1:
tr = argDict["timeRange"]
try:
trName = argDict["timeRangeName"]
except:
trName = ""
if tr is not None:
rawRanges.append((tr, trName))
elif len(dfRanges) == 0:
pass
else:
import TimeRangeUtils
forecast = TimeRangeUtils.TimeRangeUtils()
for rangeName in dfRanges:
rawRange = forecast.getTimeRange(rangeName, argDict)
rawRanges.append((rawRange, rangeName))
argDict["rawRanges"] = rawRanges
#print("rawRanges", rawRanges)
# Row Label
areaType = self.__ut.set(forecastDef, "areaType", "")
rowVariable = self.__ut.set(forecastDef, "rowVariable", "EditArea")
if rowVariable == "EditArea":
rowLabel = areaType
elif rowVariable == "WeatherElement":
rowLabel = "Weather Element"
else:
rowLabel = "Time Period"
argDict["heading"] = rowLabel
def process(self):
import TimeRange
# get list of edit areas that are part of the Zones/FireWx group
from com.raytheon.viz.gfe.smarttool import TextFileUtil, GridCycler
textID = TextFileUtil.getTextFile('Zones', 'editAreaGroups')
zoneList = []
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
zoneList.append(line.rstrip())
textFile.close()
textID = TextFileUtil.getTextFile('FireWxZones', 'editAreaGroups')
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
zoneList.append(line.rstrip())
textFile.close()
refMgr = self.__dataMgr.getRefManager()
# make the basic edit areas that are required, go sequentially through
# the zoneList
requiredEA = [
"west_half", "east_half", "east_one_third", "west_one_third",
"east_two_thirds", "west_two_thirds", "east_one_quarter",
"west_one_quarter", "east_three_quarters", "west_three_quarters",
"Superior"
]
for x in xrange(len(requiredEA)):
refData = refMgr.loadRefSet(ReferenceID(zoneList[x]))
ea = ReferenceData(refData)
ea.setId(ReferenceID(requiredEA[x]))
refMgr.saveRefSet(ea)
#ea = self.__client.getEditAreaPolygons(zoneList[x])
#self.__client.saveEditArea(requiredEA[x], ea)
LogStream.logEvent("Saved ", zoneList[x], "under", requiredEA[x])
# special EAs (source,destination)
special = [("ISC_Send_Area", "FireArea"), ("ISC_Send_Area", "area3")]
for s in special:
refData = refMgr.loadRefSet(ReferenceID(s[0]))
ea = ReferenceData(refData)
ea.setId(ReferenceID(s[1]))
refMgr.saveRefSet(ea)
#ea = self.__client.getEditAreaPolygons(s[0])
#self.__client.saveEditArea(s[1], ea)
LogStream.logEvent("Saved ", s[0], "under", s[1])
# topography simulated based edit areas
# area3 = whole area, AboveElev, BelowElev
LogStream.logEvent("Calculating topo-dependent edit areas...")
topo = self.__dataMgr.getParmManager().getParmInExpr("Topo", True)
topogrid = GridCycler.getInstance().getCorrespondingResult(
topo,
TimeRange.allTimes().toJavaObj(), "TimeWtAverage")
topogrid = topogrid[0].getGridSlice().getNDArray()
iscSend = ReferenceID('ISC_Send_Area')
#wholeGrid = self.__client.getEditArea("ISC_Send_Area")
wholeGrid = refMgr.loadRefSet(iscSend).getGrid().getNDArray()
topoAve = 0
count = 0
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
for x in range(minx, maxx):
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
count = count + 1
topoAve = topoAve + topogrid[y, x]
topoAve = topoAve / count
aboveGrid = wholeGrid * 0
belowGrid = wholeGrid * 0
for x in xrange(topogrid.shape[1]):
for y in xrange(topogrid.shape[0]):
if wholeGrid[y, x] == 1:
if topogrid[y, x] > topoAve:
aboveGrid[y, x] = 1
else:
belowGrid[y, x] = 1
# area1 and area2 need to be "BelowElev", but should be different
# than area3
desiredCount = 2000
count = 0
area1 = wholeGrid * 0
area2 = wholeGrid * 0
for x in xrange(topogrid.shape[1]):
if count < desiredCount:
for y in xrange(topogrid.shape[0]):
if wholeGrid[y, x] == 0 and topogrid[y, x] < topoAve:
area1[y, x] = 1
belowGrid[y, x] = 1
count = count + 1
count = 0
for x in xrange(topogrid.shape[1]):
if count < desiredCount:
for y in xrange(topogrid.shape[0]):
if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve and \
area1[y,x] == 0:
area2[y, x] = 1
belowGrid[y, x] = 1
count = count + 1
# save all topography-dependent edit areas
self.__saveEA("area1", area1)
LogStream.logEvent("Saved area1 based on area2, area3, and topo <",
topoAve)
self.__saveEA("area2", area2)
LogStream.logEvent("Saved area2 based on area1, area3, and topo <",
topoAve)
self.__saveEA("AboveElev", aboveGrid)
LogStream.logEvent("Saved AboveElev based on area3 > ", topoAve)
self.__saveEA("BelowElev", belowGrid)
LogStream.logEvent("Saved BelowElev based on area3 <= ", topoAve)
self.__saveEA("Ridges", aboveGrid)
LogStream.logEvent("Saved Ridges based on area3 > ", topoAve)
self.__saveEA("Valleys", belowGrid)
LogStream.logEvent("Saved Valleys based on area3 < ", topoAve)
self.__saveEA("Inland", aboveGrid)
LogStream.logEvent("Saved Ridges based on area3 > ", topoAve)
self.__saveEA("Coastal", belowGrid)
LogStream.logEvent("Saved Valleys based on area3 < ", topoAve)
#city areas, which are a small part of other edit areas
cityBased = [("area1", ["city1", "city2"]), ("area2", ["city3"]),
("area3", ["city4", "area3_pt"])]
for baseArea, cityAreas in cityBased:
#wholeGrid = self.__client.getEditArea(baseArea)
wholeGrid = refMgr.loadRefSet(
ReferenceID(baseArea)).getGrid().getNDArray()
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
cNumber = 0
print minx, maxx, miny, maxy, wholeGrid.shape
for x in range(minx, maxx):
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
if cNumber >= len(cityAreas):
break
cityGrid = numpy.logical_and(wholeGrid, 0)
cityGrid[y, x] = 1
self.__saveEA(cityAreas[cNumber],
cityGrid.astype('int8'))
LogStream.logEvent("Saved ", cityAreas[cNumber],
"based on:", baseArea)
cNumber = cNumber + 1
# special for ISC areas for CCF database source test
#txt = self.__eagdb["ISC"]
#iscList = cPickle.loads(txt)
textID = TextFileUtil.getTextFile('ISC', 'editAreaGroups')
iscList = []
textFile = open(textID.getFile().getPath())
textFile.readline()
for line in textFile:
iscList.append(line.rstrip())
textFile.close()
count = 0
while count < 6:
for i in iscList:
if i == "ISC_Send_Area" or i == "ISC_Tool_Area":
continue
wholeGrid = refMgr.loadRefSet(
ReferenceID(i)).getGrid().getNDArray()
minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid)
if minx == -1:
continue
ok = 1
print minx, maxx, miny, maxy, wholeGrid.shape
for x in range(minx, maxx):
if ok:
for y in range(miny, maxy):
if wholeGrid[y, x] == 1:
ptGrid = numpy.logical_and(wholeGrid, 0)
ptGrid[y, x] = 1
name = "isc" + ` count `
self.__saveEA(name, ptGrid.astype('int8'))
requiredEA.append(name)
LogStream.logEvent("Saved ", name, "based on ",
i)
ok = 0
break
else:
break
count = count + 1
if count > 6:
break
# store an edit area group with all of the generated edit areas
requiredEA.append("FireArea")
requiredEA.append("AboveElev")
requiredEA.append("BelowElev")
requiredEA.append("Valleys")
requiredEA.append("Ridges")
requiredEA.append("Inland")
requiredEA.append("Coastal")
requiredEA.append("city1")
requiredEA.append("city2")
requiredEA.append("city3")
requiredEA.append("city4")
requiredEA.append("area3")
requiredEA.append("area2")
requiredEA.append("area1")
refMgr.saveGroup("GFETest", JUtil.pylistToJavaStringList(requiredEA))
time.sleep(.5)
def getEditArea(self, name):
rs = self._refmgr.loadRefSet(ReferenceID(name))
if rs.isQuery():
return rs.getQuery()
return rs.getGrid().getNDArray()