def getStatDict(self, sampler, analysisList, timeRange, area, componentName=None): # Get Statistic dictionary for given analysisList, timeRange, # and area # Area can be ReferenceData or text string # Convert area to ID if necessary if area is None: return None if isinstance(area, str): area = ReferenceID(area) else: area = area.getId() return Analysis.Analysis(sampler).createStats( analysisList, area, timeRange, componentName)
def getUnion(self, argDict, areaLabels, areaPrefix): GridLoc = self.getIFPClient().getDBGridLocation() area = None for areaLabel in areaLabels: newArea = self.getEditArea(areaLabel, argDict) if areaLabels.index(areaLabel) == 0: comboNumber = self.getComboNumber() label = areaPrefix + ` int(time.time()) ` + ` comboNumber ` refId = ReferenceID(label) from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData CoordinateType = ReferenceData.CoordinateType coordType = CoordinateType.valueOf('LATLON') area = ReferenceData(GridLoc, refId, newArea.getPolygons(coordType), coordType) # randerso: I don't think this is necessary # area.convertToAWIPS() area = self.unionAreas(label, area, newArea) return area
def getSampler(self, argDict, sampleInfo, sampleFromServer=0): # Get a HistoSampler given # sampleInfo, which is a list of tuples, or just a single tuple # of tuples ([elements], [periods], [areas]) # the elements are [(name, method)] -- basically the analysis list # the periods [(timeRange, label)] # areas [(name,label)] or [(refData, label)] or [(refID, label)] ifpClient = argDict["ifpClient"] databaseID = argDict["databaseID"] from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceID, ReferenceData from com.raytheon.viz.gfe.sampler import SamplerRequest, HistoSampler from java.util import ArrayList # convert input sampleInfo to samplerRequests samplerRequests = ArrayList() if type(sampleInfo) == tuple: sampleInfo = [sampleInfo] for si in sampleInfo: elements, periods, areas = si for e in elements: parmID = self.getParmID(e[0], databaseID) for p in periods: for a in areas: if type(a[0]) is str: samplerRequests.add(SamplerRequest( \ parmID, ReferenceID(a[0]), p[0].toJavaObj())) elif str(type(a[0])) == "<type 'PyJobject'>": samplerRequests.add(SamplerRequest( \ parmID, a[0], p[0].toJavaObj())) else: raise Exception, "area specification incorrect" # do sampling if sampleFromServer: sampler = ifpClient.sampleRequest(samplerRequests) else: sampler = HistoSampler(ifpClient.getJavaClient(), samplerRequests) if sampler.isValid() != 1: print "Cannot Sample: Check for invalid Weather Elements, ",\ "Invalid Areas", str(samplerRequests) return None #print "sampler ", sampler return sampler
def __createArea(self, latLonTuple, argDict): # Return a ReferenceData created for the given lat,lon and dimension # If dim is zero, make edit area of the one grid # point closest to the lat/lon value. lat, lon, dim = latLonTuple name = self.__getLatLonAreaName((lat, lon, dim)) #print("\ncreateArea", lat, lon, dim, name) if dim != 0: for x in range(100): points = makeSquare(lat, lon, dim) pointList = [] for point in points: pointList.append(makePoint(point)) refData = makeArea(self.ifpClient.getDBGridLocation(), pointList, refname=name) # Make sure we have at least one grid point in # the edit area if refData.getGrid().isAnyBitsSet(): #print("returning", dim) return refData # Increment dim and try again #print("iterating", dim) dim += 0.25 msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n" self.log.warning(msg) return None else: from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit # Get grid cell coordinates for lat/lon gridLoc = self.ifpClient.getDBGridLocation() nx = int(gridLoc.getNx()) ny = int(gridLoc.getNy()) cc2D = gridLoc.gridCell(float(lat), float(lon)) # convert grid cell to Grid2DBit with single bit set grid2Dbit = Grid2DBit(nx, ny) if (nx > cc2D.x >= 0 and ny > cc2D.y >= 0): grid2Dbit.set(int(cc2D.x), int(cc2D.y)) #refData = GridLoc.convertToReferenceData(grid2Dbit) refID = ReferenceID(name) refData = ReferenceData(gridLoc, refID, grid2Dbit) return refData
def getEditAreas(self, argDict, areas): # Get the ReferenceArea given # a name or ReferenceID or ReferenceData(just return) refDataList = [] for area in areas: if type(area) is str: # Get area from name id = ReferenceID(area) jlist = ArrayList() jlist.add(id) tmp = argDict["ifpClient"].getReferenceData(jlist) refDataList.append(tmp.get(0)) else: if str(area).find('Id') > -1: refDataList.append(area) else: jlist = ArrayList() jlist.add(id) tmp = argDict["ifpClient"].getReferenceData(jlist) refDataList.append(tmp.get(0)) return refDataList
def __init__(self, imageWidth=None, imageHeight=None, expandLeft=25.0, expandRight=25.0, expandTop=25.0, expandBottom=25.0, mask=None, wholeDomain=0, bgColor=None): # Create empty display and data manager for display display = GFEMapRenderableDisplay() self.dataMgr = DataManagerOffscreenFactory.getInstance(display) self.refId = None envelope = None gloc = self.dataMgr.getParmManager().compositeGridLocation() if mask is not None: from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData CoordinateType = ReferenceData.CoordinateType self.refId = ReferenceID(mask) if wholeDomain == 0: envelope = self.dataMgr.getRefManager().loadRefSet( self.refId).overallDomain(CoordinateType.GRID) if imageWidth is not None: imageWidth = Integer(int(imageWidth)) if imageHeight is not None: imageHeight = Integer(int(imageHeight)) geom = GfeImageUtil.getLocationGeometry( gloc, envelope, imageWidth, imageHeight, expandLeft / 100.0, expandRight / 100.0, expandTop / 100.0, expandBottom / 100.0) # Create descriptor for display desc = MapDescriptor(geom) display.setDescriptor(desc) VizPainter.VizPainter.__init__(self, display, backgroundColor=bgColor) gfeSystem = GFESystemResource(self.dataMgr) self.addVizResource(gfeSystem) desc.getResourceList().getProperties(gfeSystem).setSystemResource(True) self.primaryRsc = None
def makeArea(gridLoc, pointList, refname=None): " Make a Reference Area with a unique ReferenceID" geomFactory = GeometryFactory() import jep size = len(pointList) if pointList[0] != pointList[size - 1]: # closing the loop pointList.append(pointList[0]) pointArray = jep.jarray(len(pointList), Coordinate) for i in range(len(pointList)): pointArray[i] = pointList[i] lr = geomFactory.createLinearRing(pointArray) poly = geomFactory.createPolygon(lr, jep.jarray(0, LinearRing)) polyArray = jep.jarray(1, Polygon) polyArray[0] = poly region = geomFactory.createMultiPolygon(polyArray) if refname is None: refname = "Ref" + getTime() refId = ReferenceID(refname) refData = ReferenceData(gridLoc, refId, region, CoordinateType.LATLON) # randerso: I don't think this is necessary # refData.convertToAWIPS() return refData
def createLatLonArea(self, lat, lon, dim): # Create a square edit area given a latitude, longitude, # and kilometer dimension for the sides. # Example: # area = self.createLatLonArea(40.93, -106.26, 5) # # If dim is zero, make edit area of the one grid # point closest to the lat/lon value. # name = self.getLatLonAreaName((lat, lon, dim)) if dim != 0: for x in range(100): points = self.makeSquare(lat, lon, dim) pointList = [] for point in points: pointList.append(self.makePoint(point)) refData = self.makeArea(pointList, refname=name) # Make sure we have at least one grid point in # the edit area if refData.getGrid().isAnyBitsSet(): return refData # Increment dim and try again dim += 0.25 msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n" self.log.warning(msg) return None else: # Get grid cell coordinates for lat/lon gridLoc = self.getGridLoc() cc2D = gridLoc.gridCell(lat, lon) # convert grid cell to Grid2DBit with single bit set grid2Dbit = JavaGrid2DBit(gridLoc.gridSize().x, gridLoc.gridSize().y) grid2Dbit.set(int(cc2D.x), int(cc2D.y)) #refData = gridLoc.convertToReferenceData(grid2Dbit) refID = ReferenceID(name) refData = ReferenceData(gridLoc, refID, grid2Dbit) #refData.setId(refID) return refData
def executeIscExtract(parmNames, databaseName, startTime, endTime, irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost, ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None): startT = time.time() parms = parmNames dbid = databaseName startTR = startTime endTR = endTime xmlDestinations = destinations ancf = irtTableAddressA bncf = irtTableAddressB xmtScript = transmitScript serverHost = ourServerHost serverPort = ourServerPort serverProtocol = ourServerProtocol mhsid = ourMHSid siteID = ourSiteID myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType() #-------------------------------------------------------------------- # determine the list of destination servers #-------------------------------------------------------------------- try: nowT = time.time() #current time useUntilTime = None #cached use until time cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract" cachedXmlDestinations = None #if xmlDestinations is None: #destinations not on command line # # check the cache # try: # fd = open(cacheFilename, 'rb') # buf = fd.read() # fd.close() # useUntilTime, cachedXmlDestinations = cPickle.loads(buf) # nowT = time.time() #current time # if nowT > useUntilTime: # xmlDestinations = None #cache is too old # useUntilTime = None # else: # logEvent('Using xmlDestinations cache') # xmlDestinations = cachedXmlDestinations # except: # pass # need to contact IRT to get destinations irt = IrtAccess.IrtAccess(ancf, bncf) if xmlDestinations is None: logEvent('contacting IRT to get destinations') count = 1 while True: status, xmlDestinations = irt.getSendAddrs(siteID) logEvent('IRT getSendAddrs status:', status) if status: # if we obtained XML destinations from IRT, then decode # the useUntilTime field try: d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) dE = d.getroot() for e in dE: if e.tag == "useuntil": isoTimeStr = e.text idx = isoTimeStr.find(".") if idx != - 1: isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds useUntilTime = time.mktime(time.strptime(isoTimeStr, "%Y-%m-%dT%H:%M:%S")) logEvent("Use Until: ", isoTimeStr) except: logProblem("Malformed XML on getSendAddrs()") logProblem("XML=", xmlDestinations) return if useUntilTime is None: useUntilTime = time.time() + 180.0 #3 minutes default logEvent("Using default 180 second useUntilTime") # write the cache fd = open(cacheFilename, 'wb') buf = cPickle.dumps((useUntilTime, xmlDestinations)) fd.write(buf) fd.close() break #success from the irt else: # try again and again for 10 minutes, then use cache # if available and alert GFE users if time.time() - nowT > 600.00: logProblem("Unable to access IRT for send addrs") if cachedXmlDestinations is None: s = "Unable to access IRT for send addrs. Previous" + \ " cache not available." logProblem(s) return # use cached value, even if out of date else: xmlDestinations = cachedXmlDestinations if useUntilTime is not None: s = time.asctime(time.gmtime(useUntilTime)) else: s = "Unknown" logProblem("Using expired cache. Date=", s) #determine when we issued our last GFE alert #we alert every 30 minutes. try: fd = open(cacheFilename + "-warn", 'rb') buf = fd.read() fd.close() lastAlertTime = cPickle.loads(buf) except: lastAlertTime = 0 #for way long ago if time.time() - lastAlertTime > 1800.0: logProblem("Sending GFE notification") msg = """ Contact NCF. ifpServer is unable to contact IRT central server. ISC traffic routing information is old and possibly incorrect.""" os.system("sendGfeMessage -u -c GFE -m '" + \ msg + "'") fd = open(cacheFilename + "-warn", 'wb') fd.write(cPickle.dumps(time.time())) fd.close() break time.sleep(15.0) #sleep awhile and then try again count = count + 1 logProblem("Retrying to getSendAddrs()", count) # qc the XML try: destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) destE = destTree.getroot() except: logProblem("Malformed XML on getSendAddrs() or provided xmlDest") logProblem("XML=", xmlDestinations) return #-------------------------------------------------------------------- # determine how many transmissions are necessary #-------------------------------------------------------------------- xmt = [] logEvent("XML dest:", xmlDestinations) if destE.tag != "destinations": logProblem("Destinations packet missing from web service") return # create list of individual transmissions (before attempting to combine doClip = 1 #0 to send entire domain, 1 to do destination clipping (default) destStr = "Destination Servers:\n" for addressE in destE: if addressE.tag == "doclip": for name, value in addressE.items(): if name == "clip": if value == "1": doClip = 1 elif value == "0": doClip = 0 logEvent("Clipping State: ", doClip) for addressE in destE: if addressE.tag != "address": continue # find destination server info and domain information serverInfo = irt.decodeXMLAddress(addressE) if doClip == 0: serverInfo['domain'] = None keycheckfail = False for key in ['mhsid', 'host', 'port', 'protocol', 'site']: if not serverInfo.has_key(key): logProblem("Fail to decode XML. Skipping serverInfo:", serverInfo) keycheckfail = True continue if keycheckfail: continue #skipping this destination due to insufficient info # get the destination office type try: siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site']) destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex)) except: logProblem("Unknown site id to get office type. ", "Skipping serverInfo:", serverInfo) continue #skipping this destination due to unknown site id # find weather elements that remote ifpServer wants # that is available in our server and in the -p parm switches any = False for parm in serverInfo['parms']: p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists # translation of parm name needed, also if no office type, then # not wanted from this office. # example: changes QPFwfo to QPF if we are wfo # example: discards T if we are wfo and site is non-wfo if myOfficeType != destOfficeType: if p1.find(myOfficeType) != - 1: p1 = string.replace(p1, myOfficeType, "") #remove type else: continue #no type, so not intended for our type # see if parm was listed in the command line switches if parms.contains(p1): xmt.append({'serverInfo':[serverInfo], 'parms':[p1], 'domain': serverInfo['domain'], 'area': serverInfo['area']}) if not any: destStr += irt.printServerInfo(serverInfo) + "\n" any = True logEvent(destStr) # now combine transmissions # find same domains, same parms, to combine servers/destinations i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['parms'] == xmt[j]['parms']: for si in xmt[j]['serverInfo']: if si not in xmt[i]['serverInfo']: dests = xmt[i]['serverInfo'] dests.append(si) xmt[j]['serverInfo'] = dests del xmt[j] #delete the entry j = j - 1 #redo this entry index next loop j = j + 1 i = i + 1 # now try to combine common parm lists (same domain, same servers/destinations) i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['serverInfo'] == xmt[j]['serverInfo'] : iparms = xmt[i]['parms'] for p in xmt[j]['parms']: if p not in iparms: iparms.append(p) xmt[i]['parms'] = iparms del xmt[j] #delete the entry j = j - 1 #redo this entry index for next loop j = j + 1 i = i + 1 # if doClip, gather some required information if doClip: #get the isc send area and grid domain from the ifpServer iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID) sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain() iscSendAreaGrid.setGloc(sourceDomain) iscSendAreaGrid = iscSendAreaGrid.getGrid() #-------------------------------------------------------------------- # prepare output files #-------------------------------------------------------------------- for dest in xmt: s = "Processing Xmt Pass:\n" for sv in dest['serverInfo']: s += irt.printServerInfo(sv) + '\n' s += "Domain:" + `dest['domain']` + '\n' s += "Area:" + `dest['area']` + '\n' s += "Parms:" + `dest['parms']` + '\n\n' logEvent(s) # extract the data using ifpnetCDF if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False: os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC") tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC" fname = tempfile.mktemp(".isc") # Determine domain edit area. if doClip == 1 and dest['domain'] is not None and \ dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID(): #make a GridLocation for our domain gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy()))) origin = sourceDomain.getOrigin() extent = sourceDomain.getExtent() domain = CartDomain2D(origin, extent) gloc = sourceDomain #make a GridLocation covering the area for the destination, expanded #by 1/2 grid cell dd = dest['domain'] da = dest['area'] cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0) cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0) originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0, float(dd['origy']) - cellsizeY / 2.0) extentD = Coordinate(float(dd['extx']) + cellsizeX, float(dd['exty']) + cellsizeY) domainD = CartDomain2D(originD, extentD) #check for overlap if not domainD.overlaps(domain): logEvent("No intersection of domain box, skipping....") continue #no bits set in the resulting mask, no intersect domainD.trim(domain) #trim it to just the overlapping section gridSize = Point(int(da['xdim']),int(da['ydim'])) destGridLocation = GridLocation("Dest",sourceDomain.getProjection(), gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT") # make a Reference Set refid = ReferenceID("jibberish") refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON) # convert destination site's domain to gridpoints iscMask = refSet.getGrid() # "and" it with our ISC_Send_Area iscMask.andEquals(iscSendAreaGrid) if not iscMask.isAnyBitsSet(): logEvent("No intersection of domain points, skipping....") continue #no bits set in the resulting mask, no intersect # store the grid back into the ifpServer maskName = "iscExtract" + `time.time()` refSet.setGrid(iscMask) iscUtil.saveEditAreaGrid(maskName, refSet, siteID) else: #no clipping, or different projection maskName = "ISC_Send_Area" # Run ifpnetCDF for the data argv = {"outputFilename": fname, "parmList": dest['parms'], "databaseID": dbid, "startTime": startTR, "endTime": endTR, "mask": maskName, "geoInfo": False, "compressFileFlag": True, "configFileName": "iscSendSampleDef", "compressFileFactor": 6, "trim": True, "krunch": True, "userID": "iscExtract", "logFileName": None, "siteIdOverride": None} ifpnetCDF.main(**argv) fname = fname + '.gz' size = os.stat(fname)[stat.ST_SIZE] endT = time.time() logEvent('File Size: ', size) logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) # create XML destinations file for this output iscE = Element('isc') #create the XML tree root sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort, 'protocol': serverProtocol, 'site': siteID} irt.addSourceXML(iscE, sourceServer) irt.addDestinationXML(iscE, dest['serverInfo']) #get the unique list of mhs sites mhsSites = [] for si in dest['serverInfo']: if si['mhsid'] not in mhsSites: mhsSites.append(si['mhsid']) # create the XML file fnameXML = tempfile.mktemp(".xml") fd = open(fnameXML, 'wb') fd.write(ElementTree.tostring(iscE)) fd.close() # Transmit files - do string substitution irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript) # Delete temporary files if maskName != "ISC_Send_Area": iscUtil.deleteEditArea(maskName,siteID) endT = time.time() logEvent('After transmission pass, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) except: logProblem("Failure", traceback.format_exc())
def __getRunTimeVariables(self, fcstName, forecastDef, fcstType, module, argDict): # Input variables can come from various sources: # varDict from command line # command line switch e.g. -l language (will be in argDict) # definition section of product # We must check all three in that order. varDict = argDict["varDict"] #print("varDict", varDict) for item, default in [ ("language", "english"), ("appendFile", None), ("lineLength", 69), # no command line option ("timePeriod", 3), ]: try: # Try the varDict #print("trying varDict", item) argDict[item] = varDict[item] #print("got it in varDict") except: try: # Try argDict i.e. from command line switch # If so, argDict is already set #print("trying argDict", item) argValue = argDict[item] #print("got it in argDict", argValue) except: argValue = None # Try #print("getting from definition", item) if argValue is None: argDict[item] = self.__ut.set(forecastDef, item, default) #print("value from definition", argDict[item]) # These need to be integers for item in ["lineLength", "timePeriod"]: if argDict[item] is not None: argDict[item] = int(argDict[item]) # Edit Areas and Time Ranges # # Set up these argDict values: # editAreas -- list of (refData, label) pairs # timeRanges -- list of named ranges # rawRanges -- list of (rawRange, rangeName) pairs # # As we eventually loop through the product, these values will be set: # editArea -- current edit area pair # timeRange -- current raw time range # issueTime -- first raw time range # timeRangeName -- current time range name (if available) # combinations -- list of (editAreaList, comboLabel) tuples # where editAreaList is the list of edit areas composing # the combination # Set up Edit Areas # May be from these sources: # "AreaLabel" # varDict from command line -- list of names # command line as named reference areas # defaultEditAreas # list of (name, label) pairs # "LatLon" # OR list of ((lat,lon,dimension), label) # "Combinations" # OR A Combinations file # We may have had edit areas entered from the command line # or from the Interfaces "generateForecast" command # If so, argDict["editAreas"] will be a list of either # (name, label) or (refData, label) pairs editAreaType = "AreaLabel" # We may have had edit areas entered from the command line # If so, argDict["editAreas"] will be a list of either # (name, label) or (refData, label) pairs if len(argDict["editAreas"]) == 0: dfEditAreas = self.__ut.set(forecastDef, "defaultEditAreas", []) try: # Try the varDict chosenAreas = varDict["Choose Edit Areas"] # Turn the list of chosen areas into (name, label) pairs # using the defaultEditAreas list dfEditAreas = self.__pairAreaWithLabel(chosenAreas, dfEditAreas) except: pass # Set up edit areas as ReferenceData's for AreaLabel and LatLon areas if type(dfEditAreas) == types.StringType: editAreaType = "Combinations" # Get edit areas from file with format: # Combinations = [ # ([editArea1, editArea2,...],label) # ... # ] # For example: # Combinations = [ # (["Zones48", "Zones49", "Zones50"],"/48/49/50"), # (["Zones37","Zones38"], "/37/38"),"/37/38"), # (["Zones57","Zones58","Zones59"],"57/58/59") # ] comboName = dfEditAreas for retryCount in xrange(MAX_TRIES): accessor = ModuleAccessor.ModuleAccessor() dfEditAreas = accessor.variable(comboName, "Combinations") if dfEditAreas is None: if sys.modules.has_key(comboName): comboMod = sys.modules[comboName] if comboMod.__file__.endswith(".pyo"): os.remove(comboMod.__file__) comboMod = None del sys.modules[comboName] # if not last try, log and try again if retryCount < MAX_TRIES - 1: # log but don't pop up self.log.error( "Error loading combinations file: %s, retrying", comboName) else: return "COMBINATION FILE NOT FOUND: " + \ self.__ut.set(forecastDef, "defaultEditAreas", []) else: break elif len(dfEditAreas) > 0: refDataList = [] tempRefData = [] for area, label in dfEditAreas: if type(area) is types.TupleType: # Create a referenceData given lat, lon, dim refData = self.__createArea(area, argDict) tempRefData.append(refData) else: # Get named Reference Data refId = ReferenceID(area) refData = self.getEditArea(area, argDict) if refData is None: return "EDIT AREA NOT FOUND: " + str(refId) refDataList.append((refData, label)) argDict["editAreas"] = refDataList storeReferenceData(self.dataMgr.getRefManager(), tempRefData) # Set up HazardsTable # Product must be: # --Type "smart" # --Have an "filterMethod" method if fcstType == "smart": product = module.TextProduct() # Test Code: Uncomment to test #allowedHazards = product.allowedHazards() #filterMethod = product.filterMethod #print("allowedHazards", allowedHazards) try: allowedHazards = product.allowedHazards() filterMethod = product.filterMethod except: allowedHazards = None if allowedHazards is not None and allowedHazards != []: # Set up editAreas as a list of combinations # Cases: # lat/lon or (area, label) pairs -- call HazardsTable, # but the edit areas will not change # Combinations -- call HazardsTable and check for changed combinations # Set up edit areas as list of lists editAreas = [] for area, label in dfEditAreas: if type(area) is types.ListType: editAreas.append(area) elif type(area) is types.TupleType: #LatLon editAreas.append([self.__getLatLonAreaName(area)]) else: editAreas.append([area]) # if Definition['separateByTimeZone'] set to "effectiveTZ" # or "actualTZ", change the set of edit areas to ensure # that time zones are same in each grouping. separateByTZ = product.Definition.get('separateByTimeZone', None) if separateByTZ is not None: areaDictName = product.Definition.get( 'areaDictionary', "AreaDictionary") editAreas = self._separateByTimeZone( editAreas, areaDictName, argDict['creationTime'], effectiveTZ=separateByTZ) accurateCities = product.Definition.get('accurateCities', 0) cityRefData = [] if accurateCities: cityLocationName = product.Definition.get( 'cityLocation', "CityLocation") accessor = ModuleAccessor.ModuleAccessor() citydict = accessor.variable(cityLocationName, "CityLocation") cityEAs = [] if citydict is None: msg = "CityLocation dictionary module was not found for"\ " city location:" self.log.error(msg + repr(cityLocationName)) else: for ea in editAreas: for ean in ea: if ean not in citydict: msg = "CityLocation dictionary does not "\ "contain entry for edit area: " self.log.error(msg + repr(ean)) continue for city, llrec in citydict[ean].iteritems(): # Create a referenceData given lat, lon, dim area = (llrec[0], llrec[1], 0) refData = self.__createArea(area, argDict) cityEAs.append(refData) cityRefData.append((refData, city)) # Store temporary reference data in the server storeReferenceData(self.dataMgr.getRefManager(), cityEAs) # Get Product ID and other info for HazardsTable pil = self.__ut.set(forecastDef, "pil", None) stationID4 = product.Definition['fullStationID'] productCategory = pil[0:3] #part of the pil sampleThreshold = product.Definition.get(\ "hazardSamplingThreshold", (10, None)) # Process the hazards import HazardsTable hazards = HazardsTable.HazardsTable( argDict["ifpClient"], editAreas, productCategory, filterMethod, argDict["databaseID"], stationID4, argDict["vtecActiveTable"], argDict["vtecMode"], sampleThreshold, creationTime=argDict["creationTime"], dataMgr=self.dataMgr, accurateCities=accurateCities, cityEditAreas=cityRefData) # Store hazards object for later use argDict["hazards"] = hazards # Get revised combinations if editAreaType == "Combinations": reorganizeCombos = product.Definition.get( "reorganizeCombinations", 1) if reorganizeCombos: hazardAreas = hazards.getHazardAreaCombinations() # Add a bogus label newCombos = [] for combo in hazardAreas: newCombos.append((combo, "")) # Re-assign new combinations dfEditAreas = newCombos # Set up Combinations as ReferenceDatas if editAreaType == "Combinations": argDict["editAreas"], dfEditAreas = self.getCombinations( dfEditAreas, argDict) argDict["combinations"] = dfEditAreas # Set up Time Ranges # May be from these sources: # varDict from command line # defaultTimeRanges # command line as named time ranges # command line as start and end times OR # from argDict already set up by Interfaces::generateProduct # In these cases "useRawTR" will be set to 1 if len(argDict["timeRanges"]) > 0: # Use named time ranges from command line dfRanges = argDict["timeRanges"] else: try: # Try the varDict dfRanges = varDict["Choose Time Ranges"] except: dfRanges = self.__ut.set(forecastDef, "defaultRanges", []) argDict["timeRanges"] = dfRanges rawRanges = [] argDict["rawRanges"] = rawRanges if argDict["useRawTR"] == 1: tr = argDict["timeRange"] try: trName = argDict["timeRangeName"] except: trName = "" if tr is not None: rawRanges.append((tr, trName)) elif len(dfRanges) == 0: pass else: import TimeRangeUtils forecast = TimeRangeUtils.TimeRangeUtils() for rangeName in dfRanges: rawRange = forecast.getTimeRange(rangeName, argDict) rawRanges.append((rawRange, rangeName)) argDict["rawRanges"] = rawRanges #print("rawRanges", rawRanges) # Row Label areaType = self.__ut.set(forecastDef, "areaType", "") rowVariable = self.__ut.set(forecastDef, "rowVariable", "EditArea") if rowVariable == "EditArea": rowLabel = areaType elif rowVariable == "WeatherElement": rowLabel = "Weather Element" else: rowLabel = "Time Period" argDict["heading"] = rowLabel
def process(self): import TimeRange # get list of edit areas that are part of the Zones/FireWx group from com.raytheon.viz.gfe.smarttool import TextFileUtil, GridCycler textID = TextFileUtil.getTextFile('Zones', 'editAreaGroups') zoneList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() textID = TextFileUtil.getTextFile('FireWxZones', 'editAreaGroups') textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() refMgr = self.__dataMgr.getRefManager() # make the basic edit areas that are required, go sequentially through # the zoneList requiredEA = [ "west_half", "east_half", "east_one_third", "west_one_third", "east_two_thirds", "west_two_thirds", "east_one_quarter", "west_one_quarter", "east_three_quarters", "west_three_quarters", "Superior" ] for x in xrange(len(requiredEA)): refData = refMgr.loadRefSet(ReferenceID(zoneList[x])) ea = ReferenceData(refData) ea.setId(ReferenceID(requiredEA[x])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(zoneList[x]) #self.__client.saveEditArea(requiredEA[x], ea) LogStream.logEvent("Saved ", zoneList[x], "under", requiredEA[x]) # special EAs (source,destination) special = [("ISC_Send_Area", "FireArea"), ("ISC_Send_Area", "area3")] for s in special: refData = refMgr.loadRefSet(ReferenceID(s[0])) ea = ReferenceData(refData) ea.setId(ReferenceID(s[1])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(s[0]) #self.__client.saveEditArea(s[1], ea) LogStream.logEvent("Saved ", s[0], "under", s[1]) # topography simulated based edit areas # area3 = whole area, AboveElev, BelowElev LogStream.logEvent("Calculating topo-dependent edit areas...") topo = self.__dataMgr.getParmManager().getParmInExpr("Topo", True) topogrid = GridCycler.getInstance().getCorrespondingResult( topo, TimeRange.allTimes().toJavaObj(), "TimeWtAverage") topogrid = topogrid[0].getGridSlice().getNDArray() iscSend = ReferenceID('ISC_Send_Area') #wholeGrid = self.__client.getEditArea("ISC_Send_Area") wholeGrid = refMgr.loadRefSet(iscSend).getGrid().getNDArray() topoAve = 0 count = 0 minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y, x] == 1: count = count + 1 topoAve = topoAve + topogrid[y, x] topoAve = topoAve / count aboveGrid = wholeGrid * 0 belowGrid = wholeGrid * 0 for x in xrange(topogrid.shape[1]): for y in xrange(topogrid.shape[0]): if wholeGrid[y, x] == 1: if topogrid[y, x] > topoAve: aboveGrid[y, x] = 1 else: belowGrid[y, x] = 1 # area1 and area2 need to be "BelowElev", but should be different # than area3 desiredCount = 2000 count = 0 area1 = wholeGrid * 0 area2 = wholeGrid * 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y, x] == 0 and topogrid[y, x] < topoAve: area1[y, x] = 1 belowGrid[y, x] = 1 count = count + 1 count = 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve and \ area1[y,x] == 0: area2[y, x] = 1 belowGrid[y, x] = 1 count = count + 1 # save all topography-dependent edit areas self.__saveEA("area1", area1) LogStream.logEvent("Saved area1 based on area2, area3, and topo <", topoAve) self.__saveEA("area2", area2) LogStream.logEvent("Saved area2 based on area1, area3, and topo <", topoAve) self.__saveEA("AboveElev", aboveGrid) LogStream.logEvent("Saved AboveElev based on area3 > ", topoAve) self.__saveEA("BelowElev", belowGrid) LogStream.logEvent("Saved BelowElev based on area3 <= ", topoAve) self.__saveEA("Ridges", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Valleys", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) self.__saveEA("Inland", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Coastal", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) #city areas, which are a small part of other edit areas cityBased = [("area1", ["city1", "city2"]), ("area2", ["city3"]), ("area3", ["city4", "area3_pt"])] for baseArea, cityAreas in cityBased: #wholeGrid = self.__client.getEditArea(baseArea) wholeGrid = refMgr.loadRefSet( ReferenceID(baseArea)).getGrid().getNDArray() minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) cNumber = 0 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y, x] == 1: if cNumber >= len(cityAreas): break cityGrid = numpy.logical_and(wholeGrid, 0) cityGrid[y, x] = 1 self.__saveEA(cityAreas[cNumber], cityGrid.astype('int8')) LogStream.logEvent("Saved ", cityAreas[cNumber], "based on:", baseArea) cNumber = cNumber + 1 # special for ISC areas for CCF database source test #txt = self.__eagdb["ISC"] #iscList = cPickle.loads(txt) textID = TextFileUtil.getTextFile('ISC', 'editAreaGroups') iscList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: iscList.append(line.rstrip()) textFile.close() count = 0 while count < 6: for i in iscList: if i == "ISC_Send_Area" or i == "ISC_Tool_Area": continue wholeGrid = refMgr.loadRefSet( ReferenceID(i)).getGrid().getNDArray() minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) if minx == -1: continue ok = 1 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): if ok: for y in range(miny, maxy): if wholeGrid[y, x] == 1: ptGrid = numpy.logical_and(wholeGrid, 0) ptGrid[y, x] = 1 name = "isc" + ` count ` self.__saveEA(name, ptGrid.astype('int8')) requiredEA.append(name) LogStream.logEvent("Saved ", name, "based on ", i) ok = 0 break else: break count = count + 1 if count > 6: break # store an edit area group with all of the generated edit areas requiredEA.append("FireArea") requiredEA.append("AboveElev") requiredEA.append("BelowElev") requiredEA.append("Valleys") requiredEA.append("Ridges") requiredEA.append("Inland") requiredEA.append("Coastal") requiredEA.append("city1") requiredEA.append("city2") requiredEA.append("city3") requiredEA.append("city4") requiredEA.append("area3") requiredEA.append("area2") requiredEA.append("area1") refMgr.saveGroup("GFETest", JUtil.pylistToJavaStringList(requiredEA)) time.sleep(.5)
def getEditArea(self, name): rs = self._refmgr.loadRefSet(ReferenceID(name)) if rs.isQuery(): return rs.getQuery() return rs.getGrid().getNDArray()