def getCombinations(self, combinations, argDict): editAreas = [] newCombinations = [] for comboList, areaLabel in combinations: newComboList = [] for editArea in comboList: #print("Get edit area TF: get edit area combo", editArea) #print("TF2: Get Edit Area set up edit areas", editArea) newArea = self.getEditArea(editArea, argDict) if comboList.index(editArea) == 0: comboNumber = self.getComboNumber() label = "Combo" + repr(comboNumber) refId = ReferenceID(label) #global GridLoc #GridLoc = newArea.getGloc() area = ReferenceData(newArea) area.setId(refId) #GridLoc, refId, newArea.getPolygons("LATLON"), "LATLON") # randerso: I don't think this is necessary # area.convertToAWIPS() newComboList.append(newArea.getId().getName()) area = self.unionAreas(label, area, newArea) if argDict["fcstType"] == "table": # Allow user-supplied area labels to be used for simple tables editAreas.append((area, areaLabel)) else: editAreas.append((area, label)) newCombinations.append((newComboList, label)) return editAreas, newCombinations
def intersectAreas(self, name, area1, area2): # AND the areas (ReferenceData objects) # together and return a ReferenceData object cpy = ReferenceData(area1) refData = cpy.andEquals(area2) #refData.convertToLatLon() refData.setId(ReferenceID(name)) refData.getGrid() return refData
def intersectAreas(self, name, area1, area2): # AND the areas (ReferenceData objects) # together and return a ReferenceData object cpy = ReferenceData(area1) refData = cpy.andEquals(area2) #refData.convertToLatLon() refData.setId(ReferenceID(name)) refData.getGrid() return refData
def saveEditAreaGrid(maskName, iscMask, siteID): iscMask.getPolygons(CoordinateType.LATLON) pathMgr = PathManagerFactory.getPathManager() commonStaticConfig = pathMgr.getContext(LocalizationType.COMMON_STATIC, LocalizationLevel.CONFIGURED) commonStaticConfig.setContextName(siteID) sitePath = pathMgr.getFile(commonStaticConfig, "gfe/editAreas").getPath() editAreaPath = str(sitePath) + "/" + maskName + ".xml" ReferenceData.getJAXBManager().marshalToXmlFile(iscMask, editAreaPath)
def makeArea(gridLoc, pointList, refname=None): " Make a Reference Area with a unique ReferenceID" from com.vividsolutions.jts.geom import GeometryFactory, LinearRing, Coordinate, Polygon from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData CoordinateType = ReferenceData.CoordinateType geomFactory = GeometryFactory() import jep size = len(pointList) if pointList[0] != pointList[size - 1]: # closing the loop pointList.append(pointList[0]) pointArray = jep.jarray(len(pointList), Coordinate) for i in range(len(pointList)): pointArray[i] = pointList[i] lr = geomFactory.createLinearRing(pointArray) poly = geomFactory.createPolygon(lr, jep.jarray(0, LinearRing)) polyArray = jep.jarray(1, Polygon) polyArray[0] = poly region = geomFactory.createMultiPolygon(polyArray) if refname is None: refname = "Ref" + getTime() refId = ReferenceID(refname) refData = ReferenceData(gridLoc, refId, region, CoordinateType.LATLON) # randerso: I don't think this is necessary # refData.convertToAWIPS() return refData
def getEditArea(name, siteID): pathMgr = PathManagerFactory.getPathManager() commonStaticConfig = pathMgr.getContext(LocalizationType.COMMON_STATIC, LocalizationLevel.SITE) commonStaticConfig.setContextName(siteID) file = pathMgr.getFile(commonStaticConfig, "gfe/editAreas" + File.separator + name + ".xml") if not os.path.exists(file.getPath()): commonStaticConfig = pathMgr.getContext(LocalizationType.COMMON_STATIC, LocalizationLevel.CONFIGURED) commonStaticConfig.setContextName(siteID) file = pathMgr.getFile( commonStaticConfig, "gfe/editAreas" + File.separator + name + ".xml") refData = None try: if os.path.exists(file.getPath()): refData = ReferenceData.getJAXBManager().unmarshalFromXmlFile( file.getPath()) else: LogStream.logProblem("EDIT AREA NOT FOUND: ", name, " for site ", siteID) except: LogStream.logProblem("Unable to unmarshal " + name + " in iscExtract") return refData
def evaluate(self, expression, timeInfluence=None): # if no expression, return the empty ref set if len(expression) == 0: return self.__dm.getRefManager().emptyRefSet() if timeInfluence is None: timeInfluence = self.__dm.getSpatialDisplayManager().getSpatialEditorTime() if not isinstance(timeInfluence, AbsTime.AbsTime): timeInfluence = AbsTime.AbsTime(timeInfluence) self._query.setTime(timeInfluence.unixTime()) grid = self._query.eval(expression) if type(grid) != type(numpy.array([])) or grid.shape != self._shape: raise TypeError("query did not eval to a grid of shape: " + `self._shape`) from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData, ReferenceID from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit if grid.dtype == 'bool': grid = numpy.array(grid, 'byte') bits = Grid2DBit.createBitGrid(grid.shape[1], grid.shape[0], grid) return ReferenceData(self.gloc(), ReferenceID(expression), expression, bits)
def __saveEA(self, name, grid): #save edit area from a grid gloc = self.__dataMgr.getParmManager().compositeGridLocation() id = ReferenceID(name) # convert grid to polygons grid2d = Grid2DBit.createBitGrid(int(gloc.getNx()), int(gloc.getNy()), grid) refdata = ReferenceData(gloc, id, grid2d) # save the edit area self.__dataMgr.getRefManager().saveRefSet(refdata)
def getUnion(self, argDict, areaLabels, areaPrefix): GridLoc = self.getIFPClient().getDBGridLocation() area = None for areaLabel in areaLabels: newArea = self.getEditArea(areaLabel, argDict) if areaLabels.index(areaLabel) == 0: comboNumber = self.getComboNumber() label = areaPrefix + ` int(time.time()) ` + ` comboNumber ` refId = ReferenceID(label) from com.raytheon.uf.common.dataplugin.gfe.reference import ReferenceData CoordinateType = ReferenceData.CoordinateType coordType = CoordinateType.valueOf('LATLON') area = ReferenceData(GridLoc, refId, newArea.getPolygons(coordType), coordType) # randerso: I don't think this is necessary # area.convertToAWIPS() area = self.unionAreas(label, area, newArea) return area
def __createArea(self, latLonTuple, argDict): # Return a ReferenceData created for the given lat,lon and dimension # If dim is zero, make edit area of the one grid # point closest to the lat/lon value. lat, lon, dim = latLonTuple name = self.__getLatLonAreaName((lat, lon, dim)) #print("\ncreateArea", lat, lon, dim, name) if dim != 0: for x in range(100): points = makeSquare(lat, lon, dim) pointList = [] for point in points: pointList.append(makePoint(point)) refData = makeArea(self.ifpClient.getDBGridLocation(), pointList, refname=name) # Make sure we have at least one grid point in # the edit area if refData.getGrid().isAnyBitsSet(): #print("returning", dim) return refData # Increment dim and try again #print("iterating", dim) dim += 0.25 msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n" self.log.warning(msg) return None else: from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DBit # Get grid cell coordinates for lat/lon gridLoc = self.ifpClient.getDBGridLocation() nx = int(gridLoc.getNx()) ny = int(gridLoc.getNy()) cc2D = gridLoc.gridCell(float(lat), float(lon)) # convert grid cell to Grid2DBit with single bit set grid2Dbit = Grid2DBit(nx, ny) if (nx > cc2D.x >= 0 and ny > cc2D.y >= 0): grid2Dbit.set(int(cc2D.x), int(cc2D.y)) #refData = GridLoc.convertToReferenceData(grid2Dbit) refID = ReferenceID(name) refData = ReferenceData(gridLoc, refID, grid2Dbit) return refData
def createLatLonArea(self, lat, lon, dim): # Create a square edit area given a latitude, longitude, # and kilometer dimension for the sides. # Example: # area = self.createLatLonArea(40.93, -106.26, 5) # # If dim is zero, make edit area of the one grid # point closest to the lat/lon value. # name = self.getLatLonAreaName((lat, lon, dim)) if dim != 0: for x in range(100): points = self.makeSquare(lat, lon, dim) pointList = [] for point in points: pointList.append(self.makePoint(point)) refData = self.makeArea(pointList, refname=name) # Make sure we have at least one grid point in # the edit area if refData.getGrid().isAnyBitsSet(): return refData # Increment dim and try again dim += 0.25 msg = "\nWARNING!!! EMPTY EDIT AREA. INCREASE LAT/LON AREA DIMENSION!!\n" self.log.warning(msg) return None else: # Get grid cell coordinates for lat/lon gridLoc = self.getGridLoc() cc2D = gridLoc.gridCell(lat, lon) # convert grid cell to Grid2DBit with single bit set grid2Dbit = JavaGrid2DBit(gridLoc.gridSize().x, gridLoc.gridSize().y) grid2Dbit.set(int(cc2D.x), int(cc2D.y)) #refData = gridLoc.convertToReferenceData(grid2Dbit) refID = ReferenceID(name) refData = ReferenceData(gridLoc, refID, grid2Dbit) #refData.setId(refID) return refData
def executeIscExtract(parmNames, databaseName, startTime, endTime, irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost, ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None): startT = time.time() parms = parmNames dbid = databaseName startTR = startTime endTR = endTime xmlDestinations = destinations ancf = irtTableAddressA bncf = irtTableAddressB xmtScript = transmitScript serverHost = ourServerHost serverPort = ourServerPort serverProtocol = ourServerProtocol mhsid = ourMHSid siteID = ourSiteID myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType() #-------------------------------------------------------------------- # determine the list of destination servers #-------------------------------------------------------------------- try: nowT = time.time() #current time useUntilTime = None #cached use until time cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract" cachedXmlDestinations = None #if xmlDestinations is None: #destinations not on command line # # check the cache # try: # fd = open(cacheFilename, 'rb') # buf = fd.read() # fd.close() # useUntilTime, cachedXmlDestinations = cPickle.loads(buf) # nowT = time.time() #current time # if nowT > useUntilTime: # xmlDestinations = None #cache is too old # useUntilTime = None # else: # logEvent('Using xmlDestinations cache') # xmlDestinations = cachedXmlDestinations # except: # pass # need to contact IRT to get destinations irt = IrtAccess.IrtAccess(ancf, bncf) if xmlDestinations is None: logEvent('contacting IRT to get destinations') count = 1 while True: status, xmlDestinations = irt.getSendAddrs(siteID) logEvent('IRT getSendAddrs status:', status) if status: # if we obtained XML destinations from IRT, then decode # the useUntilTime field try: d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) dE = d.getroot() for e in dE: if e.tag == "useuntil": isoTimeStr = e.text idx = isoTimeStr.find(".") if idx != - 1: isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds useUntilTime = time.mktime(time.strptime(isoTimeStr, "%Y-%m-%dT%H:%M:%S")) logEvent("Use Until: ", isoTimeStr) except: logProblem("Malformed XML on getSendAddrs()") logProblem("XML=", xmlDestinations) return if useUntilTime is None: useUntilTime = time.time() + 180.0 #3 minutes default logEvent("Using default 180 second useUntilTime") # write the cache fd = open(cacheFilename, 'wb') buf = cPickle.dumps((useUntilTime, xmlDestinations)) fd.write(buf) fd.close() break #success from the irt else: # try again and again for 10 minutes, then use cache # if available and alert GFE users if time.time() - nowT > 600.00: logProblem("Unable to access IRT for send addrs") if cachedXmlDestinations is None: s = "Unable to access IRT for send addrs. Previous" + \ " cache not available." logProblem(s) return # use cached value, even if out of date else: xmlDestinations = cachedXmlDestinations if useUntilTime is not None: s = time.asctime(time.gmtime(useUntilTime)) else: s = "Unknown" logProblem("Using expired cache. Date=", s) #determine when we issued our last GFE alert #we alert every 30 minutes. try: fd = open(cacheFilename + "-warn", 'rb') buf = fd.read() fd.close() lastAlertTime = cPickle.loads(buf) except: lastAlertTime = 0 #for way long ago if time.time() - lastAlertTime > 1800.0: logProblem("Sending GFE notification") msg = """ Contact NCF. ifpServer is unable to contact IRT central server. ISC traffic routing information is old and possibly incorrect.""" os.system("sendGfeMessage -u -c GFE -m '" + \ msg + "'") fd = open(cacheFilename + "-warn", 'wb') fd.write(cPickle.dumps(time.time())) fd.close() break time.sleep(15.0) #sleep awhile and then try again count = count + 1 logProblem("Retrying to getSendAddrs()", count) # qc the XML try: destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) destE = destTree.getroot() except: logProblem("Malformed XML on getSendAddrs() or provided xmlDest") logProblem("XML=", xmlDestinations) return #-------------------------------------------------------------------- # determine how many transmissions are necessary #-------------------------------------------------------------------- xmt = [] logEvent("XML dest:", xmlDestinations) if destE.tag != "destinations": logProblem("Destinations packet missing from web service") return # create list of individual transmissions (before attempting to combine doClip = 1 #0 to send entire domain, 1 to do destination clipping (default) destStr = "Destination Servers:\n" for addressE in destE: if addressE.tag == "doclip": for name, value in addressE.items(): if name == "clip": if value == "1": doClip = 1 elif value == "0": doClip = 0 logEvent("Clipping State: ", doClip) for addressE in destE: if addressE.tag != "address": continue # find destination server info and domain information serverInfo = irt.decodeXMLAddress(addressE) if doClip == 0: serverInfo['domain'] = None keycheckfail = False for key in ['mhsid', 'host', 'port', 'protocol', 'site']: if not serverInfo.has_key(key): logProblem("Fail to decode XML. Skipping serverInfo:", serverInfo) keycheckfail = True continue if keycheckfail: continue #skipping this destination due to insufficient info # get the destination office type try: siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site']) destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex)) except: logProblem("Unknown site id to get office type. ", "Skipping serverInfo:", serverInfo) continue #skipping this destination due to unknown site id # find weather elements that remote ifpServer wants # that is available in our server and in the -p parm switches any = False for parm in serverInfo['parms']: p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists # translation of parm name needed, also if no office type, then # not wanted from this office. # example: changes QPFwfo to QPF if we are wfo # example: discards T if we are wfo and site is non-wfo if myOfficeType != destOfficeType: if p1.find(myOfficeType) != - 1: p1 = string.replace(p1, myOfficeType, "") #remove type else: continue #no type, so not intended for our type # see if parm was listed in the command line switches if parms.contains(p1): xmt.append({'serverInfo':[serverInfo], 'parms':[p1], 'domain': serverInfo['domain'], 'area': serverInfo['area']}) if not any: destStr += irt.printServerInfo(serverInfo) + "\n" any = True logEvent(destStr) # now combine transmissions # find same domains, same parms, to combine servers/destinations i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['parms'] == xmt[j]['parms']: for si in xmt[j]['serverInfo']: if si not in xmt[i]['serverInfo']: dests = xmt[i]['serverInfo'] dests.append(si) xmt[j]['serverInfo'] = dests del xmt[j] #delete the entry j = j - 1 #redo this entry index next loop j = j + 1 i = i + 1 # now try to combine common parm lists (same domain, same servers/destinations) i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['serverInfo'] == xmt[j]['serverInfo'] : iparms = xmt[i]['parms'] for p in xmt[j]['parms']: if p not in iparms: iparms.append(p) xmt[i]['parms'] = iparms del xmt[j] #delete the entry j = j - 1 #redo this entry index for next loop j = j + 1 i = i + 1 # if doClip, gather some required information if doClip: #get the isc send area and grid domain from the ifpServer iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID) sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain() iscSendAreaGrid.setGloc(sourceDomain) iscSendAreaGrid = iscSendAreaGrid.getGrid() #-------------------------------------------------------------------- # prepare output files #-------------------------------------------------------------------- for dest in xmt: s = "Processing Xmt Pass:\n" for sv in dest['serverInfo']: s += irt.printServerInfo(sv) + '\n' s += "Domain:" + `dest['domain']` + '\n' s += "Area:" + `dest['area']` + '\n' s += "Parms:" + `dest['parms']` + '\n\n' logEvent(s) # extract the data using ifpnetCDF if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False: os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC") tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC" fname = tempfile.mktemp(".isc") # Determine domain edit area. if doClip == 1 and dest['domain'] is not None and \ dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID(): #make a GridLocation for our domain gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy()))) origin = sourceDomain.getOrigin() extent = sourceDomain.getExtent() domain = CartDomain2D(origin, extent) gloc = sourceDomain #make a GridLocation covering the area for the destination, expanded #by 1/2 grid cell dd = dest['domain'] da = dest['area'] cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0) cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0) originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0, float(dd['origy']) - cellsizeY / 2.0) extentD = Coordinate(float(dd['extx']) + cellsizeX, float(dd['exty']) + cellsizeY) domainD = CartDomain2D(originD, extentD) #check for overlap if not domainD.overlaps(domain): logEvent("No intersection of domain box, skipping....") continue #no bits set in the resulting mask, no intersect domainD.trim(domain) #trim it to just the overlapping section gridSize = Point(int(da['xdim']),int(da['ydim'])) destGridLocation = GridLocation("Dest",sourceDomain.getProjection(), gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT") # make a Reference Set refid = ReferenceID("jibberish") refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON) # convert destination site's domain to gridpoints iscMask = refSet.getGrid() # "and" it with our ISC_Send_Area iscMask.andEquals(iscSendAreaGrid) if not iscMask.isAnyBitsSet(): logEvent("No intersection of domain points, skipping....") continue #no bits set in the resulting mask, no intersect # store the grid back into the ifpServer maskName = "iscExtract" + `time.time()` refSet.setGrid(iscMask) iscUtil.saveEditAreaGrid(maskName, refSet, siteID) else: #no clipping, or different projection maskName = "ISC_Send_Area" # Run ifpnetCDF for the data argv = {"outputFilename": fname, "parmList": dest['parms'], "databaseID": dbid, "startTime": startTR, "endTime": endTR, "mask": maskName, "geoInfo": False, "compressFileFlag": True, "configFileName": "iscSendSampleDef", "compressFileFactor": 6, "trim": True, "krunch": True, "userID": "iscExtract", "logFileName": None, "siteIdOverride": None} ifpnetCDF.main(**argv) fname = fname + '.gz' size = os.stat(fname)[stat.ST_SIZE] endT = time.time() logEvent('File Size: ', size) logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) # create XML destinations file for this output iscE = Element('isc') #create the XML tree root sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort, 'protocol': serverProtocol, 'site': siteID} irt.addSourceXML(iscE, sourceServer) irt.addDestinationXML(iscE, dest['serverInfo']) #get the unique list of mhs sites mhsSites = [] for si in dest['serverInfo']: if si['mhsid'] not in mhsSites: mhsSites.append(si['mhsid']) # create the XML file fnameXML = tempfile.mktemp(".xml") fd = open(fnameXML, 'wb') fd.write(ElementTree.tostring(iscE)) fd.close() # Transmit files - do string substitution irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript) # Delete temporary files if maskName != "ISC_Send_Area": iscUtil.deleteEditArea(maskName,siteID) endT = time.time() logEvent('After transmission pass, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) except: logProblem("Failure", traceback.format_exc())
def process(self): import TimeRange # get list of edit areas that are part of the Zones/FireWx group from com.raytheon.viz.gfe.smarttool import TextFileUtil, GridCycler textID = TextFileUtil.getTextFile('Zones', 'editAreaGroups') zoneList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() textID = TextFileUtil.getTextFile('FireWxZones', 'editAreaGroups') textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() refMgr = self.__dataMgr.getRefManager() # make the basic edit areas that are required, go sequentially through # the zoneList requiredEA = ["west_half","east_half","east_one_third", "west_one_third", "east_two_thirds","west_two_thirds", "east_one_quarter", "west_one_quarter", "east_three_quarters", "west_three_quarters","SUPERIOR"] for x in xrange(len(requiredEA)): refData = refMgr.loadRefSet(ReferenceID(zoneList[x])) ea = ReferenceData(refData) ea.setId(ReferenceID(requiredEA[x])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(zoneList[x]) #self.__client.saveEditArea(requiredEA[x], ea) LogStream.logEvent("Saved ", zoneList[x], "under", requiredEA[x]) # special EAs (source,destination) special = [("ISC_Send_Area","FireArea"), ("ISC_Send_Area", "area3")] for s in special: refData = refMgr.loadRefSet(ReferenceID(s[0])) ea = ReferenceData(refData) ea.setId(ReferenceID(s[1])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(s[0]) #self.__client.saveEditArea(s[1], ea) LogStream.logEvent("Saved ", s[0], "under", s[1]) # topography simulated based edit areas # area3 = whole area, AboveElev, BelowElev LogStream.logEvent("Calculating topo-dependent edit areas...") topo = self.__dataMgr.getParmManager().getParmInExpr("Topo", True) topogrid = GridCycler.getInstance().getCorrespondingResult( topo, TimeRange.allTimes().toJavaObj(), "TimeWtAverage") topogrid = topogrid[0].getGridSlice().__numpy__[0] iscSend = ReferenceID('ISC_Send_Area') #wholeGrid = self.__client.getEditArea("ISC_Send_Area") wholeGrid = refMgr.loadRefSet(iscSend).getGrid().__numpy__[0] topoAve = 0 count = 0 minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y,x] == 1: count = count + 1 topoAve = topoAve + topogrid[y,x] topoAve = topoAve / count aboveGrid = wholeGrid * 0 belowGrid = wholeGrid * 0 for x in xrange(topogrid.shape[1]): for y in xrange(topogrid.shape[0]): if wholeGrid[y,x] == 1: if topogrid[y,x] > topoAve: aboveGrid[y,x] = 1 else: belowGrid[y,x] = 1 # area1 and area2 need to be "BelowElev", but should be different # than area3 desiredCount = 2000 count = 0 area1 = wholeGrid * 0 area2 = wholeGrid * 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve: area1[y,x] = 1 belowGrid[y,x] = 1 count = count + 1 count = 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve and \ area1[y,x] == 0: area2[y,x] = 1 belowGrid[y,x] = 1 count = count + 1 # save all topography-dependent edit areas self.__saveEA("area1", area1) LogStream.logEvent("Saved area1 based on area2, area3, and topo <", topoAve) self.__saveEA("area2", area2) LogStream.logEvent("Saved area2 based on area1, area3, and topo <", topoAve) self.__saveEA("AboveElev", aboveGrid) LogStream.logEvent("Saved AboveElev based on area3 > ", topoAve) self.__saveEA("BelowElev", belowGrid) LogStream.logEvent("Saved BelowElev based on area3 <= ", topoAve) self.__saveEA("Ridges", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Valleys", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) self.__saveEA("Inland", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Coastal", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) #city areas, which are a small part of other edit areas cityBased = [("area1",["city1","city2"]), ("area2", ["city3"]), ("area3",["city4", "area3_pt"])] for baseArea,cityAreas in cityBased: #wholeGrid = self.__client.getEditArea(baseArea) wholeGrid = refMgr.loadRefSet(ReferenceID(baseArea)).getGrid().__numpy__[0] minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) cNumber = 0 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y,x] == 1: if cNumber >= len(cityAreas): break cityGrid = numpy.logical_and(wholeGrid, 0) cityGrid[y,x] = 1 self.__saveEA(cityAreas[cNumber], cityGrid.astype('int8')) LogStream.logEvent("Saved ", cityAreas[cNumber], "based on:", baseArea) cNumber = cNumber + 1 # special for ISC areas for CCF database source test #txt = self.__eagdb["ISC"] #iscList = cPickle.loads(txt) textID = TextFileUtil.getTextFile('ISC', 'editAreaGroups') iscList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: iscList.append(line.rstrip()) textFile.close() count = 0 while count < 6: for i in iscList: if i == "ISC_Send_Area" or i == "ISC_Tool_Area": continue wholeGrid = refMgr.loadRefSet(ReferenceID(i)).getGrid().__numpy__[0] minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) if minx == -1: continue ok = 1 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): if ok: for y in range(miny, maxy): if wholeGrid[y,x] == 1: ptGrid = numpy.logical_and(wholeGrid, 0) ptGrid[y,x] = 1 name = "isc" + `count` self.__saveEA(name, ptGrid.astype('int8')) requiredEA.append(name) LogStream.logEvent("Saved ", name, "based on ", i) ok = 0 break else: break count = count + 1 if count > 6: break # store an edit area group with all of the generated edit areas requiredEA.append("FireArea") requiredEA.append("AboveElev") requiredEA.append("BelowElev") requiredEA.append("Valleys") requiredEA.append("Ridges") requiredEA.append("Inland") requiredEA.append("Coastal") requiredEA.append("city1") requiredEA.append("city2") requiredEA.append("city3") requiredEA.append("city4") requiredEA.append("area3") requiredEA.append("area2") requiredEA.append("area1") refMgr.saveGroup("GFETest", JUtil.pylistToJavaStringList(requiredEA)) time.sleep(.5)
def executeIscExtract(parmNames, databaseName, startTime, endTime, irtTableAddressA, irtTableAddressB, transmitScript, ourServerHost, ourServerPort, ourServerProtocol, ourMHSid, ourSiteID, destinations=None): startT = time.time() parms = parmNames dbid = databaseName startTR = startTime endTR = endTime xmlDestinations = destinations ancf = irtTableAddressA bncf = irtTableAddressB xmtScript = transmitScript serverHost = ourServerHost serverPort = ourServerPort serverProtocol = ourServerProtocol mhsid = ourMHSid siteID = ourSiteID myOfficeType = IFPServerConfigManager.getServerConfig(siteID).officeType() #-------------------------------------------------------------------- # determine the list of destination servers #-------------------------------------------------------------------- try: nowT = time.time() #current time useUntilTime = None #cached use until time cacheFilename = "/tmp/" + serverHost + serverPort + ".iscExtract" cachedXmlDestinations = None #if xmlDestinations is None: #destinations not on command line # # check the cache # try: # fd = open(cacheFilename, 'rb') # buf = fd.read() # fd.close() # useUntilTime, cachedXmlDestinations = cPickle.loads(buf) # nowT = time.time() #current time # if nowT > useUntilTime: # xmlDestinations = None #cache is too old # useUntilTime = None # else: # logEvent('Using xmlDestinations cache') # xmlDestinations = cachedXmlDestinations # except: # pass # need to contact IRT to get destinations irt = IrtAccess.IrtAccess(ancf, bncf) if xmlDestinations is None: logEvent('contacting IRT to get destinations') count = 1 while True: status, xmlDestinations = irt.getSendAddrs(siteID) logEvent('IRT getSendAddrs status:', status) if status: # if we obtained XML destinations from IRT, then decode # the useUntilTime field try: d = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) dE = d.getroot() for e in dE: if e.tag == "useuntil": isoTimeStr = e.text idx = isoTimeStr.find(".") if idx != - 1: isoTimeStr = isoTimeStr[0:idx] #eliminate subseconds useUntilTime = time.mktime(time.strptime(isoTimeStr, "%Y-%m-%dT%H:%M:%S")) logEvent("Use Until: ", isoTimeStr) except: logProblem("Malformed XML on getSendAddrs()") logProblem("XML=", xmlDestinations) return if useUntilTime is None: useUntilTime = time.time() + 180.0 #3 minutes default logEvent("Using default 180 second useUntilTime") # write the cache fd = open(cacheFilename, 'wb') buf = cPickle.dumps((useUntilTime, xmlDestinations)) fd.write(buf) fd.close() break #success from the irt else: # try again and again for 10 minutes, then use cache # if available and alert GFE users if time.time() - nowT > 600.00: logProblem("Unable to access IRT for send addrs") if cachedXmlDestinations is None: s = "Unable to access IRT for send addrs. Previous" + \ " cache not available." logProblem(s) return # use cached value, even if out of date else: xmlDestinations = cachedXmlDestinations if useUntilTime is not None: s = time.asctime(time.gmtime(useUntilTime)) else: s = "Unknown" logProblem("Using expired cache. Date=", s) #determine when we issued our last GFE alert #we alert every 30 minutes. try: fd = open(cacheFilename + "-warn", 'rb') buf = fd.read() fd.close() lastAlertTime = cPickle.loads(buf) except: lastAlertTime = 0 #for way long ago if time.time() - lastAlertTime > 1800.0: logProblem("Sending GFE notification") msg = """ Contact NCF. ifpServer is unable to contact IRT central server. ISC traffic routing information is old and possibly incorrect.""" os.system("sendGfeMessage -u -c GFE -m '" + \ msg + "'") fd = open(cacheFilename + "-warn", 'wb') fd.write(cPickle.dumps(time.time())) fd.close() break time.sleep(15.0) #sleep awhile and then try again count = count + 1 logProblem("Retrying to getSendAddrs()", count) # qc the XML try: destTree = ElementTree.ElementTree(ElementTree.XML(xmlDestinations)) destE = destTree.getroot() except: logProblem("Malformed XML on getSendAddrs() or provided xmlDest") logProblem("XML=", xmlDestinations) return #-------------------------------------------------------------------- # determine how many transmissions are necessary #-------------------------------------------------------------------- xmt = [] logEvent("XML dest:", xmlDestinations) if destE.tag != "destinations": logProblem("Destinations packet missing from web service") return # create list of individual transmissions (before attempting to combine doClip = 1 #0 to send entire domain, 1 to do destination clipping (default) destStr = "Destination Servers:\n" for addressE in destE: if addressE.tag == "doclip": for name, value in addressE.items(): if name == "clip": if value == "1": doClip = 1 elif value == "0": doClip = 0 logEvent("Clipping State: ", doClip) for addressE in destE: if addressE.tag != "address": continue # find destination server info and domain information serverInfo = irt.decodeXMLAddress(addressE) if doClip == 0: serverInfo['domain'] = None keycheckfail = False for key in ['mhsid', 'host', 'port', 'protocol', 'site']: if not serverInfo.has_key(key): logProblem("Fail to decode XML. Skipping serverInfo:", serverInfo) keycheckfail = True continue if keycheckfail: continue #skipping this destination due to insufficient info # get the destination office type try: siteIndex = IFPServerConfigManager.getServerConfig(siteID).allSites().indexOf(serverInfo['site']) destOfficeType = str(IFPServerConfigManager.getServerConfig(siteID).officeTypes().get(siteIndex)) except: logProblem("Unknown site id to get office type. ", "Skipping serverInfo:", serverInfo) continue #skipping this destination due to unknown site id # find weather elements that remote ifpServer wants # that is available in our server and in the -p parm switches any = False for parm in serverInfo['parms']: p1 = string.replace(parm, "_SFC", "") #remove _SFC if exists # translation of parm name needed, also if no office type, then # not wanted from this office. # example: changes QPFwfo to QPF if we are wfo # example: discards T if we are wfo and site is non-wfo if myOfficeType != destOfficeType: if p1.find(myOfficeType) != - 1: p1 = string.replace(p1, myOfficeType, "") #remove type else: continue #no type, so not intended for our type # see if parm was listed in the command line switches if parms.contains(p1): xmt.append({'serverInfo':[serverInfo], 'parms':[p1], 'domain': serverInfo['domain'], 'area': serverInfo['area']}) if not any: destStr += irt.printServerInfo(serverInfo) + "\n" any = True logEvent(destStr) # now combine transmissions # find same domains, same parms, to combine servers/destinations i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['parms'] == xmt[j]['parms']: for si in xmt[j]['serverInfo']: if si not in xmt[i]['serverInfo']: dests = xmt[i]['serverInfo'] dests.append(si) xmt[j]['serverInfo'] = dests del xmt[j] #delete the entry j = j - 1 #redo this entry index next loop j = j + 1 i = i + 1 # now try to combine common parm lists (same domain, same servers/destinations) i = 0 while i < len(xmt): j = i + 1 while j < len(xmt): if xmt[i]['domain'] == xmt[j]['domain'] and \ xmt[i]['area'] == xmt[j]['area'] and \ xmt[i]['serverInfo'] == xmt[j]['serverInfo'] : iparms = xmt[i]['parms'] for p in xmt[j]['parms']: if p not in iparms: iparms.append(p) xmt[i]['parms'] = iparms del xmt[j] #delete the entry j = j - 1 #redo this entry index for next loop j = j + 1 i = i + 1 # if doClip, gather some required information if doClip: #get the isc send area and grid domain from the ifpServer iscSendAreaGrid = iscUtil.getEditArea("ISC_Send_Area",siteID) sourceDomain = IFPServerConfigManager.getServerConfig(siteID).dbDomain() iscSendAreaGrid.setGloc(sourceDomain) iscSendAreaGrid = iscSendAreaGrid.getGrid() #-------------------------------------------------------------------- # prepare output files #-------------------------------------------------------------------- for dest in xmt: s = "Processing Xmt Pass:\n" for sv in dest['serverInfo']: s += irt.printServerInfo(sv) + '\n' s += "Domain:" + `dest['domain']` + '\n' s += "Area:" + `dest['area']` + '\n' s += "Parms:" + `dest['parms']` + '\n\n' logEvent(s) # extract the data using ifpnetCDF if os.path.exists(siteConfig.GFESUITE_HOME + "/products/ISC") == False: os.makedirs(siteConfig.GFESUITE_HOME + "/products/ISC") tempfile.tempdir = siteConfig.GFESUITE_HOME + "/products/ISC" fname = tempfile.mktemp(".isc") # Determine domain edit area. if doClip == 1 and dest['domain'] is not None and \ dest['domain']['proj'] == sourceDomain.getProjection().getProjectionID(): #make a GridLocation for our domain gridSize = Coordinate(float(str(sourceDomain.getNx())), float(str(sourceDomain.getNy()))) origin = sourceDomain.getOrigin() extent = sourceDomain.getExtent() domain = CartDomain2D(origin, extent) gloc = sourceDomain #make a GridLocation covering the area for the destination, expanded #by 1/2 grid cell dd = dest['domain'] da = dest['area'] cellsizeX = float(dd['extx']) / (float(da['xdim']) - 1.0) cellsizeY = float(dd['exty']) / (float(da['ydim']) - 1.0) originD = Coordinate(float(dd['origx']) - cellsizeX / 2.0, float(dd['origy']) - cellsizeY / 2.0) extentD = Coordinate(float(dd['extx']) + cellsizeX, float(dd['exty']) + cellsizeY) domainD = CartDomain2D(originD, extentD) #check for overlap if not domainD.overlaps(domain): logEvent("No intersection of domain box, skipping....") continue #no bits set in the resulting mask, no intersect domainD.trim(domain) #trim it to just the overlapping section gridSize = Point(int(da['xdim']),int(da['ydim'])) destGridLocation = GridLocation("Dest",sourceDomain.getProjection(), gridSize,domainD.getOrigin(),domainD.getExtent(),"GMT") # make a Reference Set refid = ReferenceID("jibberish") refSet = ReferenceData(gloc, refid, destGridLocation.getGeometry(), CoordinateType.LATLON) # convert destination site's domain to gridpoints iscMask = refSet.getGrid() # "and" it with our ISC_Send_Area iscMask.andEquals(iscSendAreaGrid) if not iscMask.isAnyBitsSet(): logEvent("No intersection of domain points, skipping....") continue #no bits set in the resulting mask, no intersect # store the grid back into the ifpServer maskName = "iscExtract" + `time.time()` refSet.setGrid(iscMask) iscUtil.saveEditAreaGrid(maskName, refSet, siteID) else: #no clipping, or different projection maskName = "ISC_Send_Area" # Run ifpnetCDF for the data argv = {"outputFilename": fname, "parmList": dest['parms'], "databaseID": dbid, "startTime": startTR, "endTime": endTR, "mask": maskName, "geoInfo": False, "compressFileFlag": True, "configFileName": "iscSendSampleDef", "compressFileFactor": 6, "trim": True, "krunch": True, "userID": "iscExtract", "logFileName": None} ifpnetCDF.main(**argv) fname = fname + '.gz' size = os.stat(fname)[stat.ST_SIZE] endT = time.time() logEvent('File Size: ', size) logEvent('After ifpnetCDF, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) # create XML destinations file for this output iscE = Element('isc') #create the XML tree root sourceServer = {'mhsid': mhsid, 'host': serverHost, 'port': serverPort, 'protocol': serverProtocol, 'site': siteID} irt.addSourceXML(iscE, sourceServer) irt.addDestinationXML(iscE, dest['serverInfo']) #get the unique list of mhs sites mhsSites = [] for si in dest['serverInfo']: if si['mhsid'] not in mhsSites: mhsSites.append(si['mhsid']) # create the XML file fnameXML = tempfile.mktemp(".xml") fd = open(fnameXML, 'wb') fd.write(ElementTree.tostring(iscE)) fd.close() # Transmit files - do string substitution irt.transmitFiles("ISCGRIDS2", mhsSites, mhsid, [fname,fnameXML], xmtScript) # Delete temporary files if maskName != "ISC_Send_Area": iscUtil.deleteEditArea(maskName,siteID) endT = time.time() logEvent('After transmission pass, ,wctime:', "%-6.2f" % (endT - startT), ',cputime:', "%-6.2f" % time.clock()) except: logProblem("Failure", traceback.format_exc())
def process(self): import TimeRange # get list of edit areas that are part of the Zones/FireWx group from com.raytheon.viz.gfe.smarttool import TextFileUtil, GridCycler textID = TextFileUtil.getTextFile('Zones', 'editAreaGroups') zoneList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() textID = TextFileUtil.getTextFile('FireWxZones', 'editAreaGroups') textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: zoneList.append(line.rstrip()) textFile.close() refMgr = self.__dataMgr.getRefManager() # make the basic edit areas that are required, go sequentially through # the zoneList requiredEA = [ "west_half", "east_half", "east_one_third", "west_one_third", "east_two_thirds", "west_two_thirds", "east_one_quarter", "west_one_quarter", "east_three_quarters", "west_three_quarters", "Superior" ] for x in xrange(len(requiredEA)): refData = refMgr.loadRefSet(ReferenceID(zoneList[x])) ea = ReferenceData(refData) ea.setId(ReferenceID(requiredEA[x])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(zoneList[x]) #self.__client.saveEditArea(requiredEA[x], ea) LogStream.logEvent("Saved ", zoneList[x], "under", requiredEA[x]) # special EAs (source,destination) special = [("ISC_Send_Area", "FireArea"), ("ISC_Send_Area", "area3")] for s in special: refData = refMgr.loadRefSet(ReferenceID(s[0])) ea = ReferenceData(refData) ea.setId(ReferenceID(s[1])) refMgr.saveRefSet(ea) #ea = self.__client.getEditAreaPolygons(s[0]) #self.__client.saveEditArea(s[1], ea) LogStream.logEvent("Saved ", s[0], "under", s[1]) # topography simulated based edit areas # area3 = whole area, AboveElev, BelowElev LogStream.logEvent("Calculating topo-dependent edit areas...") topo = self.__dataMgr.getParmManager().getParmInExpr("Topo", True) topogrid = GridCycler.getInstance().getCorrespondingResult( topo, TimeRange.allTimes().toJavaObj(), "TimeWtAverage") topogrid = topogrid[0].getGridSlice().getNDArray() iscSend = ReferenceID('ISC_Send_Area') #wholeGrid = self.__client.getEditArea("ISC_Send_Area") wholeGrid = refMgr.loadRefSet(iscSend).getGrid().getNDArray() topoAve = 0 count = 0 minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y, x] == 1: count = count + 1 topoAve = topoAve + topogrid[y, x] topoAve = topoAve / count aboveGrid = wholeGrid * 0 belowGrid = wholeGrid * 0 for x in xrange(topogrid.shape[1]): for y in xrange(topogrid.shape[0]): if wholeGrid[y, x] == 1: if topogrid[y, x] > topoAve: aboveGrid[y, x] = 1 else: belowGrid[y, x] = 1 # area1 and area2 need to be "BelowElev", but should be different # than area3 desiredCount = 2000 count = 0 area1 = wholeGrid * 0 area2 = wholeGrid * 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y, x] == 0 and topogrid[y, x] < topoAve: area1[y, x] = 1 belowGrid[y, x] = 1 count = count + 1 count = 0 for x in xrange(topogrid.shape[1]): if count < desiredCount: for y in xrange(topogrid.shape[0]): if wholeGrid[y,x] == 0 and topogrid[y,x] < topoAve and \ area1[y,x] == 0: area2[y, x] = 1 belowGrid[y, x] = 1 count = count + 1 # save all topography-dependent edit areas self.__saveEA("area1", area1) LogStream.logEvent("Saved area1 based on area2, area3, and topo <", topoAve) self.__saveEA("area2", area2) LogStream.logEvent("Saved area2 based on area1, area3, and topo <", topoAve) self.__saveEA("AboveElev", aboveGrid) LogStream.logEvent("Saved AboveElev based on area3 > ", topoAve) self.__saveEA("BelowElev", belowGrid) LogStream.logEvent("Saved BelowElev based on area3 <= ", topoAve) self.__saveEA("Ridges", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Valleys", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) self.__saveEA("Inland", aboveGrid) LogStream.logEvent("Saved Ridges based on area3 > ", topoAve) self.__saveEA("Coastal", belowGrid) LogStream.logEvent("Saved Valleys based on area3 < ", topoAve) #city areas, which are a small part of other edit areas cityBased = [("area1", ["city1", "city2"]), ("area2", ["city3"]), ("area3", ["city4", "area3_pt"])] for baseArea, cityAreas in cityBased: #wholeGrid = self.__client.getEditArea(baseArea) wholeGrid = refMgr.loadRefSet( ReferenceID(baseArea)).getGrid().getNDArray() minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) cNumber = 0 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): for y in range(miny, maxy): if wholeGrid[y, x] == 1: if cNumber >= len(cityAreas): break cityGrid = numpy.logical_and(wholeGrid, 0) cityGrid[y, x] = 1 self.__saveEA(cityAreas[cNumber], cityGrid.astype('int8')) LogStream.logEvent("Saved ", cityAreas[cNumber], "based on:", baseArea) cNumber = cNumber + 1 # special for ISC areas for CCF database source test #txt = self.__eagdb["ISC"] #iscList = cPickle.loads(txt) textID = TextFileUtil.getTextFile('ISC', 'editAreaGroups') iscList = [] textFile = open(textID.getFile().getPath()) textFile.readline() for line in textFile: iscList.append(line.rstrip()) textFile.close() count = 0 while count < 6: for i in iscList: if i == "ISC_Send_Area" or i == "ISC_Tool_Area": continue wholeGrid = refMgr.loadRefSet( ReferenceID(i)).getGrid().getNDArray() minx, maxx, miny, maxy = self.__extremaOfSetBits(wholeGrid) if minx == -1: continue ok = 1 print minx, maxx, miny, maxy, wholeGrid.shape for x in range(minx, maxx): if ok: for y in range(miny, maxy): if wholeGrid[y, x] == 1: ptGrid = numpy.logical_and(wholeGrid, 0) ptGrid[y, x] = 1 name = "isc" + ` count ` self.__saveEA(name, ptGrid.astype('int8')) requiredEA.append(name) LogStream.logEvent("Saved ", name, "based on ", i) ok = 0 break else: break count = count + 1 if count > 6: break # store an edit area group with all of the generated edit areas requiredEA.append("FireArea") requiredEA.append("AboveElev") requiredEA.append("BelowElev") requiredEA.append("Valleys") requiredEA.append("Ridges") requiredEA.append("Inland") requiredEA.append("Coastal") requiredEA.append("city1") requiredEA.append("city2") requiredEA.append("city3") requiredEA.append("city4") requiredEA.append("area3") requiredEA.append("area2") requiredEA.append("area1") refMgr.saveGroup("GFETest", JUtil.pylistToJavaStringList(requiredEA)) time.sleep(.5)