def testFilterOnInvalidLocationThrowsIncompatibleRequestException(self): req = DAL.newDataRequest() req.setDatatype(self.datatype) req.setParameters('id') req.setLocationNames(') and 0=1') with self.assertRaises(Exception) as cm: DAL.getGeometryData(req) self.assertIn('IncompatibleRequestException', str(cm.exception))
def getPolygonDict(self, tablename, siteID, locationField="name", columns=[]): req = DataAccessLayer.newDataRequest() req.setDatatype("maps") table = "mapdata." + tablename req.addIdentifier("table", table) req.addIdentifier("geomField", "the_geom") req.addIdentifier("locationField", locationField) req.addIdentifier("cwa", siteID) if columns: req.setParameters(columns) retGeoms = DataAccessLayer.getGeometryData(req) retDict = {} for retGeom in retGeoms: id = retGeom.getLocationName() if id: poly = self._extract_poly_coords(retGeom.getGeometry()) formattedPoly = [list(c) for c in poly['exterior_coords']] retDict[id] = formattedPoly return retDict
def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") req.setLocationNames(params.OBS_STATION) req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") print("Testing getGeometryData()") geomData = DAL.getGeometryData(req) print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: print("level=" + record.getLevel(), end="") # One dimensional parameters are reported on the 0.0UNKNOWN level. # 2D parameters are reported on MB levels from pressure. if record.getLevel() == "0.0UNKNOWN": print(" sfcPress=" + record.getString("sfcPress") + record.getUnit("sfcPress"), end="") print(" temp2=" + record.getString("temp2") + record.getUnit("temp2"), end="") print(" q2=" + record.getString("q2") + record.getUnit("q2"), end="") else: print(" pressure=" + record.getString("pressure") + record.getUnit("pressure"), end="") print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="") print(" specHum=" + record.getString("specHum") + record.getUnit("specHum"), end="") print(" geometry=" + str(record.getGeometry())) print("getGeometryData() complete\n\n")
def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) req.setParameters("temperature", "pressure", "uComponent", "vComponent") print("Testing getGeometryData()") geomData = DAL.getGeometryData(req) self.assertIsNotNone(geomData) print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: print("level:", record.getLevel(), end="") # One dimensional parameters are reported on the 0.0UNKNOWN level. # 2D parameters are reported on MB levels from pressure. if record.getLevel() == "0.0UNKNOWN": print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="") print(" pressure=" + record.getString("pressure") + record.getUnit("pressure"), end="") else: print(" uComponent=" + record.getString("uComponent") + record.getUnit("uComponent"), end="") print(" vComponent=" + record.getString("vComponent") + record.getUnit("vComponent"), end="") print(" geometry:", record.getGeometry()) print("getGeometryData() complete\n\n")
def getQPEValues(self): ''' Strategy method for reading and accumulating data from preprocessed FFMP QPE datasets. ''' request = DataAccessLayer.newDataRequest() request.setDatatype(FFMP_KEY) request.setParameters(self._sourceName) request.addIdentifier(WFO_KEY, self.currentSite) request.addIdentifier(SITE_KEY, self._siteKey) request.addIdentifier(DATA_KEY, self._dataKey) request.addIdentifier(HUC_KEY, ALL_HUC) availableTimes = DataAccessLayer.getAvailableTimes(request) # determine correct times latestTime = 0 for time in availableTimes: tm = time.getRefTime().getTime() if tm > latestTime: latestTime = tm timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000 usedTimes = [] for time in availableTimes: if time.getRefTime().getTime() >= timedelta: usedTimes.append(time) basins = [] if usedTimes: geometries = DataAccessLayer.getGeometryData(request, usedTimes) for geometry in geometries: self.__storeQpe(geometry.getLocationName(), geometry.getNumber(self._sourceName), geometry) return True return False
def testGetGeometryDataWithEnvelope(self): req = DAL.newDataRequest(self.datatype) req.setParameters("temperature", "windSpeed", "hazardType", "turbType") req.setEnvelope(params.ENVELOPE) print("Testing getGeometryData()") data = DAL.getGeometryData(req) for item in data: self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testNoDuplicateData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_monthly') req.setLocationNames('KOMA') req.setParameters('maxtemp_day1') rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5]) for i in range(len(rows)): for j in range(len(rows)): if i != j: self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testGetGeometryDataWithEnvelope(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") req.setEnvelope(params.ENVELOPE) req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") print("Testing getGeometryData()") data = DAL.getGeometryData(req) for item in data: self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testFilterOnLocationName(self): allLocationNames = self._getLocationNames() if len(allLocationNames) == 0: errmsg = "No {0} data exists on {1}. Try again with {0} data." raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST)) testCount = 3 # number of different location names to test for locationName in allLocationNames[:testCount]: req = DAL.newDataRequest() req.setDatatype(self.datatype) req.setParameters('id') req.setLocationNames(locationName) geomData = DAL.getGeometryData(req) for geom in geomData: self.assertEqual(geom.getLocationName(), locationName)
def getSounding(modelName, weatherElements, levels, samplePoint, refTime=None, timeRange=None): """" Performs a series of Data Access Framework requests to retrieve a sounding object based on the specified request parameters. Args: modelName: the grid model datasetid to use as the basis of the sounding. weatherElements: a list of parameters to return in the sounding. levels: a list of levels to sample the given weather elements at samplePoint: a lat/lon pair to perform the sampling of data at. refTime: (optional) the grid model reference time to use for the sounding. If not specified, the latest ref time in the system will be used. timeRange: (optional) a TimeRange to specify which forecast hours to use. If not specified, will default to all forecast hours. Returns: A _SoundingCube instance, which acts a 3-tiered dictionary, keyed by DataTime, then by level and finally by weather element. If no data is available for the given request parameters, None is returned. """ (locationNames, parameters, levels, envelope, refTime, timeRange) = \ __sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange) requestArgs = { 'datatype': 'grid', 'locationNames': locationNames, 'parameters': parameters, 'levels': levels, 'envelope': envelope, } req = DataAccessLayer.newDataRequest(**requestArgs) forecastHours = __determineForecastHours(req, refTime, timeRange) if not forecastHours: return None response = DataAccessLayer.getGeometryData(req, forecastHours) soundingObject = _SoundingCube(response) return soundingObject
def runGeometryDataTestWithTimeRange(self, req, timeRange): """ Test that we are able to successfully retrieve geometry data for the given request. """ geomData = DAL.getGeometryData(req, timeRange) self.assertIsNotNone(geomData) if not geomData: raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: self.assertGreaterEqual( record.getDataTime().getRefTime().getTime(), timeRange.getStartInMillis()) self.assertLessEqual(record.getDataTime().getRefTime().getTime(), timeRange.getEndInMillis()) print("geometry=" + str(record.getGeometry()), end="") for p in req.getParameters(): print(" " + p + "=" + record.getString(p), end="") print() return geomData
def getGuidanceValues(self): ''' Retrieves the guidance values from the correct FFMP source ''' ffmpSourceConfigManager = FFMPSourceConfigurationManager.getInstance() if self._guidanceName == 'RFCFFG': if self.accumulationHours <= 1: guidanceSource = ffgGuidances[0] elif self.accumulationHours <= 3: guidanceSource = ffgGuidances[1] else: guidanceSource = ffgGuidances[2] else: guidanceSource = self._guidanceName request = DataAccessLayer.newDataRequest() request.setDatatype(FFMP_KEY) request.setParameters(guidanceSource) request.addIdentifier(WFO_KEY, self.currentSite) request.addIdentifier(SITE_KEY, self._siteKey) request.addIdentifier(HUC_KEY, ALL_HUC) request.addIdentifier(ACCUM_HRS, self.accumulationHours) availableTimes = DataAccessLayer.getAvailableTimes(request) if availableTimes: time = availableTimes[-1] geometries = DataAccessLayer.getGeometryData(request, [time]) for geometry in geometries: if isinstance(guidanceSource, list): geometry = self._interpolateGuidance(geometry) self.__storeGuidance(geometry.getLocationName(), geometry.getNumber(guidanceSource), geometry.getGeometry()) return True return False
def runGeometryDataTest(self, req, checkDataTimes=True): """ Test that we are able to successfully retrieve geometry data for the given request. """ times = DafTestCase.getTimesIfSupported(req) geomData = DAL.getGeometryData(req, times[:self.numTimesToLimit]) self.assertIsNotNone(geomData) if times: self.assertNotEqual(len(geomData), 0) if not geomData: raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: if (checkDataTimes and times and "PERIOD_USED" not in record.getDataTime().getUtilityFlags()): self.assertIn(record.getDataTime(), times[:self.numTimesToLimit]) print("geometry=" + str(record.getGeometry()), end="") for p in req.getParameters(): print(" " + p + "=" + record.getString(p), end="") print() return geomData
def _getAllRecords(self): req = DAL.newDataRequest() req.setDatatype(self.datatype) req.setParameters('id') return DAL.getGeometryData(req)
plt.ioff() # Initialize figure size (48,27) for 4K and (24,15) for 1080p plt.figure(num=1, figsize=(8, 5), dpi=80) m.drawmapboundary() # Look for cities list, if not create it... create it in file so you can # comment out cities if os.path.exists(homeDir + 'bin/' + rda + 'cities.pck') == False: cityList = [] req = DataAccessLayer.newDataRequest(datatype='maps', table='mapdata.city', geomField='the_geom', parameters=['name', 'lon', 'lat'], envelope=envelope) response = DataAccessLayer.getGeometryData(req, times=None) for city in response: cityInfo = str(city.getString('name')) + "," + str( city.getNumber('lon')) + "," + str(city.getNumber('lat')) cityList.append(cityInfo) pickle.dump(cityList, open(homeDir + 'bin/' + rda + 'cities.pck', 'w')) else: cityList = pickle.load(open(homeDir + 'bin/' + rda + 'cities.pck', 'r')) for city in cityList: (name, lon, lat) = city.split(',') if name[0] != "#": #for postediting to comment out lon, lat = m(lon, lat) if path.contains_point([lon, lat]): plt.text(lon,
from numpy import linspace from numpy import meshgrid import matplotlib.pyplot as plt DataAccessLayer.changeEDEXHost("edex.unidata.ucar.edu") gridTimeIndex = -1 # EDEX Data Access Framework request = DataAccessLayer.newDataRequest() request.setDatatype("maps") request.setParameters("cwa","wfo") request.addIdentifier("locationField","wfo") request.addIdentifier("geomField","the_geom") request.addIdentifier("table","mapdata.cwa") request.setLocationNames("BOU") response = DataAccessLayer.getGeometryData(request, None) # Now set area request = DataAccessLayer.newDataRequest() request.setEnvelope(response[0].getGeometry()) # Now query grid request.setDatatype("grid") request.setLocationNames("HRRR") request.setParameters("T") request.setLevels("500MB") t = DataAccessLayer.getAvailableTimes(request) print t exit() response = DataAccessLayer.getGridData(request, [t[gridTimeIndex]]) data = response[0]
def testFilterOnNonexistentLocationReturnsEmpty(self): req = DAL.newDataRequest() req.setDatatype(self.datatype) req.setParameters('id') req.setLocationNames('ZZZZ') self.assertEqual(len(DAL.getGeometryData(req)), 0)