def getPolygonDict(self,
tablename,
siteID,
locationField="name",
columns=[]):
req = DataAccessLayer.newDataRequest()
req.setDatatype("maps")
table = "mapdata." + tablename
req.addIdentifier("table", table)
req.addIdentifier("geomField", "the_geom")
req.addIdentifier("locationField", locationField)
req.addIdentifier("cwa", siteID)
if columns:
req.setParameters(columns)
retGeoms = DataAccessLayer.getGeometryData(req)
retDict = {}
for retGeom in retGeoms:
id = retGeom.getLocationName()
if id:
poly = self._extract_poly_coords(retGeom.getGeometry())
formattedPoly = [list(c) for c in poly['exterior_coords']]
retDict[id] = formattedPoly
return retDict
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "ETA")
req.setLocationNames(params.OBS_STATION)
req.setParameters("temperature", "pressure", "specHum", "sfcPress",
"temp2", "q2")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=" + record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" sfcPress=" + record.getString("sfcPress") +
record.getUnit("sfcPress"),
end="")
print(" temp2=" + record.getString("temp2") +
record.getUnit("temp2"),
end="")
print(" q2=" + record.getString("q2") + record.getUnit("q2"),
end="")
else:
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" specHum=" + record.getString("specHum") +
record.getUnit("specHum"),
end="")
print(" geometry=" + str(record.getGeometry()))
print("getGeometryData() complete\n\n")
def runGetIdValuesTest(self, identifiers):
for id in identifiers:
if id.lower() == 'datauri':
continue
req = DAL.newDataRequest(self.datatype)
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "pressure", "uComponent",
"vComponent")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
self.assertIsNotNone(geomData)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level:", record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
else:
print(" uComponent=" + record.getString("uComponent") +
record.getUnit("uComponent"),
end="")
print(" vComponent=" + record.getString("vComponent") +
record.getUnit("vComponent"),
end="")
print(" geometry:", record.getGeometry())
print("getGeometryData() complete\n\n")
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not (times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(),
'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(),
'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(),
'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(),
'Wind Direction should be different from Wind Speed')
def getQPEValues(self):
'''
Strategy method for reading and accumulating data
from preprocessed FFMP QPE datasets.
'''
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(self._sourceName)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(DATA_KEY, self._dataKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
availableTimes = DataAccessLayer.getAvailableTimes(request)
# determine correct times
latestTime = 0
for time in availableTimes:
tm = time.getRefTime().getTime()
if tm > latestTime:
latestTime = tm
timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000
usedTimes = []
for time in availableTimes:
if time.getRefTime().getTime() >= timedelta:
usedTimes.append(time)
basins = []
if usedTimes:
geometries = DataAccessLayer.getGeometryData(request, usedTimes)
for geometry in geometries:
self.__storeQpe(geometry.getLocationName(),
geometry.getNumber(self._sourceName), geometry)
return True
return False
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
req.setEnvelope(params.ENVELOPE)
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testFilterOnInvalidLocationThrowsIncompatibleRequestException(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(') and 0=1')
with self.assertRaises(Exception) as cm:
DAL.getGeometryData(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetGeometryDataWithLocationSpecified(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'fcstheight')
locs = DAL.getAvailableLocationNames(req)
if locs:
req.setLocationNames(locs[0])
req.setParameters('probability', 'value')
data = self.runGeometryDataTest(req)
self.assertNotEqual(len(data), 0)
def getPolygonNames(self, tablename, locationField="name", columns=[]):
req = DataAccessLayer.newDataRequest()
req.setDatatype("maps")
table = "mapdata." + tablename
req.addIdentifier("table", table)
req.addIdentifier("geomField", "the_geom")
req.addIdentifier("locationField", locationField)
locNames = DataAccessLayer.getAvailableLocationNames(req)
return list(locNames)
def setEDEXHost(host):
"""
Changes the EDEX host the Data Access Framework is communicating with.
Args:
host: the EDEX host to connect to
"""
if host:
DataAccessLayer.changeEDEXHost(str(host))
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "ETA")
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "pressure", "specHum", "sfcPress",
"temp2", "q2")
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testRequestingTooMuchDataThrowsResponseTooLargeException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
req.addIdentifier("dataset", "full")
points = ((-180, 90), (180, 90), (180, -90), (-180, -90))
poly = shapely.geometry.LinearRing(points)
req.setEnvelope(poly)
with self.assertRaises(ThriftRequestException) as cm:
DAL.getGridData(req)
self.assertIn('ResponseTooLargeException', str(cm.exception))
def testGetDataWithTimeRange(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
times = DAL.getAvailableTimes(req)
limitTimes = times[-self.numTimesToLimit:]
startTime = datetime.datetime.utcfromtimestamp(
limitTimes[0].getRefTime().getTime() / 1000)
endTime = datetime.datetime.utcnow()
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', 'ENSEMBLE')
req.setLevels('2FHAG')
req.setParameters('T')
idValues = DAL.getIdentifierValues(req, 'info.ensembleId')
self.assertTrue(hasattr(idValues, '__iter__'))
if idValues:
self.assertIn('ctl1', idValues)
self.assertIn('p1', idValues)
self.assertIn('n1', idValues)
else:
raise unittest.SkipTest("no data available")
def testFilterOnLocationName(self):
allLocationNames = self._getLocationNames()
if len(allLocationNames) == 0:
errmsg = "No {0} data exists on {1}. Try again with {0} data."
raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST))
testCount = 3 # number of different location names to test
for locationName in allLocationNames[:testCount]:
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(locationName)
geomData = DAL.getGeometryData(req)
for geom in geomData:
self.assertEqual(geom.getLocationName(), locationName)
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
optionalIds = set(DAL.getOptionalIdentifiers(req))
requiredIds = set(DAL.getRequiredIdentifiers(req))
ids = requiredIds | optionalIds
for id in ids:
req = DAL.newDataRequest(self.datatype)
if id == 'accumHrs':
req.setParameters('ARI6H2YR')
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', self.location)
req.addIdentifier('huc', 'ALL')
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
print(id + " values: " + str(idValues))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setLocationNames(params.SITE_ID)
req.setParameters('T')
return self.runGridDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters('dataURI')
req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK')
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "seaLevelPress", "dewpoint")
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testSuccessfulQuery(self):
req = DAL.newDataRequest('grid')
req.setLocationNames('RAP13')
req.setParameters('T', 'GH')
req.setLevels('300MB', '500MB', '700MB')
times = CTQ.getAvailableTimes(req)
self.assertNotEqual(len(times), 0)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters(*self.data_params)
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters("temperature", "reportType")
req.setLocationNames(params.OBS_STATION)
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters("Imager 11 micron IR")
req.setLocationNames("West CONUS")
return self.runGridDataTest(req)
def runGridDataTest(self, req, testSameShape=True):
"""
Test that we are able to successfully retrieve grid data for the given
request.
Args:
testSameShape: whether or not to verify that all the retrieved data
have the same shape (most data don't change shape)
"""
times = DafTestCase.getTimesIfSupported(req)
gridData = DAL.getGridData(req, times[:self.numTimesToLimit])
self.assertIsNotNone(gridData)
if not gridData:
raise unittest.SkipTest("No data available")
print("Number of grid records: " + str(len(gridData)))
if len(gridData) > 0:
print("Sample grid data shape:\n" +
str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
print("Sample lat-lon data:\n" +
str(gridData[0].getLatLonCoords()) + "\n")
if testSameShape:
correctGridShape = gridData[0].getLatLonCoords()[0].shape
for record in gridData:
rawData = record.getRawData()
self.assertIsNotNone(rawData)
self.assertEqual(rawData.shape, correctGridShape)
return gridData
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'cli_asos_monthly')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('station_code', 'avg_daily_max')
return self.runGeometryDataTest(req)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setLocationNames(self.radarLoc)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
self.runGridDataTest(req, testSameShape=False)
def runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
return self.runGridDataTest(req, testSameShape=False)
def getSounding(modelName,
weatherElements,
levels,
samplePoint,
refTime=None,
timeRange=None):
""""
Performs a series of Data Access Framework requests to retrieve a sounding object
based on the specified request parameters.
Args:
modelName: the grid model datasetid to use as the basis of the sounding.
weatherElements: a list of parameters to return in the sounding.
levels: a list of levels to sample the given weather elements at
samplePoint: a lat/lon pair to perform the sampling of data at.
refTime: (optional) the grid model reference time to use for the sounding.
If not specified, the latest ref time in the system will be used.
timeRange: (optional) a TimeRange to specify which forecast hours to use.
If not specified, will default to all forecast hours.
Returns:
A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
by DataTime, then by level and finally by weather element. If no
data is available for the given request parameters, None is returned.
"""
(locationNames, parameters, levels, envelope, refTime, timeRange) = \
__sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
requestArgs = {
'datatype': 'grid',
'locationNames': locationNames,
'parameters': parameters,
'levels': levels,
'envelope': envelope,
}
req = DataAccessLayer.newDataRequest(**requestArgs)
forecastHours = __determineForecastHours(req, refTime, timeRange)
if not forecastHours:
return None
response = DataAccessLayer.getGeometryData(req, forecastHours)
soundingObject = _SoundingCube(response)
return soundingObject
import string
import cherrypy
from ufpy.dataaccess import DataAccessLayer
import matplotlib.pyplot as plt
from matplotlib.transforms import offset_copy
#import cartopy.crs as ccrs
#import cartopy.io.img_tiles as cimgt
from mpl_toolkits.basemap import Basemap, cm
# requires netcdf4-python (netcdf4-python.googlecode.com)
#from netCDF4 import Dataset as NetCDFFile
import numpy as np
from numpy import linspace
from numpy import meshgrid
import matplotlib.pyplot as plt
DataAccessLayer.changeEDEXHost("edex.unidata.ucar.edu")
gridTimeIndex = -1
# EDEX Data Access Framework
request = DataAccessLayer.newDataRequest()
request.setDatatype("maps")
request.setParameters("cwa","wfo")
request.addIdentifier("locationField","wfo")
request.addIdentifier("geomField","the_geom")
request.addIdentifier("table","mapdata.cwa")
request.setLocationNames("BOU")
response = DataAccessLayer.getGeometryData(request, None)
# Now set area
request = DataAccessLayer.newDataRequest()
request.setEnvelope(response[0].getGeometry())