def testFilterOnInvalidLocationThrowsIncompatibleRequestException(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(') and 0=1')
with self.assertRaises(Exception) as cm:
DAL.getGeometryData(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def getPolygonDict(self,
tablename,
siteID,
locationField="name",
columns=[]):
req = DataAccessLayer.newDataRequest()
req.setDatatype("maps")
table = "mapdata." + tablename
req.addIdentifier("table", table)
req.addIdentifier("geomField", "the_geom")
req.addIdentifier("locationField", locationField)
req.addIdentifier("cwa", siteID)
if columns:
req.setParameters(columns)
retGeoms = DataAccessLayer.getGeometryData(req)
retDict = {}
for retGeom in retGeoms:
id = retGeom.getLocationName()
if id:
poly = self._extract_poly_coords(retGeom.getGeometry())
formattedPoly = [list(c) for c in poly['exterior_coords']]
retDict[id] = formattedPoly
return retDict
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "ETA")
req.setLocationNames(params.OBS_STATION)
req.setParameters("temperature", "pressure", "specHum", "sfcPress",
"temp2", "q2")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=" + record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" sfcPress=" + record.getString("sfcPress") +
record.getUnit("sfcPress"),
end="")
print(" temp2=" + record.getString("temp2") +
record.getUnit("temp2"),
end="")
print(" q2=" + record.getString("q2") + record.getUnit("q2"),
end="")
else:
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" specHum=" + record.getString("specHum") +
record.getUnit("specHum"),
end="")
print(" geometry=" + str(record.getGeometry()))
print("getGeometryData() complete\n\n")
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "pressure", "uComponent",
"vComponent")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
self.assertIsNotNone(geomData)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level:", record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
else:
print(" uComponent=" + record.getString("uComponent") +
record.getUnit("uComponent"),
end="")
print(" vComponent=" + record.getString("vComponent") +
record.getUnit("vComponent"),
end="")
print(" geometry:", record.getGeometry())
print("getGeometryData() complete\n\n")
def getQPEValues(self):
'''
Strategy method for reading and accumulating data
from preprocessed FFMP QPE datasets.
'''
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(self._sourceName)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(DATA_KEY, self._dataKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
availableTimes = DataAccessLayer.getAvailableTimes(request)
# determine correct times
latestTime = 0
for time in availableTimes:
tm = time.getRefTime().getTime()
if tm > latestTime:
latestTime = tm
timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000
usedTimes = []
for time in availableTimes:
if time.getRefTime().getTime() >= timedelta:
usedTimes.append(time)
basins = []
if usedTimes:
geometries = DataAccessLayer.getGeometryData(request, usedTimes)
for geometry in geometries:
self.__storeQpe(geometry.getLocationName(),
geometry.getNumber(self._sourceName), geometry)
return True
return False
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
req.setEnvelope(params.ENVELOPE)
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "ETA")
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "pressure", "specHum", "sfcPress",
"temp2", "q2")
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testFilterOnLocationName(self):
allLocationNames = self._getLocationNames()
if len(allLocationNames) == 0:
errmsg = "No {0} data exists on {1}. Try again with {0} data."
raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST))
testCount = 3 # number of different location names to test
for locationName in allLocationNames[:testCount]:
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(locationName)
geomData = DAL.getGeometryData(req)
for geom in geomData:
self.assertEqual(geom.getLocationName(), locationName)
def getSounding(modelName,
weatherElements,
levels,
samplePoint,
refTime=None,
timeRange=None):
""""
Performs a series of Data Access Framework requests to retrieve a sounding object
based on the specified request parameters.
Args:
modelName: the grid model datasetid to use as the basis of the sounding.
weatherElements: a list of parameters to return in the sounding.
levels: a list of levels to sample the given weather elements at
samplePoint: a lat/lon pair to perform the sampling of data at.
refTime: (optional) the grid model reference time to use for the sounding.
If not specified, the latest ref time in the system will be used.
timeRange: (optional) a TimeRange to specify which forecast hours to use.
If not specified, will default to all forecast hours.
Returns:
A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
by DataTime, then by level and finally by weather element. If no
data is available for the given request parameters, None is returned.
"""
(locationNames, parameters, levels, envelope, refTime, timeRange) = \
__sanitizeInputs(modelName, weatherElements, levels, samplePoint, refTime, timeRange)
requestArgs = {
'datatype': 'grid',
'locationNames': locationNames,
'parameters': parameters,
'levels': levels,
'envelope': envelope,
}
req = DataAccessLayer.newDataRequest(**requestArgs)
forecastHours = __determineForecastHours(req, refTime, timeRange)
if not forecastHours:
return None
response = DataAccessLayer.getGeometryData(req, forecastHours)
soundingObject = _SoundingCube(response)
return soundingObject
def runGeometryDataTestWithTimeRange(self, req, timeRange):
"""
Test that we are able to successfully retrieve geometry data for the
given request.
"""
geomData = DAL.getGeometryData(req, timeRange)
self.assertIsNotNone(geomData)
if not geomData:
raise unittest.SkipTest("No data available")
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
self.assertGreaterEqual(
record.getDataTime().getRefTime().getTime(),
timeRange.getStartInMillis())
self.assertLessEqual(record.getDataTime().getRefTime().getTime(),
timeRange.getEndInMillis())
print("geometry=" + str(record.getGeometry()), end="")
for p in req.getParameters():
print(" " + p + "=" + record.getString(p), end="")
print()
return geomData
def getGuidanceValues(self):
'''
Retrieves the guidance values from the correct FFMP source
'''
ffmpSourceConfigManager = FFMPSourceConfigurationManager.getInstance()
if self._guidanceName == 'RFCFFG':
if self.accumulationHours <= 1:
guidanceSource = ffgGuidances[0]
elif self.accumulationHours <= 3:
guidanceSource = ffgGuidances[1]
else:
guidanceSource = ffgGuidances[2]
else:
guidanceSource = self._guidanceName
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(guidanceSource)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
request.addIdentifier(ACCUM_HRS, self.accumulationHours)
availableTimes = DataAccessLayer.getAvailableTimes(request)
if availableTimes:
time = availableTimes[-1]
geometries = DataAccessLayer.getGeometryData(request, [time])
for geometry in geometries:
if isinstance(guidanceSource, list):
geometry = self._interpolateGuidance(geometry)
self.__storeGuidance(geometry.getLocationName(),
geometry.getNumber(guidanceSource),
geometry.getGeometry())
return True
return False
def runGeometryDataTest(self, req, checkDataTimes=True):
"""
Test that we are able to successfully retrieve geometry data for the
given request.
"""
times = DafTestCase.getTimesIfSupported(req)
geomData = DAL.getGeometryData(req, times[:self.numTimesToLimit])
self.assertIsNotNone(geomData)
if times:
self.assertNotEqual(len(geomData), 0)
if not geomData:
raise unittest.SkipTest("No data available")
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
if (checkDataTimes and times and "PERIOD_USED"
not in record.getDataTime().getUtilityFlags()):
self.assertIn(record.getDataTime(),
times[:self.numTimesToLimit])
print("geometry=" + str(record.getGeometry()), end="")
for p in req.getParameters():
print(" " + p + "=" + record.getString(p), end="")
print()
return geomData
def _getAllRecords(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
return DAL.getGeometryData(req)
plt.ioff()
# Initialize figure size (48,27) for 4K and (24,15) for 1080p
plt.figure(num=1, figsize=(8, 5), dpi=80)
m.drawmapboundary()
# Look for cities list, if not create it... create it in file so you can
# comment out cities
if os.path.exists(homeDir + 'bin/' + rda + 'cities.pck') == False:
cityList = []
req = DataAccessLayer.newDataRequest(datatype='maps',
table='mapdata.city',
geomField='the_geom',
parameters=['name', 'lon', 'lat'],
envelope=envelope)
response = DataAccessLayer.getGeometryData(req, times=None)
for city in response:
cityInfo = str(city.getString('name')) + "," + str(
city.getNumber('lon')) + "," + str(city.getNumber('lat'))
cityList.append(cityInfo)
pickle.dump(cityList, open(homeDir + 'bin/' + rda + 'cities.pck', 'w'))
else:
cityList = pickle.load(open(homeDir + 'bin/' + rda + 'cities.pck',
'r'))
for city in cityList:
(name, lon, lat) = city.split(',')
if name[0] != "#": #for postediting to comment out
lon, lat = m(lon, lat)
if path.contains_point([lon, lat]):
plt.text(lon,
from numpy import linspace
from numpy import meshgrid
import matplotlib.pyplot as plt
DataAccessLayer.changeEDEXHost("edex.unidata.ucar.edu")
gridTimeIndex = -1
# EDEX Data Access Framework
request = DataAccessLayer.newDataRequest()
request.setDatatype("maps")
request.setParameters("cwa","wfo")
request.addIdentifier("locationField","wfo")
request.addIdentifier("geomField","the_geom")
request.addIdentifier("table","mapdata.cwa")
request.setLocationNames("BOU")
response = DataAccessLayer.getGeometryData(request, None)
# Now set area
request = DataAccessLayer.newDataRequest()
request.setEnvelope(response[0].getGeometry())
# Now query grid
request.setDatatype("grid")
request.setLocationNames("HRRR")
request.setParameters("T")
request.setLevels("500MB")
t = DataAccessLayer.getAvailableTimes(request)
print t
exit()
response = DataAccessLayer.getGridData(request, [t[gridTimeIndex]])
data = response[0]
def testFilterOnNonexistentLocationReturnsEmpty(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames('ZZZZ')
self.assertEqual(len(DAL.getGeometryData(req)), 0)
