def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
optionalIds = set(DAL.getOptionalIdentifiers(req))
requiredIds = set(DAL.getRequiredIdentifiers(req))
ids = requiredIds | optionalIds
for id in ids:
req = DAL.newDataRequest(self.datatype)
if id == 'accumHrs':
req.setParameters('ARI6H2YR')
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', self.location)
req.addIdentifier('huc', 'ALL')
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
print(id + " values: " + str(idValues))
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not (times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(),
'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(),
'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(),
'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(),
'Wind Direction should be different from Wind Speed')
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setLocationNames(params.SITE_ID)
req.setParameters('T')
return self.runGridDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters("Imager 11 micron IR")
req.setLocationNames("West CONUS")
return self.runGridDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters("temperature", "reportType")
req.setLocationNames(params.OBS_STATION)
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'cli_asos_monthly')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('station_code', 'avg_daily_max')
return self.runGeometryDataTest(req)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setLocationNames(self.radarLoc)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
self.runGridDataTest(req, testSameShape=False)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters('dataURI')
req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK')
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "ETA")
req.setLocationNames(params.OBS_STATION)
req.setParameters("temperature", "pressure", "specHum", "sfcPress",
"temp2", "q2")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=" + record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" sfcPress=" + record.getString("sfcPress") +
record.getUnit("sfcPress"),
end="")
print(" temp2=" + record.getString("temp2") +
record.getUnit("temp2"),
end="")
print(" q2=" + record.getString("q2") + record.getUnit("q2"),
end="")
else:
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" specHum=" + record.getString("specHum") +
record.getUnit("specHum"),
end="")
print(" geometry=" + str(record.getGeometry()))
print("getGeometryData() complete\n\n")
def testSuccessfulQuery(self):
req = DAL.newDataRequest('grid')
req.setLocationNames('RAP13')
req.setParameters('T', 'GH')
req.setLevels('300MB', '500MB', '700MB')
times = CTQ.getAvailableTimes(req)
self.assertNotEqual(len(times), 0)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "pressure", "uComponent",
"vComponent")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
self.assertIsNotNone(geomData)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level:", record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"),
end="")
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"),
end="")
else:
print(" uComponent=" + record.getString("uComponent") +
record.getUnit("uComponent"),
end="")
print(" vComponent=" + record.getString("vComponent") +
record.getUnit("vComponent"),
end="")
print(" geometry:", record.getGeometry())
print("getGeometryData() complete\n\n")
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "seaLevelPress", "dewpoint")
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters(*self.data_params)
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def getQPEValues(self):
'''
Strategy method for reading and accumulating data
from preprocessed FFMP QPE datasets.
'''
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(self._sourceName)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(DATA_KEY, self._dataKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
availableTimes = DataAccessLayer.getAvailableTimes(request)
# determine correct times
latestTime = 0
for time in availableTimes:
tm = time.getRefTime().getTime()
if tm > latestTime:
latestTime = tm
timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000
usedTimes = []
for time in availableTimes:
if time.getRefTime().getTime() >= timedelta:
usedTimes.append(time)
basins = []
if usedTimes:
geometries = DataAccessLayer.getGeometryData(request, usedTimes)
for geometry in geometries:
self.__storeQpe(geometry.getLocationName(),
geometry.getNumber(self._sourceName), geometry)
return True
return False
def runGetIdValuesTest(self, identifiers):
for id in identifiers:
if id.lower() == 'datauri':
continue
req = DAL.newDataRequest(self.datatype)
idValues = DAL.getIdentifierValues(req, id)
self.assertTrue(hasattr(idValues, '__iter__'))
def getPolygonDict(self,
tablename,
siteID,
locationField="name",
columns=[]):
req = DataAccessLayer.newDataRequest()
req.setDatatype("maps")
table = "mapdata." + tablename
req.addIdentifier("table", table)
req.addIdentifier("geomField", "the_geom")
req.addIdentifier("locationField", locationField)
req.addIdentifier("cwa", siteID)
if columns:
req.setParameters(columns)
retGeoms = DataAccessLayer.getGeometryData(req)
retDict = {}
for retGeom in retGeoms:
id = retGeom.getLocationName()
if id:
poly = self._extract_poly_coords(retGeom.getGeometry())
formattedPoly = [list(c) for c in poly['exterior_coords']]
retDict[id] = formattedPoly
return retDict
def runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
return self.runGridDataTest(req, testSameShape=False)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('huc', 'ALL')
req.setParameters('QPFSCAN')
return self.runGeometryDataTest(req, checkDataTimes=False)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
req.setEnvelope(params.ENVELOPE)
print("Testing getGeometryData()")
data = DAL.getGeometryData(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self):
# Test that accumHrs identifier is not required for guidance data
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('wfo', params.SITE_ID)
req.addIdentifier('siteKey', self.location)
req.addIdentifier('huc', 'ALL')
req.setParameters('FFG0124hr')
self.runGeometryDataTest(req, checkDataTimes=False)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
return self.runGeometryDataTest(req)
def __cloneRequest(request):
return DataAccessLayer.newDataRequest(
datatype=request.getDatatype(),
parameters=request.getParameters(),
levels=request.getLevels(),
locationNames=request.getLocationNames(),
envelope=request.getEnvelope(),
**request.getIdentifiers())
def testGetGeometryDataInvalidParamRaisesIncompatibleRequestException(
self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("source", "NLDN")
req.setParameters('blahblahblah')
with self.assertRaises(ThriftRequestException) as cm:
self.runGeometryDataTest(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertTrue(params.ENVELOPE.intersects(item.getGeometry()))
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('T')
gridDatas = self.runGridDataTest(req)
for gridData in gridDatas:
self.assertEqual(gridData.getAttribute('parmId.dbId.dbType'), '')
def testGetGeometryDataWithEnvelopeThrowsException(self):
# Envelope is not used
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setParameters('maxtemp_mon', 'min_sea_press')
req.setEnvelope(params.ENVELOPE)
with self.assertRaises(Exception):
data = self.runGeometryDataTest(req)
def testGetGeometryDataWithLocNamesAndGroupsThrowException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
req.addIdentifier(self.groupKey,
RequestConstraint.new('in', self.groups))
with self.assertRaises(ThriftRequestException):
self.runGeometryDataTest(req)
def testGetGeometryDataWithLocNames(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertIn(item.getLocationName(), self.editAreaNames)
def testFilterOnInvalidLocationThrowsIncompatibleRequestException(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(') and 0=1')
with self.assertRaises(Exception) as cm:
DAL.getGeometryData(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'mapdata.ffmp_basins')
req.addIdentifier('geomField', 'the_geom')
req.addIdentifier('cwa', 'OAX')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('state', 'reservoir', 'area_sq_mi')
return self.runGeometryDataTest(req, checkDataTimes=False)
from matplotlib.transforms import offset_copy
#import cartopy.crs as ccrs
#import cartopy.io.img_tiles as cimgt
from mpl_toolkits.basemap import Basemap, cm
# requires netcdf4-python (netcdf4-python.googlecode.com)
#from netCDF4 import Dataset as NetCDFFile
import numpy as np
from numpy import linspace
from numpy import meshgrid
import matplotlib.pyplot as plt
DataAccessLayer.changeEDEXHost("edex.unidata.ucar.edu")
gridTimeIndex = -1
# EDEX Data Access Framework
request = DataAccessLayer.newDataRequest()
request.setDatatype("maps")
request.setParameters("cwa","wfo")
request.addIdentifier("locationField","wfo")
request.addIdentifier("geomField","the_geom")
request.addIdentifier("table","mapdata.cwa")
request.setLocationNames("BOU")
response = DataAccessLayer.getGeometryData(request, None)
# Now set area
request = DataAccessLayer.newDataRequest()
request.setEnvelope(response[0].getGeometry())
# Now query grid
request.setDatatype("grid")
request.setLocationNames("HRRR")
