def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not (times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(),
'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(),
'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(),
'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(),
'Wind Direction should be different from Wind Speed')
def getQPEValues(self):
'''
Strategy method for reading and accumulating data
from preprocessed FFMP QPE datasets.
'''
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(self._sourceName)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(DATA_KEY, self._dataKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
availableTimes = DataAccessLayer.getAvailableTimes(request)
# determine correct times
latestTime = 0
for time in availableTimes:
tm = time.getRefTime().getTime()
if tm > latestTime:
latestTime = tm
timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000
usedTimes = []
for time in availableTimes:
if time.getRefTime().getTime() >= timedelta:
usedTimes.append(time)
basins = []
if usedTimes:
geometries = DataAccessLayer.getGeometryData(request, usedTimes)
for geometry in geometries:
self.__storeQpe(geometry.getLocationName(),
geometry.getNumber(self._sourceName), geometry)
return True
return False
def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
locations = request.getLocationNames()
if locations:
times = None
for location in locations:
specificRequest = __cloneRequest(request)
specificRequest.setLocationNames(location)
specificTimes = DataAccessLayer.getAvailableTimes(
specificRequest, refTimeOnly)
if times is None:
times = set(specificTimes)
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def getTimesIfSupported(req):
"""Return available times for req. If req refers to a time-agnostic
datatype, return an empty list instead.
"""
times = []
try:
times = DAL.getAvailableTimes(req)
except ThriftRequestException as e:
if not 'TimeAgnosticDataException' in str(e):
raise
return times
def testGetDataWithTimeRange(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
times = DAL.getAvailableTimes(req)
limitTimes = times[-self.numTimesToLimit:]
startTime = datetime.datetime.utcfromtimestamp(
limitTimes[0].getRefTime().getTime() / 1000)
endTime = datetime.datetime.utcnow()
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def __determineForecastHours(request, refTime, timeRange):
dataTimes = DataAccessLayer.getAvailableTimes(request, False)
timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime)
for dataTime in dataTimes]
dataTimesMap = defaultdict(list)
for baseTime, dataTime in timesGen:
dataTimesMap[baseTime].append(dataTime)
if refTime is None:
refTime = max(dataTimesMap.keys())
forecastHours = dataTimesMap[refTime]
if timeRange is None:
return forecastHours
else:
return [
forecastHour for forecastHour in forecastHours
if timeRange.contains(forecastHour.getValidPeriod())
]
def getGuidanceValues(self):
'''
Retrieves the guidance values from the correct FFMP source
'''
ffmpSourceConfigManager = FFMPSourceConfigurationManager.getInstance()
if self._guidanceName == 'RFCFFG':
if self.accumulationHours <= 1:
guidanceSource = ffgGuidances[0]
elif self.accumulationHours <= 3:
guidanceSource = ffgGuidances[1]
else:
guidanceSource = ffgGuidances[2]
else:
guidanceSource = self._guidanceName
request = DataAccessLayer.newDataRequest()
request.setDatatype(FFMP_KEY)
request.setParameters(guidanceSource)
request.addIdentifier(WFO_KEY, self.currentSite)
request.addIdentifier(SITE_KEY, self._siteKey)
request.addIdentifier(HUC_KEY, ALL_HUC)
request.addIdentifier(ACCUM_HRS, self.accumulationHours)
availableTimes = DataAccessLayer.getAvailableTimes(request)
if availableTimes:
time = availableTimes[-1]
geometries = DataAccessLayer.getGeometryData(request, [time])
for geometry in geometries:
if isinstance(guidanceSource, list):
geometry = self._interpolateGuidance(geometry)
self.__storeGuidance(geometry.getLocationName(),
geometry.getNumber(guidanceSource),
geometry.getGeometry())
return True
return False
if parse(i.split('.')[0]) < cutoff:
print('removing old image{}'.format(i))
os.remove(homeDir + 'images/' + i)
############################################################################
# Initialize new request
req = DataAccessLayer.newDataRequest()
req.setDatatype('grid')
# Set desired data to pull
req.setLocationNames('MRMS_1000')
req.setParameters('MergedReflectivityQCComposite')
req.setLevels('500.0FH')
# Pull available times
times = DataAccessLayer.getAvailableTimes(req)
# Get the lat/lon data
#response = DataAccessLayer.getGridData(req, [times[-1]])
#lon, lat = response[0].getLatLonCoords()
#lon, lat = m(lon, lat)
############################################################################
# Turn plotting off
plt.ioff()
# Make images... each frame is two minutes. So 30 for an hour, 60 for two, etc.
for x in range(-240, 0, 1):
# Filename
filename = times[x].validPeriod.end.strftime('%Y-%m-%d-%H%M') + '.png'
def runTimeAgnosticTest(self, req):
with self.assertRaises(ThriftRequestException) as cm:
times = DAL.getAvailableTimes(req)
self.assertIn('TimeAgnosticDataException', str(cm.exception))
def runTimesTest(self, req):
times = DAL.getAvailableTimes(req)
self.assertIsNotNone(times)
print("Number of times: " + str(len(times)))
strTimes = [str(t) for t in times[:self.sampleDataLimit]]
print("Sample times:\n" + str(strTimes))
request.addIdentifier("locationField","wfo")
request.addIdentifier("geomField","the_geom")
request.addIdentifier("table","mapdata.cwa")
request.setLocationNames("BOU")
response = DataAccessLayer.getGeometryData(request, None)
# Now set area
request = DataAccessLayer.newDataRequest()
request.setEnvelope(response[0].getGeometry())
# Now query grid
request.setDatatype("grid")
request.setLocationNames("HRRR")
request.setParameters("T")
request.setLevels("500MB")
t = DataAccessLayer.getAvailableTimes(request)
print t
exit()
response = DataAccessLayer.getGridData(request, [t[gridTimeIndex]])
data = response[0]
fig = plt.figure(figsize=(8,8))
ax = fig.add_axes([0.1,0.1,0.8,0.8])
lons,lats = data.getLatLonCoords()
lat_min = min(lats[-1])
lat_max = max(lats[0])
lon_min = min(lons[0])
lon_max = max(lons[-1])
