def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('parmId.dbId.modelName', 'Fcst') req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not (times): raise unittest.SkipTest('No Wind Data available for testing') gridData = DAL.getGridData(req, [times[0]]) rawWind = None rawDir = None for grid in gridData: if grid.getParameter() == 'Wind': self.assertEqual(grid.getUnit(), 'kts') rawWind = grid.getRawData() elif grid.getParameter() == 'WindDirection': self.assertEqual(grid.getUnit(), 'deg') rawDir = grid.getRawData() self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present') self.assertIsNotNone(rawDir, 'Wind Direction grid is not present') # rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays. self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values') self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values') self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360') self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def getQPEValues(self): ''' Strategy method for reading and accumulating data from preprocessed FFMP QPE datasets. ''' request = DataAccessLayer.newDataRequest() request.setDatatype(FFMP_KEY) request.setParameters(self._sourceName) request.addIdentifier(WFO_KEY, self.currentSite) request.addIdentifier(SITE_KEY, self._siteKey) request.addIdentifier(DATA_KEY, self._dataKey) request.addIdentifier(HUC_KEY, ALL_HUC) availableTimes = DataAccessLayer.getAvailableTimes(request) # determine correct times latestTime = 0 for time in availableTimes: tm = time.getRefTime().getTime() if tm > latestTime: latestTime = tm timedelta = latestTime - self.accumulationHours * 60 * 60 * 1000 usedTimes = [] for time in availableTimes: if time.getRefTime().getTime() >= timedelta: usedTimes.append(time) basins = [] if usedTimes: geometries = DataAccessLayer.getGeometryData(request, usedTimes) for geometry in geometries: self.__storeQpe(geometry.getLocationName(), geometry.getNumber(self._sourceName), geometry) return True return False
def __getAvailableTimesForEachLocation(request, refTimeOnly=False): locations = request.getLocationNames() if locations: times = None for location in locations: specificRequest = __cloneRequest(request) specificRequest.setLocationNames(location) specificTimes = DataAccessLayer.getAvailableTimes( specificRequest, refTimeOnly) if times is None: times = set(specificTimes) else: times.intersection_update(specificTimes) if not times: break return times else: return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def testNoDuplicateData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_monthly') req.setLocationNames('KOMA') req.setParameters('maxtemp_day1') rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5]) for i in range(len(rows)): for j in range(len(rows)): if i != j: self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def getTimesIfSupported(req): """Return available times for req. If req refers to a time-agnostic datatype, return an empty list instead. """ times = [] try: times = DAL.getAvailableTimes(req) except ThriftRequestException as e: if not 'TimeAgnosticDataException' in str(e): raise return times
def testGetDataWithTimeRange(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'height') req.addIdentifier('ts', 'RG') req.setParameters('value', 'lid', 'quality_code') times = DAL.getAvailableTimes(req) limitTimes = times[-self.numTimesToLimit:] startTime = datetime.datetime.utcfromtimestamp( limitTimes[0].getRefTime().getTime() / 1000) endTime = datetime.datetime.utcnow() tr = TimeRange(startTime, endTime) self.runGeometryDataTestWithTimeRange(req, tr)
def __determineForecastHours(request, refTime, timeRange): dataTimes = DataAccessLayer.getAvailableTimes(request, False) timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes] dataTimesMap = defaultdict(list) for baseTime, dataTime in timesGen: dataTimesMap[baseTime].append(dataTime) if refTime is None: refTime = max(dataTimesMap.keys()) forecastHours = dataTimesMap[refTime] if timeRange is None: return forecastHours else: return [ forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod()) ]
def getGuidanceValues(self): ''' Retrieves the guidance values from the correct FFMP source ''' ffmpSourceConfigManager = FFMPSourceConfigurationManager.getInstance() if self._guidanceName == 'RFCFFG': if self.accumulationHours <= 1: guidanceSource = ffgGuidances[0] elif self.accumulationHours <= 3: guidanceSource = ffgGuidances[1] else: guidanceSource = ffgGuidances[2] else: guidanceSource = self._guidanceName request = DataAccessLayer.newDataRequest() request.setDatatype(FFMP_KEY) request.setParameters(guidanceSource) request.addIdentifier(WFO_KEY, self.currentSite) request.addIdentifier(SITE_KEY, self._siteKey) request.addIdentifier(HUC_KEY, ALL_HUC) request.addIdentifier(ACCUM_HRS, self.accumulationHours) availableTimes = DataAccessLayer.getAvailableTimes(request) if availableTimes: time = availableTimes[-1] geometries = DataAccessLayer.getGeometryData(request, [time]) for geometry in geometries: if isinstance(guidanceSource, list): geometry = self._interpolateGuidance(geometry) self.__storeGuidance(geometry.getLocationName(), geometry.getNumber(guidanceSource), geometry.getGeometry()) return True return False
if parse(i.split('.')[0]) < cutoff: print('removing old image{}'.format(i)) os.remove(homeDir + 'images/' + i) ############################################################################ # Initialize new request req = DataAccessLayer.newDataRequest() req.setDatatype('grid') # Set desired data to pull req.setLocationNames('MRMS_1000') req.setParameters('MergedReflectivityQCComposite') req.setLevels('500.0FH') # Pull available times times = DataAccessLayer.getAvailableTimes(req) # Get the lat/lon data #response = DataAccessLayer.getGridData(req, [times[-1]]) #lon, lat = response[0].getLatLonCoords() #lon, lat = m(lon, lat) ############################################################################ # Turn plotting off plt.ioff() # Make images... each frame is two minutes. So 30 for an hour, 60 for two, etc. for x in range(-240, 0, 1): # Filename filename = times[x].validPeriod.end.strftime('%Y-%m-%d-%H%M') + '.png'
def runTimeAgnosticTest(self, req): with self.assertRaises(ThriftRequestException) as cm: times = DAL.getAvailableTimes(req) self.assertIn('TimeAgnosticDataException', str(cm.exception))
def runTimesTest(self, req): times = DAL.getAvailableTimes(req) self.assertIsNotNone(times) print("Number of times: " + str(len(times))) strTimes = [str(t) for t in times[:self.sampleDataLimit]] print("Sample times:\n" + str(strTimes))
request.addIdentifier("locationField","wfo") request.addIdentifier("geomField","the_geom") request.addIdentifier("table","mapdata.cwa") request.setLocationNames("BOU") response = DataAccessLayer.getGeometryData(request, None) # Now set area request = DataAccessLayer.newDataRequest() request.setEnvelope(response[0].getGeometry()) # Now query grid request.setDatatype("grid") request.setLocationNames("HRRR") request.setParameters("T") request.setLevels("500MB") t = DataAccessLayer.getAvailableTimes(request) print t exit() response = DataAccessLayer.getGridData(request, [t[gridTimeIndex]]) data = response[0] fig = plt.figure(figsize=(8,8)) ax = fig.add_axes([0.1,0.1,0.8,0.8]) lons,lats = data.getLatLonCoords() lat_min = min(lats[-1]) lat_max = max(lats[0]) lon_min = min(lons[0]) lon_max = max(lons[-1])