def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('parmId.dbId.modelName', 'Fcst') req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not(times): raise unittest.SkipTest('No Wind Data available for testing') gridData = DAL.getGridData(req, [times[0]]) rawWind = None rawDir = None for grid in gridData: if grid.getParameter() == 'Wind': self.assertEqual(grid.getUnit(),'kts') rawWind = grid.getRawData() elif grid.getParameter() == 'WindDirection': self.assertEqual(grid.getUnit(),'deg') rawDir = grid.getRawData() self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present') self.assertIsNotNone(rawDir, 'Wind Direction grid is not present') # rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays. self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values') self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values') self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360') self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('modelName', 'Fcst') req.addIdentifier('siteId', 'OAX') req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not (times): raise unittest.SkipTest('No Wind Data available for testing') gridData = DAL.getGridData(req, [times[0]]) rawWind = None rawDir = None for grid in gridData: if grid.getParameter() == 'Wind': self.assertEqual(grid.getUnit(), 'kts') rawWind = grid.getRawData() elif grid.getParameter() == 'WindDirection': self.assertEqual(grid.getUnit(), 'deg') rawDir = grid.getRawData() self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present') self.assertIsNotNone(rawDir, 'Wind Direction grid is not present') # rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays. self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values') self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values') self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360') self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def __getAvailableTimesForEachLocation(request, refTimeOnly=False): locations = request.getLocationNames() if locations: times = None for location in locations: specificRequest = __cloneRequest(request) specificRequest.setLocationNames(location) specificTimes = DataAccessLayer.getAvailableTimes(specificRequest, refTimeOnly) if times is None: times = set(specificTimes) else: times.intersection_update(specificTimes) if not times: break return times else: return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def fcst_times(self, interval=3600, max_forecast=None): ''' Name: awips_fcst_times Purpose: A function to get forecast times for latest model run Inputs: request : A DataAccessLayer request object Outputs: Returns a list of forecast times Keywords: interval : Time step between forecast times in seconds Default is 3600s (1 hour) max_forecast : Maximum forecast time to get, in seconds. Default is last available time ''' cycles = DAL.getAvailableTimes(self._request, True) # Get forecast cycles times = DAL.getAvailableTimes(self._request) # Get forecast times times = DAL.getForecastRun(cycles[-1], times) # Get forecast times in latest cycle if max_forecast is None: max_forecast = times[-1].getFcstTime() # Set max_forecast value default based on model nTimes = max_forecast // interval + 1 # Number of forecast steps to get based on inteval flt_times = [[] for i in range(nTimes) ] # Initialized list of empty lists for forecast times for time in times: # Iterate over all times fcstTime = time.getFcstTime() # Get the valid forecast time if ((fcstTime % interval) == 0) and ( fcstTime < max_forecast ): # If the forecast hour falls on the interval requested AND is before the max forecast time fcstDur = time.getValidPeriod().duration() # Get duration of the forecast period if (fcstDur == 0) or ( fcstDur == interval ): # If instantaneous forecast period OR period covers requested interval index = fcstTime // interval # Index for the times flt_times array flt_times[index].append(time) # Append the time to the list at index return flt_times
def main(): user_args = get_args() if user_args.host: DataAccessLayer.changeEDEXHost(user_args.host) req = DataAccessLayer.newDataRequest("grid") if not user_args.srcId: print >> sys.stderr, "srcId not provided" return req.addIdentifier("info.datasetId", user_args.srcId) if user_args.varAbrev: req.setParameters(user_args.varAbrev) if user_args.lvlName is not None: req.addIdentifier("info.level.masterLevel.name", user_args.lvlName) if user_args.lvlOne is not None: req.addIdentifier("info.level.levelonevalue", numpy.float64(user_args.lvlOne)) if user_args.lvlTwo is not None: req.addIdentifier("info.level.leveltwovalue", numpy.float64(user_args.lvlTwo)) mode = user_args.mode if mode not in ["time", "plane", "field", "fieldplane"]: print >> sys.stderr, "mode must be one of time, plane, field, or fieldplane." return msg = "" if mode == "time": times = DataAccessLayer.getAvailableTimes(req) for time in times: timeStr = str(time) if "--" in timeStr: timeStr = timeStr[0:-22] + ".0" + timeStr[ -22:-1] + ".0" + timeStr[-1] msg += timeStr[0:19] + ".0" + timeStr[19:] + "\n" elif mode == "plane": levels = DataAccessLayer.getAvailableLevels(req) for level in levels: msg += level_to_string(level) + "\n" elif mode == "field": params = DataAccessLayer.getAvailableParameters(req) msg = "\n".join(params) else: #fieldplane params = DataAccessLayer.getAvailableParameters(req) for param in params: msg += param + ":\n" req.setParameters(param) levels = DataAccessLayer.getAvailableLevels(req) if levels: levelStr = [] for level in levels: levelStr.append(level_to_string(level)) msg += " ".join(levelStr) + " \n" print msg.strip("\n")
def download(site: str): # %% SITE site = site.lower() DataAccessLayer.changeEDEXHost('edex-cloud.unidata.ucar.edu') request = DataAccessLayer.newDataRequest() request.setDatatype('radar') request.setLocationNames(site) # %% TIME times = DataAccessLayer.getAvailableTimes(request) times = [parse(str(t)) for t in times] timerange = TimeRange(times[0], times[-1]) # %% REQUEST client = ThriftClient.ThriftClient('edex-cloud.unidata.ucar.edu') request = GetRadarDataRecordRequest() request.setTimeRange(timerange) request.setRadarId(site) code = 'N0Q' request.setProductCode(nexrad[code]['id']) request.setPrimaryElevationAngle(nexrad[code]['elev']) response = client.sendRequest(request) records = response.getData() print(f'found {len(records)} records at {site}') if not response.getData(): raise OSError(f'data not available {timerange}') for rec in records: idra = rec.getHdf5Data() rdat, azdat, depVals, threshVals = RadarCommon.get_hdf5_data(idra) # dim = rdat.getDimension() lat, lon = float(rec.getLatitude()), float(rec.getLongitude()) radials, rangeGates = rdat.getSizes() # Convert raw byte to pixel value array = np.array(rdat.getByteData()) array[array < 0] = array[array < 0] + 256 if azdat: azVals = azdat.getFloatData() az = np.array(RadarCommon.encode_radial(azVals)) # dattyp = RadarCommon.get_data_type(azdat) az = np.append(az, az[-1]) # header = RadarCommon.get_header(rec, format, rangeGates, radials, azdat, 'description') rng = np.linspace(0, rangeGates, rangeGates + 1) * nexrad[code]['res'] lats = np.empty((rng.size, az.size)) lons = np.empty_like(lats) for i, a in enumerate(az): lats, lons, _ = vreckon(lat, lon, rng, a)
def testNoDuplicateData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_monthly') req.setLocationNames('KOMA') req.setParameters('maxtemp_day1') rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5]) for i in range(len(rows)): for j in range(len(rows)): if i != j: self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testGetDataWithTimeRange(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'height') req.addIdentifier('ts', 'RG') req.setParameters('value', 'lid', 'quality_code') times = DAL.getAvailableTimes(req) limitTimes = times[-self.numTimesToLimit:] startTime = datetime.datetime.utcfromtimestamp(limitTimes[0].getRefTime().getTime()/1000) endTime = datetime.datetime.utcnow() tr = TimeRange(startTime, endTime) self.runGeometryDataTestWithTimeRange(req, tr)
def getTimesIfSupported(req): """Return available times for req. If req refers to a time-agnostic datatype, return an empty list instead. """ times = [] try: times = DAL.getAvailableTimes(req) except ThriftRequestException as e: if 'TimeAgnosticDataException' not in str(e): raise return times
def getTimesIfSupported(req): """Return available times for req. If req refers to a time-agnostic datatype, return an empty list instead. """ times = [] try: times = DAL.getAvailableTimes(req) except ThriftRequestException as e: if not 'TimeAgnosticDataException' in str(e): raise return times
def testGetDataWithTimeRange(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'height') req.addIdentifier('ts', 'RG') req.setParameters('value', 'lid', 'quality_code') times = DAL.getAvailableTimes(req) limitTimes = times[-self.numTimesToLimit:] startTime = datetime.datetime.utcfromtimestamp( limitTimes[0].getRefTime().getTime() / 1000) endTime = datetime.datetime.utcnow() tr = TimeRange(startTime, endTime) self.runGeometryDataTestWithTimeRange(req, tr)
def json(self, name="", parm="", level="",time=""): # # Need point query results for time-series # Cross-sections # Time-height # Var vs. Height # request = DataAccessLayer.newDataRequest() request.setDatatype("grid") if name != "": request.setLocationNames(name) if parm != "": request.setParameters(parm) if level != "": request.setLevels(level) cycles = DataAccessLayer.getAvailableTimes(request, True) t = DataAccessLayer.getAvailableTimes(request) fcstRun = [] #for time in t: # if str(time)[:19] == str(cycles[-1]): # fcstRun.append(time) fcstRun.append(t[0]) response = DataAccessLayer.getGridData(request, fcstRun) grid = response[0] data = grid.getRawData() lons, lats = grid.getLatLonCoords() columns = ( 'dtype', 'id', 'crs', 'dx', 'dy', 'firstgridpointcorner', 'the_geom', 'la1', 'lo1', 'name', 'nx', 'ny', 'spacingunit', 'latin1', 'latin2', 'lov', 'majoraxis', 'minoraxis', 'la2', 'latin', 'lo2', 'lad' ) return json.dumps(data, indent=2)
def __determineForecastHours(request, refTime, timeRange): dataTimes = DataAccessLayer.getAvailableTimes(request, False) timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes] dataTimesMap = defaultdict(list) for baseTime, dataTime in timesGen: dataTimesMap[baseTime].append(dataTime) if refTime is None: refTime = max(dataTimesMap.keys()) forecastHours = dataTimesMap[refTime] if timeRange is None: return forecastHours else: return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
request.setDatatype("grid") available_grids = DataAccessLayer.getAvailableLocationNames(request) available_grids.sort() list(available_grids) request.setLocationNames("NAM12") availableParms = DataAccessLayer.getAvailableParameters(request) availableParms.sort() list(availableParms) request.setParameters("T") availableLevels = DataAccessLayer.getAvailableLevels(request) request.setLevels("0.0SFC") cycles = DataAccessLayer.getAvailableTimes(request, True) times = DataAccessLayer.getAvailableTimes(request) fcstRun = DataAccessLayer.getForecastRun(cycles[-2], times) list(fcstRun) response = DataAccessLayer.getGridData(request, [fcstRun[36]]) for grid in response: data = grid.getRawData() lons, lats = grid.getLatLonCoords() print('Time :', str(grid.getDataTime().getFcstTime() / 3600)) Ta = ((data - 273.15)) DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu") dataTypes = DataAccessLayer.getSupportedDatatypes() dataTypes.sort() list(dataTypes)
def runTimeAgnosticTest(self, req): with self.assertRaises(ThriftRequestException) as cm: times = DAL.getAvailableTimes(req) self.assertIn('TimeAgnosticDataException', str(cm.exception))
def runTimesTest(self, req): times = DAL.getAvailableTimes(req) self.assertIsNotNone(times) print("Number of times: " + str(len(times))) strTimes = [str(t) for t in times[:self.sampleDataLimit]] print("Sample times:\n" + str(strTimes))
def runTimeAgnosticTest(self, req): with self.assertRaises(ThriftRequestException) as cm: DAL.getAvailableTimes(req) self.assertIn('TimeAgnosticDataException', str(cm.exception))
def make_map(bbox, projection=ccrs.PlateCarree()): fig, ax = plt.subplots(figsize=(20, 10), subplot_kw=dict(projection=projection)) ax.set_extent(bbox) gl = ax.gridlines(draw_labels=True) gl.xlabels_top = gl.ylabels_right = False gl.xformatter = LONGITUDE_FORMATTER gl.yformatter = LATITUDE_FORMATTER return fig, ax DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu") request = DataAccessLayer.newDataRequest() request.setDatatype("warning") request.setParameters('phensig') times = DataAccessLayer.getAvailableTimes(request) # Get records for last 50 available times response = DataAccessLayer.getGeometryData(request, times[-50:-1]) print("Using " + str(len(response)) + " records") # Each record will have a numpy array the length of the number of "parameters" # Default is 1 (request.setParameters('phensig')) parameters = {} for x in request.getParameters(): parameters[x] = np.array([]) print(parameters) bbox = [-125, -65, 20, 50] fig, ax = make_map(bbox=bbox)
def main(): user_args = get_args() if user_args.host: DataAccessLayer.changeEDEXHost(user_args.host) if (user_args.date and not user_args.time) or (user_args.time and not user_args.date): print >> sys.stderr, "date and time must be provided together" return # If a time range is provided, results will be filtered based on available times timeRange = None if user_args.date: midRange = datetime.strptime( user_args.date + " " + user_args.time, "%Y-%m-%d %H:%M") beginRange = midRange - timedelta(0, 60) endRange = midRange + timedelta(0, 60) timeRange = TimeRange(beginRange, endRange) req = create_request(user_args) if user_args.icao: if user_args.productCode: # retrieve available times and/or true or primary elevation angles if timeRange: tr = timeRange else: tr = None lines = set() if user_args.outputAngle: levels = DataAccessLayer.getAvailableLevels(req) for level in levels: line = "" req.setLevels(level) if user_args.outputDate: times = DataAccessLayer.getAvailableTimes(req) for time in times: if not tr or tr.contains(time.getValidPeriod()): line = str(time) + ".0" line += " " if user_args.outputAngle == "true": line += "%.1f"%level.getLeveltwovalue() else: line += "%.1f"%level.getLevelonevalue() lines.add(line) else: if not tr or data_in_time_range(req, tr): if user_args.outputAngle == "true": line = "%.1f"%level.getLeveltwovalue() else: line = "%.1f"%level.getLevelonevalue() lines.add(line) else : # just output time times = DataAccessLayer.getAvailableTimes(req) for time in times: if not tr or tr.contains(time.getValidPeriod()): lines.add(str(time) + ".0") msg = "\n".join(lines) else: #retrieve available product codes unfiltered = DataAccessLayer.getAvailableParameters(req) productCodes = [] for parameter in unfiltered: #filter to just productCodes if parameter.isdigit(): productCodes.append(parameter) if timeRange: unfiltered = productCodes productCodes = [] for productCode in unfiltered: req = create_request(user_args) req.setParameters(productCode) if data_in_time_range(req, timeRange): productCodes.append(productCode) msg = "\n".join(productCodes) else: # retrieve available icaos icaos = DataAccessLayer.getAvailableLocationNames(req) if timeRange: unfiltered = icaos icaos = [] for icao in unfiltered: req = create_request(user_args) req.addIdentifier("icao", icao) if data_in_time_range(req, timeRange): icaos.append(icao) msg = "\n".join(icaos) print msg.strip()
def data_in_time_range(req, timeRange): times = DataAccessLayer.getAvailableTimes(req) for time in times: if timeRange.contains(time.getValidPeriod()): return True return False