def testRequestingTooMuchDataThrowsResponseTooLargeException(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("group", "/") req.addIdentifier("dataset", "full") points = ((-180, 90), (180, 90), (180, -90), (-180, -90)) poly = shapely.geometry.LinearRing(points) req.setEnvelope(poly) with self.assertRaises(ThriftRequestException) as cm: DAL.getGridData(req) self.assertIn('ResponseTooLargeException', str(cm.exception))
def runGridDataTest(self, req, testSameShape=True): """ Test that we are able to successfully retrieve grid data for the given request. Args: testSameShape: whether or not to verify that all the retrieved data have the same shape (most data don't change shape) """ times = DafTestCase.getTimesIfSupported(req) gridData = DAL.getGridData(req, times[:self.numTimesToLimit]) self.assertIsNotNone(gridData) print("Number of grid records: " + str(len(gridData))) if len(gridData) > 0: print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") print("Sample lat-lon data:\n" + str(gridData[0].getLatLonCoords()) + "\n") if testSameShape: correctGridShape = gridData[0].getLatLonCoords()[0].shape for record in gridData: rawData = record.getRawData() self.assertIsNotNone(rawData) self.assertEqual(rawData.shape, correctGridShape) return gridData
def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('modelName', 'Fcst') req.addIdentifier('siteId', 'OAX') req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not (times): raise unittest.SkipTest('No Wind Data available for testing') gridData = DAL.getGridData(req, [times[0]]) rawWind = None rawDir = None for grid in gridData: if grid.getParameter() == 'Wind': self.assertEqual(grid.getUnit(), 'kts') rawWind = grid.getRawData() elif grid.getParameter() == 'WindDirection': self.assertEqual(grid.getUnit(), 'deg') rawDir = grid.getRawData() self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present') self.assertIsNotNone(rawDir, 'Wind Direction grid is not present') # rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays. self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values') self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values') self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360') self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('parmId.dbId.modelName', 'Fcst') req.addIdentifier('parmId.dbId.siteId', params.SITE_ID) req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not(times): raise unittest.SkipTest('No Wind Data available for testing') gridData = DAL.getGridData(req, [times[0]]) rawWind = None rawDir = None for grid in gridData: if grid.getParameter() == 'Wind': self.assertEqual(grid.getUnit(),'kts') rawWind = grid.getRawData() elif grid.getParameter() == 'WindDirection': self.assertEqual(grid.getUnit(),'deg') rawDir = grid.getRawData() self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present') self.assertIsNotNone(rawDir, 'Wind Direction grid is not present') # rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays. self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values') self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values') self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360') self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def runGridDataTest(self, req, testSameShape=True): """ Test that we are able to successfully retrieve grid data for the given request. Args: req: the grid request testSameShape: whether or not to verify that all the retrieved data have the same shape (most data don't change shape) """ times = DafTestCase.getTimesIfSupported(req) gridData = DAL.getGridData(req, times[:self.numTimesToLimit]) self.assertIsNotNone(gridData) if not gridData: raise unittest.SkipTest("No data available") print("Number of grid records: " + str(len(gridData))) if len(gridData) > 0: print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") print("Sample lat-lon data:\n" + str(gridData[0].getLatLonCoords()) + "\n") if testSameShape: correctGridShape = gridData[0].getLatLonCoords()[0].shape for record in gridData: rawData = record.getRawData() self.assertIsNotNone(rawData) self.assertEqual(rawData.shape, correctGridShape) return gridData
def testGetGridData(self): print("defaultTopo") req = DAL.newDataRequest(self.datatype) req.addIdentifier("group", "/") req.addIdentifier("dataset", "full") poly = shapely.geometry.LinearRing(((-70, 40), (-71, 40), (-71, 42), (-70, 42))) req.setEnvelope(poly) gridData = DAL.getGridData(req) self.assertIsNotNone(gridData) print("Number of grid records: " + str(len(gridData))) print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") for topoFile in ["gmted2010", "gtopo30"]: print("\n" + topoFile) req.addIdentifier("topoFile", topoFile) gridData = DAL.getGridData(req) self.assertIsNotNone(gridData) print("Number of grid records: " + str(len(gridData))) print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
def testGetGridData(self): print("defaultTopo") req = DAL.newDataRequest(self.datatype) req.addIdentifier("group", "/") req.addIdentifier("dataset", "full") poly = shapely.geometry.LinearRing(((-70, 40), (-71, 40), (-71, 42), (-70, 42))) req.setEnvelope(poly) gridData = DAL.getGridData(req) self.assertIsNotNone(gridData) print("Number of grid records: " + str(len(gridData))) print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") for topoFile in ["gtopo30"]: print("\n" + topoFile) req.addIdentifier("topoFile", topoFile) gridData = DAL.getGridData(req) self.assertIsNotNone(gridData) print("Number of grid records: " + str(len(gridData))) print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
def json(self, name="", parm="", level="",time=""): # # Need point query results for time-series # Cross-sections # Time-height # Var vs. Height # request = DataAccessLayer.newDataRequest() request.setDatatype("grid") if name != "": request.setLocationNames(name) if parm != "": request.setParameters(parm) if level != "": request.setLevels(level) cycles = DataAccessLayer.getAvailableTimes(request, True) t = DataAccessLayer.getAvailableTimes(request) fcstRun = [] #for time in t: # if str(time)[:19] == str(cycles[-1]): # fcstRun.append(time) fcstRun.append(t[0]) response = DataAccessLayer.getGridData(request, fcstRun) grid = response[0] data = grid.getRawData() lons, lats = grid.getLatLonCoords() columns = ( 'dtype', 'id', 'crs', 'dx', 'dy', 'firstgridpointcorner', 'the_geom', 'la1', 'lo1', 'name', 'nx', 'ny', 'spacingunit', 'latin1', 'latin2', 'lov', 'majoraxis', 'minoraxis', 'la2', 'latin', 'lo2', 'lad' ) return json.dumps(data, indent=2)
def getData(self, time, model_vars, mdl2stnd, previous_data=None): ''' Name: awips_model_base Purpose: A function to get data from NAM40 model to create HDWX products Inputs: request : A DataAccessLayer request object time : List of datatime(s) for data to grab model_vars : Dictionary with variables/levels to get mdl2stnd : Dictionary to convert from model variable names to standardized names Outputs: Returns a dictionary containing all data Keywords: previous_data : Dictionary with data from previous time step ''' log = logging.getLogger(__name__) # Set up function for logger initTime, fcstTime = get_init_fcst_times(time[0]) data = { 'model': self._request.getLocationNames()[0], 'initTime': initTime, 'fcstTime': fcstTime } # Initialize empty dictionary log.info('Attempting to download {} data'.format(data['model'])) for var in model_vars: # Iterate over variables in the vars list log.debug('Getting: {}'.format(var)) self._request.setParameters(*model_vars[var]['parameters']) # Set parameters for the download request self._request.setLevels(*model_vars[var]['levels']) # Set levels for the download request response = DAL.getGridData(self._request, time) # Request the data for res in response: # Iterate over all data request responses varName = res.getParameter() # Get name of the variable in the response varLvl = res.getLevel() # Get level of the variable in the response varName = mdl2stnd[varName] # Convert variable name to local standarized name if varName not in data: data[varName] = {} # If variable name NOT in data dictionary, initialize new dictionary under key data[varName][varLvl] = res.getRawData() # Add data under level name try: # Try to unit = units(res.getUnit()) # Get units and convert to MetPy units except: # On exception unit = '?' # Set units to ? else: # If get units success data[varName][varLvl] *= unit # Get data and create MetPy quantity by multiplying by units log.debug( 'Got data for:\n Var: {}\n Lvl: {}\n Unit: {}'.format( varName, varLvl, unit)) data['lon'], data['lat'] = res.getLatLonCoords() # Get latitude and longitude values data['lon'] *= units('degree') # Add units of degree to longitude data['lat'] *= units('degree') # Add units of degree to latitude # Absolute vorticity dx, dy = lat_lon_grid_deltas(data['lon'], data['lat']) # Get grid spacing in x and y uTag = mdl2stnd[model_vars['wind']['parameters'][0]] # Get initial tag name for u-wind vTag = mdl2stnd[model_vars['wind']['parameters'][1]] # Get initial tag name for v-wind if (uTag in data) and ( vTag in data): # If both tags are in the data structure data['abs_vort'] = {} # Add absolute vorticity key for lvl in model_vars['wind'][ 'levels']: # Iterate over all leves in the wind data if (lvl in data[uTag]) and ( lvl in data[vTag] ): # If given level in both u- and v-wind dictionaries log.debug('Computing absolute vorticity at {}'.format(lvl)) data['abs_vort'][ lvl ] = \ absolute_vorticity( data[uTag][lvl], data[vTag][lvl], dx, dy, data['lat'] ) # Compute absolute vorticity # 1000 MB equivalent potential temperature if ('temperature' in data) and ( 'dewpoint' in data): # If temperature AND depoint data were downloaded data['theta_e'] = {} T, Td = 'temperature', 'dewpoint' if ('1000.0MB' in data[T]) and ( '1000.0MB' in data[Td] ): # If temperature AND depoint data were downloaded log.debug( 'Computing equivalent potential temperature at 1000 hPa') data['theta_e']['1000.0MB'] = equivalent_potential_temperature( 1000.0 * units('hPa'), data[T]['1000.0MB'], data[Td]['1000.0MB']) return data # MLCAPE log.debug('Computing mixed layer CAPE') T_lvl = list(data[T].keys()) Td_lvl = list(data[Td].keys()) levels = list(set(T_lvl).intersection(Td_lvl)) levels = [float(lvl.replace('MB', '')) for lvl in levels] levels = sorted(levels, reverse=True) nLvl = len(levels) if nLvl > 0: log.debug( 'Found {} matching levels in temperature and dewpoint data' .format(nLvl)) nLat, nLon = data['lon'].shape data['MLCAPE'] = np.zeros(( nLat, nLon, ), dtype=np.float32) * units('J/kg') TT = np.zeros(( nLvl, nLat, nLon, ), dtype=np.float32) * units('degC') TTd = np.zeros(( nLvl, nLat, nLon, ), dtype=np.float32) * units('degC') log.debug('Sorting temperature and dewpoint data by level') for i in range(nLvl): key = '{:.1f}MB'.format(levels[i]) TT[i, :, :] = data[T][key].to('degC') TTd[i, :, :] = data[Td][key].to('degC') levels = np.array(levels) * units.hPa depth = 100.0 * units.hPa log.debug('Iterating over grid boxes to compute MLCAPE') for j in range(nLat): for i in range(nLon): try: _, T_parc, Td_parc = mixed_parcel( levels, TT[:, j, i], TTd[:, j, i], depth=depth, interpolate=False, ) profile = parcel_profile(levels, T_parc, Td_parc) cape, cin = cape_cin(levels, TT[:, j, i], TTd[:, j, i], profile) except: log.warning( 'Failed to compute MLCAPE for lon/lat: {}; {}'. format(data['lon'][j, i], data['lat'][j, i])) else: data['MLCAPE'][j, i] = cape return data
def do_request(user_args): if user_args.host: DataAccessLayer.changeEDEXHost(user_args.host) srcId = user_args.srcId varAbrev = user_args.varAbrev if not srcId or not varAbrev: raise Exception("srcId or varAbrev not provided") return date = user_args.date hour = user_args.hour fcst = user_args.fcst if not date or not hour or not fcst: raise Exception("date, hour, or fcst not provided") return dt = datetime.strptime( str(date) + " " + str(hour) + ":00:00.0", "%Y-%m-%d %H:%M:%S.%f") # check for and build date range if necessary daterange = None if varAbrev.endswith("hr"): import re matches = re.findall(r'\d+', varAbrev) if matches: from datetime import timedelta hourRange = int(matches[-1]) endDate = dt + timedelta(hours=int(fcst)) beginDate = endDate - timedelta(hours=hourRange) daterange = TimeRange(beginDate, endDate) # convert hours to seconds because DataTime does the reverse time = DataTime(dt, int(fcst) * 3600, daterange) req = DataAccessLayer.newDataRequest("grid") req.setParameters(varAbrev) req.addIdentifier("info.datasetId", srcId) # To handle partial level matches, use identifiers instead of a Level. if user_args.lvlName is not None: req.addIdentifier("info.level.masterLevel.name", user_args.lvlName) if user_args.lvlOne is not None: req.addIdentifier("info.level.levelonevalue", numpy.float64(user_args.lvlOne)) if user_args.lvlTwo is not None: req.addIdentifier("info.level.leveltwovalue", numpy.float64(user_args.lvlTwo)) times = [time] # If fcst is 0, also query for times with FCST_USED flag if fcst == '0': time = DataTime(dt, int(fcst) * 3600, daterange) time.utilityFlags.add("FCST_USED") times.append(time) grids = DataAccessLayer.getGridData(req, times) if not grids: # print "Data not available" raise Exception("") grid = grids[0] rawData = grid.getRawData() yLen, xLen = rawData.shape return grid, xLen, yLen
def main(): user_args = get_args() if user_args.host: DataAccessLayer.changeEDEXHost(user_args.host) slop = user_args.slop dateTimeStr = user_args.datetime if not dateTimeStr: print >> sys.stderr, "DateTime not provided" return physicalElement = user_args.physical if not physicalElement: print >> sys.stderr, "PhysicalElement not provided" return sectorID = user_args.sector if not sectorID: print >> sys.stderr, "SectorID not provided" return creatingEntity = user_args.entity part = user_args.partition encoding = user_args.encoding dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M") beginRange = dateTime - timedelta(0, slop) endRange = dateTime + timedelta(0, slop) timerange = TimeRange(beginRange, endRange) req = DataAccessLayer.newDataRequest("satellite") req.setParameters(physicalElement) req.setLocationNames(sectorID) if creatingEntity: req.addIdentifier("creatingEntity", creatingEntity) grids = DataAccessLayer.getGridData(req, timerange) if not grids: # print "Data not available" return grid = grids[0] data = grid.getRawData() myent = grid.getAttribute("creatingEntity") mytime = a2dafcommon.datatime_to_string(grid.getDataTime()) + ".0" if data is None or len(data) == 0: # print "No data." return data[numpy.isnan(data)] = 0 yLen, xLen = data.shape plus = " ghijklmnopqrstuvwxyz" minus = " GHIJKLMNOPQRSTUVWXYZ" limit = 10000000 if encoding == 1: limit = limit / 2 elif encoding == 0: limit = limit / 8 k = xLen * (yLen / 4) j = 0 nxy = yLen * xLen if part == "D": j = k + k + k elif part == "C": j = k + k nxy = j + k elif part == "B": j = k nxy = j + k elif part == "A" or nxy > limit: nxy = k msg = "" if part <= "A": msg += str(xLen) + " " + str(yLen) + " " msg += mytime + " " + myent + "\n" i = 0 kk = None while j < yLen: i = 0 kk = int(data[j, i]) if kk < 0: kk += 256 if encoding == 0: msg += str(kk) elif encoding == 1: msg += "%2.2x" % kk elif kk == 0: msg += "@" elif kk == 255: msg += "#" else: msg += "%2.2x" % kk i += 1 while i < xLen: k = int(data[j, i]) if k < 0: k += 256 if encoding == 0: msg += " " + str(k) elif encoding == 1: msg += "%2.2x" % k elif k == 0: msg += "@" elif k == 255: msg += "#" elif k == kk: msg += "." elif k > kk + 20 or k < kk - 20: msg += "%2.2x" % k elif k > kk: msg += plus[k - kk] else: msg += minus[kk - k] kk = k i += 1 msg += "\n" j += 1 print msg.strip()
request.setLocationNames("NAM12") availableParms = DataAccessLayer.getAvailableParameters(request) availableParms.sort() list(availableParms) request.setParameters("T") availableLevels = DataAccessLayer.getAvailableLevels(request) request.setLevels("0.0SFC") cycles = DataAccessLayer.getAvailableTimes(request, True) times = DataAccessLayer.getAvailableTimes(request) fcstRun = DataAccessLayer.getForecastRun(cycles[-2], times) list(fcstRun) response = DataAccessLayer.getGridData(request, [fcstRun[36]]) for grid in response: data = grid.getRawData() lons, lats = grid.getLatLonCoords() print('Time :', str(grid.getDataTime().getFcstTime() / 3600)) Ta = ((data - 273.15)) DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu") dataTypes = DataAccessLayer.getSupportedDatatypes() dataTypes.sort() list(dataTypes) request = DataAccessLayer.newDataRequest() request.setDatatype("grid") available_grids = DataAccessLayer.getAvailableLocationNames(request) available_grids.sort()