def testRequestingTooMuchDataThrowsResponseTooLargeException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
req.addIdentifier("dataset", "full")
points = ((-180, 90), (180, 90), (180, -90), (-180, -90))
poly = shapely.geometry.LinearRing(points)
req.setEnvelope(poly)
with self.assertRaises(ThriftRequestException) as cm:
DAL.getGridData(req)
self.assertIn('ResponseTooLargeException', str(cm.exception))
def testRequestingTooMuchDataThrowsResponseTooLargeException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
req.addIdentifier("dataset", "full")
points = ((-180, 90), (180, 90), (180, -90), (-180, -90))
poly = shapely.geometry.LinearRing(points)
req.setEnvelope(poly)
with self.assertRaises(ThriftRequestException) as cm:
DAL.getGridData(req)
self.assertIn('ResponseTooLargeException', str(cm.exception))
def runGridDataTest(self, req, testSameShape=True):
"""
Test that we are able to successfully retrieve grid data for the given
request.
Args:
testSameShape: whether or not to verify that all the retrieved data
have the same shape (most data don't change shape)
"""
times = DafTestCase.getTimesIfSupported(req)
gridData = DAL.getGridData(req, times[:self.numTimesToLimit])
self.assertIsNotNone(gridData)
print("Number of grid records: " + str(len(gridData)))
if len(gridData) > 0:
print("Sample grid data shape:\n" +
str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
print("Sample lat-lon data:\n" +
str(gridData[0].getLatLonCoords()) + "\n")
if testSameShape:
correctGridShape = gridData[0].getLatLonCoords()[0].shape
for record in gridData:
rawData = record.getRawData()
self.assertIsNotNone(rawData)
self.assertEqual(rawData.shape, correctGridShape)
return gridData
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('modelName', 'Fcst')
req.addIdentifier('siteId', 'OAX')
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not (times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(),
'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(),
'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(),
'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(),
'Wind Direction should be different from Wind Speed')
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not(times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(),'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(),'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def runGridDataTest(self, req, testSameShape=True):
"""
Test that we are able to successfully retrieve grid data for the given
request.
Args:
req: the grid request
testSameShape: whether or not to verify that all the retrieved data
have the same shape (most data don't change shape)
"""
times = DafTestCase.getTimesIfSupported(req)
gridData = DAL.getGridData(req, times[:self.numTimesToLimit])
self.assertIsNotNone(gridData)
if not gridData:
raise unittest.SkipTest("No data available")
print("Number of grid records: " + str(len(gridData)))
if len(gridData) > 0:
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
print("Sample lat-lon data:\n" + str(gridData[0].getLatLonCoords()) + "\n")
if testSameShape:
correctGridShape = gridData[0].getLatLonCoords()[0].shape
for record in gridData:
rawData = record.getRawData()
self.assertIsNotNone(rawData)
self.assertEqual(rawData.shape, correctGridShape)
return gridData
def testGetGridData(self):
print("defaultTopo")
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
req.addIdentifier("dataset", "full")
poly = shapely.geometry.LinearRing(((-70, 40), (-71, 40), (-71, 42), (-70, 42)))
req.setEnvelope(poly)
gridData = DAL.getGridData(req)
self.assertIsNotNone(gridData)
print("Number of grid records: " + str(len(gridData)))
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
for topoFile in ["gmted2010", "gtopo30"]:
print("\n" + topoFile)
req.addIdentifier("topoFile", topoFile)
gridData = DAL.getGridData(req)
self.assertIsNotNone(gridData)
print("Number of grid records: " + str(len(gridData)))
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
def testGetGridData(self):
print("defaultTopo")
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("group", "/")
req.addIdentifier("dataset", "full")
poly = shapely.geometry.LinearRing(((-70, 40), (-71, 40), (-71, 42), (-70, 42)))
req.setEnvelope(poly)
gridData = DAL.getGridData(req)
self.assertIsNotNone(gridData)
print("Number of grid records: " + str(len(gridData)))
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
for topoFile in ["gtopo30"]:
print("\n" + topoFile)
req.addIdentifier("topoFile", topoFile)
gridData = DAL.getGridData(req)
self.assertIsNotNone(gridData)
print("Number of grid records: " + str(len(gridData)))
print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
def json(self, name="", parm="", level="",time=""):
#
# Need point query results for time-series
# Cross-sections
# Time-height
# Var vs. Height
#
request = DataAccessLayer.newDataRequest()
request.setDatatype("grid")
if name != "": request.setLocationNames(name)
if parm != "": request.setParameters(parm)
if level != "": request.setLevels(level)
cycles = DataAccessLayer.getAvailableTimes(request, True)
t = DataAccessLayer.getAvailableTimes(request)
fcstRun = []
#for time in t:
# if str(time)[:19] == str(cycles[-1]):
# fcstRun.append(time)
fcstRun.append(t[0])
response = DataAccessLayer.getGridData(request, fcstRun)
grid = response[0]
data = grid.getRawData()
lons, lats = grid.getLatLonCoords()
columns = (
'dtype', 'id', 'crs', 'dx', 'dy', 'firstgridpointcorner', 'the_geom',
'la1', 'lo1', 'name', 'nx', 'ny', 'spacingunit', 'latin1', 'latin2', 'lov',
'majoraxis', 'minoraxis', 'la2', 'latin', 'lo2', 'lad'
)
return json.dumps(data, indent=2)
def getData(self, time, model_vars, mdl2stnd, previous_data=None):
'''
Name:
awips_model_base
Purpose:
A function to get data from NAM40 model to create HDWX products
Inputs:
request : A DataAccessLayer request object
time : List of datatime(s) for data to grab
model_vars : Dictionary with variables/levels to get
mdl2stnd : Dictionary to convert from model variable names
to standardized names
Outputs:
Returns a dictionary containing all data
Keywords:
previous_data : Dictionary with data from previous time step
'''
log = logging.getLogger(__name__)
# Set up function for logger
initTime, fcstTime = get_init_fcst_times(time[0])
data = {
'model': self._request.getLocationNames()[0],
'initTime': initTime,
'fcstTime': fcstTime
}
# Initialize empty dictionary
log.info('Attempting to download {} data'.format(data['model']))
for var in model_vars: # Iterate over variables in the vars list
log.debug('Getting: {}'.format(var))
self._request.setParameters(*model_vars[var]['parameters'])
# Set parameters for the download request
self._request.setLevels(*model_vars[var]['levels'])
# Set levels for the download request
response = DAL.getGridData(self._request, time) # Request the data
for res in response: # Iterate over all data request responses
varName = res.getParameter()
# Get name of the variable in the response
varLvl = res.getLevel()
# Get level of the variable in the response
varName = mdl2stnd[varName]
# Convert variable name to local standarized name
if varName not in data:
data[varName] = {}
# If variable name NOT in data dictionary, initialize new dictionary under key
data[varName][varLvl] = res.getRawData()
# Add data under level name
try: # Try to
unit = units(res.getUnit())
# Get units and convert to MetPy units
except: # On exception
unit = '?'
# Set units to ?
else: # If get units success
data[varName][varLvl] *= unit
# Get data and create MetPy quantity by multiplying by units
log.debug(
'Got data for:\n Var: {}\n Lvl: {}\n Unit: {}'.format(
varName, varLvl, unit))
data['lon'], data['lat'] = res.getLatLonCoords()
# Get latitude and longitude values
data['lon'] *= units('degree')
# Add units of degree to longitude
data['lat'] *= units('degree')
# Add units of degree to latitude
# Absolute vorticity
dx, dy = lat_lon_grid_deltas(data['lon'], data['lat'])
# Get grid spacing in x and y
uTag = mdl2stnd[model_vars['wind']['parameters'][0]]
# Get initial tag name for u-wind
vTag = mdl2stnd[model_vars['wind']['parameters'][1]]
# Get initial tag name for v-wind
if (uTag in data) and (
vTag in data): # If both tags are in the data structure
data['abs_vort'] = {}
# Add absolute vorticity key
for lvl in model_vars['wind'][
'levels']: # Iterate over all leves in the wind data
if (lvl in data[uTag]) and (
lvl in data[vTag]
): # If given level in both u- and v-wind dictionaries
log.debug('Computing absolute vorticity at {}'.format(lvl))
data['abs_vort'][ lvl ] = \
absolute_vorticity( data[uTag][lvl], data[vTag][lvl],
dx, dy, data['lat'] )
# Compute absolute vorticity
# 1000 MB equivalent potential temperature
if ('temperature' in data) and (
'dewpoint'
in data): # If temperature AND depoint data were downloaded
data['theta_e'] = {}
T, Td = 'temperature', 'dewpoint'
if ('1000.0MB' in data[T]) and (
'1000.0MB' in data[Td]
): # If temperature AND depoint data were downloaded
log.debug(
'Computing equivalent potential temperature at 1000 hPa')
data['theta_e']['1000.0MB'] = equivalent_potential_temperature(
1000.0 * units('hPa'), data[T]['1000.0MB'],
data[Td]['1000.0MB'])
return data
# MLCAPE
log.debug('Computing mixed layer CAPE')
T_lvl = list(data[T].keys())
Td_lvl = list(data[Td].keys())
levels = list(set(T_lvl).intersection(Td_lvl))
levels = [float(lvl.replace('MB', '')) for lvl in levels]
levels = sorted(levels, reverse=True)
nLvl = len(levels)
if nLvl > 0:
log.debug(
'Found {} matching levels in temperature and dewpoint data'
.format(nLvl))
nLat, nLon = data['lon'].shape
data['MLCAPE'] = np.zeros((
nLat,
nLon,
), dtype=np.float32) * units('J/kg')
TT = np.zeros((
nLvl,
nLat,
nLon,
), dtype=np.float32) * units('degC')
TTd = np.zeros((
nLvl,
nLat,
nLon,
), dtype=np.float32) * units('degC')
log.debug('Sorting temperature and dewpoint data by level')
for i in range(nLvl):
key = '{:.1f}MB'.format(levels[i])
TT[i, :, :] = data[T][key].to('degC')
TTd[i, :, :] = data[Td][key].to('degC')
levels = np.array(levels) * units.hPa
depth = 100.0 * units.hPa
log.debug('Iterating over grid boxes to compute MLCAPE')
for j in range(nLat):
for i in range(nLon):
try:
_, T_parc, Td_parc = mixed_parcel(
levels,
TT[:, j, i],
TTd[:, j, i],
depth=depth,
interpolate=False,
)
profile = parcel_profile(levels, T_parc, Td_parc)
cape, cin = cape_cin(levels, TT[:, j, i],
TTd[:, j, i], profile)
except:
log.warning(
'Failed to compute MLCAPE for lon/lat: {}; {}'.
format(data['lon'][j, i], data['lat'][j, i]))
else:
data['MLCAPE'][j, i] = cape
return data
def do_request(user_args):
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
srcId = user_args.srcId
varAbrev = user_args.varAbrev
if not srcId or not varAbrev:
raise Exception("srcId or varAbrev not provided")
return
date = user_args.date
hour = user_args.hour
fcst = user_args.fcst
if not date or not hour or not fcst:
raise Exception("date, hour, or fcst not provided")
return
dt = datetime.strptime(
str(date) + " " + str(hour) + ":00:00.0", "%Y-%m-%d %H:%M:%S.%f")
# check for and build date range if necessary
daterange = None
if varAbrev.endswith("hr"):
import re
matches = re.findall(r'\d+', varAbrev)
if matches:
from datetime import timedelta
hourRange = int(matches[-1])
endDate = dt + timedelta(hours=int(fcst))
beginDate = endDate - timedelta(hours=hourRange)
daterange = TimeRange(beginDate, endDate)
# convert hours to seconds because DataTime does the reverse
time = DataTime(dt, int(fcst) * 3600, daterange)
req = DataAccessLayer.newDataRequest("grid")
req.setParameters(varAbrev)
req.addIdentifier("info.datasetId", srcId)
# To handle partial level matches, use identifiers instead of a Level.
if user_args.lvlName is not None:
req.addIdentifier("info.level.masterLevel.name", user_args.lvlName)
if user_args.lvlOne is not None:
req.addIdentifier("info.level.levelonevalue",
numpy.float64(user_args.lvlOne))
if user_args.lvlTwo is not None:
req.addIdentifier("info.level.leveltwovalue",
numpy.float64(user_args.lvlTwo))
times = [time]
# If fcst is 0, also query for times with FCST_USED flag
if fcst == '0':
time = DataTime(dt, int(fcst) * 3600, daterange)
time.utilityFlags.add("FCST_USED")
times.append(time)
grids = DataAccessLayer.getGridData(req, times)
if not grids:
# print "Data not available"
raise Exception("")
grid = grids[0]
rawData = grid.getRawData()
yLen, xLen = rawData.shape
return grid, xLen, yLen
def main():
user_args = get_args()
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
slop = user_args.slop
dateTimeStr = user_args.datetime
if not dateTimeStr:
print >> sys.stderr, "DateTime not provided"
return
physicalElement = user_args.physical
if not physicalElement:
print >> sys.stderr, "PhysicalElement not provided"
return
sectorID = user_args.sector
if not sectorID:
print >> sys.stderr, "SectorID not provided"
return
creatingEntity = user_args.entity
part = user_args.partition
encoding = user_args.encoding
dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M")
beginRange = dateTime - timedelta(0, slop)
endRange = dateTime + timedelta(0, slop)
timerange = TimeRange(beginRange, endRange)
req = DataAccessLayer.newDataRequest("satellite")
req.setParameters(physicalElement)
req.setLocationNames(sectorID)
if creatingEntity:
req.addIdentifier("creatingEntity", creatingEntity)
grids = DataAccessLayer.getGridData(req, timerange)
if not grids:
# print "Data not available"
return
grid = grids[0]
data = grid.getRawData()
myent = grid.getAttribute("creatingEntity")
mytime = a2dafcommon.datatime_to_string(grid.getDataTime()) + ".0"
if data is None or len(data) == 0:
# print "No data."
return
data[numpy.isnan(data)] = 0
yLen, xLen = data.shape
plus = " ghijklmnopqrstuvwxyz"
minus = " GHIJKLMNOPQRSTUVWXYZ"
limit = 10000000
if encoding == 1:
limit = limit / 2
elif encoding == 0:
limit = limit / 8
k = xLen * (yLen / 4)
j = 0
nxy = yLen * xLen
if part == "D":
j = k + k + k
elif part == "C":
j = k + k
nxy = j + k
elif part == "B":
j = k
nxy = j + k
elif part == "A" or nxy > limit:
nxy = k
msg = ""
if part <= "A":
msg += str(xLen) + " " + str(yLen) + " "
msg += mytime + " " + myent + "\n"
i = 0
kk = None
while j < yLen:
i = 0
kk = int(data[j, i])
if kk < 0: kk += 256
if encoding == 0:
msg += str(kk)
elif encoding == 1:
msg += "%2.2x" % kk
elif kk == 0:
msg += "@"
elif kk == 255:
msg += "#"
else:
msg += "%2.2x" % kk
i += 1
while i < xLen:
k = int(data[j, i])
if k < 0: k += 256
if encoding == 0:
msg += " " + str(k)
elif encoding == 1:
msg += "%2.2x" % k
elif k == 0:
msg += "@"
elif k == 255:
msg += "#"
elif k == kk:
msg += "."
elif k > kk + 20 or k < kk - 20:
msg += "%2.2x" % k
elif k > kk:
msg += plus[k - kk]
else:
msg += minus[kk - k]
kk = k
i += 1
msg += "\n"
j += 1
print msg.strip()
request.setLocationNames("NAM12")
availableParms = DataAccessLayer.getAvailableParameters(request)
availableParms.sort()
list(availableParms)
request.setParameters("T")
availableLevels = DataAccessLayer.getAvailableLevels(request)
request.setLevels("0.0SFC")
cycles = DataAccessLayer.getAvailableTimes(request, True)
times = DataAccessLayer.getAvailableTimes(request)
fcstRun = DataAccessLayer.getForecastRun(cycles[-2], times)
list(fcstRun)
response = DataAccessLayer.getGridData(request, [fcstRun[36]])
for grid in response:
data = grid.getRawData()
lons, lats = grid.getLatLonCoords()
print('Time :', str(grid.getDataTime().getFcstTime() / 3600))
Ta = ((data - 273.15))
DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu")
dataTypes = DataAccessLayer.getSupportedDatatypes()
dataTypes.sort()
list(dataTypes)
request = DataAccessLayer.newDataRequest()
request.setDatatype("grid")
available_grids = DataAccessLayer.getAvailableLocationNames(request)
available_grids.sort()
