def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not(times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(),'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(),'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('modelName', 'Fcst')
req.addIdentifier('siteId', 'OAX')
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not (times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(), 'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(), 'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(),
'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(),
'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(),
'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(),
'Wind Direction should be different from Wind Speed')
def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
locations = request.getLocationNames()
if locations:
times = None
for location in locations:
specificRequest = __cloneRequest(request)
specificRequest.setLocationNames(location)
specificTimes = DataAccessLayer.getAvailableTimes(specificRequest, refTimeOnly)
if times is None:
times = set(specificTimes)
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def fcst_times(self, interval=3600, max_forecast=None):
'''
Name:
awips_fcst_times
Purpose:
A function to get forecast times for latest model run
Inputs:
request : A DataAccessLayer request object
Outputs:
Returns a list of forecast times
Keywords:
interval : Time step between forecast times in seconds
Default is 3600s (1 hour)
max_forecast : Maximum forecast time to get, in seconds.
Default is last available time
'''
cycles = DAL.getAvailableTimes(self._request, True)
# Get forecast cycles
times = DAL.getAvailableTimes(self._request) # Get forecast times
times = DAL.getForecastRun(cycles[-1], times)
# Get forecast times in latest cycle
if max_forecast is None:
max_forecast = times[-1].getFcstTime()
# Set max_forecast value default based on model
nTimes = max_forecast // interval + 1
# Number of forecast steps to get based on inteval
flt_times = [[] for i in range(nTimes)
] # Initialized list of empty lists for forecast times
for time in times: # Iterate over all times
fcstTime = time.getFcstTime()
# Get the valid forecast time
if ((fcstTime % interval) == 0) and (
fcstTime < max_forecast
): # If the forecast hour falls on the interval requested AND is before the max forecast time
fcstDur = time.getValidPeriod().duration()
# Get duration of the forecast period
if (fcstDur == 0) or (
fcstDur == interval
): # If instantaneous forecast period OR period covers requested interval
index = fcstTime // interval
# Index for the times flt_times array
flt_times[index].append(time)
# Append the time to the list at index
return flt_times
def __getAvailableTimesForEachLocation(request, refTimeOnly=False):
locations = request.getLocationNames()
if locations:
times = None
for location in locations:
specificRequest = __cloneRequest(request)
specificRequest.setLocationNames(location)
specificTimes = DataAccessLayer.getAvailableTimes(specificRequest, refTimeOnly)
if times is None:
times = set(specificTimes)
else:
times.intersection_update(specificTimes)
if not times:
break
return times
else:
return DataAccessLayer.getAvailableTimes(request, refTimeOnly)
def main():
user_args = get_args()
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
req = DataAccessLayer.newDataRequest("grid")
if not user_args.srcId:
print >> sys.stderr, "srcId not provided"
return
req.addIdentifier("info.datasetId", user_args.srcId)
if user_args.varAbrev:
req.setParameters(user_args.varAbrev)
if user_args.lvlName is not None:
req.addIdentifier("info.level.masterLevel.name", user_args.lvlName)
if user_args.lvlOne is not None:
req.addIdentifier("info.level.levelonevalue",
numpy.float64(user_args.lvlOne))
if user_args.lvlTwo is not None:
req.addIdentifier("info.level.leveltwovalue",
numpy.float64(user_args.lvlTwo))
mode = user_args.mode
if mode not in ["time", "plane", "field", "fieldplane"]:
print >> sys.stderr, "mode must be one of time, plane, field, or fieldplane."
return
msg = ""
if mode == "time":
times = DataAccessLayer.getAvailableTimes(req)
for time in times:
timeStr = str(time)
if "--" in timeStr:
timeStr = timeStr[0:-22] + ".0" + timeStr[
-22:-1] + ".0" + timeStr[-1]
msg += timeStr[0:19] + ".0" + timeStr[19:] + "\n"
elif mode == "plane":
levels = DataAccessLayer.getAvailableLevels(req)
for level in levels:
msg += level_to_string(level) + "\n"
elif mode == "field":
params = DataAccessLayer.getAvailableParameters(req)
msg = "\n".join(params)
else: #fieldplane
params = DataAccessLayer.getAvailableParameters(req)
for param in params:
msg += param + ":\n"
req.setParameters(param)
levels = DataAccessLayer.getAvailableLevels(req)
if levels:
levelStr = []
for level in levels:
levelStr.append(level_to_string(level))
msg += " ".join(levelStr) + " \n"
print msg.strip("\n")
def download(site: str):
# %% SITE
site = site.lower()
DataAccessLayer.changeEDEXHost('edex-cloud.unidata.ucar.edu')
request = DataAccessLayer.newDataRequest()
request.setDatatype('radar')
request.setLocationNames(site)
# %% TIME
times = DataAccessLayer.getAvailableTimes(request)
times = [parse(str(t)) for t in times]
timerange = TimeRange(times[0], times[-1])
# %% REQUEST
client = ThriftClient.ThriftClient('edex-cloud.unidata.ucar.edu')
request = GetRadarDataRecordRequest()
request.setTimeRange(timerange)
request.setRadarId(site)
code = 'N0Q'
request.setProductCode(nexrad[code]['id'])
request.setPrimaryElevationAngle(nexrad[code]['elev'])
response = client.sendRequest(request)
records = response.getData()
print(f'found {len(records)} records at {site}')
if not response.getData():
raise OSError(f'data not available {timerange}')
for rec in records:
idra = rec.getHdf5Data()
rdat, azdat, depVals, threshVals = RadarCommon.get_hdf5_data(idra)
# dim = rdat.getDimension()
lat, lon = float(rec.getLatitude()), float(rec.getLongitude())
radials, rangeGates = rdat.getSizes()
# Convert raw byte to pixel value
array = np.array(rdat.getByteData())
array[array < 0] = array[array < 0] + 256
if azdat:
azVals = azdat.getFloatData()
az = np.array(RadarCommon.encode_radial(azVals))
# dattyp = RadarCommon.get_data_type(azdat)
az = np.append(az, az[-1])
# header = RadarCommon.get_header(rec, format, rangeGates, radials, azdat, 'description')
rng = np.linspace(0, rangeGates, rangeGates + 1) * nexrad[code]['res']
lats = np.empty((rng.size, az.size))
lons = np.empty_like(lats)
for i, a in enumerate(az):
lats, lons, _ = vreckon(lat, lon, rng, a)
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testNoDuplicateData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setLocationNames('KOMA')
req.setParameters('maxtemp_day1')
rows = DAL.getGeometryData(req, DAL.getAvailableTimes(req)[0:5])
for i in range(len(rows)):
for j in range(len(rows)):
if i != j:
self.assertNotEqual(rows[i].__dict__, rows[j].__dict__)
def testGetDataWithTimeRange(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
times = DAL.getAvailableTimes(req)
limitTimes = times[-self.numTimesToLimit:]
startTime = datetime.datetime.utcfromtimestamp(limitTimes[0].getRefTime().getTime()/1000)
endTime = datetime.datetime.utcnow()
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def getTimesIfSupported(req):
"""Return available times for req. If req refers to a time-agnostic
datatype, return an empty list instead.
"""
times = []
try:
times = DAL.getAvailableTimes(req)
except ThriftRequestException as e:
if 'TimeAgnosticDataException' not in str(e):
raise
return times
def getTimesIfSupported(req):
"""Return available times for req. If req refers to a time-agnostic
datatype, return an empty list instead.
"""
times = []
try:
times = DAL.getAvailableTimes(req)
except ThriftRequestException as e:
if not 'TimeAgnosticDataException' in str(e):
raise
return times
def testGetDataWithTimeRange(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
times = DAL.getAvailableTimes(req)
limitTimes = times[-self.numTimesToLimit:]
startTime = datetime.datetime.utcfromtimestamp(
limitTimes[0].getRefTime().getTime() / 1000)
endTime = datetime.datetime.utcnow()
tr = TimeRange(startTime, endTime)
self.runGeometryDataTestWithTimeRange(req, tr)
def json(self, name="", parm="", level="",time=""):
#
# Need point query results for time-series
# Cross-sections
# Time-height
# Var vs. Height
#
request = DataAccessLayer.newDataRequest()
request.setDatatype("grid")
if name != "": request.setLocationNames(name)
if parm != "": request.setParameters(parm)
if level != "": request.setLevels(level)
cycles = DataAccessLayer.getAvailableTimes(request, True)
t = DataAccessLayer.getAvailableTimes(request)
fcstRun = []
#for time in t:
# if str(time)[:19] == str(cycles[-1]):
# fcstRun.append(time)
fcstRun.append(t[0])
response = DataAccessLayer.getGridData(request, fcstRun)
grid = response[0]
data = grid.getRawData()
lons, lats = grid.getLatLonCoords()
columns = (
'dtype', 'id', 'crs', 'dx', 'dy', 'firstgridpointcorner', 'the_geom',
'la1', 'lo1', 'name', 'nx', 'ny', 'spacingunit', 'latin1', 'latin2', 'lov',
'majoraxis', 'minoraxis', 'la2', 'latin', 'lo2', 'lad'
)
return json.dumps(data, indent=2)
def __determineForecastHours(request, refTime, timeRange):
dataTimes = DataAccessLayer.getAvailableTimes(request, False)
timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
dataTimesMap = defaultdict(list)
for baseTime, dataTime in timesGen:
dataTimesMap[baseTime].append(dataTime)
if refTime is None:
refTime = max(dataTimesMap.keys())
forecastHours = dataTimesMap[refTime]
if timeRange is None:
return forecastHours
else:
return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
def __determineForecastHours(request, refTime, timeRange):
dataTimes = DataAccessLayer.getAvailableTimes(request, False)
timesGen = [(DataTime(refTime=dataTime.getRefTime()), dataTime) for dataTime in dataTimes]
dataTimesMap = defaultdict(list)
for baseTime, dataTime in timesGen:
dataTimesMap[baseTime].append(dataTime)
if refTime is None:
refTime = max(dataTimesMap.keys())
forecastHours = dataTimesMap[refTime]
if timeRange is None:
return forecastHours
else:
return [forecastHour for forecastHour in forecastHours if timeRange.contains(forecastHour.getValidPeriod())]
request.setDatatype("grid")
available_grids = DataAccessLayer.getAvailableLocationNames(request)
available_grids.sort()
list(available_grids)
request.setLocationNames("NAM12")
availableParms = DataAccessLayer.getAvailableParameters(request)
availableParms.sort()
list(availableParms)
request.setParameters("T")
availableLevels = DataAccessLayer.getAvailableLevels(request)
request.setLevels("0.0SFC")
cycles = DataAccessLayer.getAvailableTimes(request, True)
times = DataAccessLayer.getAvailableTimes(request)
fcstRun = DataAccessLayer.getForecastRun(cycles[-2], times)
list(fcstRun)
response = DataAccessLayer.getGridData(request, [fcstRun[36]])
for grid in response:
data = grid.getRawData()
lons, lats = grid.getLatLonCoords()
print('Time :', str(grid.getDataTime().getFcstTime() / 3600))
Ta = ((data - 273.15))
DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu")
dataTypes = DataAccessLayer.getSupportedDatatypes()
dataTypes.sort()
list(dataTypes)
def runTimeAgnosticTest(self, req):
with self.assertRaises(ThriftRequestException) as cm:
times = DAL.getAvailableTimes(req)
self.assertIn('TimeAgnosticDataException', str(cm.exception))
def runTimesTest(self, req):
times = DAL.getAvailableTimes(req)
self.assertIsNotNone(times)
print("Number of times: " + str(len(times)))
strTimes = [str(t) for t in times[:self.sampleDataLimit]]
print("Sample times:\n" + str(strTimes))
def runTimesTest(self, req):
times = DAL.getAvailableTimes(req)
self.assertIsNotNone(times)
print("Number of times: " + str(len(times)))
strTimes = [str(t) for t in times[:self.sampleDataLimit]]
print("Sample times:\n" + str(strTimes))
def runTimeAgnosticTest(self, req):
with self.assertRaises(ThriftRequestException) as cm:
DAL.getAvailableTimes(req)
self.assertIn('TimeAgnosticDataException', str(cm.exception))
def make_map(bbox, projection=ccrs.PlateCarree()):
fig, ax = plt.subplots(figsize=(20, 10),
subplot_kw=dict(projection=projection))
ax.set_extent(bbox)
gl = ax.gridlines(draw_labels=True)
gl.xlabels_top = gl.ylabels_right = False
gl.xformatter = LONGITUDE_FORMATTER
gl.yformatter = LATITUDE_FORMATTER
return fig, ax
DataAccessLayer.changeEDEXHost("edex-cloud.unidata.ucar.edu")
request = DataAccessLayer.newDataRequest()
request.setDatatype("warning")
request.setParameters('phensig')
times = DataAccessLayer.getAvailableTimes(request)
# Get records for last 50 available times
response = DataAccessLayer.getGeometryData(request, times[-50:-1])
print("Using " + str(len(response)) + " records")
# Each record will have a numpy array the length of the number of "parameters"
# Default is 1 (request.setParameters('phensig'))
parameters = {}
for x in request.getParameters():
parameters[x] = np.array([])
print(parameters)
bbox = [-125, -65, 20, 50]
fig, ax = make_map(bbox=bbox)
def main():
user_args = get_args()
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
if (user_args.date and not user_args.time) or (user_args.time and not user_args.date):
print >> sys.stderr, "date and time must be provided together"
return
# If a time range is provided, results will be filtered based on available times
timeRange = None
if user_args.date:
midRange = datetime.strptime( user_args.date + " " + user_args.time, "%Y-%m-%d %H:%M")
beginRange = midRange - timedelta(0, 60)
endRange = midRange + timedelta(0, 60)
timeRange = TimeRange(beginRange, endRange)
req = create_request(user_args)
if user_args.icao:
if user_args.productCode: # retrieve available times and/or true or primary elevation angles
if timeRange:
tr = timeRange
else:
tr = None
lines = set()
if user_args.outputAngle:
levels = DataAccessLayer.getAvailableLevels(req)
for level in levels:
line = ""
req.setLevels(level)
if user_args.outputDate:
times = DataAccessLayer.getAvailableTimes(req)
for time in times:
if not tr or tr.contains(time.getValidPeriod()):
line = str(time) + ".0"
line += " "
if user_args.outputAngle == "true":
line += "%.1f"%level.getLeveltwovalue()
else:
line += "%.1f"%level.getLevelonevalue()
lines.add(line)
else:
if not tr or data_in_time_range(req, tr):
if user_args.outputAngle == "true":
line = "%.1f"%level.getLeveltwovalue()
else:
line = "%.1f"%level.getLevelonevalue()
lines.add(line)
else : # just output time
times = DataAccessLayer.getAvailableTimes(req)
for time in times:
if not tr or tr.contains(time.getValidPeriod()):
lines.add(str(time) + ".0")
msg = "\n".join(lines)
else: #retrieve available product codes
unfiltered = DataAccessLayer.getAvailableParameters(req)
productCodes = []
for parameter in unfiltered: #filter to just productCodes
if parameter.isdigit():
productCodes.append(parameter)
if timeRange:
unfiltered = productCodes
productCodes = []
for productCode in unfiltered:
req = create_request(user_args)
req.setParameters(productCode)
if data_in_time_range(req, timeRange):
productCodes.append(productCode)
msg = "\n".join(productCodes)
else: # retrieve available icaos
icaos = DataAccessLayer.getAvailableLocationNames(req)
if timeRange:
unfiltered = icaos
icaos = []
for icao in unfiltered:
req = create_request(user_args)
req.addIdentifier("icao", icao)
if data_in_time_range(req, timeRange):
icaos.append(icao)
msg = "\n".join(icaos)
print msg.strip()
def data_in_time_range(req, timeRange):
times = DataAccessLayer.getAvailableTimes(req)
for time in times:
if timeRange.contains(time.getValidPeriod()):
return True
return False
