def runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
return self.runGridDataTest(req, testSameShape=False)
def testGetGeometryDataWithLocNamesAndGroupsThrowException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
with self.assertRaises(ThriftRequestException):
self.runGeometryDataTest(req)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setParameters("temperature", "seaLevelPress", "dewpoint")
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testGetGeometryDataInvalidParamRaisesIncompatibleRequestException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('source', self.source)
req.setParameters('blahblahblah')
with self.assertRaises(ThriftRequestException) as cm:
self.runGeometryDataTest(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def runGetIdValuesTest(self, identifiers):
for identifier in identifiers:
if identifier.lower() == 'datauri':
continue
req = DAL.newDataRequest(self.datatype)
idValues = DAL.getIdentifierValues(req, identifier)
self.assertTrue(hasattr(idValues, '__iter__'))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setLocationNames(params.SITE_ID)
req.setParameters('T')
return self.runGridDataTest(req)
def getSounding(modelName, weatherElements, levels, samplePoint, timeRange=None):
"""
Performs a series of Data Access Framework requests to retrieve a sounding object
based on the specified request parameters.
Args:
modelName: the grid model datasetid to use as the basis of the sounding.
weatherElements: a list of parameters to return in the sounding.
levels: a list of levels to sample the given weather elements at
samplePoint: a lat/lon pair to perform the sampling of data at.
timeRange: (optional) a list of times, or a TimeRange to specify
which forecast hours to use. If not specified, will default to all forecast hours.
Returns:
A _SoundingCube instance, which acts a 3-tiered dictionary, keyed
by DataTime, then by level and finally by weather element. If no
data is available for the given request parameters, None is returned.
"""
(locationNames, parameters, levels, envelope, timeRange) = \
__sanitizeInputs(modelName, weatherElements, levels, samplePoint, timeRange)
requestArgs = {'datatype': 'grid', 'locationNames': locationNames,
'parameters': parameters, 'levels': levels, 'envelope': envelope}
req = DataAccessLayer.newDataRequest(**requestArgs)
response = DataAccessLayer.getGeometryData(req, timeRange)
soundingObject = _SoundingCube(response)
return soundingObject
def testGetVectorGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('Wind')
times = DAL.getAvailableTimes(req)
if not(times):
raise unittest.SkipTest('No Wind Data available for testing')
gridData = DAL.getGridData(req, [times[0]])
rawWind = None
rawDir = None
for grid in gridData:
if grid.getParameter() == 'Wind':
self.assertEqual(grid.getUnit(),'kts')
rawWind = grid.getRawData()
elif grid.getParameter() == 'WindDirection':
self.assertEqual(grid.getUnit(),'deg')
rawDir = grid.getRawData()
self.assertIsNotNone(rawWind, 'Wind Magnitude grid is not present')
self.assertIsNotNone(rawDir, 'Wind Direction grid is not present')
# rawWind and rawDir are numpy.ndarrays so comparison will result in boolean ndarrays.
self.assertTrue((rawWind >= 0).all(), 'Wind Speed should not contain negative values')
self.assertTrue((rawDir >= 0).all(), 'Wind Direction should not contain negative values')
self.assertTrue((rawDir <= 360).all(), 'Wind Direction should be less than or equal to 360')
self.assertFalse((rawDir == rawWind).all(), 'Wind Direction should be different from Wind Speed')
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters("temperature", "reportType")
req.setLocationNames(params.OBS_STATION)
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("staName", "rptType")
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
def testSuccessfulQuery(self):
req = DAL.newDataRequest('grid')
req.setLocationNames('RUC130')
req.setParameters('T','GH')
req.setLevels('300MB', '500MB','700MB')
times = CTQ.getAvailableTimes(req);
self.assertNotEqual(len(times), 0)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
req.setLocationNames(self.radarLoc)
req.setParameters(*self.parameterList)
# Don't test shapes since they may differ.
self.runGridDataTest(req, testSameShape=False)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", self.reporttype)
req.setLocationNames(params.OBS_STATION)
req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=" + record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" sfcPress=" + record.getString("sfcPress") +
record.getUnit("sfcPress"), end="")
print(" temp2=" + record.getString("temp2") +
record.getUnit("temp2"), end="")
print(" q2=" + record.getString("q2") +
record.getUnit("q2"), end="")
else:
print(" pressure=" + record.getString("pressure") +
record.getUnit("pressure"), end="")
print(" temperature=" + record.getString("temperature") +
record.getUnit("temperature"), end="")
print(" specHum=" + record.getString("specHum") +
record.getUnit("specHum"), end="")
print(" geometry=" + str(record.getGeometry()))
print("getGeometryData() complete\n\n")
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters('dataURI')
req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK')
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters("Imager 11 micron IR")
req.setLocationNames("West CONUS")
return self.runGridDataTest(req)
def __cloneRequest(request):
return DataAccessLayer.newDataRequest(datatype = request.getDatatype(),
parameters = request.getParameters(),
levels = request.getLevels(),
locationNames = request.getLocationNames(),
envelope = request.getEnvelope(),
**request.getIdentifiers())
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames(self.location)
req.addIdentifier("reportType", "2020")
req.setParameters("sfcPressure", "staName", "rptType", "tdMan")
print("Testing getGeometryData()")
geomData = DAL.getGeometryData(req)
self.assertIsNotNone(geomData)
print("Number of geometry records: " + str(len(geomData)))
print("Sample geometry data:")
for record in geomData[:self.sampleDataLimit]:
print("level=", record.getLevel(), end="")
# One dimensional parameters are reported on the 0.0UNKNOWN level.
# 2D parameters are reported on MB levels from pressure.
if record.getLevel() == "0.0UNKNOWN":
print(" sfcPressure=" + record.getString("sfcPressure") + record.getUnit("sfcPressure"), end="")
print(" staName=" + record.getString("staName"), end="")
print(" rptType=" + record.getString("rptType") + record.getUnit("rptType"), end="")
else:
print(" tdMan=" + str(record.getNumber("tdMan")) + record.getUnit("tdMan"), end="")
print(" geometry=", record.getGeometry())
print("getGeometryData() complete\n\n")
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'cli_asos_monthly')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('station_code', 'avg_daily_max')
return self.runGeometryDataTest(req)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('parmId.dbId.modelName', 'Fcst')
req.addIdentifier('parmId.dbId.siteId', params.SITE_ID)
req.setParameters('T')
gridDatas = self.runGridDataTest(req)
for gridData in gridDatas:
self.assertEqual(gridData.getAttribute('parmId.dbId.dbType'), '')
def testGetGeometryDataWithEnvelope(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setEnvelope(params.ENVELOPE)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertTrue(params.ENVELOPE.intersects(item.getGeometry()))
def testGetGeometryDataWithEnvelopeThrowsException(self):
# Envelope is not used
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setParameters('maxtemp_mon', 'min_sea_press')
req.setEnvelope(params.ENVELOPE)
with self.assertRaises(Exception):
data = self.runGeometryDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('table', 'height')
req.addIdentifier('ts', 'RG')
req.setParameters('value', 'lid', 'quality_code')
return self.runGeometryDataTest(req)
def testGetGeometryDataWithGroups(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.addIdentifier(self.groupKey, RequestConstraint.new('in', self.groups))
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertIn(item.getAttribute(self.groupKey), self.groups)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
# As an identifier it is "reportType" but as a parameter it is
# "rptType"... this is weird...
req.setParameters("staName", "rptType")
return self.runGeometryDataTest(req)
def testGetGeometryDataWithLocNames(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier(self.siteIdKey, params.SITE_ID)
req.setLocationNames(*self.editAreaNames)
data = self.runGeometryDataTest(req)
for item in data:
self.assertEqual(params.SITE_ID, item.getAttribute(self.siteIdKey))
self.assertIn(item.getLocationName(), self.editAreaNames)
def testFilterOnInvalidLocationThrowsIncompatibleRequestException(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
req.setLocationNames(') and 0=1')
with self.assertRaises(Exception) as cm:
DAL.getGeometryData(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.addIdentifier('info.datasetId', self.model)
req.addIdentifier('info.level.masterLevel.name', 'FHAG')
req.addIdentifier('info.level.leveltwovalue', 3000.0)
req.setParameters('T')
return self.runGridDataTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'mapdata.ffmp_basins')
req.addIdentifier('geomField', 'the_geom')
req.addIdentifier('cwa', 'OAX')
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('state', 'reservoir', 'area_sq_mi')
return self.runGeometryDataTest(req, checkDataTimes=False)
def testGetGeometryDataWithLocationSpecified(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'fcstheight')
locs = DAL.getAvailableLocationNames(req)
if locs:
req.setLocationNames(locs[0])
req.setParameters('probability', 'value')
data = self.runGeometryDataTest(req)
self.assertNotEqual(len(data), 0)
def runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters('166')
# TODO: Cannot check datatimes on the result because the times returned
# by getAvailableTimes have level = -1.0, while the time on the actual
# data has the correct level set (>= 0.0).
return self.runGeometryDataTest(req, checkDataTimes=False)
def testGetAvailableLocationsWithInvalidTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.boolean_values')
with self.assertRaises(ThriftRequestException) as cm:
DAL.getAvailableLocationNames(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames("KOMA")
req.setParameters("forecastHr", "maxTempDay")
self.runGeometryDataTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
req.setParameters("PRTM")
self.runGeometryDataTest(req, checkDataTimes=False)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames("West CONUS")
self.runTimesTest(req)
def testGetAvailableLevels(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', self.model)
self.runLevelsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
self.runTimesTest(req)
def testGetGeometryData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("flightLevel", "reportType")
self.runGeometryDataTest(req)
def testGetIdentifierValues(self):
req = DAL.newDataRequest(self.datatype)
optionalIds = set(DAL.getOptionalIdentifiers(req))
self.runGetIdValuesTest(optionalIds)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.setParameters("flightLevel", "reportType")
req.addIdentifier(key, constraint)
return self.runGeometryDataTest(req)
def testGetAvailableLocationsForRptTable(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.rpt')
self.runLocationsTest(req)
def _getAllRecords(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
req.setParameters('id')
return DAL.getGeometryData(req)
def testGetAvailableLocationsForStationId(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.day_climate_norm')
self.runLocationsTest(req)
def _getLocationNames(self):
req = DAL.newDataRequest()
req.setDatatype(self.datatype)
return DAL.getAvailableLocationNames(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
self.runLocationsTest(req)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.setParameters("Imager 11 micron IR")
req.setLocationNames("West CONUS")
self.runGridDataTest(req)
def testGetAvailableParameters(self):
req = DAL.newDataRequest(self.datatype)
self.runParametersTest(req)
def testGetGeometryDataEmptyResult(self):
req = DAL.newDataRequest(self.datatype)
self.addIdentifiers(req)
req.setParameters("blah blah blah") # force 0 records returned
result = self.runGeometryDataTest(req, checkDataTimes=False)
self.assertEqual(len(result), 0)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.setLocationNames(self.location)
req.addIdentifier("reportType", "2020")
self.runTimesTest(req)
def main():
user_args = get_args()
if user_args.host:
DataAccessLayer.changeEDEXHost(user_args.host)
slop = user_args.slop
dateTimeStr = user_args.datetime
if not dateTimeStr:
print >> sys.stderr, "DateTime not provided"
return
physicalElement = user_args.physical
if not physicalElement:
print >> sys.stderr, "PhysicalElement not provided"
return
sectorID = user_args.sector
if not sectorID:
print >> sys.stderr, "SectorID not provided"
return
creatingEntity = user_args.entity
part = user_args.partition
encoding = user_args.encoding
dateTime = datetime.strptime(dateTimeStr, "%Y-%m-%d %H:%M")
beginRange = dateTime - timedelta(0, slop)
endRange = dateTime + timedelta(0, slop)
timerange = TimeRange(beginRange, endRange)
req = DataAccessLayer.newDataRequest("satellite")
req.setParameters(physicalElement)
req.setLocationNames(sectorID)
if creatingEntity:
req.addIdentifier("creatingEntity", creatingEntity)
grids = DataAccessLayer.getGridData(req, timerange)
if not grids:
# print "Data not available"
return
grid = grids[0]
data = grid.getRawData()
myent = grid.getAttribute("creatingEntity")
mytime = a2dafcommon.datatime_to_string(grid.getDataTime()) + ".0"
if data is None or len(data) == 0:
# print "No data."
return
data[numpy.isnan(data)] = 0
yLen, xLen = data.shape
plus = " ghijklmnopqrstuvwxyz"
minus = " GHIJKLMNOPQRSTUVWXYZ"
limit = 10000000
if encoding == 1:
limit = limit / 2
elif encoding == 0:
limit = limit / 8
k = xLen * (yLen / 4)
j = 0
nxy = yLen * xLen
if part == "D":
j = k + k + k
elif part == "C":
j = k + k
nxy = j + k
elif part == "B":
j = k
nxy = j + k
elif part == "A" or nxy > limit:
nxy = k
msg = ""
if part <= "A":
msg += str(xLen) + " " + str(yLen) + " "
msg += mytime + " " + myent + "\n"
i = 0
kk = None
while j < yLen:
i = 0
kk = int(data[j, i])
if kk < 0: kk += 256
if encoding == 0:
msg += str(kk)
elif encoding == 1:
msg += "%2.2x" % kk
elif kk == 0:
msg += "@"
elif kk == 255:
msg += "#"
else:
msg += "%2.2x" % kk
i += 1
while i < xLen:
k = int(data[j, i])
if k < 0: k += 256
if encoding == 0:
msg += " " + str(k)
elif encoding == 1:
msg += "%2.2x" % k
elif k == 0:
msg += "@"
elif k == 255:
msg += "#"
elif k == kk:
msg += "."
elif k > kk + 20 or k < kk - 20:
msg += "%2.2x" % k
elif k > kk:
msg += plus[k - kk]
else:
msg += minus[kk - k]
kk = k
i += 1
msg += "\n"
j += 1
print msg.strip()
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier("reportType", "2020")
self.runLocationsTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.setParameters('maxtemp_mon', 'min_sea_press')
self.runTimesTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.setEnvelope(params.ENVELOPE)
self.runTimesTest(req)
def testGetAvailableLocationsWithConstraints(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
req.addIdentifier('maxtemp_mon', RequestConstraint.new('>', 95))
self.runLocationsTest(req)
def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('level.one.field', 'invalidLevelField')
with self.assertRaises(ThriftRequestException) as cm:
self.runLevelsTest(req)
self.assertIn('IncompatibleRequestException', str(cm.exception))
def testGetAvailableLocationsForInformId(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_mon_season_yr')
self.runLocationsTest(req)
def testGetAvailableLevels(self):
req = DAL.newDataRequest(self.datatype)
self.runLevelsTest(req)
def testGetGridData(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', self.model)
req.setLevels('2FHAG')
req.setParameters('T')
self.runGridDataTest(req)
def testGetAvailableTimes(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('info.datasetId', self.model)
req.setLevels('2FHAG')
self.runTimesTest(req)
def testGetAvailableLocations(self):
req = DAL.newDataRequest(self.datatype)
req.addIdentifier('table', 'public.cli_asos_monthly')
self.runLocationsTest(req)
def _runConstraintTest(self, key, operator, value):
req = DAL.newDataRequest(self.datatype)
constraint = RequestConstraint.new(operator, value)
req.addIdentifier(key, constraint)
req.setParameters("etn", "wmoid", "sig")
return self.runGeometryDataTest(req)