def testVariableMatcher2(self):
# First let's creates the mask (it is the same for NCEP and ECMWF since they are on the same grid).
refmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_dnm.nc')
M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.])
# Reference
ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml')
Ref = cdutil.VariableConditioner(ref, weightsMaker=M)
Ref.variable = 'tas'
Ref.id = 'D1'
Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')}
# Test
tstmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_ccsr.nc')
M = cdutil.WeightsMaker(tstmsk, var='sftlf_ccsr', values=[1.])
tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml')
Tst = cdutil.VariableConditioner(tst, weightsMaker=M)
Tst.variable = 'tas'
Tst.id = 'D2'
# External Variable (for the mask)
ext = ref
EV = cdutil.VariableConditioner(ext)
EV.variable = 'tas'
EV.id = 'OUT'
# Final Grid
# We need a mask for the final grid
fgmask = ext
M2 = cdutil.WeightsMaker(source=refmsk,
var='sftlf_dnm',
values=[["input", 100.]])
FG = cdutil.WeightedGridMaker(weightsMaker=M2)
FG.longitude.n = 36
FG.longitude.first = 0.
FG.longitude.delta = 10.
FG.latitude.n = 18
FG.latitude.first = -85.
FG.latitude.delta = 10.
# Now creates the compare object
print("REF:", Ref())
c = cdutil.VariablesMatcher(Ref,
Tst,
weightedGridMaker=FG,
externalVariableConditioner=EV)
# And gets it
(ref, reffrc), (test, tfrc) = c()
print('Shapes:', test.shape, ref.shape)
self.assertEqual(test.shape, ref.shape)
self.assertEqual(test.shape, (12, 1, 18, 36))
def testVariableMatcher1(self):
ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml')
# Reference
Ref = cdutil.VariableConditioner(ref)
Ref.variable = 'tas'
Ref.id = 'JONES' # optional
# Test
tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml')
Tst = cdutil.VariableConditioner(tst)
Tst.variable = 'tas'
Tst.id = 'NCEP' #optional
# Final Grid
FG = cdutil.WeightedGridMaker()
FG.longitude.n = 36
FG.longitude.first = 0.
FG.longitude.delta = 10.
FG.latitude.n = 18
FG.latitude.first = -85.
FG.latitude.delta = 10.
# Now creates the compare object.
c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG)
# And get it (3 different ways).
(ref, ref_frac), (test, test_frac) = c.get()
ref, test = c.get(returnTuple=0)
self.assertEqual(ref.shape, (72, 1, 18, 36))
self.assertEqual(test.shape, (72, 1, 18, 36))
self.assertTrue(numpy.allclose(ref.min(), 194.175))
self.assertTrue(numpy.allclose(test.min(), 210.83))
self.assertTrue(numpy.allclose(ref.max(), 309.541))
self.assertTrue(numpy.allclose(test.max(), 309.483))
ref, test = c(returnTuple=0)
self.assertEqual(ref.shape, (72, 1, 18, 36))
self.assertEqual(test.shape, (72, 1, 18, 36))
self.assertTrue(numpy.allclose(ref.min(), 194.175))
self.assertTrue(numpy.allclose(test.min(), 210.83))
self.assertTrue(numpy.allclose(ref.max(), 309.541))
self.assertTrue(numpy.allclose(test.max(), 309.483))
#!/usr/bin/env python import cdutil, os, sys, numpy ref = os.path.join(sys.prefix, 'sample_data', 'tas_dnm-95a.xml') # Reference Ref = cdutil.VariableConditioner(ref) Ref.var = 'tas' Ref.id = 'JONES' # optional # Test tst = os.path.join(sys.prefix, 'sample_data', 'tas_ccsr-95a.xml') Tst = cdutil.VariableConditioner(tst) Tst.var = 'tas' Tst.id = 'NCEP' #optional # Final Grid FG = cdutil.WeightedGridMaker() FG.longitude.n = 36 FG.longitude.first = 0. FG.longitude.delta = 10. FG.latitude.n = 18 FG.latitude.first = -85. FG.latitude.delta = 10. # Now creates the compare object. c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG) # And get it (3 different ways). (ref, ref_frac), (test, test_frac) = c.get() ref, test = c.get(returnTuple=0) assert (ref.shape == (72, 1, 18, 36)) assert (test.shape == (72, 1, 18, 36)) assert (numpy.allclose(ref.min(), 194.175)) assert (numpy.allclose(test.min(), 210.83)) assert (numpy.allclose(ref.max(), 309.541))
"""
import cdutil, MV2 as MV, os, sys
# First let's create the mask (it is the same for NCEP and ECMWF since they are on the same grid)
refmsk = os.path.join(sys.prefix, 'sample_data', 'sftlf_dnm.nc')
M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.])
# Reference
ref = os.path.join(sys.prefix, 'sample_data', 'tas_dnm-95a.xml')
Ref = cdutil.VariableConditioner(ref, weightsMaker=M)
Ref.var = 'tas'
Ref.id = 'ECMWF'
Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')}
# Ok now the final grid for this variable is a T63, masked where temperatures are not between 280K and 300K
ECMWFGrid = cdutil.WeightedGridMaker(source=refmsk, var='sftlf_dnm')
ECMWFinalMask = cdutil.WeightsMaker()
ECMWFinalMask.values = [('input', 280.), ('input', 300.)]
ECMWFinalMask.actions = [MV.less, MV.greater]
# Associate the mask with the grid
ECMWFGrid.weightsMaker = ECMWFinalMask
# Now associates the grid with the variable.
Ref.weightedGridMaker = ECMWFGrid
# Test
tstmsk = os.path.join(sys.prefix, 'sample_data', 'sftlf_ccsr.nc')
M = cdutil.WeightsMaker(tstmsk, var='sftlf_ccsr', values=[1.])
tst = os.path.join(cdutil.__path__[0], '..', '..', '..', '..', 'sample_data',
'tas_ccsr-95a.xml')
Tst = cdutil.VariableConditioner(tst, weightsMaker=M)
tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml')
Tst = cdutil.VariableConditioner(tst, weightsMaker=M)
Tst.var = 'tas'
Tst.id = 'D2'
# External Variable (for the mask)
ext = ref
EV = cdutil.VariableConditioner(ext)
EV.var = 'tas'
EV.id = 'OUT'
# Final Grid
# We need a mask for the final grid
fgmask = ext
M2 = cdutil.WeightsMaker(source=refmsk,
var='sftlf_dnm',
values=[["input", 100.]])
FG = cdutil.WeightedGridMaker(weightsMaker=M2)
FG.longitude.n = 36
FG.longitude.first = 0.
FG.longitude.delta = 10.
FG.latitude.n = 18
FG.latitude.first = -85.
FG.latitude.delta = 10.
# Now creates the compare object
c = cdutil.VariablesMatcher(Ref,
Tst,
weightedGridMaker=FG,
externalVariableConditioner=EV)
# And gets it
(ref, reffrc), (test, tfrc) = c()
print 'Shapes:', test.shape, ref.shape
class CDMSDatasetRecord():
def __init__(self, id, dataset=None, dataFile=None):
self.id = id
self.lev = None
self.dataset = dataset
self.cdmsFile = dataFile
# self.cachedFileVariables = {}
def getTimeValues(self, dsid):
return self.dataset['time'].getValue()
def getVariable(self, varName):
return self.dataset[varName]
# def clearDataCache( self ):
# self.cachedFileVariables = {}
def getLevAxis(self):
for axis in self.dataset.axes.values():
if axis.isLevel() or PlotType.isLevelAxis(axis): return axis
return None
def getLevBounds(self, levaxis):
levbounds = None
if levaxis:
values = levaxis.getValue()
ascending_values = (values[-1] > values[0])
if levaxis:
if levaxis.attributes.get('positive',
'') == 'down' and ascending_values:
levbounds = slice(None, None, -1)
elif levaxis.attributes.get(
'positive', '') == 'up' and not ascending_values:
levbounds = slice(None, None, -1)
return levbounds
def getVarDataTimeSlice(self,
varName,
timeValue,
gridBounds,
decimation,
referenceVar=None,
referenceLev=None):
"""
This method extracts a CDMS variable object (varName) and then cuts out a data slice with the correct axis ordering (returning a NumPy masked array).
"""
# cachedFileVariableRec = self.cachedFileVariables.get( varName )
# if cachedFileVariableRec:
# cachedTimeVal = cachedFileVariableRec[ 0 ]
# if cachedTimeVal.value == timeValue.value:
# return cachedFileVariableRec[ 1 ]
rv = CDMSDataset.NullVariable
varData = self.dataset[varName]
# print "Reading Variable %s, attributes: %s" % ( varName, str(varData.attributes) )
refFile = self.cdmsFile
refVar = varName
refGrid = None
if referenceVar:
referenceData = referenceVar.split('*')
refDsid = referenceData[0]
refFileRelPath = referenceData[1]
refVar = referenceData[2]
try:
refFile = getFullPath(refFileRelPath)
f = cdms2.open(refFile)
refGrid = f[refVar].getGrid()
except cdms2.error.CDMSError, err:
print >> sys.stderr, " --- Error[1] opening dataset file %s: %s " % (
refFile, str(err))
if not refGrid: refGrid = varData.getGrid()
if not refGrid:
mb = QtGui.QMessageBox.warning(
None, "DV3D Error",
"CDAT is unable to create a grid for this dataset.")
return None
refLat = refGrid.getLatitude()
refLon = refGrid.getLongitude()
nRefLat, nRefLon = len(refLat) - 1, len(refLon) - 1
LatMin, LatMax = float(refLat[0]), float(refLat[-1])
LonMin, LonMax = float(refLon[0]), float(refLon[-1])
if LatMin > LatMax:
tmpLatMin = LatMin
LatMin = LatMax
LatMax = tmpLatMin
args1 = {}
gridMaker = None
decimationFactor = 1
if decimation: decimationFactor = decimation[0] + 1
# try:
args1['time'] = timeValue
if gridBounds[0] < LonMin and gridBounds[0] + 360.0 < LonMax:
gridBounds[0] = gridBounds[0] + 360.0
if gridBounds[2] < LonMin and gridBounds[2] + 360.0 < LonMax:
gridBounds[2] = gridBounds[2] + 360.0
if gridBounds[0] > LonMax and gridBounds[0] - 360.0 > LonMin:
gridBounds[0] = gridBounds[0] - 360.0
if gridBounds[2] > LonMax and gridBounds[2] - 360.0 > LonMin:
gridBounds[2] = gridBounds[2] - 360.0
if decimationFactor == 1:
args1['lon'] = (gridBounds[0], gridBounds[2])
args1['lat'] = (gridBounds[1], gridBounds[3])
else:
varGrid = varData.getGrid()
if varGrid:
varLonInt = varGrid.getLongitude().mapIntervalExt(
[gridBounds[0], gridBounds[2]])
latAxis = varGrid.getLatitude()
latVals = latAxis.getValue()
latBounds = [gridBounds[3], gridBounds[1]
] if latVals[0] > latVals[1] else [
gridBounds[1], gridBounds[3]
]
varLatInt = latAxis.mapIntervalExt(latBounds)
args1['lon'] = slice(varLonInt[0], varLonInt[1],
decimationFactor)
args1['lat'] = slice(varLatInt[0], varLatInt[1],
decimationFactor)
print " ---- Decimate(%d) grid %s: varLonInt=%s, varLatInt=%s, lonSlice=%s, latSlice=%s" % (
decimationFactor, str(gridBounds), str(varLonInt),
str(varLatInt), str(args1['lon']), str(args1['lat']))
# args1['squeeze'] = 1
start_t = time.time()
# if (gridMaker == None) or ( gridMaker.grid == varData.getGrid() ):
if ((referenceVar == None) or
((referenceVar[0] == self.cdmsFile) and
(referenceVar[1] == varName))) and (decimationFactor == 1):
levbounds = self.getLevBounds(referenceLev)
if levbounds: args1['lev'] = levbounds
args1['order'] = 'xyz'
rv = varData(**args1)
else:
refDelLat = (LatMax - LatMin) / nRefLat
refDelLon = (LonMax - LonMin) / nRefLon
# nodataMask = cdutil.WeightsMaker( source=self.cdmsFile, var=varName, actions=[ MV2.not_equal ], values=[ nodata_value ] ) if nodata_value else None
gridMaker = cdutil.WeightedGridMaker(
flat=LatMin,
flon=LonMin,
nlat=int(nRefLat / decimationFactor),
nlon=int(nRefLon / decimationFactor),
dellat=(refDelLat * decimationFactor),
dellon=(refDelLon *
decimationFactor)) # weightsMaker=nodataMask )
vc = cdutil.VariableConditioner(source=self.cdmsFile,
var=varName,
cdmsKeywords=args1,
weightedGridMaker=gridMaker)
regridded_var_slice = vc.get(returnTuple=0)
if referenceLev:
regridded_var_slice = regridded_var_slice.pressureRegrid(
referenceLev)
args2 = {'order': 'xyz', 'squeeze': 1}
levbounds = self.getLevBounds(referenceLev)
if levbounds: args2['lev'] = levbounds
rv = regridded_var_slice(**args2)
try:
rv = MV2.masked_equal(rv, rv.fill_value)
except:
pass
# max_values = [ regridded_var_slice.max(), rv.max() ]
# print " Regrid variable %s: max values = %s " % ( varName, str(max_values) )
end_t = time.time()
# self.cachedFileVariables[ varName ] = ( timeValue, rv )
# print "Reading variable %s, shape = %s, base shape = %s, time = %s (%s), args = %s, slice duration = %.4f sec." % ( varName, str(rv.shape), str(varData.shape), str(timeValue), str(timeValue.tocomp()), str(args1), end_t-start_t )
# except Exception, err:
# print>>sys.stderr, ' Exception getting var slice: %s ' % str( err )
return rv
if levBounds <> None:
args1['lev'] = levBounds[0] if (len(levBounds)
== 1) else levBounds
else:
levBounds = self.getLevBounds(referenceLev)
if levBounds: args1['lev'] = levBounds
args1['order'] = order
rv = varData(**args1)
else:
refDelLat = (LatMax - LatMin) / nRefLat
refDelLon = (LonMax - LonMin) / nRefLon
# nodataMask = cdutil.WeightsMaker( source=self.cdmsFile, var=varName, actions=[ MV2.not_equal ], values=[ nodata_value ] ) if nodata_value else None
gridMaker = cdutil.WeightedGridMaker(
flat=LatMin,
flon=LonMin,
nlat=int(nRefLat / decimationFactor),
nlon=int(nRefLon / decimationFactor),
dellat=(refDelLat * decimationFactor),
dellon=(refDelLon *
decimationFactor)) # weightsMaker=nodataMask )
# from packages.vtDV3D.CDMS_DatasetReaders import getRelativeTimeValues
# time_values, dt, time_units = getRelativeTimeValues ( cdms2.open( self.cdmsFile ) )
vc = cdutil.VariableConditioner(source=self.cdmsFile,
var=varName,
cdmsKeywords=args1,
weightedGridMaker=gridMaker)
print " regridded_var_slice(%s:%s): %s " % (self.dataset.id,
varName, str(args1))
regridded_var_slice = vc.get(returnTuple=0)
# if (referenceLev <> None) and ( referenceLev.shape[0] <> currentLevel.shape[0] ):
def testVariableMatcher3(self):
"""This is a very complicated example that shows MOST of the options and power of VariablesMatcher.
Once again we retrieve NCEP and ECMWF (for 1981), but this time, they are both masked for land first.
ECMWF is then regridded to a T63 grid and NCEP to a T42 grid. There they are masked where the temperatures are less than 280K or more than 300K (in two different ways).
The JONES external variable is then applied for additional external masking (with a personal land mask).
Finally, everything is put on the 10x10 grid and masked for land.
Also a 'selector' for Northern Hemisphere is applied (see cdutil.region documentation)
"""
# First let's create the mask (it is the same for NCEP and ECMWF since they are on the same grid)
refmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_dnm.nc')
M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.])
# Reference
ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml')
Ref = cdutil.VariableConditioner(ref, weightsMaker=M)
Ref.variable = 'tas'
Ref.id = 'ECMWF'
Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')}
# Ok now the final grid for this variable is a T63, masked where temperatures are not between 280K and 300K
ECMWFGrid = cdutil.WeightedGridMaker(source=refmsk, var='sftlf_dnm')
ECMWFinalMask = cdutil.WeightsMaker()
ECMWFinalMask.values = [('input', 280.), ('input', 300.)]
ECMWFinalMask.actions = [MV2.less, MV2.greater]
# Associate the mask with the grid
ECMWFGrid.weightsMaker = ECMWFinalMask
# Now associates the grid with the variable.
Ref.weightedGridMaker = ECMWFGrid
# Test
tstmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_ccsr.nc')
M = cdutil.WeightsMaker(tstmsk, var='sftlf_ccsr', values=[1.])
tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml')
Tst = cdutil.VariableConditioner(tst, weightsMaker=M)
Tst.variable = 'tas'
Tst.id = 'NCEP'
# Ok now the final grid for this variable is a T42, masked where temperatures are not between 280K and 300K
NCEPGrid = cdutil.WeightedGridMaker()
NCEPGrid.latitude.n = 64
NCEPGrid.latitude.type = 'gaussian'
# Ok now let's create a function to return the mask
def myMakeMask(array, range):
"""Returns the input array masked where the values are not between range[0] and range[1]"""
m1 = MV2.less(array,
range[0]) # mask where it is less than the 1st value
m2 = MV2.greater(
array, range[1]) # mask where it is more than the 2nd value
return MV2.logical_or(m1, m2)
# And associate the mask with the grid
NCEPGrid.weightsMaker.values = [('input', (280., 300.))]
NCEPGrid.weightsMaker.actions = [myMakeMask]
# Now associates the grid with the variable.
Tst.weightedGridMaker = NCEPGrid
# External Variable
ext = ref
extmask = refmsk
EMask = cdutil.WeightsMaker(source=extmask, var='sftlf_dnm')
ED = cdutil.VariableConditioner(ext, weightsMaker=EMask)
ED.variable = 'tas'
ED.id = 'JONES'
# Final Grid
# We need a mask for the final grid
fgmask = os.path.join(cdat_info.get_sampledata_path(),
'sftlf_10x10.nc')
M2 = cdutil.WeightsMaker(source=fgmask, var='sftlf', values=[100.])
FG = cdutil.WeightedGridMaker(weightsMaker=M2)
FG.longitude.n = 36
FG.longitude.first = 0.
FG.longitude.delta = 10.
FG.latitude.n = 18
FG.latitude.first = -85.
FG.latitude.delta = 10.
# Now creates the compare object
c = cdutil.VariablesMatcher(Ref,
Tst,
weightedGridMaker=FG,
externalVariableConditioner=ED)
c.cdmsArguments = [cdutil.region.NH]
#print c
# And gets it
(ref, reffrc), (test, tfrc) = c()
print('Shapes:', test.shape, ref.shape)
print('Shapes:', tfrc.shape, reffrc.shape)
self.assertEqual(ref.shape, reffrc.shape)
self.assertEqual(test.shape, tfrc.shape)
self.assertEqual(test.shape, ref.shape)
self.assertEqual(test.shape, (12, 1, 9, 36))
