def testVariableMatcher2(self): # First let's creates the mask (it is the same for NCEP and ECMWF since they are on the same grid). refmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_dnm.nc') M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.]) # Reference ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml') Ref = cdutil.VariableConditioner(ref, weightsMaker=M) Ref.variable = 'tas' Ref.id = 'D1' Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')} # Test tstmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_ccsr.nc') M = cdutil.WeightsMaker(tstmsk, var='sftlf_ccsr', values=[1.]) tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml') Tst = cdutil.VariableConditioner(tst, weightsMaker=M) Tst.variable = 'tas' Tst.id = 'D2' # External Variable (for the mask) ext = ref EV = cdutil.VariableConditioner(ext) EV.variable = 'tas' EV.id = 'OUT' # Final Grid # We need a mask for the final grid fgmask = ext M2 = cdutil.WeightsMaker(source=refmsk, var='sftlf_dnm', values=[["input", 100.]]) FG = cdutil.WeightedGridMaker(weightsMaker=M2) FG.longitude.n = 36 FG.longitude.first = 0. FG.longitude.delta = 10. FG.latitude.n = 18 FG.latitude.first = -85. FG.latitude.delta = 10. # Now creates the compare object print("REF:", Ref()) c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG, externalVariableConditioner=EV) # And gets it (ref, reffrc), (test, tfrc) = c() print('Shapes:', test.shape, ref.shape) self.assertEqual(test.shape, ref.shape) self.assertEqual(test.shape, (12, 1, 18, 36))
def getVarDataTimeSlices(self, varList, timeValue): """ This method extracts a CDMS variable object (varName) and then cuts out a data slice with the correct axis ordering (returning a NumPy masked array). """ timeSlices, condTimeSlices = [], [] vc0 = None for (dsid, varName) in varList: varTimeSlice = self.getVarDataTimeSlice(dsid, varName, timeValue) if not vc0: vc0 = cdutil.VariableConditioner(varTimeSlice) else: timeSlices.append(varTimeSlice) for varTimeSlice in timeSlices: vc1 = cdutil.VariableConditioner(varTimeSlice) VM = cdutil.VariablesMatcher(vc0, vc1) condTimeSlice0, condTimeSlice1 = VM.get(returnTuple=0) if not condTimeSlices: condTimeSlices.append(condTimeSlice0) condTimeSlices.append(condTimeSlice1) return condTimeSlices
def testVariableMatcher1(self): ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml') # Reference Ref = cdutil.VariableConditioner(ref) Ref.variable = 'tas' Ref.id = 'JONES' # optional # Test tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml') Tst = cdutil.VariableConditioner(tst) Tst.variable = 'tas' Tst.id = 'NCEP' #optional # Final Grid FG = cdutil.WeightedGridMaker() FG.longitude.n = 36 FG.longitude.first = 0. FG.longitude.delta = 10. FG.latitude.n = 18 FG.latitude.first = -85. FG.latitude.delta = 10. # Now creates the compare object. c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG) # And get it (3 different ways). (ref, ref_frac), (test, test_frac) = c.get() ref, test = c.get(returnTuple=0) self.assertEqual(ref.shape, (72, 1, 18, 36)) self.assertEqual(test.shape, (72, 1, 18, 36)) self.assertTrue(numpy.allclose(ref.min(), 194.175)) self.assertTrue(numpy.allclose(test.min(), 210.83)) self.assertTrue(numpy.allclose(ref.max(), 309.541)) self.assertTrue(numpy.allclose(test.max(), 309.483)) ref, test = c(returnTuple=0) self.assertEqual(ref.shape, (72, 1, 18, 36)) self.assertEqual(test.shape, (72, 1, 18, 36)) self.assertTrue(numpy.allclose(ref.min(), 194.175)) self.assertTrue(numpy.allclose(test.min(), 210.83)) self.assertTrue(numpy.allclose(ref.max(), 309.541)) self.assertTrue(numpy.allclose(test.max(), 309.483))
#!/usr/bin/env python import cdutil, os, sys, numpy ref = os.path.join(sys.prefix, 'sample_data', 'tas_dnm-95a.xml') # Reference Ref = cdutil.VariableConditioner(ref) Ref.var = 'tas' Ref.id = 'JONES' # optional # Test tst = os.path.join(sys.prefix, 'sample_data', 'tas_ccsr-95a.xml') Tst = cdutil.VariableConditioner(tst) Tst.var = 'tas' Tst.id = 'NCEP' #optional # Final Grid FG = cdutil.WeightedGridMaker() FG.longitude.n = 36 FG.longitude.first = 0. FG.longitude.delta = 10. FG.latitude.n = 18 FG.latitude.first = -85. FG.latitude.delta = 10. # Now creates the compare object. c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG) # And get it (3 different ways). (ref, ref_frac), (test, test_frac) = c.get() ref, test = c.get(returnTuple=0) assert (ref.shape == (72, 1, 18, 36)) assert (test.shape == (72, 1, 18, 36)) assert (numpy.allclose(ref.min(), 194.175)) assert (numpy.allclose(test.min(), 210.83)) assert (numpy.allclose(ref.max(), 309.541))
"""This is a very complicated example that shows MOST of the options and power of VariablesMatcher. Once again we retrieve NCEP and ECMWF (for 1981), but this time, they are both masked for land first. ECMWF is then regridded to a T63 grid and NCEP to a T42 grid. There they are masked where the temperatures are less than 280K or more than 300K (in two different ways). The JONES external variable is then applied for additional external masking (with a personal land mask). Finally, everything is put on the 10x10 grid and masked for land. Also a 'selector' for Northern Hemisphere is applied (see cdutil.region documentation) """ import cdutil, MV2 as MV, os, sys # First let's create the mask (it is the same for NCEP and ECMWF since they are on the same grid) refmsk = os.path.join(sys.prefix, 'sample_data', 'sftlf_dnm.nc') M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.]) # Reference ref = os.path.join(sys.prefix, 'sample_data', 'tas_dnm-95a.xml') Ref = cdutil.VariableConditioner(ref, weightsMaker=M) Ref.var = 'tas' Ref.id = 'ECMWF' Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')} # Ok now the final grid for this variable is a T63, masked where temperatures are not between 280K and 300K ECMWFGrid = cdutil.WeightedGridMaker(source=refmsk, var='sftlf_dnm') ECMWFinalMask = cdutil.WeightsMaker() ECMWFinalMask.values = [('input', 280.), ('input', 300.)] ECMWFinalMask.actions = [MV.less, MV.greater] # Associate the mask with the grid ECMWFGrid.weightsMaker = ECMWFinalMask # Now associates the grid with the variable. Ref.weightedGridMaker = ECMWFGrid
class CDMSDatasetRecord(): def __init__(self, id, dataset=None, dataFile=None): self.id = id self.lev = None self.dataset = dataset self.cdmsFile = dataFile # self.cachedFileVariables = {} def getTimeValues(self, dsid): return self.dataset['time'].getValue() def getVariable(self, varName): return self.dataset[varName] # def clearDataCache( self ): # self.cachedFileVariables = {} def getLevAxis(self): for axis in self.dataset.axes.values(): if axis.isLevel() or PlotType.isLevelAxis(axis): return axis return None def getLevBounds(self, levaxis): levbounds = None if levaxis: values = levaxis.getValue() ascending_values = (values[-1] > values[0]) if levaxis: if levaxis.attributes.get('positive', '') == 'down' and ascending_values: levbounds = slice(None, None, -1) elif levaxis.attributes.get( 'positive', '') == 'up' and not ascending_values: levbounds = slice(None, None, -1) return levbounds def getVarDataTimeSlice(self, varName, timeValue, gridBounds, decimation, referenceVar=None, referenceLev=None): """ This method extracts a CDMS variable object (varName) and then cuts out a data slice with the correct axis ordering (returning a NumPy masked array). """ # cachedFileVariableRec = self.cachedFileVariables.get( varName ) # if cachedFileVariableRec: # cachedTimeVal = cachedFileVariableRec[ 0 ] # if cachedTimeVal.value == timeValue.value: # return cachedFileVariableRec[ 1 ] rv = CDMSDataset.NullVariable varData = self.dataset[varName] # print "Reading Variable %s, attributes: %s" % ( varName, str(varData.attributes) ) refFile = self.cdmsFile refVar = varName refGrid = None if referenceVar: referenceData = referenceVar.split('*') refDsid = referenceData[0] refFileRelPath = referenceData[1] refVar = referenceData[2] try: refFile = getFullPath(refFileRelPath) f = cdms2.open(refFile) refGrid = f[refVar].getGrid() except cdms2.error.CDMSError, err: print >> sys.stderr, " --- Error[1] opening dataset file %s: %s " % ( refFile, str(err)) if not refGrid: refGrid = varData.getGrid() if not refGrid: mb = QtGui.QMessageBox.warning( None, "DV3D Error", "CDAT is unable to create a grid for this dataset.") return None refLat = refGrid.getLatitude() refLon = refGrid.getLongitude() nRefLat, nRefLon = len(refLat) - 1, len(refLon) - 1 LatMin, LatMax = float(refLat[0]), float(refLat[-1]) LonMin, LonMax = float(refLon[0]), float(refLon[-1]) if LatMin > LatMax: tmpLatMin = LatMin LatMin = LatMax LatMax = tmpLatMin args1 = {} gridMaker = None decimationFactor = 1 if decimation: decimationFactor = decimation[0] + 1 # try: args1['time'] = timeValue if gridBounds[0] < LonMin and gridBounds[0] + 360.0 < LonMax: gridBounds[0] = gridBounds[0] + 360.0 if gridBounds[2] < LonMin and gridBounds[2] + 360.0 < LonMax: gridBounds[2] = gridBounds[2] + 360.0 if gridBounds[0] > LonMax and gridBounds[0] - 360.0 > LonMin: gridBounds[0] = gridBounds[0] - 360.0 if gridBounds[2] > LonMax and gridBounds[2] - 360.0 > LonMin: gridBounds[2] = gridBounds[2] - 360.0 if decimationFactor == 1: args1['lon'] = (gridBounds[0], gridBounds[2]) args1['lat'] = (gridBounds[1], gridBounds[3]) else: varGrid = varData.getGrid() if varGrid: varLonInt = varGrid.getLongitude().mapIntervalExt( [gridBounds[0], gridBounds[2]]) latAxis = varGrid.getLatitude() latVals = latAxis.getValue() latBounds = [gridBounds[3], gridBounds[1] ] if latVals[0] > latVals[1] else [ gridBounds[1], gridBounds[3] ] varLatInt = latAxis.mapIntervalExt(latBounds) args1['lon'] = slice(varLonInt[0], varLonInt[1], decimationFactor) args1['lat'] = slice(varLatInt[0], varLatInt[1], decimationFactor) print " ---- Decimate(%d) grid %s: varLonInt=%s, varLatInt=%s, lonSlice=%s, latSlice=%s" % ( decimationFactor, str(gridBounds), str(varLonInt), str(varLatInt), str(args1['lon']), str(args1['lat'])) # args1['squeeze'] = 1 start_t = time.time() # if (gridMaker == None) or ( gridMaker.grid == varData.getGrid() ): if ((referenceVar == None) or ((referenceVar[0] == self.cdmsFile) and (referenceVar[1] == varName))) and (decimationFactor == 1): levbounds = self.getLevBounds(referenceLev) if levbounds: args1['lev'] = levbounds args1['order'] = 'xyz' rv = varData(**args1) else: refDelLat = (LatMax - LatMin) / nRefLat refDelLon = (LonMax - LonMin) / nRefLon # nodataMask = cdutil.WeightsMaker( source=self.cdmsFile, var=varName, actions=[ MV2.not_equal ], values=[ nodata_value ] ) if nodata_value else None gridMaker = cdutil.WeightedGridMaker( flat=LatMin, flon=LonMin, nlat=int(nRefLat / decimationFactor), nlon=int(nRefLon / decimationFactor), dellat=(refDelLat * decimationFactor), dellon=(refDelLon * decimationFactor)) # weightsMaker=nodataMask ) vc = cdutil.VariableConditioner(source=self.cdmsFile, var=varName, cdmsKeywords=args1, weightedGridMaker=gridMaker) regridded_var_slice = vc.get(returnTuple=0) if referenceLev: regridded_var_slice = regridded_var_slice.pressureRegrid( referenceLev) args2 = {'order': 'xyz', 'squeeze': 1} levbounds = self.getLevBounds(referenceLev) if levbounds: args2['lev'] = levbounds rv = regridded_var_slice(**args2) try: rv = MV2.masked_equal(rv, rv.fill_value) except: pass # max_values = [ regridded_var_slice.max(), rv.max() ] # print " Regrid variable %s: max values = %s " % ( varName, str(max_values) ) end_t = time.time() # self.cachedFileVariables[ varName ] = ( timeValue, rv ) # print "Reading variable %s, shape = %s, base shape = %s, time = %s (%s), args = %s, slice duration = %.4f sec." % ( varName, str(rv.shape), str(varData.shape), str(timeValue), str(timeValue.tocomp()), str(args1), end_t-start_t ) # except Exception, err: # print>>sys.stderr, ' Exception getting var slice: %s ' % str( err ) return rv
refDelLon = (LonMax - LonMin) / nRefLon # nodataMask = cdutil.WeightsMaker( source=self.cdmsFile, var=varName, actions=[ MV2.not_equal ], values=[ nodata_value ] ) if nodata_value else None gridMaker = cdutil.WeightedGridMaker( flat=LatMin, flon=LonMin, nlat=int(nRefLat / decimationFactor), nlon=int(nRefLon / decimationFactor), dellat=(refDelLat * decimationFactor), dellon=(refDelLon * decimationFactor)) # weightsMaker=nodataMask ) # from packages.vtDV3D.CDMS_DatasetReaders import getRelativeTimeValues # time_values, dt, time_units = getRelativeTimeValues ( cdms2.open( self.cdmsFile ) ) vc = cdutil.VariableConditioner(source=self.cdmsFile, var=varName, cdmsKeywords=args1, weightedGridMaker=gridMaker) print " regridded_var_slice(%s:%s): %s " % (self.dataset.id, varName, str(args1)) regridded_var_slice = vc.get(returnTuple=0) # if (referenceLev <> None) and ( referenceLev.shape[0] <> currentLevel.shape[0] ): # regridded_var_slice = regridded_var_slice.pressureRegrid( referenceLev ) args2 = {'order': order, 'squeeze': 1} if levBounds <> None: args2['lev'] = levBounds[0] if (len(levBounds) == 1) else levBounds else: levBounds = self.getLevBounds(currentLevel) if levBounds: args2['lev'] = levBounds rv = regridded_var_slice(**args2)
def testVariableMatcher3(self): """This is a very complicated example that shows MOST of the options and power of VariablesMatcher. Once again we retrieve NCEP and ECMWF (for 1981), but this time, they are both masked for land first. ECMWF is then regridded to a T63 grid and NCEP to a T42 grid. There they are masked where the temperatures are less than 280K or more than 300K (in two different ways). The JONES external variable is then applied for additional external masking (with a personal land mask). Finally, everything is put on the 10x10 grid and masked for land. Also a 'selector' for Northern Hemisphere is applied (see cdutil.region documentation) """ # First let's create the mask (it is the same for NCEP and ECMWF since they are on the same grid) refmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_dnm.nc') M = cdutil.WeightsMaker(refmsk, var='sftlf_dnm', values=[1.]) # Reference ref = os.path.join(cdat_info.get_sampledata_path(), 'tas_dnm-95a.xml') Ref = cdutil.VariableConditioner(ref, weightsMaker=M) Ref.variable = 'tas' Ref.id = 'ECMWF' Ref.cdmsKeywords = {'time': ('1979', '1980', 'co')} # Ok now the final grid for this variable is a T63, masked where temperatures are not between 280K and 300K ECMWFGrid = cdutil.WeightedGridMaker(source=refmsk, var='sftlf_dnm') ECMWFinalMask = cdutil.WeightsMaker() ECMWFinalMask.values = [('input', 280.), ('input', 300.)] ECMWFinalMask.actions = [MV2.less, MV2.greater] # Associate the mask with the grid ECMWFGrid.weightsMaker = ECMWFinalMask # Now associates the grid with the variable. Ref.weightedGridMaker = ECMWFGrid # Test tstmsk = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_ccsr.nc') M = cdutil.WeightsMaker(tstmsk, var='sftlf_ccsr', values=[1.]) tst = os.path.join(cdat_info.get_sampledata_path(), 'tas_ccsr-95a.xml') Tst = cdutil.VariableConditioner(tst, weightsMaker=M) Tst.variable = 'tas' Tst.id = 'NCEP' # Ok now the final grid for this variable is a T42, masked where temperatures are not between 280K and 300K NCEPGrid = cdutil.WeightedGridMaker() NCEPGrid.latitude.n = 64 NCEPGrid.latitude.type = 'gaussian' # Ok now let's create a function to return the mask def myMakeMask(array, range): """Returns the input array masked where the values are not between range[0] and range[1]""" m1 = MV2.less(array, range[0]) # mask where it is less than the 1st value m2 = MV2.greater( array, range[1]) # mask where it is more than the 2nd value return MV2.logical_or(m1, m2) # And associate the mask with the grid NCEPGrid.weightsMaker.values = [('input', (280., 300.))] NCEPGrid.weightsMaker.actions = [myMakeMask] # Now associates the grid with the variable. Tst.weightedGridMaker = NCEPGrid # External Variable ext = ref extmask = refmsk EMask = cdutil.WeightsMaker(source=extmask, var='sftlf_dnm') ED = cdutil.VariableConditioner(ext, weightsMaker=EMask) ED.variable = 'tas' ED.id = 'JONES' # Final Grid # We need a mask for the final grid fgmask = os.path.join(cdat_info.get_sampledata_path(), 'sftlf_10x10.nc') M2 = cdutil.WeightsMaker(source=fgmask, var='sftlf', values=[100.]) FG = cdutil.WeightedGridMaker(weightsMaker=M2) FG.longitude.n = 36 FG.longitude.first = 0. FG.longitude.delta = 10. FG.latitude.n = 18 FG.latitude.first = -85. FG.latitude.delta = 10. # Now creates the compare object c = cdutil.VariablesMatcher(Ref, Tst, weightedGridMaker=FG, externalVariableConditioner=ED) c.cdmsArguments = [cdutil.region.NH] #print c # And gets it (ref, reffrc), (test, tfrc) = c() print('Shapes:', test.shape, ref.shape) print('Shapes:', tfrc.shape, reffrc.shape) self.assertEqual(ref.shape, reffrc.shape) self.assertEqual(test.shape, tfrc.shape) self.assertEqual(test.shape, ref.shape) self.assertEqual(test.shape, (12, 1, 9, 36))