def test2_tasWithLevelsUsingLibCF(self):
test = 'Time test using regrid2 gsRegrid'
r2 = regrid2.GenericRegrid(self.gGrid, self.fGrid,
dtype = self.gtas.dtype,
regridMethod='linear',
regridTool='libcf')
r2.computeWeights()
fShape = list(self.gtas.shape[:-2]) + list(self.fGrid[0].shape)
ftas = np.ma.masked_array(np.ones(fShape, self.gtas.dtype)*self.gtas.missing_value,
mask = np.zeros(fShape))
ftas.missing_value = self.gtas.missing_value
r2.apply(self.gtas, ftas)
if PLOT:
vmin = self.gtas.min()
vmax = self.gtas.max()
pylab.subplot(1,4,1)
pylab.pcolor(self.gtas[-1,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('self.gtas[-1,...]')
pylab.subplot(1,4,2)
pylab.pcolor(ftasOld[-1,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('ftasOld[-1,...]')
pylab.subplot(1,4,3)
pylab.pcolor(ftas[0,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('ftas[-1,...]')
pylab.subplot(1,4,4)
pylab.pcolor(ftas[-1,...] - ftasOld[-1,...], vmin=-1, vmax=1)
pylab.colorbar()
pylab.title('ftas[-1,...] - ftasOld[-1,...]')
self.assertEqual(ftas[0,...].shape, self.fclt[0,...].shape)
self.assertEqual(ftas.shape[0], self.gtas.shape[0])
self.assertGreater(abs(ftas[ftas>0].mean()-self.gtas.mean()), 1)
def Xtest4_ESMF_Conservative_2D_clt(self):
"""
Out, ESMF, Conservative metric
"""
per = 1
coordSys = 'spherical degrees'
grid = [self.cltGrid[0], self.cltGrid[1]]
mask = numpy.array(self.clt > 90, dtype = numpy.int32)
newclt = numpy.ones(self.clt.shape) * self.clt
newclt[numpy.where(self.clt>75)] = self.clt.missing_value
ro = regrid2.GenericRegrid(grid, grid, self.clt.dtype,
regridMethod = 'conserv',
regridTool = 'esMf',
periodicity = per,
coordSys = coordSys)
ro.computeWeights()
print dir(ro.computeWeights())
ro.apply(numpy.array(newclt), self.cltInterp,
srcMissingValue = self.clt.missing_value, rootPe = 0)
nCell = numpy.array(self.clt.shape).prod()
if self.rank == 0:
avgDiffInterp = (abs(self.clt - self.cltInterp)).sum()/float(nCell)
avgDiffInterpInterp = abs(self.clt - self.cltInterpInterp).sum()/float(nCell)
#self.assertLess(avgDiffInterp, 50)
if PLOT:
pl.figure(4)
pl.subplot(3,2,1)
pl.pcolor(grid[1], grid[0], self.clt)
pl.title('clt')
pl.colorbar()
pl.subplot(3,2,2)
pl.pcolor(grid[1], grid[0], (newclt == self.clt.missing_value)+mask,
vmin = 0, vmax = 2)
pl.title('newclt == self.clt.missing_value')
pl.colorbar()
pl.subplot(3,2,3)
pl.pcolor(grid[1], grid[0], self.cltInterp)
mn, mx = self.cltInterp.min(), self.cltInterp.max()
pl.title('newMask %5.2f %5.2f' % (mn,mx))
pl.colorbar()
pl.subplot(3,2,4)
pl.pcolor(grid[1],grid[0], mask+self.cltInterp)
pl.colorbar()
pl.title('mask')
pl.subplot(3,2,5)
pl.pcolor(grid[1],grid[0], mask)
pl.colorbar()
pl.title('(newclt==self.clt.missing_value) - self.cltInterp')
pl.subplot(3,2,6)
pl.pcolor(grid[1],grid[0], newclt == self.clt.missing_value)
pl.colorbar()
pl.title('newclt-cltInterp')
string0 = "ESMF coordSys = %s, " % coordSys
string1 = "periodicity = %d, " % (per)
string2 = "MPI COMM size = %d" % (self.size)
pl.suptitle(string0 + string1 + string2)
def test2_ESMF_clt(self):
"""
Out and back, same grid using mvGenericRegrid -> ESMF, linear
"""
per = 1
coordSys = 'spherical degrees'
grid = [self.cltGrid[0], self.cltGrid[1]]
ro = regrid2.GenericRegrid(grid, grid,
dtype=self.clt.dtype,
regridMethod='linear',
regridTool = 'esMf', periodicity = per,
coordSys = coordSys)
ro.computeWeights()
ro.apply(numpy.array(self.clt), self.cltInterp, rootPe = 0)
self.cltInterp = self.comm.bcast(self.cltInterp, root = 0)
ro.apply(self.cltInterp, self.cltInterpInterp, rootPe = 0)
nCell = numpy.array(self.clt.shape).prod()
if self.rank == 0:
avgDiffInterp = (abs(self.clt - self.cltInterp)).sum()/float(nCell)
avgDiffInterpInterp = abs(self.clt - self.cltInterpInterp).sum()/float(nCell)
self.assertLess(avgDiffInterp, self.tol)
if self.size > 1:
self.assertLess(avgDiffInterpInterp, 600)
else:
self.assertLess(avgDiffInterpInterp, self.tol)
if PLOT:
pl.figure(2)
pl.subplot(3,2,1)
pl.pcolor(grid[1], grid[0], self.clt)
pl.title('clt')
pl.colorbar()
pl.subplot(3,2,2)
pl.pcolor(grid[1], grid[0], self.cltInterp,
vmin = 0, vmax = 100)
pl.title('Interp')
pl.colorbar()
pl.subplot(3,2,3)
pl.pcolor(grid[1], grid[0], self.cltInterpInterp,
vmin = 0, vmax = 100)
pl.title('InterpInterp')
pl.colorbar()
pl.subplot(3,2,4)
pl.pcolor(grid[1],grid[0], self.clt-self.cltInterp)
pl.colorbar()
pl.title('clt-cltInterp')
pl.subplot(3,2,5)
pl.pcolor(grid[1],grid[0], self.clt-self.cltInterpInterp)
pl.colorbar()
pl.title('clt-cltInterpInterp')
pl.subplot(3,2,6)
pl.pcolor(grid[1],grid[0], grid[1])
pl.colorbar()
pl.title('Longitude')
string0 = "ESMF coordSys = %s, " % coordSys
string1 = "periodicity = %d, " % (per)
string2 = "MPI COMM size = %d" % (self.size)
pl.suptitle(string0 + string1 + string2)
def Xtest6_ESMF_Conserve_LevelTime_clt(self):
"""
Interpolate over level/time in addition to lat-lon
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
# mask
srcGridMask = numpy.array(v[0, 0, ...] == v.missing_value, numpy.int32)
# v onto the ctl grid
srcGrd, srcNDims = regrid2.gsRegrid.makeCurvilinear(
[v.getLatitude(), v.getLongitude()])
dstGrd, dstNDims = regrid2.gsRegrid.makeCurvilinear(
[clt.getLatitude(), clt.getLongitude()])
ro = regrid2.GenericRegrid(srcGrd,
dstGrd,
regridMethod='conserve',
regridTool='esmp',
periodicity=1,
srcGridMask=srcGridMask)
ro.computeWeights()
vInterp = numpy.ones(
list(v.shape[:-2]) + list(clt.shape[-2:]),
v.dtype) * v.missing_value
ro.apply(numpy.array(v), vInterp, rootPe=0)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
nTimes = v.shape[0]
nLevels = v.shape[1]
for el in range(nTimes):
for k in range(nLevels):
pl.figure()
pl.subplot(1, 2, 1)
pl.pcolor(srcGrd[1],
srcGrd[0],
v[el, k, ...],
vmin=-20,
vmax=20)
pl.title('test6: v[%d, %d,...]' % (el, k))
pl.colorbar()
pl.subplot(1, 2, 2)
pl.pcolor(dstGrd[1],
dstGrd[0],
vInterp[el, k, ...],
vmin=-20,
vmax=20)
pl.title('test6: vInterp[%d, %d,...]' % (el, k))
pl.colorbar()
def test1_LibCF_clt(self):
"""
Out and back, same grid using mvGenericRegrid -> LibCF, linear
"""
ro = regrid2.GenericRegrid(self.cltGrid, self.cltGrid, self.clt.dtype,
regridMethod='linear', regridTool='libcf')
ro.computeWeights()
ro.apply(self.clt, self.cltInterp)
ro.apply(self.cltInterp, self.cltInterpInterp)
nCell = numpy.array(self.clt.shape).prod()
avgDiffInterp = (abs(self.clt - self.cltInterp)).sum()/float(nCell)
avgDiffInterpInterp = abs(self.clt - self.cltInterpInterp).sum()/float(nCell)
self.assertLess(avgDiffInterp, self.tol)
self.assertLess(avgDiffInterpInterp, self.tol)
if self.rank == 0:
avgDiffInterp = (abs(self.clt - self.cltInterp)).sum()/float(nCell)
avgDiffInterpInterp = abs(self.clt - self.cltInterpInterp).sum()/float(nCell)
self.assertLess(avgDiffInterp, self.tol)
self.assertLess(avgDiffInterpInterp, self.tol)
if PLOT:
pl.figure(1)
pl.subplot(3,2,1)
pl.pcolor(self.cltGrid[1], self.cltGrid[0], self.clt)
pl.title('clt')
pl.colorbar()
pl.subplot(3,2,2)
pl.pcolor(self.cltGrid[1], self.cltGrid[0], self.cltInterp,
vmin = 0, vmax = 100)
pl.title('Interp')
pl.colorbar()
pl.subplot(3,2,3)
pl.pcolor(self.cltGrid[1], self.cltGrid[0], self.cltInterpInterp,
vmin = 0, vmax = 100)
pl.title('InterpInterp')
pl.colorbar()
pl.subplot(3,2,4)
pl.pcolor(self.cltGrid[1],self.cltGrid[0], self.clt-self.cltInterp)
pl.colorbar()
pl.title('clt-cltInterp')
pl.subplot(3,2,5)
pl.pcolor(self.cltGrid[1],self.cltGrid[0], self.clt-self.cltInterpInterp)
pl.colorbar()
pl.title('clt-cltInterpInterp')
pl.subplot(3,2,6)
pl.pcolor(self.cltGrid[1],self.cltGrid[0], self.cltGrid[1])
pl.colorbar()
pl.title('Longitude')
per = 0
string0 = "LibCF coordSys = Bilinear, "
string1 = "periodicity = %d, " % (per)
string2 = "MPI COMM size = %d" % (self.size)
pl.suptitle(string0 + string1 + string2)
def test0_mvGeneric_dstMaskFloat_salinity(self):
"""
Check that the number of returned masked values is ok
"""
ro = regrid2.GenericRegrid(self.soGrid, self.cltGrid, dtype = self.so.dtype,
regridMethod='linear', regridTool='libcf')
ro.computeWeights()
ro.apply(self.so[0,0,:,:], self.soInterp, missingValue = self.so.missing_value)
self.assertLess(abs(self.so[0,0,...].mask.sum()/float(self.so[0,0,...].size) - 0.35),
0.05)
soInterpMask = numpy.array(self.soInterp == self.so.missing_value,
numpy.int64)
self.assertLess(abs(soInterpMask.sum()/float(soInterpMask.size) - 0.39),
0.05)
def test4_gsRegrid_Masking(self):
# Masking
test = 'test mask attached to data for gsRegrid'
r3 = regrid2.GenericRegrid(self.hGrid, self.fGrid,
dtype = self.hso.dtype,
regridMethod='linear',
regridTool='libcf')
r3.computeWeights()
gShape = list(self.hso.shape[:-2]) + list(self.fGrid[0].shape)
gso = np.ma.masked_array(np.zeros(gShape, self.hso.dtype), mask = np.zeros(gShape))
r3.apply(self.hso, gso)
maxv = gso <= self.hso.max()+1
aso = gso > 0
bb = (np.array(maxv,np.int32) + np.array(aso,np.int32))==2
self.assertLess(abs(gso[bb].mean()-self.hso.mean()), 1)
self.assertFalse(gso.mask[10,10])
self.assertFalse(gso.mask[3,3])
def test3_ESMF_Masking(self):
"""
Out, ESMF, Masking in __init__, Bilinear
"""
per = 1
coordSys = 'spherical degrees'
grid = [self.cltGrid[0], self.cltGrid[1]]
mask = numpy.array(self.clt > 90, dtype=numpy.int32)
ro = regrid2.GenericRegrid(grid,
grid,
self.clt.dtype,
regridMethod='linear',
regridTool='esMf',
periodicity=per,
coordSys=coordSys,
srcGridMask=mask)
ro.computeWeights()
ro.apply(numpy.array(self.clt), self.cltInterp, rootPe=0)
nCell = numpy.array(self.clt.shape).prod()
if self.rank == 0:
avgDiffInterp = (abs(self.clt -
self.cltInterp)).sum() / float(nCell)
avgDiffInterpInterp = abs(
self.clt - self.cltInterpInterp).sum() / float(nCell)
# we're expecting some ver large values because of the masking
#self.assertLess(avgDiffInterp, 50)
if False:
pl.figure(3)
pl.subplot(1, 3, 1)
pl.pcolor(grid[1], grid[0], self.clt)
pl.title('clt')
pl.colorbar()
pl.subplot(1, 3, 2)
pl.pcolor(grid[1], grid[0], self.cltInterp, vmin=0, vmax=100)
pl.title('Interp')
pl.colorbar()
pl.subplot(1, 3, 3)
pl.pcolor(grid[1], grid[0], self.clt - self.cltInterp)
pl.colorbar()
pl.title('clt-cltInterp')
string0 = "ESMF coordSys = %s, " % coordSys
string1 = "periodicity = %d, " % (per)
string2 = "MPI COMM size = %d" % (self.size)
pl.suptitle(string0 + string1 + string2)
def test5_gsRegrid_set_mask_before_regridding(self):
test = 'test set mask and compute before regridding'
so = self.hso
sGrid = [so.getLatitude(), so.getLongitude()]
r4 = regrid2.GenericRegrid(self.hGrid, self.fGrid,
dtype = so.dtype,
regridMethod = 'linear',
regridTool = 'libcf',
srcGridMask = self.hso.mask)
r4.computeWeights()
gShape = list(self.hso.shape[:-2]) + list(self.fGrid[0].shape)
gso = np.ma.masked_array(np.zeros(gShape, self.hso.dtype), mask = np.zeros(gShape))
r4.apply(so, gso)
maxv = gso <= self.hso.max()+1
aso = gso > 0
bb = (np.array(maxv,np.int32) + np.array(aso,np.int32))==2
self.assertLess(gso[bb].mean()-self.hso.mean(), 1)
self.assertFalse(gso.mask[10,10])
self.assertFalse(gso.mask[3,3])
def test5_LibCF_LevelTime(self):
"""
Interpolate over one level/time
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
# mask
srcGridMask = numpy.array(v[0, 0, ...] == v.missing_value, numpy.int32)
# v onto the ctl grid
srcGrd, srcNDims = regrid2.gsRegrid.makeCurvilinear(
[v.getLatitude(), v.getLongitude()])
dstGrd, dstNDims = regrid2.gsRegrid.makeCurvilinear(
[clt.getLatitude(), clt.getLongitude()])
ro = regrid2.GenericRegrid(srcGrd,
dstGrd,
clt.dtype,
regridMethod='linear',
regridTool='esmf',
periodicity=1,
coordSys='cart',
srcGridMask=srcGridMask)
ro.computeWeights()
vInterp = numpy.ones(clt.shape[-2:], v.dtype) * v.missing_value
ro.apply(numpy.array(v[0, 0, ...]), vInterp, rootPe=0)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
pl.figure()
pl.subplot(1, 2, 1)
pl.pcolor(srcGrd[1], srcGrd[0], v[0, 0, ...], vmin=-20, vmax=20)
pl.title('test5: v[0, 0,...]')
pl.colorbar()
pl.subplot(1, 2, 2)
pl.pcolor(dstGrd[1], dstGrd[0], vInterp, vmin=-20, vmax=20)
pl.title('test5: vInterp')
pl.colorbar()
def __init__(self, srcGrid, dstGrid, dtype,
regridMethod = 'linear', regridTool = 'libCF',
srcGridMask = None, srcGridAreas = None,
dstGridMask = None, dstGridAreas = None,
**args):
"""
Establish which regridding method to use, handle CDMS variables before
handing off to regridder. See specific tool for more information.
@param srcGrid CDMS source grid
@param dstGrid CDMS destination grid
@param dtype numpy data type for src and dst data
@param regridMethod linear (all tools - bi, tri),
conserve (ESMF Only)
patch (ESMF Only)
@param regridTool LibCF, ESMF, ...
@param srcGridMask array source mask, interpolation
coefficients will not be computed for masked
points/cells.
@param srcGridAreas array destination cell areas, only needed for
conservative regridding
@param dstGridMask array destination mask, interpolation
coefficients will not be computed for masked
points/cells.
@param dstGridAreas array destination cell areas, only needed for
conservative regridding
@param **args additional, tool dependent arguments
"""
srcBounds = None
dstBounds = None
self.srcGrid = srcGrid
self.dstGrid = dstGrid
srcCoords = _getCoordList(srcGrid)
dstCoords = _getCoordList(dstGrid)
# retrieve and build a bounds list for conservative from the grids
# We can't use the coords lists because if they are converted to
# curvilinear
# Set the tool to esmf if conservative selected. This overrides the
# regridTool selection
self.regridMethod = regridMethod
if re.search( 'conserv', regridMethod.lower()):
srcBadCellIndices = []
srcBounds = getBoundList(srcCoords, srcGridMask,
args.get('fixSrcBounds', False),
badCellIndices = srcBadCellIndices)
# mask out the bad src cells
if len(srcBadCellIndices) > 0:
if srcGridMask is None:
srcGridMask = numpy.zeros(srcCoords[0].shape, numpy.bool)
for inds in srcBadCellIndices:
srcGridMask[inds] = 1 # True mean invalid
dstBadCellIndices = []
dstBounds = getBoundList(dstCoords, dstGridMask,
args.get('fixDstBounds', False),
badCellIndices = dstBadCellIndices)
# mask out the bad dst cells
if len(dstBadCellIndices) > 0:
if dstGridMask is None:
dstGridMask = numpy.zeros(dstCoords[0].shape, numpy.bool)
for inds in dstBadCellIndices:
dstGridMask[inds] = 1 # True means invalid
for c, b in zip(srcBounds, srcCoords):
if c.min() == b.min() or c.max() == b.max():
print """
WARNING: Edge bounds are the same. The results of conservative regridding will not conserve.
coordMin = %7.2f, boundMin = %7.2f, coordMax = %7.2f, boundMax = %7.2f
""" % (c.min(), b.min(), c.max(), b.max())
if srcBounds[0].min() < -90 or srcBounds[0].max() > 90 or \
dstBounds[0].min() < -90 or dstBounds[0].max() > 90:
print "WARNING: Bounds exceed +/-90 degree latitude: min/max lats = %g/%g" % \
(srcBounds[0].min(), srcBounds[0].max())
if not re.search('esmp', regridTool.lower()):
regridTool = 'esmf'
# If LibCF handleCut is True, the bounds are needed to extend the grid
# close the cut at the top
if re.search('LibCF', regridTool, re.I) and args.has_key('handleCut'):
if args['handleCut']: srcBounds = getBoundList(srcCoords)
srcCoordsArrays = [numpy.array(sc) for sc in srcCoords]
dstCoordsArrays = [numpy.array(dc) for dc in dstCoords]
self.regridObj = regrid2.GenericRegrid(srcCoordsArrays, dstCoordsArrays,
regridMethod = regridMethod,
regridTool = regridTool,
dtype = dtype,
srcGridMask = srcGridMask,
srcBounds = srcBounds,
srcGridAreas = srcGridAreas,
dstGridMask = dstGridMask,
dstBounds = dstBounds,
dstGridAreas = dstGridAreas,
**args )
self.regridObj.computeWeights(**args)
def __init__(self,
srcGrid,
dstGrid,
dtype,
regridMethod='linear',
regridTool='libCF',
srcGridMask=None,
srcGridAreas=None,
dstGridMask=None,
dstGridAreas=None,
**args):
"""
"""
srcBounds = None
dstBounds = None
self.srcGrid = srcGrid
self.dstGrid = dstGrid
srcCoords = _getCoordList(srcGrid)
dstCoords = _getCoordList(dstGrid)
regridTool = str(regridTool) # force string if unicode or byte
regridMethod = str(regridMethod)
# retrieve and build a bounds list for conservative from the grids
# We can't use the coords lists because if they are converted to
# curvilinear
# Set the tool to esmf if conservative selected. This overrides the
# regridTool selection
self.regridMethod = str(regridMethod)
if re.search('conserv', regridMethod.lower()):
srcBadCellIndices = []
srcBounds = getBoundList(srcCoords,
srcGridMask,
args.get('fixSrcBounds', False),
badCellIndices=srcBadCellIndices)
# mask out the bad src cells
if len(srcBadCellIndices) > 0:
if srcGridMask is None:
srcGridMask = numpy.zeros(srcCoords[0].shape, numpy.bool)
for inds in srcBadCellIndices:
srcGridMask[inds] = 1 # True mean invalid
dstBadCellIndices = []
dstBounds = getBoundList(dstCoords,
dstGridMask,
args.get('fixDstBounds', False),
badCellIndices=dstBadCellIndices)
# mask out the bad dst cells
if len(dstBadCellIndices) > 0:
if dstGridMask is None:
dstGridMask = numpy.zeros(dstCoords[0].shape, numpy.bool)
for inds in dstBadCellIndices:
dstGridMask[inds] = 1 # True means invalid
for c, b in zip(srcBounds, srcCoords):
if c.min() == b.min() or c.max() == b.max():
print("""
WARNING: Edge bounds are the same. The results of conservative regridding will not conserve.
coordMin = %7.2f, boundMin = %7.2f, coordMax = %7.2f, boundMax = %7.2f
""" % (c.min(), b.min(), c.max(), b.max()))
if srcBounds[0].min() < -90 or srcBounds[0].max() > 90 or \
dstBounds[0].min() < -90 or dstBounds[0].max() > 90:
print(
"WARNING: Bounds exceed +/-90 degree latitude: min/max lats = %g/%g"
% (srcBounds[0].min(), srcBounds[0].max()))
if not re.search('esmp', regridTool.lower()):
regridTool = 'esmf'
if not re.search('esmf', regridTool.lower()):
regridTool = 'esmf'
# If LibCF handleCut is True, the bounds are needed to extend the grid
# close the cut at the top
if re.search('LibCF', regridTool, re.I) and 'handleCut' in args:
if args['handleCut']:
srcBounds = getBoundList(srcCoords)
srcCoordsArrays = [numpy.array(sc) for sc in srcCoords]
dstCoordsArrays = [numpy.array(dc) for dc in dstCoords]
self.regridObj = regrid2.GenericRegrid(srcCoordsArrays,
dstCoordsArrays,
regridMethod=regridMethod,
regridTool=regridTool,
dtype=dtype,
srcGridMask=srcGridMask,
srcBounds=srcBounds,
srcGridAreas=srcGridAreas,
dstGridMask=dstGridMask,
dstBounds=dstBounds,
dstGridAreas=dstGridAreas,
**args)
self.regridObj.computeWeights(**args)
