def select_CF(sCF, wCF, idx, *i):
    if idx == 1.:
        masked_sCF = np.ceil(sCF)
        masked_wCF = np.ceil(wCF)
        s_avg_tmp = cdutil.averager(sCF, axis='yx')
        w_avg_tmp = cdutil.averager(wCF, axis='yx')
    elif idx == 2.:
        # set threshold;
        # you can change this values based on needs;
        s_thresholds = 0.26
        w_thresholds = 0.26
        # set grids with values lower than the threshold to 0;
        sCF[sCF < s_thresholds] = 0.
        wCF[wCF < w_thresholds] = 0.
        # mask grids with a value of 0 (grids that are not needed);
        masked_sCF = set_axes(MV.masked_equal(sCF, 0.) * 0. + 1)
        masked_wCF = set_axes(MV.masked_equal(wCF, 0.) * 0. + 1)
        s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx')
        w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx')
    elif idx == 3.:
        # set threshold, top X%;
        # you can change this values based on needs;
        p_threshold_s = 0.25
        p_threshold_w = 0.25
        # sort all grids, values for grids that are
        # outside the interested region are set to -1;
        s_reshape = np.sort(np.array(MV.filled(sCF, -1)).ravel())[::-1]
        w_reshape = np.sort(np.array(MV.filled(wCF, -1)).ravel())[::-1]
        # remove grids with values of -1;
        s_no_minus1 = s_reshape[s_reshape != -1]
        w_no_minus1 = w_reshape[w_reshape != -1]
        # find the threshold for the top X%
        num_s = int(len(s_no_minus1) * p_threshold_s)
        num_w = int(len(w_no_minus1) * p_threshold_w)
        s_thresholds = s_no_minus1[num_s - 1]
        w_thresholds = w_no_minus1[num_w - 1]
        # mask grids with a value lower than the derived thresholds (grids that are not needed);
        masked_sCF = set_axes(MV.masked_less(sCF, s_thresholds) * 0. + 1)
        masked_wCF = set_axes(MV.masked_less(wCF, w_thresholds) * 0. + 1)
        s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx')
        w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx')
    print(
        f"For method {idx}, averaged solar capacity factor for the filtered grids is: {s_avg_tmp}"
    )
    print(
        f"For method {idx}, averaged wind capacity factor for the filtered grids is: {w_avg_tmp}"
    )
    print(
        f"Selected grid cells solar {np.ceil(masked_sCF).sum()}\nSelected grid cells wind {np.ceil(masked_wCF).sum()}"
    )
    return masked_sCF, masked_wCF
Ejemplo n.º 2
0
    def testContiguousRegridNANIssue(self):
        a = MV2.reshape(MV2.sin(MV2.arange(20000)), (2, 1, 100, 100))
        lon = cdms2.createAxis(MV2.arange(100) * 3.6)
        lon.designateLongitude()
        lon.units = "degrees_east"
        lon.id = "longitude"

        lat = cdms2.createAxis(MV2.arange(100) * 1.8 - 90.)
        lat.id = "latitude"
        lat.designateLatitude()
        lat.units = "degrees_north"

        lev = cdms2.createAxis([1000.])
        lev.id = "plev"
        lev.designateLevel()
        lev.units = "hPa"

        t = cdms2.createAxis([0, 31.])
        t.id = "time"
        t.designateTime()
        t.units = "days since 2014"

        cdutil.setTimeBoundsMonthly(t)
        a.setAxisList((t, lev, lat, lon))
        a = MV2.masked_less(a, .5)
        grd = cdms2.createGaussianGrid(64)

        a = a.ascontiguous()
        a = a.regrid(grd, regridTool="regrid2")
        a = cdutil.averager(a, axis='txy')
        self.assertEqual(a[0], 0.7921019540305255)
Ejemplo n.º 3
0
 def testAxismissing(self):
     data = """
     -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
     0.059503571833625334
     0.059503571833625334 0.05664014775641405 0.05193557222118004
     0.04777129850801233 0.0407139313814465 0.029382624830271705
     0.018469399844287374 0.0162382275289592 0.02646680241827459
     0.04792041732949079 0.0689138797030203 0.08167038620212037
     0.09273558459066569 0.11266293431057901 0.13663018925347364
     0.15229174546388072 0.15284435880966177 0.13423845476113883
     0.09945904378274077 0.07032267160267985 0.05551039827020481
     0.045537187647785464 0.040532491867244946 0.03577527125478327
     -999. -999. -999.
     -0.058062458673116 -0.08764922509099882 -0.11697036914487152
     -0.14836133615864944 -0.17956528904564023 -0.21109198032585794
     -0.23846429237248942 -0.2598536549218765 -0.27795672866320387
     -0.2939939095159731 -0.30541031366330024 -0.307643559333884
     -0.30078421139811795 -0.2841339526883441 -0.26485737397202497
     -0.24287299694779327 -0.22379014890999907 -0.20121548204699846
     -0.1746486732156772 -0.14585019344118372 -0.12070675757803526
     -0.0997891159111037 -0.08229393660994214 -0.06779720501287469
     -0.057213385470859794 -0.04875768191096844 -0.0402377347189964
     -0.030169328367807245 -0.017560662894847895 -0.006968922654137132
     0.0009773980274431048 0.007054306637034288 0.010472286514133042
     0.010702384151997032 0.009231553701801242 0.007544033101056543
     0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
     -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
     -999. -999. -999.
     """.split()
     data = numpy.array(data, dtype=numpy.float)
     data = MV2.masked_less(data, -900)
     d2 = cdms2.createAxis(data)
     self.assertTrue(numpy.ma.allclose(data, d2[:]))
Ejemplo n.º 4
0
 def testMaskingFunctions(self):
     xouter = MV2.outerproduct(MV2.arange(5.), [1] * 10)
     masked = MV2.masked_greater(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[2:], True))
     self.assertTrue(MV2.allequal(masked.mask[:2], False))
     masked = MV2.masked_greater_equal(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[1:], True))
     self.assertTrue(MV2.allequal(masked.mask[:1], False))
     masked = MV2.masked_less(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[:1], True))
     self.assertTrue(MV2.allequal(masked.mask[1:], False))
     masked = MV2.masked_less_equal(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[:2], True))
     self.assertTrue(MV2.allequal(masked.mask[2:], False))
     masked = MV2.masked_not_equal(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[1], False))
     self.assertTrue(MV2.allequal(masked.mask[0], True))
     self.assertTrue(MV2.allequal(masked.mask[2:], True))
     masked = MV2.masked_equal(xouter, 1)
     self.assertTrue(MV2.allequal(masked.mask[1], True))
     self.assertTrue(MV2.allequal(masked.mask[0], False))
     self.assertTrue(MV2.allequal(masked.mask[2:], False))
     masked = MV2.masked_outside(xouter, 1, 3)
     self.assertTrue(MV2.allequal(masked.mask[0:1], True))
     self.assertTrue(MV2.allequal(masked.mask[1:4], False))
     self.assertTrue(MV2.allequal(masked.mask[4:], True))
     masked = MV2.masked_where(
         MV2.logical_or(MV2.greater(xouter, 3), MV2.less(xouter, 2)),
         xouter)
     self.assertTrue(MV2.allequal(masked.mask[0:2], True))
     self.assertTrue(MV2.allequal(masked.mask[2:4], False))
     self.assertTrue(MV2.allequal(masked.mask[4:], True))
Ejemplo n.º 5
0
def bony_sorting_part1(w500, binedges):

    A, B, C = w500.shape
    dx = np.diff(binedges)[0]
    # Compute composite:
    OKwaps = nanarray(
        (A, B, C, 2 + len(binedges)))  # add 2 for the exceedances
    xx = 0
    for x in binedges:
        xx += 1
        w500_bin = MV.masked_less(w500, x)
        OKwaps[..., xx] = MV.masked_greater_equal(w500_bin, x + dx)

    # do the first wap bin:
    OKwaps[..., 0] = MV.masked_greater_equal(w500, binedges[0])
    # do the last wap bin:
    OKwaps[..., -1] = MV.masked_less(w500, binedges[-1] + dx)

    return OKwaps  # [month,lat,lon,wapbin]
Ejemplo n.º 6
0
def map_SWkern_to_lon(Ksw, albcsmap):
    """
    Map each location's clear-sky surface albedo to the correct albedo bin
    """

    albcsmap = MV.masked_greater(albcsmap, 1.0)
    albcsmap = MV.masked_less(albcsmap, 0.0)
    from scipy.interpolate import interp1d
    # Ksw is size 12,7,7,lats,3
    # albcsmap is size A,lats,lons
    albcs = np.arange(0.0, 1.5, 0.5)
    A = albcsmap.shape[0]
    TT = Ksw.shape[1]
    PP = Ksw.shape[2]
    lenlat = Ksw.shape[3]
    lenlon = albcsmap.shape[2]
    SWkernel_map = MV.zeros((A, TT, PP, lenlat, lenlon))
    SWkernel_map = MV.masked_where(SWkernel_map == 0, SWkernel_map)

    for M in range(A):
        MM = M
        while MM > 11:
            MM = MM - 12
        for LA in range(lenlat):
            alon = albcsmap[M, LA, :]
            # interp1d can't handle mask but it can deal with NaN (?)
            try:
                alon2 = MV.where(alon.mask, np.nan, alon)
            except:
                alon2 = alon
            if np.ma.count(alon2) > 1:  # at least 1 unmasked value
                if len(np.where(Ksw[MM, :, :, LA, :] > 0)) == 0:
                    SWkernel_map[M, :, :, LA, :] = 0
                else:
                    f = interp1d(albcs, Ksw[MM, :, :, LA, :], axis=2)
                    ynew = f(alon2.data)
                    ynew = MV.masked_where(alon2.mask, ynew)
                    SWkernel_map[M, :, :, LA, :] = ynew
            else:
                continue

    return SWkernel_map
Ejemplo n.º 7
0
def mmeAveMsk1D(listFiles,
                sw2d,
                years,
                inDir,
                outDir,
                outFile,
                timeInt,
                mme,
                ToeType,
                fullTS,
                debug=True):
    '''
    The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
    It ouputs the MME and a percentage of non-masked bins

    Created on Tue Nov 25 13:56:20 CET 2014

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - sw2d                   - dimension of fields to consider (1 or 2)
    - years(t1,t2)           - years for slice read
    - inDir(str)             - input directory where files are stored
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - FfllTS                 - 0/1: if 1, uses full time serie (ignores years(t1,t2))
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 25 Nov 2014   - Initial function write
    - EG  9 Dec 2014   - Add agreement on difference with init period - save as <var>Agree
    - EG 04 Oct 2016   - Add 3D files support

    TODO:
    ------

    '''

    # CDMS initialisation - netCDF compression
    comp = 1
    # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    if t2 <= 0:
        useLastYears = True
        t2 = -t2
    else:
        useLastYears = False
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    # Find dimension
    runN = len(listFiles)
    try:
        fi = cdm.open(inDir[0] + '/' + listFiles[0])
    except:
        print ' *** file not found ', inDir[0] + '/' + listFiles[0]
        sys.exit(' Abort')
    if sw2d == 1:
        ptopd0 = fi['ptopdepth']
        # Create variable handle
        latN = ptopd0.shape[2]
        basN = ptopd0.shape[1]
    elif sw2d == 2:
        ptopd0 = fi['ptopdepthxy']
        # Create variable handle
        lonN = ptopd0.shape[2]
        latN = ptopd0.shape[1]

    #timN = ptopd0.shape[0]
    timN = t2 - t1
    if fullTS:
        print '  !!! Working on full Time Serie (fullTS = True)'
        timN = ptopd0.shape[0]
        t1 = 0
        t2 = timN
    t10 = t1
    t20 = t2
    # Get grid objects
    axesList = ptopd0.getAxisList()
    # Declare and open files for writing
    if os.path.isfile(outDir + '/' + outFile):
        os.remove(outDir + '/' + outFile)
    outFile_f = cdm.open(outDir + '/' + outFile, 'w')

    print ' Number of members:', len(listFiles)

    valmask = ptopd0.missing_value

    # init time axis
    time = cdm.createAxis(npy.float32(range(timN)))
    time.id = 'time'
    time.units = 'years since 1861'
    time.designateTime()

    # loop on variables
    # init percent array

    if sw2d == 1:
        varList = [
            'ptopdepth', 'ptopsigma', 'ptopso', 'ptopthetao', 'volpers',
            'salpers', 'tempers'
        ]
        #varList = ['ptopdepth']
        varDim = [1, 1, 1, 1, 0, 0, 0]
        percent = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 0.
    elif sw2d == 2:
        varList = ['ptopdepthxy', 'ptopsigmaxy', 'ptopsoxy', 'ptopthetaoxy']
        #varList = ['ptopdepthxy']
        varDim = [2, 2, 2, 2]
        percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.

    varFill = [
        valmask, valmask, valmask, valmask, valmask, valmask, valmask, valmask,
        valmask
    ]

    axis1D = [time, axesList[1], axesList[2]]
    axis0D = [time, axesList[1]]
    print ' timN = ', timN

    # loop on 1D variables
    for iv, var in enumerate(varList):
        ti0 = timc.clock()

        # Array inits
        if varDim[iv] == 2:
            isonvar = npy.ma.ones([runN, timN, latN, lonN],
                                  dtype='float32') * valmask
            vardiff = npy.ma.ones([runN, timN, latN, lonN],
                                  dtype='float32') * valmask
            varones = npy.ma.ones([runN, timN, latN, lonN],
                                  dtype='float32') * 1.
            axisVar = axis1D
        elif varDim[iv] == 1:
            isonvar = npy.ma.ones([runN, timN, basN, latN],
                                  dtype='float32') * valmask
            vardiff = npy.ma.ones([runN, timN, basN, latN],
                                  dtype='float32') * valmask
            varones = npy.ma.ones([runN, timN, basN, latN],
                                  dtype='float32') * 1.
            axisVar = axis1D
        else:
            isonvar = npy.ma.ones([runN, timN, basN],
                                  dtype='float32') * valmask
            vardiff = npy.ma.ones([runN, timN, basN],
                                  dtype='float32') * valmask
            varones = npy.ma.ones([runN, timN, basN], dtype='float32') * 1.
            axisVar = axis0D
        print ' Variable ', iv, var, varDim[iv]
        # loop over files to fill up array
        for ic, file in enumerate(listFiles):
            ft = cdm.open(inDir[0] + '/' + file)
            timeax = ft.getAxis('time')
            try:
                tmax = timeax.shape[0]
            except:
                print ic, file, timeax
            if ic == 0:
                tmax0 = tmax
                #print ic,file, tmax
            #adapt [t1,t2] time bounds to piControl last NN years
            if useLastYears:
                t1 = tmax - t20
                t2 = tmax
            else:
                if tmax != tmax0:
                    print 'tmax <> tmax0', tmax, tmax0
                    print 'wrong time axis: exiting...'

                    return
            #print 'Time dims:',ic, t1,t2,tmax
            # read array
            computeVar = True
            allVars = ft.variables.keys()
            if 'ptopsigmaxy' in allVars:
                computeVar = False
            if (var == 'ptopsigmaxy') & computeVar:
                #print '  ic = ',ic
                # reconstruct from isondepthg and ptopdepthxy

                isond = ft('isondepthg', time=slice(t1, t2))
                #print isond.data.shape, timN*latN*lonN
                itest = 94 * 360 + 150
                axesList = isond.getAxisList()
                levs = axesList[1][:]
                levN = len(levs)
                #ti02 = timc.clock()
                levs3d0 = mv.reshape(npy.tile(levs, latN * lonN),
                                     (latN * lonN, levN))
                #ti05 = timc.clock()
                isonRead = npy.ma.ones([timN, latN, lonN],
                                       dtype='float32') * valmask
                for it in range(timN):  # loop on time to limit memory usage
                    levs3d = levs3d0 * 1.
                    depthlo = mv.reshape(vardepth[ic, it, ...], latN * lonN)
                    depth3d = npy.reshape(npy.repeat(depthlo, levN),
                                          (latN * lonN, levN))
                    isond3d = mv.reshape(
                        npy.transpose(isond.data[it, ...], (1, 2, 0)),
                        (latN * lonN, levN))
                    #print isond3d[itest,:]
                    isond3d[isond3d > valmask / 10] = 0.
                    #print isond3d[itest,:]
                    isond3dp1 = npy.roll(isond3d, -1, axis=1)
                    isond3dp1[:, -1] = isond3d[:, -1]
                    #print isond3dp1[itest,:]
                    #levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
                    #print levs3d[itest,:]
                    levs3d[(depth3d <= isond3d)] = 0.
                    #print levs3d[itest,:]
                    levs3d[(depth3d > isond3dp1)] = 0.
                    #print levs3d[itest,:]
                    #isonwrk = npy.sum(levs3d,axis=1)
                    isonwrk = npy.max(levs3d, axis=1)
                    if it < 0:
                        print ic, it
                        print depthlo[itest]
                        print isond3d[itest, :]
                        print isonwrk[itest]
                        print
                    isonRead[it, ...] = mv.reshape(isonwrk, (latN, lonN))
                # <-- end of loop on time
                del (isond3d, isond3dp1)
                gc.collect()
                # mask with depthxy and where sigmaxy = 0
                isonRead.mask = vardepth.mask[ic, ...]
                isonRead = mv.masked_where(isonRead == 0, isonRead)
                isonRead.long_name = var
                isonRead.units = 'sigma_n'
                isonRead.id = var
                del (isond, depth3d, levs3d, levs3d0, isonwrk)
                gc.collect()
                #ti3 = timc.clock()
                #print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
                #print ti3-ti0
                # write ptopsigmaxy
                if os.path.isfile(inDir[0] + '/work_ptopsigmaxy/' + file):
                    os.remove(inDir[0] + '/work_ptopsigmaxy/' + file)
                fiout = cdm.open(inDir[0] + '/work_ptopsigmaxy/' + file, 'w')
                if ic == 0:
                    print ' Creating ', inDir[0] + '/work_ptopsigmaxy/' + file
                isonsigxy = cdm.createVariable(isonRead,
                                               axes=axis1D,
                                               id='ptopsigmaxy')
                isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
                isonsigxy.units = 'sigma_n'
                fiout.write(isonsigxy.astype('float32'))
                fiout.close()
            else:
                # Direct read of variable
                isonRead = ft(var, time=slice(t1, t2))
            #print isonRead.shape, timN
            if varFill[iv] != valmask:
                isonvar[ic, ...] = isonRead.filled(varFill[iv])
            else:
                isonvar[ic, ...] = isonRead
            #print isonvar[ic,:,40,100]
            # compute percentage of non-masked points accros MME
            if iv == 0:
                maskvar = mv.masked_values(isonRead.data, valmask).mask
                percent[ic, ...] = npy.float32(npy.equal(maskvar, 0))
            if mme:
                # if mme then just average Bowl and Agree fields
                varst = var + 'Agree'
                vardiff[ic, ...] = ft(varst, time=slice(t1, t2))
            else:
                # Compute difference with average of first initN years, use mask of last month
                varinit = cdu.averager(isonvar[ic, peri1:peri2, ...], axis=0)
                for tr in range(timN):
                    vardiff[ic, tr, ...] = isonvar[ic, tr, ...] - varinit
                vardiff[ic, ...].mask = isonvar[ic, ...].mask

            ft.close()
        # <-- end of loop on files
        # TODO remove masked points at longitudes 0 or 180deg for some models
        # if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
        if var == 'ptopdepthxy':
            vardepth = isonvar
        # Compute percentage of bin presence
        # Only keep points where percent > 50%
        if iv == 0:
            percenta = (cdu.averager(percent, axis=0)) * 100.
            percenta = mv.masked_less(percenta, 50)
            percentw = cdm.createVariable(percenta,
                                          axes=axis1D,
                                          id='ptoppercent')
            percentw._FillValue = valmask
            percentw.long_name = 'percentage of MME bin'
            percentw.units = '%'
            outFile_f.write(percentw.astype('float32'))
        # Sign of difference
        if mme:
            vardiffsgSum = cdu.averager(vardiff, axis=0)
            vardiffsgSum = cdm.createVariable(vardiffsgSum,
                                              axes=axisVar,
                                              id='foo')
            vardiffsgSum = maskVal(vardiffsgSum, valmask)
            vardiffsgSum.mask = percentw.mask
        else:
            vardiffsg = npy.copysign(varones, vardiff)
            # average signs
            vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            vardiffsgSum.mask = percentw.mask
            vardiffsgSum._FillValue = valmask

        # average accross members
        isonVarAve = cdu.averager(isonvar, axis=0)
        isonVarAve = cdm.createVariable(isonVarAve, axes=axisVar, id='foo')
        # mask
        if varFill[iv] == valmask:
            isonVarAve = maskVal(isonVarAve, valmask)

        isonVarAve.mask = percentw.mask

        # Write
        isonave = cdm.createVariable(isonVarAve, axes=axisVar, id=isonRead.id)
        isonave.long_name = isonRead.long_name
        isonave.units = isonRead.units
        isonavediff = cdm.createVariable(vardiffsgSum,
                                         axes=axisVar,
                                         id=isonRead.id + 'Agree')
        isonavediff.long_name = isonRead.long_name
        isonavediff.units = isonRead.units

        outFile_f.write(isonave.astype('float32'))
        outFile_f.write(isonavediff.astype('float32'))
        tf = timc.clock()
        #print '   time var',tf-ti0
    # <--- end of loop on variables

    outFile_f.close()
    fi.close()
Ejemplo n.º 8
0
def mmeAveMsk3D(listFiles,
                years,
                inDir,
                outDir,
                outFile,
                timeInt,
                mme,
                ToeType,
                debug=True):
    '''
    The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
    It ouputs
     - the MME
     - a percentage of non-masked bins
     - the sign agreement of period2-period1 differences
     - ToE per run and for MME

    Author:    Eric Guilyardi : [email protected]

    Created on Tue Nov 21 2016

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - years(t1,t2)           - years for slice read
    - inDir[](str)           - input directory where files are stored (add histnat as inDir[1] for ToE)
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - ToeType(str)           - ToE type ('F': none, 'histnat')
                               -> requires running first mm+mme without ToE to compute Stddev
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 21 Nov 2016   - Initial function write

    - TODO :
                 - add computation of ToE per model (toe 1 and toe 2) see ticket #50
                 - add isonhtc (see ticket #48)
    '''

    # CDMS initialisation - netCDF compression
    comp = 1  # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    fi = cdm.open(inDir[0] + '/' + listFiles[0])
    # Switch if only variables below the bowl are present/treated
    nobowl = True
    if nobowl:
        isond0 = fi['isondepthgBowl']
        # Create variable handle
    else:
        isond0 = fi['isondepthg']
        # Create variable handle
    # Get grid objects
    axesList = isond0.getAxisList()
    sigmaGrd = isond0.getLevel()
    #time = isond0.getTime()
    lonN = isond0.shape[3]
    latN = isond0.shape[2]
    levN = isond0.shape[1]
    varsig = 'ptopsigmaxy'

    # Limit number of models to 3 for testing of mme
    #if mme:
    #    listFiles = listFiles[0:2]
    #    print ' !!! ### Testing 3 models ###',  listFiles

    # Declare and open files for writing
    if os.path.isfile(outDir + '/' + outFile):
        os.remove(outDir + '/' + outFile)
    outFile_f = cdm.open(outDir + '/' + outFile, 'w')

    #timN = isond0.shape[0]
    timN = t2 - t1
    runN = len(listFiles)

    print ' Number of members:', len(listFiles)

    valmask = isond0.missing_value

    varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg']
    varFill = [valmask, valmask, valmask, valmask, valmask]
    percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0.
    varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1.
    #varList = ['isondepthg']
    #print ' !!! ### Testing one variable ###', varList

    # init sigma axis
    sigma = cdm.createAxis(npy.float32(range(1)))
    sigma.id = axesList[1].id
    sigma.units = axesList[1].units
    sigma.designateTime()
    # init time axis
    time = cdm.createAxis(npy.float32(range(timN)))
    time.id = 'time'
    time.units = 'years since 1861'
    # init ensemble axis
    ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
    ensembleAxis.id = 'members'
    ensembleAxis.units = 'N'
    # Output axis
    sigmaList = [sigma, axesList[2], axesList[3]]
    # sigma, lat, lon
    sigmaTimeList = [sigma, time, axesList[2], axesList[3]]
    # sigma, time, lat, lon
    # init arrays
    isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask
    varbowl2D = npy.ma.ones([runN, timN, latN, lonN],
                            dtype='float32') * valmask
    varstd, varToE1, varToE2 = [
        npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask
        for _ in range(3)
    ]

    # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)

    delta_ib = 1
    print ' Sigma index:'
    for ib in range(levN):
        ib1 = ib + delta_ib
        print ib,
        tim0 = timc.clock()
        # loop on variables
        for iv, var in enumerate(varList):
            if nobowl:
                varb = var + 'Bowl'
            else:
                varb = var
            if ib == 0:
                print ' Variable ', iv, varb
            # loop over files to fill up array
            for i, file in enumerate(listFiles):
                tim01 = timc.clock()
                ft = cdm.open(inDir[0] + '/' + file)
                model = file.split('.')[1]
                timeax = ft.getAxis('time')
                if i == 0:
                    tmax0 = timeax.shape[0]
                tmax = timeax.shape[0]
                if tmax != tmax0:
                    print 'wrong time axis: exiting...'
                    return
                # read array
                isonRead = ft(varb, time=slice(t1, t2),
                              lev=slice(ib, ib1)).squeeze()
                if varFill[iv] != valmask:
                    isonvar[i, ...] = isonRead.filled(varFill[iv])
                else:
                    isonvar[i, ...] = isonRead
                tim02 = timc.clock()
                # compute percentage of non-masked points accros MME
                if iv == 0:
                    maskvar = mv.masked_values(isonRead.data, valmask).mask
                    percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
                tim03 = timc.clock()
                if mme:
                    # if mme then just accumulate Bowl, Agree and Std fields
                    #varst = var+'Agree'
                    #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
                    isonRead = ft(varb, time=slice(t1, t2),
                                  lev=slice(ib, ib1)).squeeze()
                    varbowl2D[i, ...] = isonRead
                else:
                    # Compute difference with average of first initN years
                    #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
                    #for t in range(timN):
                    #    vardiff[i,t,...] = isonvar[i,t,...] - varinit
                    #vardiff[i,...].mask = isonvar[i,...].mask
                    # Read bowl to truncate field above bowl
                    if ib == 0 and iv == 0:
                        varbowl[i, ...] = ft(varsig, time=slice(t1, t2))
                        #varbowl[i,...] = bowlRead
                    # Compute Stddev
                    varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
                    # Compute ToE
                    if ToeType == 'histnat':
                        toto = 1
                        # TODO
                        # Read mean and Std dev from histnat
                        #    if i == 0:
                        #        filehn  = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
                        #        #filehn = replace(outFile,'historical','historicalNat')
                        #        fthn = cdm.open(filehn)
                        #        varmeanhn = fthn(var)
                        #        varst = var+'Std'
                        #        varmaxstd = fthn(varst)
                        #    toemult = 1.
                        #    signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
                        #    noise = npy.reshape(varmaxstd,(basN*levN*latN))
                        #    varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
                        #    toemult = 2.
                        #    varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
                tim04 = timc.clock()
                ft.close()
                #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
            # <-- end of loop on files (i)

            tim1 = timc.clock()

            # Compute percentage of bin presence
            # Only keep points where percent > 50%
            if iv == 0:
                percenta = (cdu.averager(percent, axis=0)) * 100.
                percenta = mv.masked_less(percenta, 50)
                percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN])
                percentw = cdm.createVariable(percenta,
                                              axes=sigmaTimeList,
                                              id='isonpercent')
                percentw._FillValue = valmask
                percentw.long_name = 'percentage of MME bin'
                percentw.units = '%'
                outFile_f.write(percentw.astype('float32'), extend=1, index=ib)

            # Sign of difference
            #if mme:
            #    vardiffsgSum = cdu.averager(vardiff, axis=0)
            #    vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
            #    vardiffsgSum = maskVal(vardiffsgSum, valmask)
            #    vardiffsgSum.mask = percentw.mask
            #else:
            #    vardiffsg = npy.copysign(varones,vardiff)
            #    # average signs
            #    vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            #    vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            #    vardiffsgSum.mask = percentw.mask
            #    vardiffsgSum._FillValue = valmask

            # average variable accross members
            isonVarAve = cdu.averager(isonvar, axis=0)
            isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN])
            isonVarAve = cdm.createVariable(isonVarAve,
                                            axes=sigmaTimeList,
                                            id='foo')
            # mask
            if varFill[iv] == valmask:
                isonVarAve = maskVal(isonVarAve, valmask)

            isonVarAve.mask = percentw.mask
            tim2 = timc.clock()

            # Only keep points with rhon >  bowl-delta_rho
            delta_rho = 0.
            # mme case
            if mme:  # start from average of <var>Agree
                isonVarBowl = cdu.averager(varbowl2D, axis=0)
                isonVarBowl = npy.reshape(isonVarBowl,
                                          [delta_ib, timN, latN, lonN])
                isonVarBowl = cdm.createVariable(isonVarBowl,
                                                 axes=sigmaTimeList,
                                                 id='foo')
                isonVarBowl = maskVal(isonVarBowl, valmask)
                isonVarBowl.mask = percentw.mask
                # Compute intermodel stddev
                isonVarStd = statistics.std(varbowl2D, axis=0)
                isonVarStd = npy.reshape(isonVarStd,
                                         [delta_ib, timN, latN, lonN])
                isonVarStd = cdm.createVariable(isonVarStd,
                                                axes=sigmaTimeList,
                                                id='foo')
                isonVarStd = maskVal(isonVarStd, valmask)
                isonVarStd.mask = percentw.mask

                # Write
                isonvarbowlw = cdm.createVariable(isonVarBowl,
                                                  axes=sigmaTimeList,
                                                  id=isonRead.id)
                isonvarbowlw.long_name = isonRead.long_name
                isonvarbowlw.units = isonRead.units
                isonvarstdw = cdm.createVariable(isonVarStd,
                                                 axes=sigmaTimeList,
                                                 id=isonRead.id + 'Std')
                isonvarstdw.long_name = isonRead.long_name + ' intermodel std'
                isonvarstdw.units = isonRead.units

                outFile_f.write(isonvarbowlw.astype('float32'),
                                extend=1,
                                index=ib)
                outFile_f.write(isonvarstdw.astype('float32'),
                                extend=1,
                                index=ib)

                #if ib == 0 and iv == 0:
                #    # TODO review
                #    # Read multimodel sigma on bowl and average in time
                #    file1d  =  replace(outDir+'/'+outFile,'2D','1D')
                #    if os.path.isfile(file1d):
                #        f1d = cdm.open(file1d)
                #    else:
                #        print 'ERROR:',file1d,'missing (if mme, run 2D first)'
                #        sys.exit(1)
                #    bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
                #    f1d.close()
                #    siglimit = cdu.averager(bowlRead, axis=0)  - delta_rho
                # TODO: remove loop by building global array with 1/0
                #if sw2d == 1:
                #    for il in range(latN):
                #        for ib in range(basN):
                #            #if ib == 2:
                #            #    print il, siglimit[ib,il]
                #            if siglimit[ib,il] < valmask/1000.:
                #                 # if mme bowl density defined, mask above bowl
                #                index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
                #                isonVarBowl [:,ib,0:index[0],il].mask = True
                #                isonVarStd  [:,ib,0:index[0],il].mask = True
                #                vardiffsgSum[:,ib,0:index[0],il].mask = True
                #            else:
                #                # mask all points
                #                isonVarBowl [:,ib,:,il].mask = True
                #                isonVarStd  [:,ib,:,il].mask = True
                #                vardiffsgSum[:,ib,:,il].mask = True
            # mm case
            else:
                isonVarBowl = isonVarAve * 1.  # start from variable
                #isonVarStd  = isonVarAve*1. # start from variable
                if ib == 0 and iv == 0:
                    # build bowl position
                    siglimit = cdu.averager(varbowl,
                                            axis=0)  # average accross members
                    siglimit = npy.reshape(siglimit,
                                           [timN * latN * lonN]) - delta_rho
                if iv == 0:
                    sigarr = siglimit * 1.
                    sigarr[:] = sigmaGrd[ib]
                # test
                i = 60
                j = 60
                ij = j * lonN + i
                isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN])
                #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])

                isonVarBowl.mask = npy.where(sigarr < siglimit, True,
                                             isonVarBowl.mask)
                #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)

                isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN])
                #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])

                isonVarBowl = maskVal(isonVarBowl, valmask)
                #vardiffsgSum = maskVal(vardiffsgSum, valmask)
                # Find max of Std dev of all members
                isonVarStd = npy.ma.max(varstd, axis=0)
                # mask
                isonVarStd = maskVal(isonVarStd, valmask)

                # Write
                #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
                #isonave.long_name = isonRead.long_name
                #isonave.units     = isonRead.units
                #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
                #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
                #isonavediff.long_name = isonRead.long_name
                #isonavediff.units     = isonRead.units
                isonVarBowl = npy.reshape(isonVarBowl,
                                          [delta_ib, timN, latN, lonN])
                isonavebowl = cdm.createVariable(isonVarBowl,
                                                 axes=sigmaTimeList,
                                                 id=isonRead.id + 'Bowl')
                isonavebowl.long_name = isonRead.long_name
                isonavebowl.units = isonRead.units
                isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN])
                isonmaxstd = cdm.createVariable(isonVarStd,
                                                axes=sigmaList,
                                                id=isonRead.id + 'Std')
                isonmaxstd.long_name = isonRead.long_name
                isonmaxstd.units = isonRead.units

                #outFile_f.write(    isonave.astype('float32'), extend = 1, index = ib)
                #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
                outFile_f.write(isonavebowl.astype('float32'),
                                extend=1,
                                index=ib)
                outFile_f.write(isonmaxstd.astype('float32'),
                                extend=1,
                                index=ib)

            tim3 = timc.clock()

            if ToeType == 'histnat':
                isontoe1 = cdm.createVariable(
                    varToE1,
                    axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
                    id=isonRead.id + 'ToE1')
                isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
                isontoe1.units = 'Year'
                isontoe2 = cdm.createVariable(
                    varToE2,
                    axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
                    id=isonRead.id + 'ToE2')
                isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
                isontoe2.units = 'Year'
                outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib)
                outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib)

            tim4 = timc.clock()
        # <--- end of loop on variables

        #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
    # <--- end of loop on density
    print ' '

    outFile_f.close()
    fi.close()
Ejemplo n.º 9
0
def mmeAveMsk2D(listFiles,
                years,
                inDir,
                outDir,
                outFile,
                timeInt,
                mme,
                timeBowl,
                ToeType,
                debug=True):
    '''
    The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
    It ouputs
     - the MME
     - a percentage of non-masked bins
     - the sign agreement of period2-period1 differences
     - ToE per run and for MME

    Author:    Eric Guilyardi : [email protected]

    Created on Tue Nov 25 13:56:20 CET 2014

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - years(t1,t2)           - years for slice read
    - inDir[](str)           - input directory where files are stored (add histnat as inDir[1] for ToE)
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - timeBowl               - either time 'mean' or time 'max' bowl used to mask out bowl
    - ToeType(str)           - ToE type ('F': none, 'histnat')
                               -> requires running first mm+mme without ToE to compute Stddev
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 25 Nov 2014   - Initial function write
    - EG 27 Nov 2014   - Rewrite with loop on variables
    - EG 06 Dec 2014   - Added agreement on difference with init period - save as <var>Agree
    - EG 07 Dec 2014   - Read bowl to remove points above bowl - save as <var>Bowl
    - EG 19 Apr 2016   - ToE computation (just for 2D files)
    - EG 07 Oct 2016   - add 3D file support
    - EG 21 Nov 2016   - move 3D support to new function
    - EG 10 jan 2017   - added timeBowl option

    - TODO :
                 - remove loops
                 - add computation of ToE per model (toe 1 and toe 2) see ticket #50
                 - add isonhtc (see ticket #48)
    '''

    # CDMS initialisation - netCDF compression
    comp = 1  # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    if t2 <= 0:
        useLastYears = True
        t2 = -t2
    else:
        useLastYears = False
    t10 = t1
    t20 = t2
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    fi = cdm.open(inDir[0] + '/' + listFiles[0])
    isond0 = fi['isondepth']
    # Create variable handle
    # Get grid objects
    axesList = isond0.getAxisList()
    sigmaGrd = isond0.getLevel()
    latN = isond0.shape[3]
    levN = isond0.shape[2]
    basN = isond0.shape[1]
    varsig = 'ptopsigma'

    # Declare and open files for writing
    if os.path.isfile(outDir + '/' + outFile):
        os.remove(outDir + '/' + outFile)
    outFile_f = cdm.open(outDir + '/' + outFile, 'w')

    # Testing mme with less models
    #listFiles=listFiles[0:4]

    #timN = isond0.shape[0]
    timN = t2 - t1
    runN = len(listFiles)

    print ' Number of members:', len(listFiles)

    valmask = isond0.missing_value[0]
    varList = [
        'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol'
    ]
    varFill = [0., 0., valmask, valmask, 0., 0.]
    # init arrays (2D rho/lat)
    percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0.
    #minbowl  = npy.ma.ones([basN,latN], dtype='float32')*1000.
    varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1.
    #varList = ['isondepth']
    #print ' !!! ### Testing one variable ###'
    #varList = ['isonthetao']

    # init time axis
    time = cdm.createAxis(npy.float32(range(timN)))
    time.id = 'time'
    time.units = 'years since 1861'
    time.designateTime()
    # init ensemble axis
    ensembleAxis = cdm.createAxis(npy.float32(range(runN)))
    ensembleAxis.id = 'members'
    ensembleAxis.units = 'N'

    # loop on variables
    for iv, var in enumerate(varList):

        # Array inits (2D rho/lat 3D rho/lat/lon)
        #shapeR = [basN,levN,latN]
        isonvar = npy.ma.ones([runN, timN, basN, levN, latN],
                              dtype='float32') * valmask
        print('isonvar shape: ', isonvar.shape)
        vardiff, varbowl2D = [
            npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32')
            for _ in range(2)
        ]
        varstd, varToE1, varToE2 = [
            npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask
            for _ in range(3)
        ]
        varones = npy.ma.ones([runN, timN, basN, levN, latN],
                              dtype='float32') * 1.

        print ' Variable ', iv, var
        # loop over files to fill up array
        for i, file in enumerate(listFiles):
            ft = cdm.open(inDir[0] + '/' + file)
            model = file.split('.')[1]
            timeax = ft.getAxis('time')
            file1d = replace(inDir[0] + '/' + file, '2D', '1D')
            if os.path.isfile(file1d):
                f1d = cdm.open(file1d)
            else:
                print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
                sys.exit(1)
            tmax = timeax.shape[0]
            if i == 0:
                tmax0 = tmax
            #adapt [t1,t2] time bounds to piControl last NN years
            if useLastYears:
                t1 = tmax - t20
                t2 = tmax
            else:
                if tmax != tmax0:
                    print 'wrong time axis: exiting...'
                    return

            # read array
            # loop over time/density for memory management
            for it in range(timN):
                t1r = t1 + it
                t2r = t1r + 1
                isonRead = ft(var, time=slice(t1r, t2r))
                if varFill[iv] != valmask:
                    isonvar[i, it, ...] = isonRead.filled(varFill[iv])
                else:
                    isonvar[i, it, ...] = isonRead
            # compute percentage of non-masked points accros MME
            if iv == 0:
                maskvar = mv.masked_values(isonRead.data, valmask).mask
                percent[i, ...] = npy.float32(npy.equal(maskvar, 0))
            if mme:
                # if mme then just accumulate Bowl, Agree fields
                varst = var + 'Agree'
                vardiff[i, ...] = ft(varst, time=slice(t1, t2))
                varb = var + 'Bowl'
                varbowl2D[i, ...] = ft(varb, time=slice(t1, t2))
            else:
                # Compute difference with average of first initN years
                varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0)
                for t in range(timN):
                    vardiff[i, t, ...] = isonvar[i, t, ...] - varinit
                vardiff[i, ...].mask = isonvar[i, ...].mask
                # Read bowl and truncate 2D field above bowl
                if iv == 0:
                    bowlRead = f1d(varsig, time=slice(t1, t2))
                    varbowl[i, ...] = bowlRead
                # Compute Stddev
                varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0)
                # Compute ToE
                if ToeType == 'histnat':
                    # Read mean and Std dev from histnat
                    if i == 0:
                        filehn = glob.glob(inDir[1] + '/cmip5.' + model +
                                           '.*zon2D*')[0]
                        #filehn = replace(outFile,'historical','historicalNat')
                        fthn = cdm.open(filehn)
                        varmeanhn = fthn(var)
                        varst = var + 'Std'
                        varmaxstd = fthn(varst)
                    toemult = 1.
                    signal = npy.reshape(isonvar[i, ...] - varmeanhn,
                                         (timN, basN * levN * latN))
                    noise = npy.reshape(varmaxstd, (basN * levN * latN))
                    varToE1[i,
                            ...] = npy.reshape(findToE(signal, noise, toemult),
                                               (basN, levN, latN))
                    toemult = 2.
                    varToE2[i,
                            ...] = npy.reshape(findToE(signal, noise, toemult),
                                               (basN, levN, latN))
            ft.close()
            f1d.close()
        # <-- end of loop on files

        # Compute percentage of bin presence
        # Only keep points where percent > 50%
        if iv == 0:
            percenta = (cdu.averager(percent, axis=0)) * 100.
            percenta = mv.masked_less(percenta, 50)
            percentw = cdm.createVariable(
                percenta,
                axes=[time, axesList[1], axesList[2], axesList[3]],
                id='isonpercent')
            percentw._FillValue = valmask
            percentw.long_name = 'percentage of MME bin'
            percentw.units = '%'
            outFile_f.write(percentw.astype('float32'))

        # Sign of difference
        if mme:
            vardiffsgSum = cdu.averager(vardiff, axis=0)
            vardiffsgSum = cdm.createVariable(
                vardiffsgSum,
                axes=[time, axesList[1], axesList[2], axesList[3]],
                id='foo')
            vardiffsgSum = maskVal(vardiffsgSum, valmask)
            vardiffsgSum.mask = percentw.mask
        else:
            vardiffsg = npy.copysign(varones, vardiff)
            # average signs
            vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            vardiffsgSum.mask = percentw.mask
            vardiffsgSum._FillValue = valmask

        # average variable accross members
        isonVarAve = cdu.averager(isonvar, axis=0)
        isonVarAve = cdm.createVariable(
            isonVarAve,
            axes=[time, axesList[1], axesList[2], axesList[3]],
            id='foo')
        # mask
        if varFill[iv] == valmask:
            isonVarAve = maskVal(isonVarAve, valmask)

        isonVarAve.mask = percentw.mask

        # Only keep points with rhon >  bowl-delta_rho
        delta_rho = 0.
        if mme:  # start from average of <var>Agree
            isonVarBowl = cdu.averager(varbowl2D, axis=0)
            isonVarBowl = cdm.createVariable(
                isonVarBowl,
                axes=[time, axesList[1], axesList[2], axesList[3]],
                id='foo')
            isonVarBowl = maskVal(isonVarBowl, valmask)
            isonVarBowl.mask = percentw.mask
            # Compute intermodel stddev
            isonVarStd = statistics.std(varbowl2D, axis=0)
            isonVarStd = cdm.createVariable(
                isonVarStd,
                axes=[time, axesList[1], axesList[2], axesList[3]],
                id='foo')
            isonVarStd = maskVal(isonVarStd, valmask)
            isonVarStd.mask = percentw.mask
            if iv == 0:
                # Read mulitmodel sigma on bowl and average in time
                file1d = replace(outDir + '/' + outFile, '2D', '1D')
                if os.path.isfile(file1d):
                    f1d = cdm.open(file1d)
                else:
                    print 'ERROR:', file1d, 'missing (if mme, run 1D first)'
                    sys.exit(1)
                bowlRead = f1d(varsig, time=slice(t1, t2))
                f1d.close()
                siglimit = cdu.averager(bowlRead, axis=0) - delta_rho
            # TODO: remove loop by building global array with 1/0
            for il in range(latN):
                for ib in range(basN):
                    #if ib == 2:
                    #    print il, siglimit[ib,il]
                    if siglimit[ib, il] < valmask / 1000.:
                        # if mme bowl density defined, mask above bowl
                        index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]))
                        isonVarBowl[:, ib, 0:index[0], il].mask = True
                        isonVarStd[:, ib, 0:index[0], il].mask = True
                        vardiffsgSum[:, ib, 0:index[0], il].mask = True
                    else:
                        # mask all points
                        isonVarBowl[:, ib, :, il].mask = True
                        isonVarStd[:, ib, :, il].mask = True
                        vardiffsgSum[:, ib, :, il].mask = True
        else:
            isonVarBowl = isonVarAve * 1.  # start from variable
            isonVarStd = isonVarAve * 1.  # start from variable
            if iv == 0:
                siglimit = cdu.averager(varbowl,
                                        axis=0)  # average accross members
                # Average bowl in time
                if timeBowl == 'mean':
                    siglimit = cdu.averager(siglimit, axis=0) - delta_rho
                # or take largest sigma over time
                else:
                    siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
            # TODO: remove loop by building global array with 1/0
            for il in range(latN):
                for ib in range(basN):
                    if siglimit[ib, il] < valmask / 1000.:
                        # if bowl density defined, mask above bowl
                        index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])
                                 )[:, 0]  #Add [:,0] for python Yona
                        #import code
                        #code.interact(banner='index', local=dict(locals(), **globals()))
                        isonVarBowl[:, ib, 0:index[0], il].mask = True
                        vardiffsgSum[:, ib, 0:index[0], il].mask = True
                    else:
                        # mask all points
                        vardiffsgSum[:, ib, :, il].mask = True

            isonVarBowl = maskVal(isonVarBowl, valmask)
            # Find max of Std dev of all members
            isonVarStd = npy.ma.max(varstd, axis=0)
            # mask
            if varFill[iv] == valmask:
                isonVarStd = maskVal(isonVarStd, valmask)

        # Write
        isonave = cdm.createVariable(
            isonVarAve,
            axes=[time, axesList[1], axesList[2], axesList[3]],
            id=isonRead.id)
        isonave.long_name = isonRead.long_name
        isonave.units = isonRead.units
        isonavediff = cdm.createVariable(
            vardiffsgSum,
            axes=[time, axesList[1], axesList[2], axesList[3]],
            id=isonRead.id + 'Agree')
        isonavediff.long_name = isonRead.long_name
        isonavediff.units = isonRead.units
        isonavebowl = cdm.createVariable(
            isonVarBowl,
            axes=[time, axesList[1], axesList[2], axesList[3]],
            id=isonRead.id + 'Bowl')
        isonavebowl.long_name = isonRead.long_name
        isonavebowl.units = isonRead.units
        if not mme:
            isonmaxstd = cdm.createVariable(
                isonVarStd,
                axes=[axesList[1], axesList[2], axesList[3]],
                id=isonRead.id + 'Std')
            isonmaxstd.long_name = isonRead.long_name
            isonmaxstd.units = isonRead.units

        outFile_f.write(isonave.astype('float32'))
        outFile_f.write(isonavediff.astype('float32'))
        outFile_f.write(isonavebowl.astype('float32'))
        if not mme:
            outFile_f.write(isonmaxstd.astype('float32'))

        if ToeType == 'histnat':
            isontoe1 = cdm.createVariable(
                varToE1,
                axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
                id=isonRead.id + 'ToE1')
            isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name
            isontoe1.units = 'Year'
            isontoe2 = cdm.createVariable(
                varToE2,
                axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]],
                id=isonRead.id + 'ToE2')
            isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name
            isontoe2.units = 'Year'
            outFile_f.write(isontoe1.astype('float32'))
            outFile_f.write(isontoe2.astype('float32'))

        if mme:
            isonvarstd = cdm.createVariable(
                isonVarStd,
                axes=[time, axesList[1], axesList[2], axesList[3]],
                id=isonRead.id + 'ModStd')
            isonvarstd.long_name = isonRead.long_name + ' intermodel std'
            isonvarstd.units = isonRead.units
            outFile_f.write(isonvarstd.astype('float32'))

    # <--- end of loop on variables

    outFile_f.close()
    fi.close()
Ejemplo n.º 10
0
    def basicGm(self,
                gm_type,
                projtype="default",
                lat1=0,
                lat2=0,
                lon1=0,
                lon2=0,
                rg=False,
                flip=False,
                zero=False,
                transparent=False,
                mask=False,
                bigvalues=False):

        self.x.clear()
        self.x.setcolormap(None)
        cdms2.tvariable.TransientVariable.variable_count = 1
        loc = locals()
        exec("gm=vcs.create%s()" % gm_type)
        gm = loc["gm"]
        if projtype != "default":
            p = vcs.createprojection()
            try:
                ptype = int(projtype)
            except BaseException:
                ptype = projtype
            p.type = ptype
            gm.projection = p
        nm_xtra = ""
        xtra = {}
        if lat1 != lat2:
            if rg:
                if flip:
                    gm.datawc_y1 = lat2
                    gm.datawc_y2 = lat1
                    nm_xtra += "_gmflip"
                else:
                    gm.datawc_y1 = lat1
                    gm.datawc_y2 = lat2
            xtra["latitude"] = (lat1, lat2)
            if lat1 < 0:
                nm_xtra += "_SH"
            else:
                nm_xtra += "_NH"
        if lon1 != lon2:
            if rg:
                gm.datawc_x1 = lon1
                gm.datawc_x2 = lon2
            xtra["longitude"] = (lon1, lon2)
            nm_xtra += "_%i_%i" % (lon1, lon2)
        if rg:
            nm_xtra += "_via_gm"
        if gm_type == "meshfill":
            f = cdms2.open(os.path.join(vcs.sample_data,
                                        'sampleCurveGrid4.nc'))
        else:
            f = self.clt
        if gm_type == "vector":
            u = f("u", **xtra)
            v = f("v", **xtra)
            if mask:
                u = MV2.masked_greater(u, 58.)
            if zero:
                u -= u
                v -= v
        elif gm_type == "meshfill":
            s = f("sample", **xtra)
            if mask:
                s = MV2.masked_less(s, 1150.)
            elif bigvalues:
                s[s < 1150] = 1e40
            if zero:
                s -= s
        else:
            s = f("clt", **xtra)
            if mask:
                s = MV2.masked_greater(s, 78.)
            elif bigvalues:
                s[s > 78] = 1e40
            if gm_type in ["1d", "yxvsx", "xyvsy", "xvsy", "scatter"]:
                s = s(latitude=(20, 20, "cob"),
                      longitude=(112, 112, "cob"),
                      squeeze=1)
                s2 = MV2.sin(s)
                if zero:
                    s2 -= s2
            if zero:
                s -= s

        if bigvalues:
            gm.levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 1.e36]

        if transparent:
            cmap = self.x.createcolormap()
            for i in range(256):  # tweaks all colors
                cmap.setcolorcell(i, 100., 0, 0, i / 2.55)
            self.x.setcolormap(cmap)
            if gm_type == "vector":
                gm.linecolor = [100, 0, 0, 50.]
            elif gm_type in ["yxvsx", "xyvsy", "yvsx", "scatter", "1d"]:
                gm.linecolor = [100, 0, 0, 50.]
                gm.markercolor = [100, 0, 0, 50.]

        if gm_type == "vector":
            gm.scale = 4.
            self.x.plot(u, v, gm, bg=self.bg)
        elif gm_type in ["scatter", "xvsy"]:
            self.x.plot(s, s2, gm, bg=self.bg)
        else:
            self.x.plot(s, gm, bg=self.bg)
        fnm = "test_vcs_basic_%s" % gm_type.lower()
        if mask:
            fnm += "_masked"
        elif bigvalues:
            fnm += "_bigvalues"
        if projtype != "default":
            fnm += "_%s_proj" % projtype
        if zero:
            fnm += "_zero"
        if transparent:
            fnm += "_transparent"
        fnm += nm_xtra
        self.checkImage(fnm + '.png', threshold=20)
Ejemplo n.º 11
0
-0.14836133615864944 -0.17956528904564023 -0.21109198032585794
-0.23846429237248942 -0.2598536549218765 -0.27795672866320387
-0.2939939095159731 -0.30541031366330024 -0.307643559333884
-0.30078421139811795 -0.2841339526883441 -0.26485737397202497
-0.24287299694779327 -0.22379014890999907 -0.20121548204699846
-0.1746486732156772 -0.14585019344118372 -0.12070675757803526
-0.0997891159111037 -0.08229393660994214 -0.06779720501287469
-0.057213385470859794 -0.04875768191096844 -0.0402377347189964
-0.030169328367807245 -0.017560662894847895 -0.006968922654137132
0.0009773980274431048 0.007054306637034288 0.010472286514133042
0.010702384151997032 0.009231553701801242 0.007544033101056543
0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999.
""".split()
data = numpy.array(data,dtype=numpy.float)
data = MV2.masked_less(data,-900)
#yx.datawc_x1 = 0
#yx.datawc_x2 = 80
##yx.datawc_y1 =-12 
#yx.datawc_y2 = 12 


x.plot(data,yx,bg=1)
fnm = "test_vcs_1D_datawc_missing.png"
x.png(fnm)
print "fnm:",fnm
print "src:",src
ret = checkimage.check_result_image(fnm,src,0.05)
sys.exit(ret)
import sys, os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x = vcs.init()

x.setcolormap("rainbow")
gm = vcs.createmeshfill()

p = vcs.createprojection()
ptype = int('0')
p.type = ptype
gm.projection = p

xtra = {}
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc'))
s = f("sample", **xtra)
gm.mesh = True
s = MV2.masked_less(s, 1150.)
x.plot(s, gm, bg=bg)

x.png('test_vcs_basic_meshfill_masked_0_proj.png')
Ejemplo n.º 13
0
'''

import os, sys 
sys.path.append(os.path.abspath('../../'))
import cdms2, MV2, vcs
from region_utils import getAreaOfAllClosedDomains


f = cdms2.open('data/snc.nc')
data = f('snc', time=slice(1), squeeze=1)

print "Contions \n 1. Mask less than 100 \n 2. Mask less than 30 and greater than 70"
con = input('Enter Condion Option 1 or 2 : ')

if con == 1:
    mask_condition = MV2.masked_less(data, 100.)
    value = 'value equal to 100'
    dirname = 'outplots_eq_100'
elif con == 2:
    c1 = MV2.masked_greater(data, 70.)
    c2 = MV2.masked_less(data, 30.)
    mask_condition = MV2.logical_and(c1, c2)
    value = 'value equal to within 30 to 70'
    dirname = 'outplots_eq_30_to_70'
# end of if con == 1:

if not os.path.isdir(dirname):  os.mkdir(dirname)
print "Output plots will be saved in '%s' directory" % dirname

v = vcs.init()
# Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods.
Ejemplo n.º 14
0
import vcs
import cdms2
import sys
import os
import MV2

f = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc"))
s = f("clt", time=slice(0, 1), squeeze=1)
s = MV2.masked_less(s, 65.)
x = vcs.init()
gm = x.createisofill()
#gm=x.createboxfill()
gm.missing = 252
#gm.levels = [65,70,75,80,85,90,95,100]
x.plot(s, gm)
x.png(gm.g_name)
x.interact()
def obscuration_terms3(c1, c2):
    """
    USE THIS VERSION FOR DIFFERENCES OF 2 CLIMATOLOGIES (E.G. AMIP4K, 2xCO2 SLAB RUNS)
    
    Compute the components required for the obscuration-affected low cloud feedback
    These are the terms shown in Eq 4 of Scott et al (2020) DOI: 10.1175/JCLI-D-19-1028.1
    L_prime = dunobsc + dobsc + dobsc_cov, where
    dunobsc = L_R_prime * F_bar     (delta unobscured low clouds, i.e., true low cloud feedback)
    dobsc = L_R_bar * F_prime       (delta obscuration by upper level clouds)
    dobsc_cov = (L_R_prime * F_prime) - climo(L_R_prime * F_prime)  (covariance term)
    """
    # c is [mo,tau,ctp,lat,lon]
    # c is in percent

    AX = c2.getAxisList()

    c1 = MV.masked_where(c2.mask, c1)
    c2 = MV.masked_where(c1.mask, c2)

    # SPLICE c1 and c2:
    # MAKE SURE c1 and c2 are the same size!!!
    if c1.shape != c2.shape:
        raise RuntimeError('c1 and c2 are NOT the same size!!!')

    c12 = np.ma.append(c1, c2, axis=0)

    midpt = len(c1)

    U12 = MV.sum(MV.sum(c12[:, :, 2:, :], 1), 1) / 100.

    L12 = c12[:, :, :2, :] / 100.

    F12 = 1. - U12
    F12 = MV.masked_less(F12, 0)

    F12b = MV.array(np.expand_dims(np.expand_dims(F12, axis=1), axis=1))
    F12b = MV.masked_where(L12[:, :1, :1, :].mask, F12b)

    L_R12 = L12 / F12b
    sum_L_R12 = MV.sum(MV.sum(L_R12, 1), 1)
    sum_L_R12b = MV.array(
        np.expand_dims(np.expand_dims(sum_L_R12, axis=1), axis=1))
    sum_L_R12c = np.broadcast_to(sum_L_R12b, L_R12.shape)
    this = MV.masked_outside(sum_L_R12c, 0, 1)
    L_R12 = MV.masked_where(this.mask, L_R12)

    L_R12 = MV.masked_where(sum_L_R12c > 1, L_R12)

    L_R_prime, L_R_bar = monthly_anomalies(L_R12)
    F_prime, F_bar = monthly_anomalies(F12b)
    L_prime, L_bar = monthly_anomalies(L12)

    # Cannot have negative cloud fractions:
    L_R_bar[L_R_bar < 0] = 0
    F_bar[F_bar < 0] = 0

    rep_L_bar = tile_uneven(L_bar, L12)
    rep_L_R_bar = tile_uneven(L_R_bar, L_R12)
    rep_F_bar = tile_uneven(F_bar, F12b)

    # Cannot have negative cloud fractions:
    L_R_bar[L_R_bar < 0] = 0
    F_bar[F_bar < 0] = 0

    dobsc = rep_L_R_bar * F_prime
    dunobsc = L_R_prime * rep_F_bar
    prime_prime = (L_R_prime * F_prime)

    dobsc_cov, climo_prime_prime = monthly_anomalies(prime_prime)

    # Re-scale these anomalies by 2, since we have computed all anomalies w.r.t.
    # the ctl+pert average rather than w.r.t. the ctl average
    dobsc *= 2
    dunobsc *= 2
    dobsc_cov *= 2

    return (rep_L_R_bar[midpt:], dobsc[midpt:], dunobsc[midpt:],
            dobsc_cov[midpt:])
Ejemplo n.º 16
0
xouter.setAxisList([lat,lon])           # Equivalent

## masked_equal(x, value) 
##   masked_equal(x, value) = x masked where x == value
##   For floating point consider masked_values(x, value) instead.

## masked_greater(x, value) 
##   masked_greater(x, value) = x masked where x > value

## masked_greater_equal(x, value) 
##   masked_greater_equal(x, value) = x masked where x >= value
xge = MV2.masked_greater_equal(xouter, 120)

## masked_less(x, value) 
##   masked_less(x, value) = x masked where x < value
xl = MV2.masked_less(xouter, 160)

## masked_less_equal(x, value) 
##   masked_less_equal(x, value) = x masked where x <= value

## masked_not_equal(x, value) 
##   masked_not_equal(x, value) = x masked where x != value

## masked_outside(x, v1, v2) 
##   x with mask of all values of x that are outside [v1,v2]
xmo = MV2.masked_outside(xouter, 120, 160)

## count(a, axis=None) 
##   Count of the non-masked elements in a, or along a certain axis.
xcount = MV2.count(xmo,0)
xcount2 = MV2.count(xmo,1)
print 'Sum of all PFT is', pft_sum2010.shape
#x.plot(pft_sum2010[::-1,:],gm,bg=bg)
#x.png('pft_sum2010.png')
#x.clear()

#In case if sum of all PFTs is greater than 1, reduce all PFTs
re_pft2010 = MV2.masked_greater(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
    #print 'npft',npft
    pft_tmp1 = pft2010[npft, :, :]
    pft_tmp1 = MV2.choose(re_pft2010,
                          (pft_tmp1, pft_tmp1 * lndf_hurt / pft_sum2010))
    pft2010[npft, :, :] = pft_tmp1

#In case if sum of all PFTs is less than 1, increase all PFTs
inc_pft2010 = MV2.masked_less(pft_sum2010 - lndf_hurt, 0.).mask
for npft in range(19):
    #print 'npft',npft
    pft_tmp2 = pft2010[npft, :, :]
    pft_tmp2 = MV2.choose(inc_pft2010,
                          (pft_tmp2, pft_tmp2 * lndf_hurt / pft_sum2010))
    pft2010[npft, :, :] = pft_tmp2

newpftsum = MV2.sum(pft2010, axis=0) + icwtr
sys.stdout.flush()
pft_back[[nyears - 1], :, :, :] = pft2010
#x.plot(newpftsum[::-1,:],gm,bg=bg)
#x.png('newpftsum.png')
#x.clear()
#if (pft_sum2010.all()==1):
#   print 'Sum of all PFTs is equal to 1'
lon=cdms2.createAxis(MV2.arange(100)*3.6)
lon.designateLongitude()
lon.units="degrees_east"
lon.id="longitude"

lat = cdms2.createAxis(MV2.arange(100)*1.8-90.)
lat.id="latitude"
lat.designateLatitude()
lat.units="degrees_north"

lev = cdms2.createAxis([1000.])
lev.id="plev"
lev.designateLevel()
lev.units="hPa"

t=cdms2.createAxis([0,31.])
t.id="time"
t.designateTime()
t.units="days since 2014"

cdutil.setTimeBoundsMonthly(t)
a.setAxisList((t,lev,lat,lon))
a=MV2.masked_less(a,.5)
grd=cdms2.createGaussianGrid(64)

a=a.ascontiguous()
a=a.regrid(grd,regridTool="regrid2")
a=cdutil.averager(a,axis='txy')
assert a[0]==0.7921019540305255
Ejemplo n.º 19
0
xouter.setAxisList([lat, lon])  # Equivalent

## masked_equal(x, value)
##   masked_equal(x, value) = x masked where x == value
##   For floating point consider masked_values(x, value) instead.

## masked_greater(x, value)
##   masked_greater(x, value) = x masked where x > value

## masked_greater_equal(x, value)
##   masked_greater_equal(x, value) = x masked where x >= value
xge = MV2.masked_greater_equal(xouter, 120)

## masked_less(x, value)
##   masked_less(x, value) = x masked where x < value
xl = MV2.masked_less(xouter, 160)

## masked_less_equal(x, value)
##   masked_less_equal(x, value) = x masked where x <= value

## masked_not_equal(x, value)
##   masked_not_equal(x, value) = x masked where x != value

## masked_outside(x, v1, v2)
##   x with mask of all values of x that are outside [v1,v2]
xmo = MV2.masked_outside(xouter, 120, 160)

## count(a, axis=None)
##   Count of the non-masked elements in a, or along a certain axis.
xcount = MV2.count(xmo, 0)
xcount2 = MV2.count(xmo, 1)
Ejemplo n.º 20
0
select = dict(lat=slice(4, -4), lon=slice(4, -4), squeeze=1)
slp = f('pslvl', **select)
rain = f('rain', **select)
u = f('u10m', **select)
v = f('v10m', **select)
f.close()

# Pression de surface
slp[:] = generic2d(slp*0.01, 5)
map2(slp, fill=False, contour=True, show=False,  figsize=(6, 5.5),
    title='WRF Bretagne',
    fillcontinents=True, zorder=5, projection='merc', right=1, bottom=.07,
    lowhighs=True, lowhighs_smooth=9, lowhighs_zorder=5, fillcontinents_color='.95',
    lowhighs_color=(0, .5, 0), contour_colors=[(0, .5, 0)], drawcoastlines_linewidth=.6)

# Pluie
cmap_rain = cmap_custom([('0.9', 0), ('b', .8), ('r', 1.)])
rain[:] = MV2.masked_less(rain, 0.1)
map2(rain, cmap=cmap_rain, vmin=0.,
    fill='pcolor', fillcontinents=False, show=False,
    shadow_xoffset=4, shadow_yoffset=-4,shadow_width=4, colorbar_shrink=.7,
    alpha=.7, shadow=True, shadow_alpha=.3, contour=False, zorder=15)

# Vent
u[:] = ms2kt(u*5)
v[:] = ms2kt(v*5)
m = map2((u[::3, ::3], v[::3, ::3]), fill=False, contour=False, barbs=True, projection='merc',
    quiver_sizes=dict(height=.2, spacing=.15), quiver_linewidths=.8, zorder=10,
    shadow=True, quiver_alpha=.5, savefigs=__file__, show=False, savefigs_pdf=True,
    fillcontinents=False)
Ejemplo n.º 21
0
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True):
    '''
    The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks
    It ouputs
     - the MME
     - a percentage of non-masked bins
     - the sign agreement of period2-period1 differences
     - ToE per run and for MME

    Author:    Eric Guilyardi : [email protected]

    Created on Tue Nov 25 13:56:20 CET 2014

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - years(t1,t2)           - years for slice read
    - inDir[](str)           - input directory where files are stored (add histnat as inDir[1] for ToE)
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - timeBowl               - either time 'mean' or time 'max' bowl used to mask out bowl
    - ToeType(str)           - ToE type ('F': none, 'histnat')
                               -> requires running first mm+mme without ToE to compute Stddev
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 25 Nov 2014   - Initial function write
    - EG 27 Nov 2014   - Rewrite with loop on variables
    - EG 06 Dec 2014   - Added agreement on difference with init period - save as <var>Agree
    - EG 07 Dec 2014   - Read bowl to remove points above bowl - save as <var>Bowl
    - EG 19 Apr 2016   - ToE computation (just for 2D files)
    - EG 07 Oct 2016   - add 3D file support
    - EG 21 Nov 2016   - move 3D support to new function
    - EG 10 jan 2017   - added timeBowl option

    - TODO :
                 - remove loops
                 - add computation of ToE per model (toe 1 and toe 2) see ticket #50
                 - add isonhtc (see ticket #48)
    '''

    # CDMS initialisation - netCDF compression
    comp = 1 # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    if t2 <= 0:
        useLastYears = True
        t2 = -t2
    else:
        useLastYears = False
    t10 = t1
    t20 = t2
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    fi      = cdm.open(inDir[0]+'/'+listFiles[0])
    isond0  = fi['isondepth'] ; # Create variable handle
    # Get grid objects
    axesList = isond0.getAxisList()
    sigmaGrd = isond0.getLevel()
    latN = isond0.shape[3]
    levN = isond0.shape[2]
    basN = isond0.shape[1]
    varsig='ptopsigma'

    # Declare and open files for writing
    if os.path.isfile(outDir+'/'+outFile):
        os.remove(outDir+'/'+outFile)
    outFile_f = cdm.open(outDir+'/'+outFile,'w')

    # Testing mme with less models
    #listFiles=listFiles[0:4]

    #timN = isond0.shape[0]
    timN = t2-t1
    runN = len(listFiles)

    print ' Number of members:',len(listFiles)

    valmask = isond0.missing_value[0]
    varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol']
    varFill = [0.,0.,valmask,valmask,0.,0.]
    # init arrays (2D rho/lat)
    percent  = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0.
    #minbowl  = npy.ma.ones([basN,latN], dtype='float32')*1000.
    varbowl  = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
    #varList = ['isondepth']
    #print ' !!! ### Testing one variable ###'
    #varList = ['isonthetao']

    # init time axis
    time       = cdm.createAxis(npy.float32(range(timN)))
    time.id    = 'time'
    time.units = 'years since 1861'
    time.designateTime()
    # init ensemble axis
    ensembleAxis       = cdm.createAxis(npy.float32(range(runN)))
    ensembleAxis.id    = 'members'
    ensembleAxis.units = 'N'

    # loop on variables
    for iv,var in enumerate(varList):

        # Array inits (2D rho/lat 3D rho/lat/lon)
            #shapeR = [basN,levN,latN]
        isonvar  = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask
        vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)]
        varstd,varToE1,varToE2 =  [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)]
        varones  = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1.

        print ' Variable ',iv, var
        # loop over files to fill up array
        for i,file in enumerate(listFiles):
            ft      = cdm.open(inDir[0]+'/'+file)
            model = file.split('.')[1]
            timeax  = ft.getAxis('time')
            file1d = replace(inDir[0]+'/'+file,'2D','1D')
            if os.path.isfile(file1d):
                f1d = cdm.open(file1d)
            else:
                print 'ERROR:',file1d,'missing (if mme, run 1D first)'
                sys.exit(1)
            tmax = timeax.shape[0]
            if i == 0:
                tmax0 = tmax
            #adapt [t1,t2] time bounds to piControl last NN years
            if useLastYears:
                t1 = tmax-t20
                t2 = tmax
            else:
                if tmax != tmax0:
                    print 'wrong time axis: exiting...'
                    return

            # read array
            # loop over time/density for memory management
            for it in range(timN):
                t1r = t1 + it
                t2r = t1r + 1
                isonRead = ft(var,time = slice(t1r,t2r))
                if varFill[iv] != valmask:
                    isonvar[i,it,...] = isonRead.filled(varFill[iv])
                else:
                    isonvar[i,it,...] = isonRead
            # compute percentage of non-masked points accros MME
            if iv == 0:
                maskvar = mv.masked_values(isonRead.data,valmask).mask
                percent[i,...] = npy.float32(npy.equal(maskvar,0))
            if mme:
                # if mme then just accumulate Bowl, Agree fields
                varst = var+'Agree'
                vardiff[i,...] = ft(varst,time = slice(t1,t2))
                varb = var+'Bowl'
                varbowl2D[i,...] = ft(varb,time = slice(t1,t2))
            else:
                # Compute difference with average of first initN years
                varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
                for t in range(timN):
                    vardiff[i,t,...] = isonvar[i,t,...] - varinit
                vardiff[i,...].mask = isonvar[i,...].mask
                # Read bowl and truncate 2D field above bowl
                if iv == 0:
                    bowlRead = f1d(varsig,time = slice(t1,t2))
                    varbowl[i,...] = bowlRead
                # Compute Stddev
                varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
                # Compute ToE
                if ToeType == 'histnat':
                    # Read mean and Std dev from histnat
                    if i == 0:
                        filehn  = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
                        #filehn = replace(outFile,'historical','historicalNat')
                        fthn = cdm.open(filehn)
                        varmeanhn = fthn(var)
                        varst = var+'Std'
                        varmaxstd = fthn(varst)
                    toemult = 1.
                    signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
                    noise = npy.reshape(varmaxstd,(basN*levN*latN))
                    varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
                    toemult = 2.
                    varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
            ft.close()
            f1d.close()
        # <-- end of loop on files

        # Compute percentage of bin presence
        # Only keep points where percent > 50%
        if iv == 0:
            percenta = (cdu.averager(percent,axis=0))*100.
            percenta = mv.masked_less(percenta, 50)
            percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent')
            percentw._FillValue = valmask
            percentw.long_name = 'percentage of MME bin'
            percentw.units     = '%'
            outFile_f.write(percentw.astype('float32'))

        # Sign of difference
        if mme:
            vardiffsgSum = cdu.averager(vardiff, axis=0)
            vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
            vardiffsgSum = maskVal(vardiffsgSum, valmask)
            vardiffsgSum.mask = percentw.mask
        else:
            vardiffsg = npy.copysign(varones,vardiff)
            # average signs
            vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            vardiffsgSum.mask = percentw.mask
            vardiffsgSum._FillValue = valmask

        # average variable accross members
        isonVarAve = cdu.averager(isonvar, axis=0)
        isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
        # mask
        if varFill[iv] == valmask:
            isonVarAve = maskVal(isonVarAve, valmask)

        isonVarAve.mask = percentw.mask

        # Only keep points with rhon >  bowl-delta_rho
        delta_rho = 0.
        if mme: # start from average of <var>Agree
            isonVarBowl = cdu.averager(varbowl2D, axis=0)
            isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
            isonVarBowl = maskVal(isonVarBowl, valmask)
            isonVarBowl.mask = percentw.mask
            # Compute intermodel stddev
            isonVarStd = statistics.std(varbowl2D, axis=0)
            isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo')
            isonVarStd = maskVal(isonVarStd, valmask)
            isonVarStd.mask = percentw.mask
            if iv == 0:
                # Read mulitmodel sigma on bowl and average in time
                file1d  =  replace(outDir+'/'+outFile,'2D','1D')
                if os.path.isfile(file1d):
                    f1d = cdm.open(file1d)
                else:
                    print 'ERROR:',file1d,'missing (if mme, run 1D first)'
                    sys.exit(1)
                bowlRead = f1d(varsig,time = slice(t1,t2))
                f1d.close()
                siglimit = cdu.averager(bowlRead, axis=0)  - delta_rho
            # TODO: remove loop by building global array with 1/0
            for il in range(latN):
                for ib in range(basN):
                    #if ib == 2:
                    #    print il, siglimit[ib,il]
                    if siglimit[ib,il] < valmask/1000.:
                         # if mme bowl density defined, mask above bowl
                        index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
                        isonVarBowl [:,ib,0:index[0],il].mask = True
                        isonVarStd  [:,ib,0:index[0],il].mask = True
                        vardiffsgSum[:,ib,0:index[0],il].mask = True
                    else:
                        # mask all points
                        isonVarBowl [:,ib,:,il].mask = True
                        isonVarStd  [:,ib,:,il].mask = True
                        vardiffsgSum[:,ib,:,il].mask = True
        else:
            isonVarBowl = isonVarAve*1. # start from variable
            isonVarStd  = isonVarAve*1. # start from variable
            if iv == 0:
                siglimit = cdu.averager(varbowl, axis=0) # average accross members
                # Average bowl in time
                if timeBowl == 'mean':
                    siglimit = cdu.averager(siglimit, axis=0) - delta_rho
                # or take largest sigma over time
                else:
                    siglimit = npy.ma.max(siglimit, axis=0) - delta_rho
            # TODO: remove loop by building global array with 1/0
            for il in range(latN):
                for ib in range(basN):
                    if siglimit[ib,il] < valmask/1000.:
                        # if bowl density defined, mask above bowl
                        index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
                        isonVarBowl[:,ib,0:index[0],il].mask = True
                        vardiffsgSum[:,ib,0:index[0],il].mask = True
                    else:
                        # mask all points
                        vardiffsgSum[:,ib,:,il].mask = True

            isonVarBowl = maskVal(isonVarBowl, valmask)
            # Find max of Std dev of all members
            isonVarStd = npy.ma.max(varstd, axis=0)
            # mask
            if varFill[iv] == valmask:
                isonVarStd = maskVal(isonVarStd, valmask)

        # Write
        isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id)
        isonave.long_name = isonRead.long_name
        isonave.units     = isonRead.units
        isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree')
        isonavediff.long_name = isonRead.long_name
        isonavediff.units     = isonRead.units
        isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl')
        isonavebowl.long_name = isonRead.long_name
        isonavebowl.units     = isonRead.units
        if not mme:
            isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std')
            isonmaxstd.long_name = isonRead.long_name
            isonmaxstd.units     = isonRead.units

        outFile_f.write(    isonave.astype('float32'))
        outFile_f.write(isonavediff.astype('float32'))
        outFile_f.write(isonavebowl.astype('float32'))
        if not mme:
            outFile_f.write( isonmaxstd.astype('float32'))

        if ToeType == 'histnat':
            isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
            isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
            isontoe1.units     = 'Year'
            isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
            isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
            isontoe2.units     = 'Year'
            outFile_f.write(isontoe1.astype('float32'))
            outFile_f.write(isontoe2.astype('float32'))

        if mme:
            isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd')
            isonvarstd.long_name = isonRead.long_name+' intermodel std'
            isonvarstd.units     = isonRead.units
            outFile_f.write(isonvarstd.astype('float32'))

    # <--- end of loop on variables

    outFile_f.close()
    fi.close()
Ejemplo n.º 22
0
def plot_bathy(bathy, shadow=True, contour=True, shadow_stretch=1., shadow_shapiro=False,
    show=True, shadow_alpha=1., shadow_black=.3, white_deep=False, nmax=30,m=None, alpha=1.,
    zmin=None, zmax=None, **kwargs):
    """Plot a bathymetry

    - *lon*: Longitude range.
    - *lat*: Latitude range.
    - *show*:Display the figure [default: True]
    - *pcolor*: Use pcolor instead of contour [default: False]
    - *contour*: Add line contours [default: True]
    - *shadow*:Plot south-west shadows instead of filled contours.
    - *nmax*: Max number of levels for contours [default: 30]
    - *white_deep*: Deep contours are white [default: False]
    - All other keyword are passed to :func:`~vacumm.misc.plot.map2`
    """

    # Input
    bb = bathy
    if isinstance(bathy, GriddedBathy):
        bathy = bathy.bathy()
        if shadow:
            xxs = getattr(bb, '_xxs', None)
            yys = getattr(bb, '_yys', None)
            if xxs is None:
                lon2d = bb._lon2d
                lat2d = bb._lat2d
    elif shadow:
        xxs = yys = None
        lon2d,lat2d = meshgrid(get_axis(bathy, -1).getValue(),get_axis(bathy, -2).getValue())

    # Masking
    if 'maxdep' in kwargs:
        zmin = -maxdep
    if 'maxalt' in kwargs:
        zmax = maxalt
    if zmin is not None:
        bathy[:] = MV2.masked_less(bathy, zmin)
    if zmax is not None:
        bathy[:] = MV2.masked_greater(bathy, zmax)

    # Default arguments for map
    if hasattr(bathy, 'long_name'):
        kwargs.setdefault('title',bathy.long_name)
    if 'cmap' not in kwargs:
        vmin, vmax = minmax(bathy)
#        print 'cmap topo', vmin, vmax
        kwargs['cmap'] = auto_cmap_topo((kwargs.get('vmin', vmin), kwargs.get('vmax', vmax)))
#       kwargs.setdefault('ticklabel_size','smaller')
    kwargs.setdefault('clabel_fontsize', 8)
    kwargs.setdefault('clabel_alpha',.7*alpha)
    kwargs.setdefault('clabel_glow_alpha', kwargs['clabel_alpha'])
    kwargs.setdefault('fill', 'contourf')
    kwargs['nmax'] = nmax
    kwargs['show'] = False
    kwargs['contour'] = contour
    if shadow: kwargs.setdefault('alpha',.5*alpha)
    kwargs.setdefault('projection', 'merc')
    kwargs.setdefault('fmt', BathyFormatter())
    kwargs.setdefault('colorbar_format', BathyFormatter())
    kwargs.setdefault('units', False)
    kwargs.setdefault('levels_mode','normal')
    kwargs.setdefault('bgcolor', '0.8')
    kwargs.setdefault('contour_linestyle', '-')
    savefig = kwargs.pop('savefig', None)
    kwsavefig = kwfilter(kwargs, 'savefig_')


    # White contour when dark
    if contour and white_deep:
        levels = auto_scale(bathy,nmax=nmax)
        colors = []
        nlevel = len(levels)
        for i in range(nlevel):
            if i < old_div(nlevel,2):
                colors.append('w')
            else:
                colors.append('k')
        kwargs.setdefault('contour_colors',tuple(colors))

    # Call to map
    m = map2(bathy, m=m, **kwargs)

    # Add shadow
    if shadow:

        # Filter
        data = MV.array(bathy,'f',fill_value=0.)
        if shadow_shapiro:
            data = shapiro2d(data,fast=True).shape

        # Gradient
        grd = deriv2d(data,direction=45.,fast=True,fill_value=0.).filled(0.)
        grdn = refine(grd, 3)
        grdn = norm_atan(grdn,stretch=shadow_stretch).clip(0,1.) ; del grd

        # Grid
#           im = m.map.imshow(grdn,cmap=P.get_cmap('gist_yarg'),alpha=1) # gist_yarg , YlGnBu
        if xxs is None or yys is None:
            xx, yy = m(lon2d,lat2d)
            xxr = refine(xx, 3)
            yyr = refine(yy, 3)
            xxs, yys = meshbounds(xxr, yyr)
            if isinstance(bb, GriddedBathy):
                bb._xxs = xxs
                bb._yys = yys
            del xx, yy, xxr, yyr

        # Cmap
        cmap = cmap_custom(( ((1, )*3, 0), ((shadow_black, )*3, 1) ))

        # Plot
        pp = m.map.pcolormesh(xxs, yys, grdn,cmap=cmap)#P.get_cmap('gist_yarg'))
        pp.set_zorder(.9)
        pp.set_linewidth(0)
        pp.set_alpha(shadow_alpha*N.clip(alpha*2, 0, 1))
        del grdn

    # Show it?
    if savefig:
        m.savefig(savefig, **kwsavefig)
    if show:
        P.show()
    return m
Ejemplo n.º 23
0
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True):
    '''
    The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks
    It ouputs the MME and a percentage of non-masked bins

    Created on Tue Nov 25 13:56:20 CET 2014

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - sw2d                   - dimension of fields to consider (1 or 2)
    - years(t1,t2)           - years for slice read
    - inDir(str)             - input directory where files are stored
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - FfllTS                 - 0/1: if 1, uses full time serie (ignores years(t1,t2))
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 25 Nov 2014   - Initial function write
    - EG  9 Dec 2014   - Add agreement on difference with init period - save as <var>Agree
    - EG 04 Oct 2016   - Add 3D files support

    TODO:
    ------

    '''

    # CDMS initialisation - netCDF compression
    comp = 1 ; # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    if t2 <= 0:
        useLastYears = True
        t2 = -t2
    else:
        useLastYears = False
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    # Find dimension
    runN = len(listFiles)
    try:
        fi = cdm.open(inDir[0]+'/'+listFiles[0])
    except:
        print ' *** file not found ',inDir[0]+'/'+listFiles[0]
        sys.exit(' Abort')
    if sw2d == 1:
        ptopd0  = fi['ptopdepth'] ; # Create variable handle
        latN = ptopd0.shape[2]
        basN = ptopd0.shape[1]
    elif sw2d == 2:
        ptopd0  = fi['ptopdepthxy'] ; # Create variable handle
        lonN = ptopd0.shape[2]
        latN = ptopd0.shape[1]

    #timN = ptopd0.shape[0]
    timN = t2-t1
    if fullTS:
        print '  !!! Working on full Time Serie (fullTS = True)'
        timN = ptopd0.shape[0]
        t1=0
        t2=timN
    t10 = t1
    t20 = t2
    # Get grid objects
    axesList = ptopd0.getAxisList()
    # Declare and open files for writing
    if os.path.isfile(outDir+'/'+outFile):
        os.remove(outDir+'/'+outFile)
    outFile_f = cdm.open(outDir+'/'+outFile,'w')

    print ' Number of members:',len(listFiles)

    valmask = ptopd0.missing_value

    # init time axis
    time       = cdm.createAxis(npy.float32(range(timN)))
    time.id    = 'time'
    time.units = 'years since 1861'
    time.designateTime()

    # loop on variables
    # init percent array

    if sw2d == 1:
        varList = ['ptopdepth','ptopsigma','ptopso','ptopthetao','volpers','salpers','tempers']
        #varList = ['ptopdepth']
        varDim  = [1,1,1,1,0,0,0]
        percent  = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*0.
    elif sw2d == 2:
        varList = ['ptopdepthxy','ptopsigmaxy','ptopsoxy','ptopthetaoxy']
        #varList = ['ptopdepthxy']
        varDim  = [2,2,2,2]
        percent  = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.

    varFill = [valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask]

    axis1D = [time,axesList[1],axesList[2]]
    axis0D = [time,axesList[1]]
    print ' timN = ',timN

    # loop on 1D variables
    for iv,var in enumerate(varList):
        ti0 = timc.clock()

        # Array inits
        if varDim[iv] == 2:
            isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
            vardiff = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
            varones = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
            axisVar = axis1D
        elif varDim[iv] == 1:
            isonvar = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
            vardiff = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask
            varones = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1.
            axisVar = axis1D
        else:
            isonvar = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
            vardiff = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask
            varones = npy.ma.ones([runN,timN,basN], dtype='float32')*1.
            axisVar = axis0D
        print ' Variable ',iv, var, varDim[iv]
        # loop over files to fill up array
        for ic,file in enumerate(listFiles):
            ft      = cdm.open(inDir[0]+'/'+file)
            timeax  = ft.getAxis('time')
            try:
                tmax = timeax.shape[0]
            except:
                print ic,file, timeax
            if ic == 0:
                tmax0 = tmax
                #print ic,file, tmax
            #adapt [t1,t2] time bounds to piControl last NN years
            if useLastYears:
                t1 = tmax-t20
                t2 = tmax
            else:
                if tmax != tmax0:
                    print 'tmax <> tmax0',tmax,tmax0
                    print 'wrong time axis: exiting...'

                    return
            #print 'Time dims:',ic, t1,t2,tmax
            # read array
            computeVar = True
            allVars = ft.variables.keys()
            if 'ptopsigmaxy' in allVars:
                computeVar = False
            if (var == 'ptopsigmaxy') & computeVar:
                #print '  ic = ',ic
                # reconstruct from isondepthg and ptopdepthxy

                isond = ft('isondepthg',time = slice(t1,t2))
                #print isond.data.shape, timN*latN*lonN
                itest = 94*360+150
                axesList = isond.getAxisList()
                levs = axesList[1][:]
                levN = len(levs)
                #ti02 = timc.clock()
                levs3d0  = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN))
                #ti05 = timc.clock()
                isonRead = npy.ma.ones([timN,latN,lonN], dtype='float32')*valmask
                for it in range(timN): # loop on time to limit memory usage
                    levs3d = levs3d0*1.
                    depthlo = mv.reshape(vardepth[ic,it,...],latN*lonN)
                    depth3d = npy.reshape(npy.repeat(depthlo,levN),(latN*lonN,levN))
                    isond3d = mv.reshape(npy.transpose(isond.data[it,...],(1,2,0)),(latN*lonN,levN))
                    #print isond3d[itest,:]
                    isond3d[isond3d > valmask/10] = 0.
                    #print isond3d[itest,:]
                    isond3dp1 = npy.roll(isond3d,-1,axis=1)
                    isond3dp1[:,-1] = isond3d[:,-1]
                    #print isond3dp1[itest,:]
                    #levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points
                    #print levs3d[itest,:]
                    levs3d[(depth3d <= isond3d)] = 0.
                    #print levs3d[itest,:]
                    levs3d[(depth3d > isond3dp1)] = 0.
                    #print levs3d[itest,:]
                    #isonwrk = npy.sum(levs3d,axis=1)
                    isonwrk = npy.max(levs3d,axis=1)
                    if it < 0:
                        print ic,it
                        print depthlo[itest]
                        print isond3d[itest,:]
                        print isonwrk[itest]
                        print
                    isonRead[it,...] = mv.reshape(isonwrk,(latN,lonN))
                # <-- end of loop on time
                del (isond3d,isond3dp1); gc.collect()
                # mask with depthxy and where sigmaxy = 0
                isonRead.mask = vardepth.mask[ic,...]
                isonRead = mv.masked_where(isonRead == 0, isonRead)
                isonRead.long_name = var
                isonRead.units = 'sigma_n'
                isonRead.id = var
                del (isond,depth3d,levs3d,levs3d0,isonwrk); gc.collect()
                #ti3 = timc.clock()
                #print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2
                #print ti3-ti0
                # write ptopsigmaxy
                if os.path.isfile(inDir[0]+'/work_ptopsigmaxy/'+file):
                    os.remove(inDir[0]+'/work_ptopsigmaxy/'+file)
                fiout = cdm.open(inDir[0]+'/work_ptopsigmaxy/'+file,'w')
                if ic == 0:
                        print ' Creating ',inDir[0]+'/work_ptopsigmaxy/'+file
                isonsigxy = cdm.createVariable(isonRead, axes = axis1D, id = 'ptopsigmaxy')
                isonsigxy.long_name = 'Density of shallowest persistent ocean on ison'
                isonsigxy.units     = 'sigma_n'
                fiout.write(isonsigxy.astype('float32'))
                fiout.close()
            else:
                # Direct read of variable
                isonRead = ft(var,time = slice(t1,t2))
            #print isonRead.shape, timN
            if varFill[iv] != valmask:
                isonvar[ic,...] = isonRead.filled(varFill[iv])
            else:
                isonvar[ic,...] = isonRead
            #print isonvar[ic,:,40,100]
            # compute percentage of non-masked points accros MME
            if iv == 0:
                maskvar = mv.masked_values(isonRead.data,valmask).mask
                percent[ic,...] = npy.float32(npy.equal(maskvar,0))
            if mme:
                # if mme then just average Bowl and Agree fields
                varst = var+'Agree'
                vardiff[ic,...] = ft(varst,time = slice(t1,t2))
            else:
                # Compute difference with average of first initN years, use mask of last month
                varinit = cdu.averager(isonvar[ic,peri1:peri2,...],axis=0)
                for tr in range(timN):
                    vardiff[ic,tr,...] = isonvar[ic,tr,...] - varinit
                vardiff[ic,...].mask = isonvar[ic,...].mask

            ft.close()
        # <-- end of loop on files
        # TODO remove masked points at longitudes 0 or 180deg for some models
        # if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy)
        if var =='ptopdepthxy':
            vardepth = isonvar
        # Compute percentage of bin presence
        # Only keep points where percent > 50%
        if iv == 0:
            percenta = (cdu.averager(percent,axis=0))*100.
            percenta = mv.masked_less(percenta, 50)
            percentw = cdm.createVariable(percenta, axes = axis1D, id = 'ptoppercent')
            percentw._FillValue = valmask
            percentw.long_name = 'percentage of MME bin'
            percentw.units     = '%'
            outFile_f.write(percentw.astype('float32'))
        # Sign of difference
        if mme:
            vardiffsgSum = cdu.averager(vardiff, axis=0)
            vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = axisVar , id = 'foo')
            vardiffsgSum = maskVal(vardiffsgSum, valmask)
            vardiffsgSum.mask = percentw.mask
        else:
            vardiffsg = npy.copysign(varones,vardiff)
            # average signs
            vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            vardiffsgSum.mask = percentw.mask
            vardiffsgSum._FillValue = valmask

        # average accross members
        isonVarAve = cdu.averager(isonvar, axis=0)
        isonVarAve = cdm.createVariable(isonVarAve , axes = axisVar , id = 'foo')
        # mask
        if varFill[iv] == valmask:
            isonVarAve = maskVal(isonVarAve, valmask)

        isonVarAve.mask = percentw.mask

        # Write
        isonave = cdm.createVariable(isonVarAve, axes = axisVar, id = isonRead.id)
        isonave.long_name = isonRead.long_name
        isonave.units     = isonRead.units
        isonavediff = cdm.createVariable(vardiffsgSum, axes = axisVar, id = isonRead.id+'Agree')
        isonavediff.long_name = isonRead.long_name
        isonavediff.units     = isonRead.units

        outFile_f.write(isonave.astype('float32'))
        outFile_f.write(isonavediff.astype('float32'))
        tf = timc.clock()
        #print '   time var',tf-ti0
    # <--- end of loop on variables

    outFile_f.close()
    fi.close()
Ejemplo n.º 24
0
0.059503571833625334
0.059503571833625334 0.05664014775641405 0.05193557222118004
0.04777129850801233 0.0407139313814465 0.029382624830271705
0.018469399844287374 0.0162382275289592 0.02646680241827459
0.04792041732949079 0.0689138797030203 0.08167038620212037
0.09273558459066569 0.11266293431057901 0.13663018925347364
0.15229174546388072 0.15284435880966177 0.13423845476113883
0.09945904378274077 0.07032267160267985 0.05551039827020481
0.045537187647785464 0.040532491867244946 0.03577527125478327
-999. -999. -999.
-0.058062458673116 -0.08764922509099882 -0.11697036914487152
-0.14836133615864944 -0.17956528904564023 -0.21109198032585794
-0.23846429237248942 -0.2598536549218765 -0.27795672866320387
-0.2939939095159731 -0.30541031366330024 -0.307643559333884
-0.30078421139811795 -0.2841339526883441 -0.26485737397202497
-0.24287299694779327 -0.22379014890999907 -0.20121548204699846
-0.1746486732156772 -0.14585019344118372 -0.12070675757803526
-0.0997891159111037 -0.08229393660994214 -0.06779720501287469
-0.057213385470859794 -0.04875768191096844 -0.0402377347189964
-0.030169328367807245 -0.017560662894847895 -0.006968922654137132
0.0009773980274431048 0.007054306637034288 0.010472286514133042
0.010702384151997032 0.009231553701801242 0.007544033101056543
0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999.
-999. -999. -999.
""".split()
data = numpy.array(data, dtype=numpy.float)
data = MV2.masked_less(data, -900)
x.plot(data, yx, bg=1)
regression.run(x, "test_vcs_1D_datawc_missing.png")
Ejemplo n.º 25
0
import vcs
import cdms2
import sys
import os
import MV2

f=cdms2.open(os.path.join(sys.prefix,"sample_data","clt.nc"))
s=f("clt",time=slice(0,1),squeeze=1)
s=MV2.masked_less(s,65.)
x=vcs.init()
gm=x.createisofill()
#gm=x.createboxfill()
gm.missing = 252
#gm.levels = [65,70,75,80,85,90,95,100]
x.plot(s,gm)
x.png(gm.g_name)
x.interact()



Ejemplo n.º 26
0
if gm_type=="meshfill":
    f=cdms2.open(os.path.join(vcs.sample_data,'sampleCurveGrid4.nc'))
else:
    f=cdms2.open(os.path.join(vcs.sample_data,'clt.nc'))
if gm_type=="vector":
    u=f("u",**xtra)
    v=f("v",**xtra)
    if args.mask:
        u=MV2.masked_greater(u,58.)
    if args.zero:
      u-=u
      v-=v
elif gm_type=="meshfill":
    s=f("sample",**xtra)
    if args.mask:
        s=MV2.masked_less(s,1150.)
    elif args.bigvalues:
        s[s < 1150] = 1e40
    if args.zero:
       s-=s
else:
    s=f("clt",**xtra)
    if args.mask:
        s=MV2.masked_greater(s,78.)
    elif args.bigvalues:
        s[s > 78] = 1e40
    if gm_type in ["1d","yxvsx","xyvsy","xvsy","scatter"]:
        s = s(latitude=(20,20,"cob"),longitude=(112,112,"cob"),squeeze=1)
        s2=MV2.sin(s)
        if args.zero:
           s2-=s2
Ejemplo n.º 27
0
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True):
    '''
    The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks
    It ouputs
     - the MME
     - a percentage of non-masked bins
     - the sign agreement of period2-period1 differences
     - ToE per run and for MME

    Author:    Eric Guilyardi : [email protected]

    Created on Tue Nov 21 2016

    Inputs:
    -------
    - listFiles(str)         - the list of files to be averaged
    - years(t1,t2)           - years for slice read
    - inDir[](str)           - input directory where files are stored (add histnat as inDir[1] for ToE)
    - outDir(str)            - output directory
    - outFile(str)           - output file
    - timeInt(2xindices)     - indices of init period to compare with (e.g. [1,20])
    - mme(bool)              - multi-model mean (will read in single model ensemble stats)
    - ToeType(str)           - ToE type ('F': none, 'histnat')
                               -> requires running first mm+mme without ToE to compute Stddev
    - debug <optional>       - boolean value

    Notes:
    -----
    - EG 21 Nov 2016   - Initial function write

    - TODO :
                 - add computation of ToE per model (toe 1 and toe 2) see ticket #50
                 - add isonhtc (see ticket #48)
    '''

    # CDMS initialisation - netCDF compression
    comp = 1 # 0 for no compression
    cdm.setNetcdfShuffleFlag(comp)
    cdm.setNetcdfDeflateFlag(comp)
    cdm.setNetcdfDeflateLevelFlag(comp)
    cdm.setAutoBounds('on')
    # Numpy initialisation
    npy.set_printoptions(precision=2)

    if debug:
        debug = True
    else:
        debug = False
    # File dim and grid inits
    t1 = years[0]
    t2 = years[1]
    # Bound of period average to remove
    peri1 = timeInt[0]
    peri2 = timeInt[1]
    fi    = cdm.open(inDir[0]+'/'+listFiles[0])
    # Switch if only variables below the bowl are present/treated
    nobowl = True
    if nobowl:
        isond0 = fi['isondepthgBowl'] ; # Create variable handle
    else:
        isond0 = fi['isondepthg'] ; # Create variable handle
    # Get grid objects
    axesList = isond0.getAxisList()
    sigmaGrd = isond0.getLevel()
    #time = isond0.getTime()
    lonN = isond0.shape[3]
    latN = isond0.shape[2]
    levN = isond0.shape[1]
    varsig='ptopsigmaxy'

    # Limit number of models to 3 for testing of mme
    #if mme:
    #    listFiles = listFiles[0:2]
    #    print ' !!! ### Testing 3 models ###',  listFiles

    # Declare and open files for writing
    if os.path.isfile(outDir+'/'+outFile):
        os.remove(outDir+'/'+outFile)
    outFile_f = cdm.open(outDir+'/'+outFile,'w')

    #timN = isond0.shape[0]
    timN = t2-t1
    runN = len(listFiles)

    print ' Number of members:',len(listFiles)

    valmask = isond0.missing_value

    varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg']
    varFill = [valmask,valmask,valmask,valmask,valmask]
    percent  = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0.
    varbowl  = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1.
    #varList = ['isondepthg']
    #print ' !!! ### Testing one variable ###', varList

    # init sigma axis
    sigma = cdm.createAxis(npy.float32(range(1)))
    sigma.id = axesList[1].id
    sigma.units = axesList[1].units
    sigma.designateTime()
    # init time axis
    time       = cdm.createAxis(npy.float32(range(timN)))
    time.id    = 'time'
    time.units = 'years since 1861'
    # init ensemble axis
    ensembleAxis       = cdm.createAxis(npy.float32(range(runN)))
    ensembleAxis.id    = 'members'
    ensembleAxis.units = 'N'
    # Output axis
    sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon
    sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon
    # init arrays
    isonvar  = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
    varbowl2D  = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask
    varstd,varToE1,varToE2 =  [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)]

    # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions)

    delta_ib = 1
    print ' Sigma index:'
    for ib in range(levN):
        ib1 = ib + delta_ib
        print ib,
        tim0 = timc.clock()
        # loop on variables
        for iv,var in enumerate(varList):
            if nobowl:
                varb = var+'Bowl'
            else:
                varb = var
            if ib == 0:
                print ' Variable ',iv, varb
            # loop over files to fill up array
            for i,file in enumerate(listFiles):
                tim01 = timc.clock()
                ft      = cdm.open(inDir[0]+'/'+file)
                model = file.split('.')[1]
                timeax  = ft.getAxis('time')
                if i == 0:
                    tmax0 = timeax.shape[0]
                tmax = timeax.shape[0]
                if tmax != tmax0:
                    print 'wrong time axis: exiting...'
                    return
                # read array
                isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze()
                if varFill[iv] != valmask:
                    isonvar[i,...] = isonRead.filled(varFill[iv])
                else:
                    isonvar[i,...] = isonRead
                tim02 = timc.clock()
                # compute percentage of non-masked points accros MME
                if iv == 0:
                    maskvar = mv.masked_values(isonRead.data,valmask).mask
                    percent[i,...] = npy.float32(npy.equal(maskvar,0))
                tim03 = timc.clock()
                if mme:
                    # if mme then just accumulate Bowl, Agree and Std fields
                    #varst = var+'Agree'
                    #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
                    isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze()
                    varbowl2D[i,...] = isonRead
                else:
                    # Compute difference with average of first initN years
                    #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0)
                    #for t in range(timN):
                    #    vardiff[i,t,...] = isonvar[i,t,...] - varinit
                    #vardiff[i,...].mask = isonvar[i,...].mask
                    # Read bowl to truncate field above bowl
                    if ib == 0 and iv == 0:
                        varbowl[i,...] = ft(varsig,time = slice(t1,t2))
                        #varbowl[i,...] = bowlRead
                    # Compute Stddev
                    varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0)
                    # Compute ToE
                    if ToeType == 'histnat':
                        toto=1
                        # TODO
                        # Read mean and Std dev from histnat
                        #    if i == 0:
                        #        filehn  = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0]
                        #        #filehn = replace(outFile,'historical','historicalNat')
                        #        fthn = cdm.open(filehn)
                        #        varmeanhn = fthn(var)
                        #        varst = var+'Std'
                        #        varmaxstd = fthn(varst)
                        #    toemult = 1.
                        #    signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN))
                        #    noise = npy.reshape(varmaxstd,(basN*levN*latN))
                        #    varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
                        #    toemult = 2.
                        #    varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN))
                tim04 = timc.clock()
                ft.close()
                #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03
            # <-- end of loop on files (i)

            tim1 = timc.clock()

            # Compute percentage of bin presence
            # Only keep points where percent > 50%
            if iv == 0:
                percenta = (cdu.averager(percent,axis=0))*100.
                percenta = mv.masked_less(percenta, 50)
                percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN])
                percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent')
                percentw._FillValue = valmask
                percentw.long_name = 'percentage of MME bin'
                percentw.units     = '%'
                outFile_f.write(percentw.astype('float32'), extend = 1, index = ib)

            # Sign of difference
            #if mme:
            #    vardiffsgSum = cdu.averager(vardiff, axis=0)
            #    vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo')
            #    vardiffsgSum = maskVal(vardiffsgSum, valmask)
            #    vardiffsgSum.mask = percentw.mask
            #else:
            #    vardiffsg = npy.copysign(varones,vardiff)
            #    # average signs
            #    vardiffsgSum = cdu.averager(vardiffsg, axis=0)
            #    vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.)
            #    vardiffsgSum.mask = percentw.mask
            #    vardiffsgSum._FillValue = valmask

            # average variable accross members
            isonVarAve = cdu.averager(isonvar, axis=0)
            isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN])
            isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo')
            # mask
            if varFill[iv] == valmask:
                isonVarAve = maskVal(isonVarAve, valmask)

            isonVarAve.mask = percentw.mask
            tim2 = timc.clock()

            # Only keep points with rhon >  bowl-delta_rho
            delta_rho = 0.
            # mme case
            if mme: # start from average of <var>Agree
                isonVarBowl = cdu.averager(varbowl2D, axis=0)
                isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
                isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo')
                isonVarBowl = maskVal(isonVarBowl, valmask)
                isonVarBowl.mask = percentw.mask
                # Compute intermodel stddev
                isonVarStd = statistics.std(varbowl2D, axis=0)
                isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN])
                isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo')
                isonVarStd = maskVal(isonVarStd, valmask)
                isonVarStd.mask = percentw.mask

                # Write
                isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id)
                isonvarbowlw.long_name = isonRead.long_name
                isonvarbowlw.units     = isonRead.units
                isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std')
                isonvarstdw.long_name = isonRead.long_name+' intermodel std'
                isonvarstdw.units     = isonRead.units

                outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib)
                outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib)

                #if ib == 0 and iv == 0:
                #    # TODO review
                #    # Read multimodel sigma on bowl and average in time
                #    file1d  =  replace(outDir+'/'+outFile,'2D','1D')
                #    if os.path.isfile(file1d):
                #        f1d = cdm.open(file1d)
                #    else:
                #        print 'ERROR:',file1d,'missing (if mme, run 2D first)'
                #        sys.exit(1)
                #    bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1))
                #    f1d.close()
                #    siglimit = cdu.averager(bowlRead, axis=0)  - delta_rho
                # TODO: remove loop by building global array with 1/0
                #if sw2d == 1:
                #    for il in range(latN):
                #        for ib in range(basN):
                #            #if ib == 2:
                #            #    print il, siglimit[ib,il]
                #            if siglimit[ib,il] < valmask/1000.:
                #                 # if mme bowl density defined, mask above bowl
                #                index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il]))
                #                isonVarBowl [:,ib,0:index[0],il].mask = True
                #                isonVarStd  [:,ib,0:index[0],il].mask = True
                #                vardiffsgSum[:,ib,0:index[0],il].mask = True
                #            else:
                #                # mask all points
                #                isonVarBowl [:,ib,:,il].mask = True
                #                isonVarStd  [:,ib,:,il].mask = True
                #                vardiffsgSum[:,ib,:,il].mask = True
            # mm case
            else:
                isonVarBowl = isonVarAve*1. # start from variable
                #isonVarStd  = isonVarAve*1. # start from variable
                if ib == 0 and iv == 0:
                    # build bowl position
                    siglimit = cdu.averager(varbowl, axis=0) # average accross members
                    siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho
                if iv == 0:
                    sigarr = siglimit*1.
                    sigarr[:] = sigmaGrd[ib]
                # test
                i = 60
                j = 60
                ij = j*lonN+i
                isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN])
                #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN])

                isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask)
                #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask)

                isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN])
                #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN])

                isonVarBowl = maskVal(isonVarBowl, valmask)
                #vardiffsgSum = maskVal(vardiffsgSum, valmask)
                # Find max of Std dev of all members
                isonVarStd = npy.ma.max(varstd, axis=0)
                # mask
                isonVarStd = maskVal(isonVarStd, valmask)

                # Write
                #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id)
                #isonave.long_name = isonRead.long_name
                #isonave.units     = isonRead.units
                #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN])
                #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree')
                #isonavediff.long_name = isonRead.long_name
                #isonavediff.units     = isonRead.units
                isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN])
                isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl')
                isonavebowl.long_name = isonRead.long_name
                isonavebowl.units     = isonRead.units
                isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN])
                isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std')
                isonmaxstd.long_name = isonRead.long_name
                isonmaxstd.units     = isonRead.units

                #outFile_f.write(    isonave.astype('float32'), extend = 1, index = ib)
                #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib)
                outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib)
                outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib)

            tim3 = timc.clock()

            if ToeType == 'histnat':
                isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1')
                isontoe1.long_name = 'ToE 1 for '+isonRead.long_name
                isontoe1.units     = 'Year'
                isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2')
                isontoe2.long_name = 'ToE 2 for '+isonRead.long_name
                isontoe2.units     = 'Year'
                outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib)
                outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib)

            tim4 = timc.clock()
        # <--- end of loop on variables

        #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3
    # <--- end of loop on density
    print ' '

    outFile_f.close()
    fi.close()