def select_CF(sCF, wCF, idx, *i): if idx == 1.: masked_sCF = np.ceil(sCF) masked_wCF = np.ceil(wCF) s_avg_tmp = cdutil.averager(sCF, axis='yx') w_avg_tmp = cdutil.averager(wCF, axis='yx') elif idx == 2.: # set threshold; # you can change this values based on needs; s_thresholds = 0.26 w_thresholds = 0.26 # set grids with values lower than the threshold to 0; sCF[sCF < s_thresholds] = 0. wCF[wCF < w_thresholds] = 0. # mask grids with a value of 0 (grids that are not needed); masked_sCF = set_axes(MV.masked_equal(sCF, 0.) * 0. + 1) masked_wCF = set_axes(MV.masked_equal(wCF, 0.) * 0. + 1) s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx') w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx') elif idx == 3.: # set threshold, top X%; # you can change this values based on needs; p_threshold_s = 0.25 p_threshold_w = 0.25 # sort all grids, values for grids that are # outside the interested region are set to -1; s_reshape = np.sort(np.array(MV.filled(sCF, -1)).ravel())[::-1] w_reshape = np.sort(np.array(MV.filled(wCF, -1)).ravel())[::-1] # remove grids with values of -1; s_no_minus1 = s_reshape[s_reshape != -1] w_no_minus1 = w_reshape[w_reshape != -1] # find the threshold for the top X% num_s = int(len(s_no_minus1) * p_threshold_s) num_w = int(len(w_no_minus1) * p_threshold_w) s_thresholds = s_no_minus1[num_s - 1] w_thresholds = w_no_minus1[num_w - 1] # mask grids with a value lower than the derived thresholds (grids that are not needed); masked_sCF = set_axes(MV.masked_less(sCF, s_thresholds) * 0. + 1) masked_wCF = set_axes(MV.masked_less(wCF, w_thresholds) * 0. + 1) s_avg_tmp = cdutil.averager(masked_sCF * sCF, axis='yx') w_avg_tmp = cdutil.averager(masked_wCF * wCF, axis='yx') print( f"For method {idx}, averaged solar capacity factor for the filtered grids is: {s_avg_tmp}" ) print( f"For method {idx}, averaged wind capacity factor for the filtered grids is: {w_avg_tmp}" ) print( f"Selected grid cells solar {np.ceil(masked_sCF).sum()}\nSelected grid cells wind {np.ceil(masked_wCF).sum()}" ) return masked_sCF, masked_wCF
def testContiguousRegridNANIssue(self): a = MV2.reshape(MV2.sin(MV2.arange(20000)), (2, 1, 100, 100)) lon = cdms2.createAxis(MV2.arange(100) * 3.6) lon.designateLongitude() lon.units = "degrees_east" lon.id = "longitude" lat = cdms2.createAxis(MV2.arange(100) * 1.8 - 90.) lat.id = "latitude" lat.designateLatitude() lat.units = "degrees_north" lev = cdms2.createAxis([1000.]) lev.id = "plev" lev.designateLevel() lev.units = "hPa" t = cdms2.createAxis([0, 31.]) t.id = "time" t.designateTime() t.units = "days since 2014" cdutil.setTimeBoundsMonthly(t) a.setAxisList((t, lev, lat, lon)) a = MV2.masked_less(a, .5) grd = cdms2.createGaussianGrid(64) a = a.ascontiguous() a = a.regrid(grd, regridTool="regrid2") a = cdutil.averager(a, axis='txy') self.assertEqual(a[0], 0.7921019540305255)
def testAxismissing(self): data = """ -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. 0.059503571833625334 0.059503571833625334 0.05664014775641405 0.05193557222118004 0.04777129850801233 0.0407139313814465 0.029382624830271705 0.018469399844287374 0.0162382275289592 0.02646680241827459 0.04792041732949079 0.0689138797030203 0.08167038620212037 0.09273558459066569 0.11266293431057901 0.13663018925347364 0.15229174546388072 0.15284435880966177 0.13423845476113883 0.09945904378274077 0.07032267160267985 0.05551039827020481 0.045537187647785464 0.040532491867244946 0.03577527125478327 -999. -999. -999. -0.058062458673116 -0.08764922509099882 -0.11697036914487152 -0.14836133615864944 -0.17956528904564023 -0.21109198032585794 -0.23846429237248942 -0.2598536549218765 -0.27795672866320387 -0.2939939095159731 -0.30541031366330024 -0.307643559333884 -0.30078421139811795 -0.2841339526883441 -0.26485737397202497 -0.24287299694779327 -0.22379014890999907 -0.20121548204699846 -0.1746486732156772 -0.14585019344118372 -0.12070675757803526 -0.0997891159111037 -0.08229393660994214 -0.06779720501287469 -0.057213385470859794 -0.04875768191096844 -0.0402377347189964 -0.030169328367807245 -0.017560662894847895 -0.006968922654137132 0.0009773980274431048 0.007054306637034288 0.010472286514133042 0.010702384151997032 0.009231553701801242 0.007544033101056543 0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. """.split() data = numpy.array(data, dtype=numpy.float) data = MV2.masked_less(data, -900) d2 = cdms2.createAxis(data) self.assertTrue(numpy.ma.allclose(data, d2[:]))
def testMaskingFunctions(self): xouter = MV2.outerproduct(MV2.arange(5.), [1] * 10) masked = MV2.masked_greater(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[2:], True)) self.assertTrue(MV2.allequal(masked.mask[:2], False)) masked = MV2.masked_greater_equal(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[1:], True)) self.assertTrue(MV2.allequal(masked.mask[:1], False)) masked = MV2.masked_less(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[:1], True)) self.assertTrue(MV2.allequal(masked.mask[1:], False)) masked = MV2.masked_less_equal(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[:2], True)) self.assertTrue(MV2.allequal(masked.mask[2:], False)) masked = MV2.masked_not_equal(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[1], False)) self.assertTrue(MV2.allequal(masked.mask[0], True)) self.assertTrue(MV2.allequal(masked.mask[2:], True)) masked = MV2.masked_equal(xouter, 1) self.assertTrue(MV2.allequal(masked.mask[1], True)) self.assertTrue(MV2.allequal(masked.mask[0], False)) self.assertTrue(MV2.allequal(masked.mask[2:], False)) masked = MV2.masked_outside(xouter, 1, 3) self.assertTrue(MV2.allequal(masked.mask[0:1], True)) self.assertTrue(MV2.allequal(masked.mask[1:4], False)) self.assertTrue(MV2.allequal(masked.mask[4:], True)) masked = MV2.masked_where( MV2.logical_or(MV2.greater(xouter, 3), MV2.less(xouter, 2)), xouter) self.assertTrue(MV2.allequal(masked.mask[0:2], True)) self.assertTrue(MV2.allequal(masked.mask[2:4], False)) self.assertTrue(MV2.allequal(masked.mask[4:], True))
def bony_sorting_part1(w500, binedges): A, B, C = w500.shape dx = np.diff(binedges)[0] # Compute composite: OKwaps = nanarray( (A, B, C, 2 + len(binedges))) # add 2 for the exceedances xx = 0 for x in binedges: xx += 1 w500_bin = MV.masked_less(w500, x) OKwaps[..., xx] = MV.masked_greater_equal(w500_bin, x + dx) # do the first wap bin: OKwaps[..., 0] = MV.masked_greater_equal(w500, binedges[0]) # do the last wap bin: OKwaps[..., -1] = MV.masked_less(w500, binedges[-1] + dx) return OKwaps # [month,lat,lon,wapbin]
def map_SWkern_to_lon(Ksw, albcsmap): """ Map each location's clear-sky surface albedo to the correct albedo bin """ albcsmap = MV.masked_greater(albcsmap, 1.0) albcsmap = MV.masked_less(albcsmap, 0.0) from scipy.interpolate import interp1d # Ksw is size 12,7,7,lats,3 # albcsmap is size A,lats,lons albcs = np.arange(0.0, 1.5, 0.5) A = albcsmap.shape[0] TT = Ksw.shape[1] PP = Ksw.shape[2] lenlat = Ksw.shape[3] lenlon = albcsmap.shape[2] SWkernel_map = MV.zeros((A, TT, PP, lenlat, lenlon)) SWkernel_map = MV.masked_where(SWkernel_map == 0, SWkernel_map) for M in range(A): MM = M while MM > 11: MM = MM - 12 for LA in range(lenlat): alon = albcsmap[M, LA, :] # interp1d can't handle mask but it can deal with NaN (?) try: alon2 = MV.where(alon.mask, np.nan, alon) except: alon2 = alon if np.ma.count(alon2) > 1: # at least 1 unmasked value if len(np.where(Ksw[MM, :, :, LA, :] > 0)) == 0: SWkernel_map[M, :, :, LA, :] = 0 else: f = interp1d(albcs, Ksw[MM, :, :, LA, :], axis=2) ynew = f(alon2.data) ynew = MV.masked_where(alon2.mask, ynew) SWkernel_map[M, :, :, LA, :] = ynew else: continue return SWkernel_map
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True): ''' The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks It ouputs the MME and a percentage of non-masked bins Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - sw2d - dimension of fields to consider (1 or 2) - years(t1,t2) - years for slice read - inDir(str) - input directory where files are stored - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2)) - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree - EG 04 Oct 2016 - Add 3D files support TODO: ------ ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] # Find dimension runN = len(listFiles) try: fi = cdm.open(inDir[0] + '/' + listFiles[0]) except: print ' *** file not found ', inDir[0] + '/' + listFiles[0] sys.exit(' Abort') if sw2d == 1: ptopd0 = fi['ptopdepth'] # Create variable handle latN = ptopd0.shape[2] basN = ptopd0.shape[1] elif sw2d == 2: ptopd0 = fi['ptopdepthxy'] # Create variable handle lonN = ptopd0.shape[2] latN = ptopd0.shape[1] #timN = ptopd0.shape[0] timN = t2 - t1 if fullTS: print ' !!! Working on full Time Serie (fullTS = True)' timN = ptopd0.shape[0] t1 = 0 t2 = timN t10 = t1 t20 = t2 # Get grid objects axesList = ptopd0.getAxisList() # Declare and open files for writing if os.path.isfile(outDir + '/' + outFile): os.remove(outDir + '/' + outFile) outFile_f = cdm.open(outDir + '/' + outFile, 'w') print ' Number of members:', len(listFiles) valmask = ptopd0.missing_value # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # loop on variables # init percent array if sw2d == 1: varList = [ 'ptopdepth', 'ptopsigma', 'ptopso', 'ptopthetao', 'volpers', 'salpers', 'tempers' ] #varList = ['ptopdepth'] varDim = [1, 1, 1, 1, 0, 0, 0] percent = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 0. elif sw2d == 2: varList = ['ptopdepthxy', 'ptopsigmaxy', 'ptopsoxy', 'ptopthetaoxy'] #varList = ['ptopdepthxy'] varDim = [2, 2, 2, 2] percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0. varFill = [ valmask, valmask, valmask, valmask, valmask, valmask, valmask, valmask, valmask ] axis1D = [time, axesList[1], axesList[2]] axis0D = [time, axesList[1]] print ' timN = ', timN # loop on 1D variables for iv, var in enumerate(varList): ti0 = timc.clock() # Array inits if varDim[iv] == 2: isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask vardiff = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask varones = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1. axisVar = axis1D elif varDim[iv] == 1: isonvar = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * valmask vardiff = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * valmask varones = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1. axisVar = axis1D else: isonvar = npy.ma.ones([runN, timN, basN], dtype='float32') * valmask vardiff = npy.ma.ones([runN, timN, basN], dtype='float32') * valmask varones = npy.ma.ones([runN, timN, basN], dtype='float32') * 1. axisVar = axis0D print ' Variable ', iv, var, varDim[iv] # loop over files to fill up array for ic, file in enumerate(listFiles): ft = cdm.open(inDir[0] + '/' + file) timeax = ft.getAxis('time') try: tmax = timeax.shape[0] except: print ic, file, timeax if ic == 0: tmax0 = tmax #print ic,file, tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax - t20 t2 = tmax else: if tmax != tmax0: print 'tmax <> tmax0', tmax, tmax0 print 'wrong time axis: exiting...' return #print 'Time dims:',ic, t1,t2,tmax # read array computeVar = True allVars = ft.variables.keys() if 'ptopsigmaxy' in allVars: computeVar = False if (var == 'ptopsigmaxy') & computeVar: #print ' ic = ',ic # reconstruct from isondepthg and ptopdepthxy isond = ft('isondepthg', time=slice(t1, t2)) #print isond.data.shape, timN*latN*lonN itest = 94 * 360 + 150 axesList = isond.getAxisList() levs = axesList[1][:] levN = len(levs) #ti02 = timc.clock() levs3d0 = mv.reshape(npy.tile(levs, latN * lonN), (latN * lonN, levN)) #ti05 = timc.clock() isonRead = npy.ma.ones([timN, latN, lonN], dtype='float32') * valmask for it in range(timN): # loop on time to limit memory usage levs3d = levs3d0 * 1. depthlo = mv.reshape(vardepth[ic, it, ...], latN * lonN) depth3d = npy.reshape(npy.repeat(depthlo, levN), (latN * lonN, levN)) isond3d = mv.reshape( npy.transpose(isond.data[it, ...], (1, 2, 0)), (latN * lonN, levN)) #print isond3d[itest,:] isond3d[isond3d > valmask / 10] = 0. #print isond3d[itest,:] isond3dp1 = npy.roll(isond3d, -1, axis=1) isond3dp1[:, -1] = isond3d[:, -1] #print isond3dp1[itest,:] #levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points #print levs3d[itest,:] levs3d[(depth3d <= isond3d)] = 0. #print levs3d[itest,:] levs3d[(depth3d > isond3dp1)] = 0. #print levs3d[itest,:] #isonwrk = npy.sum(levs3d,axis=1) isonwrk = npy.max(levs3d, axis=1) if it < 0: print ic, it print depthlo[itest] print isond3d[itest, :] print isonwrk[itest] print isonRead[it, ...] = mv.reshape(isonwrk, (latN, lonN)) # <-- end of loop on time del (isond3d, isond3dp1) gc.collect() # mask with depthxy and where sigmaxy = 0 isonRead.mask = vardepth.mask[ic, ...] isonRead = mv.masked_where(isonRead == 0, isonRead) isonRead.long_name = var isonRead.units = 'sigma_n' isonRead.id = var del (isond, depth3d, levs3d, levs3d0, isonwrk) gc.collect() #ti3 = timc.clock() #print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2 #print ti3-ti0 # write ptopsigmaxy if os.path.isfile(inDir[0] + '/work_ptopsigmaxy/' + file): os.remove(inDir[0] + '/work_ptopsigmaxy/' + file) fiout = cdm.open(inDir[0] + '/work_ptopsigmaxy/' + file, 'w') if ic == 0: print ' Creating ', inDir[0] + '/work_ptopsigmaxy/' + file isonsigxy = cdm.createVariable(isonRead, axes=axis1D, id='ptopsigmaxy') isonsigxy.long_name = 'Density of shallowest persistent ocean on ison' isonsigxy.units = 'sigma_n' fiout.write(isonsigxy.astype('float32')) fiout.close() else: # Direct read of variable isonRead = ft(var, time=slice(t1, t2)) #print isonRead.shape, timN if varFill[iv] != valmask: isonvar[ic, ...] = isonRead.filled(varFill[iv]) else: isonvar[ic, ...] = isonRead #print isonvar[ic,:,40,100] # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data, valmask).mask percent[ic, ...] = npy.float32(npy.equal(maskvar, 0)) if mme: # if mme then just average Bowl and Agree fields varst = var + 'Agree' vardiff[ic, ...] = ft(varst, time=slice(t1, t2)) else: # Compute difference with average of first initN years, use mask of last month varinit = cdu.averager(isonvar[ic, peri1:peri2, ...], axis=0) for tr in range(timN): vardiff[ic, tr, ...] = isonvar[ic, tr, ...] - varinit vardiff[ic, ...].mask = isonvar[ic, ...].mask ft.close() # <-- end of loop on files # TODO remove masked points at longitudes 0 or 180deg for some models # if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy) if var == 'ptopdepthxy': vardepth = isonvar # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent, axis=0)) * 100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable(percenta, axes=axis1D, id='ptoppercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable(vardiffsgSum, axes=axisVar, id='foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones, vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable(isonVarAve, axes=axisVar, id='foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Write isonave = cdm.createVariable(isonVarAve, axes=axisVar, id=isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable(vardiffsgSum, axes=axisVar, id=isonRead.id + 'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units outFile_f.write(isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) tf = timc.clock() #print ' time var',tf-ti0 # <--- end of loop on variables outFile_f.close() fi.close()
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True): ''' The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 21 2016 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 21 Nov 2016 - Initial function write - TODO : - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0] + '/' + listFiles[0]) # Switch if only variables below the bowl are present/treated nobowl = True if nobowl: isond0 = fi['isondepthgBowl'] # Create variable handle else: isond0 = fi['isondepthg'] # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() #time = isond0.getTime() lonN = isond0.shape[3] latN = isond0.shape[2] levN = isond0.shape[1] varsig = 'ptopsigmaxy' # Limit number of models to 3 for testing of mme #if mme: # listFiles = listFiles[0:2] # print ' !!! ### Testing 3 models ###', listFiles # Declare and open files for writing if os.path.isfile(outDir + '/' + outFile): os.remove(outDir + '/' + outFile) outFile_f = cdm.open(outDir + '/' + outFile, 'w') #timN = isond0.shape[0] timN = t2 - t1 runN = len(listFiles) print ' Number of members:', len(listFiles) valmask = isond0.missing_value varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg'] varFill = [valmask, valmask, valmask, valmask, valmask] percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0. varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1. #varList = ['isondepthg'] #print ' !!! ### Testing one variable ###', varList # init sigma axis sigma = cdm.createAxis(npy.float32(range(1))) sigma.id = axesList[1].id sigma.units = axesList[1].units sigma.designateTime() # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # Output axis sigmaList = [sigma, axesList[2], axesList[3]] # sigma, lat, lon sigmaTimeList = [sigma, time, axesList[2], axesList[3]] # sigma, time, lat, lon # init arrays isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask varbowl2D = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask varstd, varToE1, varToE2 = [ npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask for _ in range(3) ] # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions) delta_ib = 1 print ' Sigma index:' for ib in range(levN): ib1 = ib + delta_ib print ib, tim0 = timc.clock() # loop on variables for iv, var in enumerate(varList): if nobowl: varb = var + 'Bowl' else: varb = var if ib == 0: print ' Variable ', iv, varb # loop over files to fill up array for i, file in enumerate(listFiles): tim01 = timc.clock() ft = cdm.open(inDir[0] + '/' + file) model = file.split('.')[1] timeax = ft.getAxis('time') if i == 0: tmax0 = timeax.shape[0] tmax = timeax.shape[0] if tmax != tmax0: print 'wrong time axis: exiting...' return # read array isonRead = ft(varb, time=slice(t1, t2), lev=slice(ib, ib1)).squeeze() if varFill[iv] != valmask: isonvar[i, ...] = isonRead.filled(varFill[iv]) else: isonvar[i, ...] = isonRead tim02 = timc.clock() # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data, valmask).mask percent[i, ...] = npy.float32(npy.equal(maskvar, 0)) tim03 = timc.clock() if mme: # if mme then just accumulate Bowl, Agree and Std fields #varst = var+'Agree' #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() isonRead = ft(varb, time=slice(t1, t2), lev=slice(ib, ib1)).squeeze() varbowl2D[i, ...] = isonRead else: # Compute difference with average of first initN years #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) #for t in range(timN): # vardiff[i,t,...] = isonvar[i,t,...] - varinit #vardiff[i,...].mask = isonvar[i,...].mask # Read bowl to truncate field above bowl if ib == 0 and iv == 0: varbowl[i, ...] = ft(varsig, time=slice(t1, t2)) #varbowl[i,...] = bowlRead # Compute Stddev varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0) # Compute ToE if ToeType == 'histnat': toto = 1 # TODO # Read mean and Std dev from histnat # if i == 0: # filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] # #filehn = replace(outFile,'historical','historicalNat') # fthn = cdm.open(filehn) # varmeanhn = fthn(var) # varst = var+'Std' # varmaxstd = fthn(varst) # toemult = 1. # signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) # noise = npy.reshape(varmaxstd,(basN*levN*latN)) # varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) # toemult = 2. # varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) tim04 = timc.clock() ft.close() #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03 # <-- end of loop on files (i) tim1 = timc.clock() # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent, axis=0)) * 100. percenta = mv.masked_less(percenta, 50) percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN]) percentw = cdm.createVariable(percenta, axes=sigmaTimeList, id='isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32'), extend=1, index=ib) # Sign of difference #if mme: # vardiffsgSum = cdu.averager(vardiff, axis=0) # vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo') # vardiffsgSum = maskVal(vardiffsgSum, valmask) # vardiffsgSum.mask = percentw.mask #else: # vardiffsg = npy.copysign(varones,vardiff) # # average signs # vardiffsgSum = cdu.averager(vardiffsg, axis=0) # vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) # vardiffsgSum.mask = percentw.mask # vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN]) isonVarAve = cdm.createVariable(isonVarAve, axes=sigmaTimeList, id='foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask tim2 = timc.clock() # Only keep points with rhon > bowl-delta_rho delta_rho = 0. # mme case if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = npy.reshape(isonVarBowl, [delta_ib, timN, latN, lonN]) isonVarBowl = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id='foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = npy.reshape(isonVarStd, [delta_ib, timN, latN, lonN]) isonVarStd = cdm.createVariable(isonVarStd, axes=sigmaTimeList, id='foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask # Write isonvarbowlw = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id=isonRead.id) isonvarbowlw.long_name = isonRead.long_name isonvarbowlw.units = isonRead.units isonvarstdw = cdm.createVariable(isonVarStd, axes=sigmaTimeList, id=isonRead.id + 'Std') isonvarstdw.long_name = isonRead.long_name + ' intermodel std' isonvarstdw.units = isonRead.units outFile_f.write(isonvarbowlw.astype('float32'), extend=1, index=ib) outFile_f.write(isonvarstdw.astype('float32'), extend=1, index=ib) #if ib == 0 and iv == 0: # # TODO review # # Read multimodel sigma on bowl and average in time # file1d = replace(outDir+'/'+outFile,'2D','1D') # if os.path.isfile(file1d): # f1d = cdm.open(file1d) # else: # print 'ERROR:',file1d,'missing (if mme, run 2D first)' # sys.exit(1) # bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1)) # f1d.close() # siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 #if sw2d == 1: # for il in range(latN): # for ib in range(basN): # #if ib == 2: # # print il, siglimit[ib,il] # if siglimit[ib,il] < valmask/1000.: # # if mme bowl density defined, mask above bowl # index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) # isonVarBowl [:,ib,0:index[0],il].mask = True # isonVarStd [:,ib,0:index[0],il].mask = True # vardiffsgSum[:,ib,0:index[0],il].mask = True # else: # # mask all points # isonVarBowl [:,ib,:,il].mask = True # isonVarStd [:,ib,:,il].mask = True # vardiffsgSum[:,ib,:,il].mask = True # mm case else: isonVarBowl = isonVarAve * 1. # start from variable #isonVarStd = isonVarAve*1. # start from variable if ib == 0 and iv == 0: # build bowl position siglimit = cdu.averager(varbowl, axis=0) # average accross members siglimit = npy.reshape(siglimit, [timN * latN * lonN]) - delta_rho if iv == 0: sigarr = siglimit * 1. sigarr[:] = sigmaGrd[ib] # test i = 60 j = 60 ij = j * lonN + i isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN]) isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask) #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask) isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN]) isonVarBowl = maskVal(isonVarBowl, valmask) #vardiffsgSum = maskVal(vardiffsgSum, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask isonVarStd = maskVal(isonVarStd, valmask) # Write #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id) #isonave.long_name = isonRead.long_name #isonave.units = isonRead.units #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN]) #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree') #isonavediff.long_name = isonRead.long_name #isonavediff.units = isonRead.units isonVarBowl = npy.reshape(isonVarBowl, [delta_ib, timN, latN, lonN]) isonavebowl = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id=isonRead.id + 'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN]) isonmaxstd = cdm.createVariable(isonVarStd, axes=sigmaList, id=isonRead.id + 'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units #outFile_f.write( isonave.astype('float32'), extend = 1, index = ib) #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib) outFile_f.write(isonavebowl.astype('float32'), extend=1, index=ib) outFile_f.write(isonmaxstd.astype('float32'), extend=1, index=ib) tim3 = timc.clock() if ToeType == 'histnat': isontoe1 = cdm.createVariable( varToE1, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE1') isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable( varToE2, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE2') isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib) outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib) tim4 = timc.clock() # <--- end of loop on variables #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3 # <--- end of loop on density print ' ' outFile_f.close() fi.close()
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True): ''' The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 27 Nov 2014 - Rewrite with loop on variables - EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree - EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl - EG 19 Apr 2016 - ToE computation (just for 2D files) - EG 07 Oct 2016 - add 3D file support - EG 21 Nov 2016 - move 3D support to new function - EG 10 jan 2017 - added timeBowl option - TODO : - remove loops - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False t10 = t1 t20 = t2 # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0] + '/' + listFiles[0]) isond0 = fi['isondepth'] # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() latN = isond0.shape[3] levN = isond0.shape[2] basN = isond0.shape[1] varsig = 'ptopsigma' # Declare and open files for writing if os.path.isfile(outDir + '/' + outFile): os.remove(outDir + '/' + outFile) outFile_f = cdm.open(outDir + '/' + outFile, 'w') # Testing mme with less models #listFiles=listFiles[0:4] #timN = isond0.shape[0] timN = t2 - t1 runN = len(listFiles) print ' Number of members:', len(listFiles) valmask = isond0.missing_value[0] varList = [ 'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol' ] varFill = [0., 0., valmask, valmask, 0., 0.] # init arrays (2D rho/lat) percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0. #minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000. varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1. #varList = ['isondepth'] #print ' !!! ### Testing one variable ###' #varList = ['isonthetao'] # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # loop on variables for iv, var in enumerate(varList): # Array inits (2D rho/lat 3D rho/lat/lon) #shapeR = [basN,levN,latN] isonvar = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * valmask print('isonvar shape: ', isonvar.shape) vardiff, varbowl2D = [ npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') for _ in range(2) ] varstd, varToE1, varToE2 = [ npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask for _ in range(3) ] varones = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 1. print ' Variable ', iv, var # loop over files to fill up array for i, file in enumerate(listFiles): ft = cdm.open(inDir[0] + '/' + file) model = file.split('.')[1] timeax = ft.getAxis('time') file1d = replace(inDir[0] + '/' + file, '2D', '1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:', file1d, 'missing (if mme, run 1D first)' sys.exit(1) tmax = timeax.shape[0] if i == 0: tmax0 = tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax - t20 t2 = tmax else: if tmax != tmax0: print 'wrong time axis: exiting...' return # read array # loop over time/density for memory management for it in range(timN): t1r = t1 + it t2r = t1r + 1 isonRead = ft(var, time=slice(t1r, t2r)) if varFill[iv] != valmask: isonvar[i, it, ...] = isonRead.filled(varFill[iv]) else: isonvar[i, it, ...] = isonRead # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data, valmask).mask percent[i, ...] = npy.float32(npy.equal(maskvar, 0)) if mme: # if mme then just accumulate Bowl, Agree fields varst = var + 'Agree' vardiff[i, ...] = ft(varst, time=slice(t1, t2)) varb = var + 'Bowl' varbowl2D[i, ...] = ft(varb, time=slice(t1, t2)) else: # Compute difference with average of first initN years varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0) for t in range(timN): vardiff[i, t, ...] = isonvar[i, t, ...] - varinit vardiff[i, ...].mask = isonvar[i, ...].mask # Read bowl and truncate 2D field above bowl if iv == 0: bowlRead = f1d(varsig, time=slice(t1, t2)) varbowl[i, ...] = bowlRead # Compute Stddev varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0) # Compute ToE if ToeType == 'histnat': # Read mean and Std dev from histnat if i == 0: filehn = glob.glob(inDir[1] + '/cmip5.' + model + '.*zon2D*')[0] #filehn = replace(outFile,'historical','historicalNat') fthn = cdm.open(filehn) varmeanhn = fthn(var) varst = var + 'Std' varmaxstd = fthn(varst) toemult = 1. signal = npy.reshape(isonvar[i, ...] - varmeanhn, (timN, basN * levN * latN)) noise = npy.reshape(varmaxstd, (basN * levN * latN)) varToE1[i, ...] = npy.reshape(findToE(signal, noise, toemult), (basN, levN, latN)) toemult = 2. varToE2[i, ...] = npy.reshape(findToE(signal, noise, toemult), (basN, levN, latN)) ft.close() f1d.close() # <-- end of loop on files # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent, axis=0)) * 100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable( percenta, axes=[time, axesList[1], axesList[2], axesList[3]], id='isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable( vardiffsgSum, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones, vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable( isonVarAve, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Only keep points with rhon > bowl-delta_rho delta_rho = 0. if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = cdm.createVariable( isonVarBowl, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = cdm.createVariable( isonVarStd, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask if iv == 0: # Read mulitmodel sigma on bowl and average in time file1d = replace(outDir + '/' + outFile, '2D', '1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:', file1d, 'missing (if mme, run 1D first)' sys.exit(1) bowlRead = f1d(varsig, time=slice(t1, t2)) f1d.close() siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): #if ib == 2: # print il, siglimit[ib,il] if siglimit[ib, il] < valmask / 1000.: # if mme bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])) isonVarBowl[:, ib, 0:index[0], il].mask = True isonVarStd[:, ib, 0:index[0], il].mask = True vardiffsgSum[:, ib, 0:index[0], il].mask = True else: # mask all points isonVarBowl[:, ib, :, il].mask = True isonVarStd[:, ib, :, il].mask = True vardiffsgSum[:, ib, :, il].mask = True else: isonVarBowl = isonVarAve * 1. # start from variable isonVarStd = isonVarAve * 1. # start from variable if iv == 0: siglimit = cdu.averager(varbowl, axis=0) # average accross members # Average bowl in time if timeBowl == 'mean': siglimit = cdu.averager(siglimit, axis=0) - delta_rho # or take largest sigma over time else: siglimit = npy.ma.max(siglimit, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): if siglimit[ib, il] < valmask / 1000.: # if bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]) )[:, 0] #Add [:,0] for python Yona #import code #code.interact(banner='index', local=dict(locals(), **globals())) isonVarBowl[:, ib, 0:index[0], il].mask = True vardiffsgSum[:, ib, 0:index[0], il].mask = True else: # mask all points vardiffsgSum[:, ib, :, il].mask = True isonVarBowl = maskVal(isonVarBowl, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask if varFill[iv] == valmask: isonVarStd = maskVal(isonVarStd, valmask) # Write isonave = cdm.createVariable( isonVarAve, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable( vardiffsgSum, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units isonavebowl = cdm.createVariable( isonVarBowl, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units if not mme: isonmaxstd = cdm.createVariable( isonVarStd, axes=[axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units outFile_f.write(isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) outFile_f.write(isonavebowl.astype('float32')) if not mme: outFile_f.write(isonmaxstd.astype('float32')) if ToeType == 'histnat': isontoe1 = cdm.createVariable( varToE1, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE1') isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable( varToE2, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE2') isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32')) outFile_f.write(isontoe2.astype('float32')) if mme: isonvarstd = cdm.createVariable( isonVarStd, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ModStd') isonvarstd.long_name = isonRead.long_name + ' intermodel std' isonvarstd.units = isonRead.units outFile_f.write(isonvarstd.astype('float32')) # <--- end of loop on variables outFile_f.close() fi.close()
def basicGm(self, gm_type, projtype="default", lat1=0, lat2=0, lon1=0, lon2=0, rg=False, flip=False, zero=False, transparent=False, mask=False, bigvalues=False): self.x.clear() self.x.setcolormap(None) cdms2.tvariable.TransientVariable.variable_count = 1 loc = locals() exec("gm=vcs.create%s()" % gm_type) gm = loc["gm"] if projtype != "default": p = vcs.createprojection() try: ptype = int(projtype) except BaseException: ptype = projtype p.type = ptype gm.projection = p nm_xtra = "" xtra = {} if lat1 != lat2: if rg: if flip: gm.datawc_y1 = lat2 gm.datawc_y2 = lat1 nm_xtra += "_gmflip" else: gm.datawc_y1 = lat1 gm.datawc_y2 = lat2 xtra["latitude"] = (lat1, lat2) if lat1 < 0: nm_xtra += "_SH" else: nm_xtra += "_NH" if lon1 != lon2: if rg: gm.datawc_x1 = lon1 gm.datawc_x2 = lon2 xtra["longitude"] = (lon1, lon2) nm_xtra += "_%i_%i" % (lon1, lon2) if rg: nm_xtra += "_via_gm" if gm_type == "meshfill": f = cdms2.open(os.path.join(vcs.sample_data, 'sampleCurveGrid4.nc')) else: f = self.clt if gm_type == "vector": u = f("u", **xtra) v = f("v", **xtra) if mask: u = MV2.masked_greater(u, 58.) if zero: u -= u v -= v elif gm_type == "meshfill": s = f("sample", **xtra) if mask: s = MV2.masked_less(s, 1150.) elif bigvalues: s[s < 1150] = 1e40 if zero: s -= s else: s = f("clt", **xtra) if mask: s = MV2.masked_greater(s, 78.) elif bigvalues: s[s > 78] = 1e40 if gm_type in ["1d", "yxvsx", "xyvsy", "xvsy", "scatter"]: s = s(latitude=(20, 20, "cob"), longitude=(112, 112, "cob"), squeeze=1) s2 = MV2.sin(s) if zero: s2 -= s2 if zero: s -= s if bigvalues: gm.levels = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 1.e36] if transparent: cmap = self.x.createcolormap() for i in range(256): # tweaks all colors cmap.setcolorcell(i, 100., 0, 0, i / 2.55) self.x.setcolormap(cmap) if gm_type == "vector": gm.linecolor = [100, 0, 0, 50.] elif gm_type in ["yxvsx", "xyvsy", "yvsx", "scatter", "1d"]: gm.linecolor = [100, 0, 0, 50.] gm.markercolor = [100, 0, 0, 50.] if gm_type == "vector": gm.scale = 4. self.x.plot(u, v, gm, bg=self.bg) elif gm_type in ["scatter", "xvsy"]: self.x.plot(s, s2, gm, bg=self.bg) else: self.x.plot(s, gm, bg=self.bg) fnm = "test_vcs_basic_%s" % gm_type.lower() if mask: fnm += "_masked" elif bigvalues: fnm += "_bigvalues" if projtype != "default": fnm += "_%s_proj" % projtype if zero: fnm += "_zero" if transparent: fnm += "_transparent" fnm += nm_xtra self.checkImage(fnm + '.png', threshold=20)
-0.14836133615864944 -0.17956528904564023 -0.21109198032585794 -0.23846429237248942 -0.2598536549218765 -0.27795672866320387 -0.2939939095159731 -0.30541031366330024 -0.307643559333884 -0.30078421139811795 -0.2841339526883441 -0.26485737397202497 -0.24287299694779327 -0.22379014890999907 -0.20121548204699846 -0.1746486732156772 -0.14585019344118372 -0.12070675757803526 -0.0997891159111037 -0.08229393660994214 -0.06779720501287469 -0.057213385470859794 -0.04875768191096844 -0.0402377347189964 -0.030169328367807245 -0.017560662894847895 -0.006968922654137132 0.0009773980274431048 0.007054306637034288 0.010472286514133042 0.010702384151997032 0.009231553701801242 0.007544033101056543 0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. """.split() data = numpy.array(data,dtype=numpy.float) data = MV2.masked_less(data,-900) #yx.datawc_x1 = 0 #yx.datawc_x2 = 80 ##yx.datawc_y1 =-12 #yx.datawc_y2 = 12 x.plot(data,yx,bg=1) fnm = "test_vcs_1D_datawc_missing.png" x.png(fnm) print "fnm:",fnm print "src:",src ret = checkimage.check_result_image(fnm,src,0.05) sys.exit(ret)
import sys, os import vcs import sys import cdms2 import vtk import os import MV2 bg = not False x = vcs.init() x.setcolormap("rainbow") gm = vcs.createmeshfill() p = vcs.createprojection() ptype = int('0') p.type = ptype gm.projection = p xtra = {} f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc')) s = f("sample", **xtra) gm.mesh = True s = MV2.masked_less(s, 1150.) x.plot(s, gm, bg=bg) x.png('test_vcs_basic_meshfill_masked_0_proj.png')
''' import os, sys sys.path.append(os.path.abspath('../../')) import cdms2, MV2, vcs from region_utils import getAreaOfAllClosedDomains f = cdms2.open('data/snc.nc') data = f('snc', time=slice(1), squeeze=1) print "Contions \n 1. Mask less than 100 \n 2. Mask less than 30 and greater than 70" con = input('Enter Condion Option 1 or 2 : ') if con == 1: mask_condition = MV2.masked_less(data, 100.) value = 'value equal to 100' dirname = 'outplots_eq_100' elif con == 2: c1 = MV2.masked_greater(data, 70.) c2 = MV2.masked_less(data, 30.) mask_condition = MV2.logical_and(c1, c2) value = 'value equal to within 30 to 70' dirname = 'outplots_eq_30_to_70' # end of if con == 1: if not os.path.isdir(dirname): os.mkdir(dirname) print "Output plots will be saved in '%s' directory" % dirname v = vcs.init() # Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods.
import vcs import cdms2 import sys import os import MV2 f = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc")) s = f("clt", time=slice(0, 1), squeeze=1) s = MV2.masked_less(s, 65.) x = vcs.init() gm = x.createisofill() #gm=x.createboxfill() gm.missing = 252 #gm.levels = [65,70,75,80,85,90,95,100] x.plot(s, gm) x.png(gm.g_name) x.interact()
def obscuration_terms3(c1, c2): """ USE THIS VERSION FOR DIFFERENCES OF 2 CLIMATOLOGIES (E.G. AMIP4K, 2xCO2 SLAB RUNS) Compute the components required for the obscuration-affected low cloud feedback These are the terms shown in Eq 4 of Scott et al (2020) DOI: 10.1175/JCLI-D-19-1028.1 L_prime = dunobsc + dobsc + dobsc_cov, where dunobsc = L_R_prime * F_bar (delta unobscured low clouds, i.e., true low cloud feedback) dobsc = L_R_bar * F_prime (delta obscuration by upper level clouds) dobsc_cov = (L_R_prime * F_prime) - climo(L_R_prime * F_prime) (covariance term) """ # c is [mo,tau,ctp,lat,lon] # c is in percent AX = c2.getAxisList() c1 = MV.masked_where(c2.mask, c1) c2 = MV.masked_where(c1.mask, c2) # SPLICE c1 and c2: # MAKE SURE c1 and c2 are the same size!!! if c1.shape != c2.shape: raise RuntimeError('c1 and c2 are NOT the same size!!!') c12 = np.ma.append(c1, c2, axis=0) midpt = len(c1) U12 = MV.sum(MV.sum(c12[:, :, 2:, :], 1), 1) / 100. L12 = c12[:, :, :2, :] / 100. F12 = 1. - U12 F12 = MV.masked_less(F12, 0) F12b = MV.array(np.expand_dims(np.expand_dims(F12, axis=1), axis=1)) F12b = MV.masked_where(L12[:, :1, :1, :].mask, F12b) L_R12 = L12 / F12b sum_L_R12 = MV.sum(MV.sum(L_R12, 1), 1) sum_L_R12b = MV.array( np.expand_dims(np.expand_dims(sum_L_R12, axis=1), axis=1)) sum_L_R12c = np.broadcast_to(sum_L_R12b, L_R12.shape) this = MV.masked_outside(sum_L_R12c, 0, 1) L_R12 = MV.masked_where(this.mask, L_R12) L_R12 = MV.masked_where(sum_L_R12c > 1, L_R12) L_R_prime, L_R_bar = monthly_anomalies(L_R12) F_prime, F_bar = monthly_anomalies(F12b) L_prime, L_bar = monthly_anomalies(L12) # Cannot have negative cloud fractions: L_R_bar[L_R_bar < 0] = 0 F_bar[F_bar < 0] = 0 rep_L_bar = tile_uneven(L_bar, L12) rep_L_R_bar = tile_uneven(L_R_bar, L_R12) rep_F_bar = tile_uneven(F_bar, F12b) # Cannot have negative cloud fractions: L_R_bar[L_R_bar < 0] = 0 F_bar[F_bar < 0] = 0 dobsc = rep_L_R_bar * F_prime dunobsc = L_R_prime * rep_F_bar prime_prime = (L_R_prime * F_prime) dobsc_cov, climo_prime_prime = monthly_anomalies(prime_prime) # Re-scale these anomalies by 2, since we have computed all anomalies w.r.t. # the ctl+pert average rather than w.r.t. the ctl average dobsc *= 2 dunobsc *= 2 dobsc_cov *= 2 return (rep_L_R_bar[midpt:], dobsc[midpt:], dunobsc[midpt:], dobsc_cov[midpt:])
xouter.setAxisList([lat,lon]) # Equivalent ## masked_equal(x, value) ## masked_equal(x, value) = x masked where x == value ## For floating point consider masked_values(x, value) instead. ## masked_greater(x, value) ## masked_greater(x, value) = x masked where x > value ## masked_greater_equal(x, value) ## masked_greater_equal(x, value) = x masked where x >= value xge = MV2.masked_greater_equal(xouter, 120) ## masked_less(x, value) ## masked_less(x, value) = x masked where x < value xl = MV2.masked_less(xouter, 160) ## masked_less_equal(x, value) ## masked_less_equal(x, value) = x masked where x <= value ## masked_not_equal(x, value) ## masked_not_equal(x, value) = x masked where x != value ## masked_outside(x, v1, v2) ## x with mask of all values of x that are outside [v1,v2] xmo = MV2.masked_outside(xouter, 120, 160) ## count(a, axis=None) ## Count of the non-masked elements in a, or along a certain axis. xcount = MV2.count(xmo,0) xcount2 = MV2.count(xmo,1)
print 'Sum of all PFT is', pft_sum2010.shape #x.plot(pft_sum2010[::-1,:],gm,bg=bg) #x.png('pft_sum2010.png') #x.clear() #In case if sum of all PFTs is greater than 1, reduce all PFTs re_pft2010 = MV2.masked_greater(pft_sum2010 - lndf_hurt, 0.).mask for npft in range(19): #print 'npft',npft pft_tmp1 = pft2010[npft, :, :] pft_tmp1 = MV2.choose(re_pft2010, (pft_tmp1, pft_tmp1 * lndf_hurt / pft_sum2010)) pft2010[npft, :, :] = pft_tmp1 #In case if sum of all PFTs is less than 1, increase all PFTs inc_pft2010 = MV2.masked_less(pft_sum2010 - lndf_hurt, 0.).mask for npft in range(19): #print 'npft',npft pft_tmp2 = pft2010[npft, :, :] pft_tmp2 = MV2.choose(inc_pft2010, (pft_tmp2, pft_tmp2 * lndf_hurt / pft_sum2010)) pft2010[npft, :, :] = pft_tmp2 newpftsum = MV2.sum(pft2010, axis=0) + icwtr sys.stdout.flush() pft_back[[nyears - 1], :, :, :] = pft2010 #x.plot(newpftsum[::-1,:],gm,bg=bg) #x.png('newpftsum.png') #x.clear() #if (pft_sum2010.all()==1): # print 'Sum of all PFTs is equal to 1'
lon=cdms2.createAxis(MV2.arange(100)*3.6) lon.designateLongitude() lon.units="degrees_east" lon.id="longitude" lat = cdms2.createAxis(MV2.arange(100)*1.8-90.) lat.id="latitude" lat.designateLatitude() lat.units="degrees_north" lev = cdms2.createAxis([1000.]) lev.id="plev" lev.designateLevel() lev.units="hPa" t=cdms2.createAxis([0,31.]) t.id="time" t.designateTime() t.units="days since 2014" cdutil.setTimeBoundsMonthly(t) a.setAxisList((t,lev,lat,lon)) a=MV2.masked_less(a,.5) grd=cdms2.createGaussianGrid(64) a=a.ascontiguous() a=a.regrid(grd,regridTool="regrid2") a=cdutil.averager(a,axis='txy') assert a[0]==0.7921019540305255
xouter.setAxisList([lat, lon]) # Equivalent ## masked_equal(x, value) ## masked_equal(x, value) = x masked where x == value ## For floating point consider masked_values(x, value) instead. ## masked_greater(x, value) ## masked_greater(x, value) = x masked where x > value ## masked_greater_equal(x, value) ## masked_greater_equal(x, value) = x masked where x >= value xge = MV2.masked_greater_equal(xouter, 120) ## masked_less(x, value) ## masked_less(x, value) = x masked where x < value xl = MV2.masked_less(xouter, 160) ## masked_less_equal(x, value) ## masked_less_equal(x, value) = x masked where x <= value ## masked_not_equal(x, value) ## masked_not_equal(x, value) = x masked where x != value ## masked_outside(x, v1, v2) ## x with mask of all values of x that are outside [v1,v2] xmo = MV2.masked_outside(xouter, 120, 160) ## count(a, axis=None) ## Count of the non-masked elements in a, or along a certain axis. xcount = MV2.count(xmo, 0) xcount2 = MV2.count(xmo, 1)
select = dict(lat=slice(4, -4), lon=slice(4, -4), squeeze=1) slp = f('pslvl', **select) rain = f('rain', **select) u = f('u10m', **select) v = f('v10m', **select) f.close() # Pression de surface slp[:] = generic2d(slp*0.01, 5) map2(slp, fill=False, contour=True, show=False, figsize=(6, 5.5), title='WRF Bretagne', fillcontinents=True, zorder=5, projection='merc', right=1, bottom=.07, lowhighs=True, lowhighs_smooth=9, lowhighs_zorder=5, fillcontinents_color='.95', lowhighs_color=(0, .5, 0), contour_colors=[(0, .5, 0)], drawcoastlines_linewidth=.6) # Pluie cmap_rain = cmap_custom([('0.9', 0), ('b', .8), ('r', 1.)]) rain[:] = MV2.masked_less(rain, 0.1) map2(rain, cmap=cmap_rain, vmin=0., fill='pcolor', fillcontinents=False, show=False, shadow_xoffset=4, shadow_yoffset=-4,shadow_width=4, colorbar_shrink=.7, alpha=.7, shadow=True, shadow_alpha=.3, contour=False, zorder=15) # Vent u[:] = ms2kt(u*5) v[:] = ms2kt(v*5) m = map2((u[::3, ::3], v[::3, ::3]), fill=False, contour=False, barbs=True, projection='merc', quiver_sizes=dict(height=.2, spacing=.15), quiver_linewidths=.8, zorder=10, shadow=True, quiver_alpha=.5, savefigs=__file__, show=False, savefigs_pdf=True, fillcontinents=False)
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True): ''' The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 27 Nov 2014 - Rewrite with loop on variables - EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree - EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl - EG 19 Apr 2016 - ToE computation (just for 2D files) - EG 07 Oct 2016 - add 3D file support - EG 21 Nov 2016 - move 3D support to new function - EG 10 jan 2017 - added timeBowl option - TODO : - remove loops - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False t10 = t1 t20 = t2 # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0]+'/'+listFiles[0]) isond0 = fi['isondepth'] ; # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() latN = isond0.shape[3] levN = isond0.shape[2] basN = isond0.shape[1] varsig='ptopsigma' # Declare and open files for writing if os.path.isfile(outDir+'/'+outFile): os.remove(outDir+'/'+outFile) outFile_f = cdm.open(outDir+'/'+outFile,'w') # Testing mme with less models #listFiles=listFiles[0:4] #timN = isond0.shape[0] timN = t2-t1 runN = len(listFiles) print ' Number of members:',len(listFiles) valmask = isond0.missing_value[0] varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol'] varFill = [0.,0.,valmask,valmask,0.,0.] # init arrays (2D rho/lat) percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0. #minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000. varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1. #varList = ['isondepth'] #print ' !!! ### Testing one variable ###' #varList = ['isonthetao'] # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # loop on variables for iv,var in enumerate(varList): # Array inits (2D rho/lat 3D rho/lat/lon) #shapeR = [basN,levN,latN] isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)] varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)] varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1. print ' Variable ',iv, var # loop over files to fill up array for i,file in enumerate(listFiles): ft = cdm.open(inDir[0]+'/'+file) model = file.split('.')[1] timeax = ft.getAxis('time') file1d = replace(inDir[0]+'/'+file,'2D','1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:',file1d,'missing (if mme, run 1D first)' sys.exit(1) tmax = timeax.shape[0] if i == 0: tmax0 = tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax-t20 t2 = tmax else: if tmax != tmax0: print 'wrong time axis: exiting...' return # read array # loop over time/density for memory management for it in range(timN): t1r = t1 + it t2r = t1r + 1 isonRead = ft(var,time = slice(t1r,t2r)) if varFill[iv] != valmask: isonvar[i,it,...] = isonRead.filled(varFill[iv]) else: isonvar[i,it,...] = isonRead # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data,valmask).mask percent[i,...] = npy.float32(npy.equal(maskvar,0)) if mme: # if mme then just accumulate Bowl, Agree fields varst = var+'Agree' vardiff[i,...] = ft(varst,time = slice(t1,t2)) varb = var+'Bowl' varbowl2D[i,...] = ft(varb,time = slice(t1,t2)) else: # Compute difference with average of first initN years varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) for t in range(timN): vardiff[i,t,...] = isonvar[i,t,...] - varinit vardiff[i,...].mask = isonvar[i,...].mask # Read bowl and truncate 2D field above bowl if iv == 0: bowlRead = f1d(varsig,time = slice(t1,t2)) varbowl[i,...] = bowlRead # Compute Stddev varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0) # Compute ToE if ToeType == 'histnat': # Read mean and Std dev from histnat if i == 0: filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] #filehn = replace(outFile,'historical','historicalNat') fthn = cdm.open(filehn) varmeanhn = fthn(var) varst = var+'Std' varmaxstd = fthn(varst) toemult = 1. signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) noise = npy.reshape(varmaxstd,(basN*levN*latN)) varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) toemult = 2. varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) ft.close() f1d.close() # <-- end of loop on files # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent,axis=0))*100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones,vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Only keep points with rhon > bowl-delta_rho delta_rho = 0. if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask if iv == 0: # Read mulitmodel sigma on bowl and average in time file1d = replace(outDir+'/'+outFile,'2D','1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:',file1d,'missing (if mme, run 1D first)' sys.exit(1) bowlRead = f1d(varsig,time = slice(t1,t2)) f1d.close() siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): #if ib == 2: # print il, siglimit[ib,il] if siglimit[ib,il] < valmask/1000.: # if mme bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) isonVarBowl [:,ib,0:index[0],il].mask = True isonVarStd [:,ib,0:index[0],il].mask = True vardiffsgSum[:,ib,0:index[0],il].mask = True else: # mask all points isonVarBowl [:,ib,:,il].mask = True isonVarStd [:,ib,:,il].mask = True vardiffsgSum[:,ib,:,il].mask = True else: isonVarBowl = isonVarAve*1. # start from variable isonVarStd = isonVarAve*1. # start from variable if iv == 0: siglimit = cdu.averager(varbowl, axis=0) # average accross members # Average bowl in time if timeBowl == 'mean': siglimit = cdu.averager(siglimit, axis=0) - delta_rho # or take largest sigma over time else: siglimit = npy.ma.max(siglimit, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): if siglimit[ib,il] < valmask/1000.: # if bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) isonVarBowl[:,ib,0:index[0],il].mask = True vardiffsgSum[:,ib,0:index[0],il].mask = True else: # mask all points vardiffsgSum[:,ib,:,il].mask = True isonVarBowl = maskVal(isonVarBowl, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask if varFill[iv] == valmask: isonVarStd = maskVal(isonVarStd, valmask) # Write isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units if not mme: isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units outFile_f.write( isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) outFile_f.write(isonavebowl.astype('float32')) if not mme: outFile_f.write( isonmaxstd.astype('float32')) if ToeType == 'histnat': isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1') isontoe1.long_name = 'ToE 1 for '+isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2') isontoe2.long_name = 'ToE 2 for '+isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32')) outFile_f.write(isontoe2.astype('float32')) if mme: isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd') isonvarstd.long_name = isonRead.long_name+' intermodel std' isonvarstd.units = isonRead.units outFile_f.write(isonvarstd.astype('float32')) # <--- end of loop on variables outFile_f.close() fi.close()
def plot_bathy(bathy, shadow=True, contour=True, shadow_stretch=1., shadow_shapiro=False, show=True, shadow_alpha=1., shadow_black=.3, white_deep=False, nmax=30,m=None, alpha=1., zmin=None, zmax=None, **kwargs): """Plot a bathymetry - *lon*: Longitude range. - *lat*: Latitude range. - *show*:Display the figure [default: True] - *pcolor*: Use pcolor instead of contour [default: False] - *contour*: Add line contours [default: True] - *shadow*:Plot south-west shadows instead of filled contours. - *nmax*: Max number of levels for contours [default: 30] - *white_deep*: Deep contours are white [default: False] - All other keyword are passed to :func:`~vacumm.misc.plot.map2` """ # Input bb = bathy if isinstance(bathy, GriddedBathy): bathy = bathy.bathy() if shadow: xxs = getattr(bb, '_xxs', None) yys = getattr(bb, '_yys', None) if xxs is None: lon2d = bb._lon2d lat2d = bb._lat2d elif shadow: xxs = yys = None lon2d,lat2d = meshgrid(get_axis(bathy, -1).getValue(),get_axis(bathy, -2).getValue()) # Masking if 'maxdep' in kwargs: zmin = -maxdep if 'maxalt' in kwargs: zmax = maxalt if zmin is not None: bathy[:] = MV2.masked_less(bathy, zmin) if zmax is not None: bathy[:] = MV2.masked_greater(bathy, zmax) # Default arguments for map if hasattr(bathy, 'long_name'): kwargs.setdefault('title',bathy.long_name) if 'cmap' not in kwargs: vmin, vmax = minmax(bathy) # print 'cmap topo', vmin, vmax kwargs['cmap'] = auto_cmap_topo((kwargs.get('vmin', vmin), kwargs.get('vmax', vmax))) # kwargs.setdefault('ticklabel_size','smaller') kwargs.setdefault('clabel_fontsize', 8) kwargs.setdefault('clabel_alpha',.7*alpha) kwargs.setdefault('clabel_glow_alpha', kwargs['clabel_alpha']) kwargs.setdefault('fill', 'contourf') kwargs['nmax'] = nmax kwargs['show'] = False kwargs['contour'] = contour if shadow: kwargs.setdefault('alpha',.5*alpha) kwargs.setdefault('projection', 'merc') kwargs.setdefault('fmt', BathyFormatter()) kwargs.setdefault('colorbar_format', BathyFormatter()) kwargs.setdefault('units', False) kwargs.setdefault('levels_mode','normal') kwargs.setdefault('bgcolor', '0.8') kwargs.setdefault('contour_linestyle', '-') savefig = kwargs.pop('savefig', None) kwsavefig = kwfilter(kwargs, 'savefig_') # White contour when dark if contour and white_deep: levels = auto_scale(bathy,nmax=nmax) colors = [] nlevel = len(levels) for i in range(nlevel): if i < old_div(nlevel,2): colors.append('w') else: colors.append('k') kwargs.setdefault('contour_colors',tuple(colors)) # Call to map m = map2(bathy, m=m, **kwargs) # Add shadow if shadow: # Filter data = MV.array(bathy,'f',fill_value=0.) if shadow_shapiro: data = shapiro2d(data,fast=True).shape # Gradient grd = deriv2d(data,direction=45.,fast=True,fill_value=0.).filled(0.) grdn = refine(grd, 3) grdn = norm_atan(grdn,stretch=shadow_stretch).clip(0,1.) ; del grd # Grid # im = m.map.imshow(grdn,cmap=P.get_cmap('gist_yarg'),alpha=1) # gist_yarg , YlGnBu if xxs is None or yys is None: xx, yy = m(lon2d,lat2d) xxr = refine(xx, 3) yyr = refine(yy, 3) xxs, yys = meshbounds(xxr, yyr) if isinstance(bb, GriddedBathy): bb._xxs = xxs bb._yys = yys del xx, yy, xxr, yyr # Cmap cmap = cmap_custom(( ((1, )*3, 0), ((shadow_black, )*3, 1) )) # Plot pp = m.map.pcolormesh(xxs, yys, grdn,cmap=cmap)#P.get_cmap('gist_yarg')) pp.set_zorder(.9) pp.set_linewidth(0) pp.set_alpha(shadow_alpha*N.clip(alpha*2, 0, 1)) del grdn # Show it? if savefig: m.savefig(savefig, **kwsavefig) if show: P.show() return m
def mmeAveMsk1D(listFiles, sw2d, years, inDir, outDir, outFile, timeInt, mme, ToeType, fullTS, debug=True): ''' The mmeAveMsk1D() function averages rhon or scalar density bined files with differing masks It ouputs the MME and a percentage of non-masked bins Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - sw2d - dimension of fields to consider (1 or 2) - years(t1,t2) - years for slice read - inDir(str) - input directory where files are stored - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - FfllTS - 0/1: if 1, uses full time serie (ignores years(t1,t2)) - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 9 Dec 2014 - Add agreement on difference with init period - save as <var>Agree - EG 04 Oct 2016 - Add 3D files support TODO: ------ ''' # CDMS initialisation - netCDF compression comp = 1 ; # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] # Find dimension runN = len(listFiles) try: fi = cdm.open(inDir[0]+'/'+listFiles[0]) except: print ' *** file not found ',inDir[0]+'/'+listFiles[0] sys.exit(' Abort') if sw2d == 1: ptopd0 = fi['ptopdepth'] ; # Create variable handle latN = ptopd0.shape[2] basN = ptopd0.shape[1] elif sw2d == 2: ptopd0 = fi['ptopdepthxy'] ; # Create variable handle lonN = ptopd0.shape[2] latN = ptopd0.shape[1] #timN = ptopd0.shape[0] timN = t2-t1 if fullTS: print ' !!! Working on full Time Serie (fullTS = True)' timN = ptopd0.shape[0] t1=0 t2=timN t10 = t1 t20 = t2 # Get grid objects axesList = ptopd0.getAxisList() # Declare and open files for writing if os.path.isfile(outDir+'/'+outFile): os.remove(outDir+'/'+outFile) outFile_f = cdm.open(outDir+'/'+outFile,'w') print ' Number of members:',len(listFiles) valmask = ptopd0.missing_value # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # loop on variables # init percent array if sw2d == 1: varList = ['ptopdepth','ptopsigma','ptopso','ptopthetao','volpers','salpers','tempers'] #varList = ['ptopdepth'] varDim = [1,1,1,1,0,0,0] percent = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*0. elif sw2d == 2: varList = ['ptopdepthxy','ptopsigmaxy','ptopsoxy','ptopthetaoxy'] #varList = ['ptopdepthxy'] varDim = [2,2,2,2] percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0. varFill = [valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask,valmask] axis1D = [time,axesList[1],axesList[2]] axis0D = [time,axesList[1]] print ' timN = ',timN # loop on 1D variables for iv,var in enumerate(varList): ti0 = timc.clock() # Array inits if varDim[iv] == 2: isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask vardiff = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask varones = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1. axisVar = axis1D elif varDim[iv] == 1: isonvar = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask vardiff = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*valmask varones = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1. axisVar = axis1D else: isonvar = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask vardiff = npy.ma.ones([runN,timN,basN], dtype='float32')*valmask varones = npy.ma.ones([runN,timN,basN], dtype='float32')*1. axisVar = axis0D print ' Variable ',iv, var, varDim[iv] # loop over files to fill up array for ic,file in enumerate(listFiles): ft = cdm.open(inDir[0]+'/'+file) timeax = ft.getAxis('time') try: tmax = timeax.shape[0] except: print ic,file, timeax if ic == 0: tmax0 = tmax #print ic,file, tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax-t20 t2 = tmax else: if tmax != tmax0: print 'tmax <> tmax0',tmax,tmax0 print 'wrong time axis: exiting...' return #print 'Time dims:',ic, t1,t2,tmax # read array computeVar = True allVars = ft.variables.keys() if 'ptopsigmaxy' in allVars: computeVar = False if (var == 'ptopsigmaxy') & computeVar: #print ' ic = ',ic # reconstruct from isondepthg and ptopdepthxy isond = ft('isondepthg',time = slice(t1,t2)) #print isond.data.shape, timN*latN*lonN itest = 94*360+150 axesList = isond.getAxisList() levs = axesList[1][:] levN = len(levs) #ti02 = timc.clock() levs3d0 = mv.reshape(npy.tile(levs,latN*lonN),(latN*lonN,levN)) #ti05 = timc.clock() isonRead = npy.ma.ones([timN,latN,lonN], dtype='float32')*valmask for it in range(timN): # loop on time to limit memory usage levs3d = levs3d0*1. depthlo = mv.reshape(vardepth[ic,it,...],latN*lonN) depth3d = npy.reshape(npy.repeat(depthlo,levN),(latN*lonN,levN)) isond3d = mv.reshape(npy.transpose(isond.data[it,...],(1,2,0)),(latN*lonN,levN)) #print isond3d[itest,:] isond3d[isond3d > valmask/10] = 0. #print isond3d[itest,:] isond3dp1 = npy.roll(isond3d,-1,axis=1) isond3dp1[:,-1] = isond3d[:,-1] #print isond3dp1[itest,:] #levs3d[levs3d > 30. ] = 0. # to distinguish bottom masked points from surface masked points #print levs3d[itest,:] levs3d[(depth3d <= isond3d)] = 0. #print levs3d[itest,:] levs3d[(depth3d > isond3dp1)] = 0. #print levs3d[itest,:] #isonwrk = npy.sum(levs3d,axis=1) isonwrk = npy.max(levs3d,axis=1) if it < 0: print ic,it print depthlo[itest] print isond3d[itest,:] print isonwrk[itest] print isonRead[it,...] = mv.reshape(isonwrk,(latN,lonN)) # <-- end of loop on time del (isond3d,isond3dp1); gc.collect() # mask with depthxy and where sigmaxy = 0 isonRead.mask = vardepth.mask[ic,...] isonRead = mv.masked_where(isonRead == 0, isonRead) isonRead.long_name = var isonRead.units = 'sigma_n' isonRead.id = var del (isond,depth3d,levs3d,levs3d0,isonwrk); gc.collect() #ti3 = timc.clock() #print ti02-ti0,ti05-ti02, ti1-ti05,ti12-ti1,ti15-ti12,ti2-ti15,ti3-ti2 #print ti3-ti0 # write ptopsigmaxy if os.path.isfile(inDir[0]+'/work_ptopsigmaxy/'+file): os.remove(inDir[0]+'/work_ptopsigmaxy/'+file) fiout = cdm.open(inDir[0]+'/work_ptopsigmaxy/'+file,'w') if ic == 0: print ' Creating ',inDir[0]+'/work_ptopsigmaxy/'+file isonsigxy = cdm.createVariable(isonRead, axes = axis1D, id = 'ptopsigmaxy') isonsigxy.long_name = 'Density of shallowest persistent ocean on ison' isonsigxy.units = 'sigma_n' fiout.write(isonsigxy.astype('float32')) fiout.close() else: # Direct read of variable isonRead = ft(var,time = slice(t1,t2)) #print isonRead.shape, timN if varFill[iv] != valmask: isonvar[ic,...] = isonRead.filled(varFill[iv]) else: isonvar[ic,...] = isonRead #print isonvar[ic,:,40,100] # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data,valmask).mask percent[ic,...] = npy.float32(npy.equal(maskvar,0)) if mme: # if mme then just average Bowl and Agree fields varst = var+'Agree' vardiff[ic,...] = ft(varst,time = slice(t1,t2)) else: # Compute difference with average of first initN years, use mask of last month varinit = cdu.averager(isonvar[ic,peri1:peri2,...],axis=0) for tr in range(timN): vardiff[ic,tr,...] = isonvar[ic,tr,...] - varinit vardiff[ic,...].mask = isonvar[ic,...].mask ft.close() # <-- end of loop on files # TODO remove masked points at longitudes 0 or 180deg for some models # if ptopdepthxy, keep for ptopsigmaxy computation (reconstruct from isondepthg and ptopdepthxy) if var =='ptopdepthxy': vardepth = isonvar # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent,axis=0))*100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable(percenta, axes = axis1D, id = 'ptoppercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = axisVar , id = 'foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones,vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable(isonVarAve , axes = axisVar , id = 'foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Write isonave = cdm.createVariable(isonVarAve, axes = axisVar, id = isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable(vardiffsgSum, axes = axisVar, id = isonRead.id+'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units outFile_f.write(isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) tf = timc.clock() #print ' time var',tf-ti0 # <--- end of loop on variables outFile_f.close() fi.close()
0.059503571833625334 0.059503571833625334 0.05664014775641405 0.05193557222118004 0.04777129850801233 0.0407139313814465 0.029382624830271705 0.018469399844287374 0.0162382275289592 0.02646680241827459 0.04792041732949079 0.0689138797030203 0.08167038620212037 0.09273558459066569 0.11266293431057901 0.13663018925347364 0.15229174546388072 0.15284435880966177 0.13423845476113883 0.09945904378274077 0.07032267160267985 0.05551039827020481 0.045537187647785464 0.040532491867244946 0.03577527125478327 -999. -999. -999. -0.058062458673116 -0.08764922509099882 -0.11697036914487152 -0.14836133615864944 -0.17956528904564023 -0.21109198032585794 -0.23846429237248942 -0.2598536549218765 -0.27795672866320387 -0.2939939095159731 -0.30541031366330024 -0.307643559333884 -0.30078421139811795 -0.2841339526883441 -0.26485737397202497 -0.24287299694779327 -0.22379014890999907 -0.20121548204699846 -0.1746486732156772 -0.14585019344118372 -0.12070675757803526 -0.0997891159111037 -0.08229393660994214 -0.06779720501287469 -0.057213385470859794 -0.04875768191096844 -0.0402377347189964 -0.030169328367807245 -0.017560662894847895 -0.006968922654137132 0.0009773980274431048 0.007054306637034288 0.010472286514133042 0.010702384151997032 0.009231553701801242 0.007544033101056543 0.004639797857203645 -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. -999. """.split() data = numpy.array(data, dtype=numpy.float) data = MV2.masked_less(data, -900) x.plot(data, yx, bg=1) regression.run(x, "test_vcs_1D_datawc_missing.png")
import vcs import cdms2 import sys import os import MV2 f=cdms2.open(os.path.join(sys.prefix,"sample_data","clt.nc")) s=f("clt",time=slice(0,1),squeeze=1) s=MV2.masked_less(s,65.) x=vcs.init() gm=x.createisofill() #gm=x.createboxfill() gm.missing = 252 #gm.levels = [65,70,75,80,85,90,95,100] x.plot(s,gm) x.png(gm.g_name) x.interact()
if gm_type=="meshfill": f=cdms2.open(os.path.join(vcs.sample_data,'sampleCurveGrid4.nc')) else: f=cdms2.open(os.path.join(vcs.sample_data,'clt.nc')) if gm_type=="vector": u=f("u",**xtra) v=f("v",**xtra) if args.mask: u=MV2.masked_greater(u,58.) if args.zero: u-=u v-=v elif gm_type=="meshfill": s=f("sample",**xtra) if args.mask: s=MV2.masked_less(s,1150.) elif args.bigvalues: s[s < 1150] = 1e40 if args.zero: s-=s else: s=f("clt",**xtra) if args.mask: s=MV2.masked_greater(s,78.) elif args.bigvalues: s[s > 78] = 1e40 if gm_type in ["1d","yxvsx","xyvsy","xvsy","scatter"]: s = s(latitude=(20,20,"cob"),longitude=(112,112,"cob"),squeeze=1) s2=MV2.sin(s) if args.zero: s2-=s2
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True): ''' The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 21 2016 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 21 Nov 2016 - Initial function write - TODO : - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0]+'/'+listFiles[0]) # Switch if only variables below the bowl are present/treated nobowl = True if nobowl: isond0 = fi['isondepthgBowl'] ; # Create variable handle else: isond0 = fi['isondepthg'] ; # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() #time = isond0.getTime() lonN = isond0.shape[3] latN = isond0.shape[2] levN = isond0.shape[1] varsig='ptopsigmaxy' # Limit number of models to 3 for testing of mme #if mme: # listFiles = listFiles[0:2] # print ' !!! ### Testing 3 models ###', listFiles # Declare and open files for writing if os.path.isfile(outDir+'/'+outFile): os.remove(outDir+'/'+outFile) outFile_f = cdm.open(outDir+'/'+outFile,'w') #timN = isond0.shape[0] timN = t2-t1 runN = len(listFiles) print ' Number of members:',len(listFiles) valmask = isond0.missing_value varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg'] varFill = [valmask,valmask,valmask,valmask,valmask] percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0. varbowl = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1. #varList = ['isondepthg'] #print ' !!! ### Testing one variable ###', varList # init sigma axis sigma = cdm.createAxis(npy.float32(range(1))) sigma.id = axesList[1].id sigma.units = axesList[1].units sigma.designateTime() # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # Output axis sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon # init arrays isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask varbowl2D = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask varstd,varToE1,varToE2 = [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)] # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions) delta_ib = 1 print ' Sigma index:' for ib in range(levN): ib1 = ib + delta_ib print ib, tim0 = timc.clock() # loop on variables for iv,var in enumerate(varList): if nobowl: varb = var+'Bowl' else: varb = var if ib == 0: print ' Variable ',iv, varb # loop over files to fill up array for i,file in enumerate(listFiles): tim01 = timc.clock() ft = cdm.open(inDir[0]+'/'+file) model = file.split('.')[1] timeax = ft.getAxis('time') if i == 0: tmax0 = timeax.shape[0] tmax = timeax.shape[0] if tmax != tmax0: print 'wrong time axis: exiting...' return # read array isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze() if varFill[iv] != valmask: isonvar[i,...] = isonRead.filled(varFill[iv]) else: isonvar[i,...] = isonRead tim02 = timc.clock() # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data,valmask).mask percent[i,...] = npy.float32(npy.equal(maskvar,0)) tim03 = timc.clock() if mme: # if mme then just accumulate Bowl, Agree and Std fields #varst = var+'Agree' #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() varbowl2D[i,...] = isonRead else: # Compute difference with average of first initN years #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) #for t in range(timN): # vardiff[i,t,...] = isonvar[i,t,...] - varinit #vardiff[i,...].mask = isonvar[i,...].mask # Read bowl to truncate field above bowl if ib == 0 and iv == 0: varbowl[i,...] = ft(varsig,time = slice(t1,t2)) #varbowl[i,...] = bowlRead # Compute Stddev varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0) # Compute ToE if ToeType == 'histnat': toto=1 # TODO # Read mean and Std dev from histnat # if i == 0: # filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] # #filehn = replace(outFile,'historical','historicalNat') # fthn = cdm.open(filehn) # varmeanhn = fthn(var) # varst = var+'Std' # varmaxstd = fthn(varst) # toemult = 1. # signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) # noise = npy.reshape(varmaxstd,(basN*levN*latN)) # varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) # toemult = 2. # varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) tim04 = timc.clock() ft.close() #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03 # <-- end of loop on files (i) tim1 = timc.clock() # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent,axis=0))*100. percenta = mv.masked_less(percenta, 50) percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN]) percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32'), extend = 1, index = ib) # Sign of difference #if mme: # vardiffsgSum = cdu.averager(vardiff, axis=0) # vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo') # vardiffsgSum = maskVal(vardiffsgSum, valmask) # vardiffsgSum.mask = percentw.mask #else: # vardiffsg = npy.copysign(varones,vardiff) # # average signs # vardiffsgSum = cdu.averager(vardiffsg, axis=0) # vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) # vardiffsgSum.mask = percentw.mask # vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN]) isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask tim2 = timc.clock() # Only keep points with rhon > bowl-delta_rho delta_rho = 0. # mme case if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN]) isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN]) isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask # Write isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id) isonvarbowlw.long_name = isonRead.long_name isonvarbowlw.units = isonRead.units isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std') isonvarstdw.long_name = isonRead.long_name+' intermodel std' isonvarstdw.units = isonRead.units outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib) outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib) #if ib == 0 and iv == 0: # # TODO review # # Read multimodel sigma on bowl and average in time # file1d = replace(outDir+'/'+outFile,'2D','1D') # if os.path.isfile(file1d): # f1d = cdm.open(file1d) # else: # print 'ERROR:',file1d,'missing (if mme, run 2D first)' # sys.exit(1) # bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1)) # f1d.close() # siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 #if sw2d == 1: # for il in range(latN): # for ib in range(basN): # #if ib == 2: # # print il, siglimit[ib,il] # if siglimit[ib,il] < valmask/1000.: # # if mme bowl density defined, mask above bowl # index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) # isonVarBowl [:,ib,0:index[0],il].mask = True # isonVarStd [:,ib,0:index[0],il].mask = True # vardiffsgSum[:,ib,0:index[0],il].mask = True # else: # # mask all points # isonVarBowl [:,ib,:,il].mask = True # isonVarStd [:,ib,:,il].mask = True # vardiffsgSum[:,ib,:,il].mask = True # mm case else: isonVarBowl = isonVarAve*1. # start from variable #isonVarStd = isonVarAve*1. # start from variable if ib == 0 and iv == 0: # build bowl position siglimit = cdu.averager(varbowl, axis=0) # average accross members siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho if iv == 0: sigarr = siglimit*1. sigarr[:] = sigmaGrd[ib] # test i = 60 j = 60 ij = j*lonN+i isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN]) isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask) #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask) isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN]) isonVarBowl = maskVal(isonVarBowl, valmask) #vardiffsgSum = maskVal(vardiffsgSum, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask isonVarStd = maskVal(isonVarStd, valmask) # Write #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id) #isonave.long_name = isonRead.long_name #isonave.units = isonRead.units #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN]) #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree') #isonavediff.long_name = isonRead.long_name #isonavediff.units = isonRead.units isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN]) isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN]) isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units #outFile_f.write( isonave.astype('float32'), extend = 1, index = ib) #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib) outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib) outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib) tim3 = timc.clock() if ToeType == 'histnat': isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1') isontoe1.long_name = 'ToE 1 for '+isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2') isontoe2.long_name = 'ToE 2 for '+isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib) outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib) tim4 = timc.clock() # <--- end of loop on variables #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3 # <--- end of loop on density print ' ' outFile_f.close() fi.close()