def spatial_std(self): """ Ecart-type pour chaque pas de temps """ from genutil import statistics import cdms2 # centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html) self.model.spa_std = list() #Initialise une liste self.obs.spa_std = list() #Initialise une liste # Estimation de la std a chaque pas de temps tps = self.obs.getTime() for i, t in enumerate(tps): self.model.spa_std.append( statistics.std(self.model[i, :, :], axis='yx')) self.obs.spa_std.append( statistics.std(self.obs[i, :, :], axis='yx')) # Transformation en variable cdms2 self.model.spa_std = cdms2.createVariable( self.model.spa_std, typecode='f', id='model_spa_std', attributes=dict(units=self.model.units)) self.obs.spa_std = cdms2.createVariable( self.obs.spa_std, typecode='f', id='obs_spa_std', attributes=dict(units=self.obs.units)) # Ajout de l'axe des temps correspondant a self... self.model.spa_std.setAxis(0, tps) self.obs.spa_std.setAxis(0, tps)
def temporal_std(self): """ Ecart-type en chaque point geographique """ from genutil import statistics from vacumm.misc.grid import get_grid, set_grid # centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html) self.model.temp_std = () #Initialise un tuple self.obs.temp_std = () #Initialise un tuple self.model.temp_std = statistics.std(self.model, axis=0) self.obs.temp_std = statistics.std(self.obs, axis=0) ggm = get_grid(self.model) set_grid(self.model.temp_std, ggm, axes=True) ggo = get_grid(self.obs) set_grid(self.obs.temp_std, ggm, axes=True)
def temporal_std(self): """ Ecart-type en chaque point geographique """ from genutil import statistics from vacumm.misc.grid import get_grid, set_grid # centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html) self.model.temp_std = () #Initialise un tuple self.obs.temp_std = () #Initialise un tuple self.model.temp_std = statistics.std(self.model, axis = 0) self.obs.temp_std = statistics.std(self.obs, axis = 0) ggm = get_grid(self.model) set_grid(self.model.temp_std, ggm, axes=True) ggo = get_grid(self.obs) set_grid(self.obs.temp_std, ggm, axes=True)
def spatial_std(self): """ Ecart-type pour chaque pas de temps """ from genutil import statistics import cdms2 # centered and biased std (cf. http://www2-pcmdi.llnl.gov/cdat/manuals/cdutil/cdat_utilities-2.html) self.model.spa_std = list() #Initialise une liste self.obs.spa_std = list() #Initialise une liste # Estimation de la std a chaque pas de temps tps = self.obs.getTime() for i, t in enumerate(tps): self.model.spa_std.append(statistics.std(self.model[i, :, :],axis='yx')) self.obs.spa_std.append(statistics.std(self.obs[i, :, :],axis='yx')) # Transformation en variable cdms2 self.model.spa_std = cdms2.createVariable(self.model.spa_std,typecode='f',id='model_spa_std', attributes=dict(units=self.model.units)) self.obs.spa_std = cdms2.createVariable(self.obs.spa_std,typecode='f',id='obs_spa_std', attributes=dict(units=self.obs.units)) # Ajout de l'axe des temps correspondant a self... self.model.spa_std.setAxis(0, tps) self.obs.spa_std.setAxis(0, tps)
gui_control.record_command( parent, s, 1 ) elif (t[ this_dim[i] ] == 'awt'): # return the altered weighted averaged dimension weight_slab = parent.panelDM.dim[i_index].new_weight tempslab = cdutil.averager(slab, axis= "(" + slab.getAxis(i_index).id + ")", weight=weight_slab.filled() ) # record sum command s = "\n# Currently, VCDAT cannot record the altered average weight command." gui_control.record_command( parent, s, 1 ) record_okay = 0 elif (t[ this_dim[i] ] == 'gtm'): # return the geometrical mean tempslab = geometricmean(slab, axis = i_index ) # record geometric mean command s = "\n%s=genutil.statistics.geometricmean( %s, axis=%d )" % (slab.id, var_name, i_index) gui_control.record_command( parent, s, 1 ) elif (t[ this_dim[i] ] == 'std'): # return the standard deviation tempslab = std(slab, axis= i_index) # record geometric mean command s = "\n%s=genutil.statistics.std( %s, axis=%d )" % (slab.id, var_name, i_index) gui_control.record_command( parent, s, 1 ) # copy the original's slab's id back to the averaged slab if record_okay == 1: tempslab.id = slab.id gui_control.record_command( parent, ("%s.id = '%s'" % (slab.id, slab.id)), 1 ) slab = tempslab else: i_index += 1 if (parent.menu.squeeze_dim_flg == 1) and (len(tempslab.shape) > 0): if record_okay == 1:
axis="(" + slab.getAxis(i_index).id + ")", weight=weight_slab.filled()) # record sum command s = "\n# Currently, VCDAT cannot record the altered average weight command." gui_control.record_command(parent, s, 1) record_okay = 0 elif (t[this_dim[i]] == 'gtm'): # return the geometrical mean tempslab = geometricmean(slab, axis=i_index) # record geometric mean command s = "\n%s=genutil.statistics.geometricmean( %s, axis=%d )" % ( slab.id, var_name, i_index) gui_control.record_command(parent, s, 1) elif (t[this_dim[i]] == 'std' ): # return the standard deviation tempslab = std(slab, axis=i_index) # record geometric mean command s = "\n%s=genutil.statistics.std( %s, axis=%d )" % ( slab.id, var_name, i_index) gui_control.record_command(parent, s, 1) # copy the original's slab's id back to the averaged slab if record_okay == 1: tempslab.id = slab.id gui_control.record_command(parent, ("%s.id = '%s'" % (slab.id, slab.id)), 1) slab = tempslab else: i_index += 1
plt.bar(np.r_[0, 4, 8], np.r_[cor11, cor12, cor13], width=1) plt.bar(np.r_[1, 5, 9], np.r_[cor21, cor22, cor23], width=1) plt.ylim(0, 1) plt.show() plt.clf() # Test taylor diagrom here: TSstd1, TSstd2 = [], [] TSrms1, TSrms2 = [], [] TScor1, TScor2 = [], [] flag = 1 for idx in range(OBSsp.shape[0]): aaa = ERAsp[idx] bbb = NOAsp[idx] ccc = OBSsp[idx] std0 = stat.std(ccc, axis='yx') std1 = stat.std(aaa, axis='yx') / std0 std2 = stat.std(bbb, axis='yx') / std0 cor1 = stat.correlation(aaa, ccc, axis='yx') cor2 = stat.correlation(bbb, ccc, axis='yx') rms1 = stat.rms(aaa, ccc, axis='yx', centered=0) rms2 = stat.rms(bbb, ccc, axis='yx', centered=0) TSstd1.append(std1) TSstd2.append(std2) TSrms1.append(rms1) TSrms2.append(rms2) TScor1.append(cor1) TScor2.append(cor2) if flag == 1: sm.taylor_diagram(np.array([1.0, np.mean(TSstd1)]), np.array([0.0, np.mean(TSrms1)]),
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True): ''' The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 21 2016 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 21 Nov 2016 - Initial function write - TODO : - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0] + '/' + listFiles[0]) # Switch if only variables below the bowl are present/treated nobowl = True if nobowl: isond0 = fi['isondepthgBowl'] # Create variable handle else: isond0 = fi['isondepthg'] # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() #time = isond0.getTime() lonN = isond0.shape[3] latN = isond0.shape[2] levN = isond0.shape[1] varsig = 'ptopsigmaxy' # Limit number of models to 3 for testing of mme #if mme: # listFiles = listFiles[0:2] # print ' !!! ### Testing 3 models ###', listFiles # Declare and open files for writing if os.path.isfile(outDir + '/' + outFile): os.remove(outDir + '/' + outFile) outFile_f = cdm.open(outDir + '/' + outFile, 'w') #timN = isond0.shape[0] timN = t2 - t1 runN = len(listFiles) print ' Number of members:', len(listFiles) valmask = isond0.missing_value varList = ['isondepthg', 'persistmxy', 'sog', 'thetaog', 'isonthickg'] varFill = [valmask, valmask, valmask, valmask, valmask] percent = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 0. varbowl = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * 1. #varList = ['isondepthg'] #print ' !!! ### Testing one variable ###', varList # init sigma axis sigma = cdm.createAxis(npy.float32(range(1))) sigma.id = axesList[1].id sigma.units = axesList[1].units sigma.designateTime() # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # Output axis sigmaList = [sigma, axesList[2], axesList[3]] # sigma, lat, lon sigmaTimeList = [sigma, time, axesList[2], axesList[3]] # sigma, time, lat, lon # init arrays isonvar = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask varbowl2D = npy.ma.ones([runN, timN, latN, lonN], dtype='float32') * valmask varstd, varToE1, varToE2 = [ npy.ma.ones([runN, latN, lonN], dtype='float32') * valmask for _ in range(3) ] # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions) delta_ib = 1 print ' Sigma index:' for ib in range(levN): ib1 = ib + delta_ib print ib, tim0 = timc.clock() # loop on variables for iv, var in enumerate(varList): if nobowl: varb = var + 'Bowl' else: varb = var if ib == 0: print ' Variable ', iv, varb # loop over files to fill up array for i, file in enumerate(listFiles): tim01 = timc.clock() ft = cdm.open(inDir[0] + '/' + file) model = file.split('.')[1] timeax = ft.getAxis('time') if i == 0: tmax0 = timeax.shape[0] tmax = timeax.shape[0] if tmax != tmax0: print 'wrong time axis: exiting...' return # read array isonRead = ft(varb, time=slice(t1, t2), lev=slice(ib, ib1)).squeeze() if varFill[iv] != valmask: isonvar[i, ...] = isonRead.filled(varFill[iv]) else: isonvar[i, ...] = isonRead tim02 = timc.clock() # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data, valmask).mask percent[i, ...] = npy.float32(npy.equal(maskvar, 0)) tim03 = timc.clock() if mme: # if mme then just accumulate Bowl, Agree and Std fields #varst = var+'Agree' #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() isonRead = ft(varb, time=slice(t1, t2), lev=slice(ib, ib1)).squeeze() varbowl2D[i, ...] = isonRead else: # Compute difference with average of first initN years #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) #for t in range(timN): # vardiff[i,t,...] = isonvar[i,t,...] - varinit #vardiff[i,...].mask = isonvar[i,...].mask # Read bowl to truncate field above bowl if ib == 0 and iv == 0: varbowl[i, ...] = ft(varsig, time=slice(t1, t2)) #varbowl[i,...] = bowlRead # Compute Stddev varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0) # Compute ToE if ToeType == 'histnat': toto = 1 # TODO # Read mean and Std dev from histnat # if i == 0: # filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] # #filehn = replace(outFile,'historical','historicalNat') # fthn = cdm.open(filehn) # varmeanhn = fthn(var) # varst = var+'Std' # varmaxstd = fthn(varst) # toemult = 1. # signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) # noise = npy.reshape(varmaxstd,(basN*levN*latN)) # varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) # toemult = 2. # varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) tim04 = timc.clock() ft.close() #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03 # <-- end of loop on files (i) tim1 = timc.clock() # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent, axis=0)) * 100. percenta = mv.masked_less(percenta, 50) percenta = npy.reshape(percenta, [delta_ib, timN, latN, lonN]) percentw = cdm.createVariable(percenta, axes=sigmaTimeList, id='isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32'), extend=1, index=ib) # Sign of difference #if mme: # vardiffsgSum = cdu.averager(vardiff, axis=0) # vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo') # vardiffsgSum = maskVal(vardiffsgSum, valmask) # vardiffsgSum.mask = percentw.mask #else: # vardiffsg = npy.copysign(varones,vardiff) # # average signs # vardiffsgSum = cdu.averager(vardiffsg, axis=0) # vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) # vardiffsgSum.mask = percentw.mask # vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = npy.reshape(isonVarAve, [delta_ib, timN, latN, lonN]) isonVarAve = cdm.createVariable(isonVarAve, axes=sigmaTimeList, id='foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask tim2 = timc.clock() # Only keep points with rhon > bowl-delta_rho delta_rho = 0. # mme case if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = npy.reshape(isonVarBowl, [delta_ib, timN, latN, lonN]) isonVarBowl = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id='foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = npy.reshape(isonVarStd, [delta_ib, timN, latN, lonN]) isonVarStd = cdm.createVariable(isonVarStd, axes=sigmaTimeList, id='foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask # Write isonvarbowlw = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id=isonRead.id) isonvarbowlw.long_name = isonRead.long_name isonvarbowlw.units = isonRead.units isonvarstdw = cdm.createVariable(isonVarStd, axes=sigmaTimeList, id=isonRead.id + 'Std') isonvarstdw.long_name = isonRead.long_name + ' intermodel std' isonvarstdw.units = isonRead.units outFile_f.write(isonvarbowlw.astype('float32'), extend=1, index=ib) outFile_f.write(isonvarstdw.astype('float32'), extend=1, index=ib) #if ib == 0 and iv == 0: # # TODO review # # Read multimodel sigma on bowl and average in time # file1d = replace(outDir+'/'+outFile,'2D','1D') # if os.path.isfile(file1d): # f1d = cdm.open(file1d) # else: # print 'ERROR:',file1d,'missing (if mme, run 2D first)' # sys.exit(1) # bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1)) # f1d.close() # siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 #if sw2d == 1: # for il in range(latN): # for ib in range(basN): # #if ib == 2: # # print il, siglimit[ib,il] # if siglimit[ib,il] < valmask/1000.: # # if mme bowl density defined, mask above bowl # index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) # isonVarBowl [:,ib,0:index[0],il].mask = True # isonVarStd [:,ib,0:index[0],il].mask = True # vardiffsgSum[:,ib,0:index[0],il].mask = True # else: # # mask all points # isonVarBowl [:,ib,:,il].mask = True # isonVarStd [:,ib,:,il].mask = True # vardiffsgSum[:,ib,:,il].mask = True # mm case else: isonVarBowl = isonVarAve * 1. # start from variable #isonVarStd = isonVarAve*1. # start from variable if ib == 0 and iv == 0: # build bowl position siglimit = cdu.averager(varbowl, axis=0) # average accross members siglimit = npy.reshape(siglimit, [timN * latN * lonN]) - delta_rho if iv == 0: sigarr = siglimit * 1. sigarr[:] = sigmaGrd[ib] # test i = 60 j = 60 ij = j * lonN + i isonVarBowl = npy.reshape(isonVarBowl, [timN * latN * lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN]) isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask) #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask) isonVarBowl = npy.reshape(isonVarBowl, [timN, latN, lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN]) isonVarBowl = maskVal(isonVarBowl, valmask) #vardiffsgSum = maskVal(vardiffsgSum, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask isonVarStd = maskVal(isonVarStd, valmask) # Write #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id) #isonave.long_name = isonRead.long_name #isonave.units = isonRead.units #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN]) #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree') #isonavediff.long_name = isonRead.long_name #isonavediff.units = isonRead.units isonVarBowl = npy.reshape(isonVarBowl, [delta_ib, timN, latN, lonN]) isonavebowl = cdm.createVariable(isonVarBowl, axes=sigmaTimeList, id=isonRead.id + 'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units isonVarStd = npy.reshape(isonVarStd, [delta_ib, latN, lonN]) isonmaxstd = cdm.createVariable(isonVarStd, axes=sigmaList, id=isonRead.id + 'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units #outFile_f.write( isonave.astype('float32'), extend = 1, index = ib) #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib) outFile_f.write(isonavebowl.astype('float32'), extend=1, index=ib) outFile_f.write(isonmaxstd.astype('float32'), extend=1, index=ib) tim3 = timc.clock() if ToeType == 'histnat': isontoe1 = cdm.createVariable( varToE1, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE1') isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable( varToE2, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE2') isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32'), extend=1, index=ib) outFile_f.write(isontoe2.astype('float32'), extend=1, index=ib) tim4 = timc.clock() # <--- end of loop on variables #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3 # <--- end of loop on density print ' ' outFile_f.close() fi.close()
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True): ''' The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 27 Nov 2014 - Rewrite with loop on variables - EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree - EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl - EG 19 Apr 2016 - ToE computation (just for 2D files) - EG 07 Oct 2016 - add 3D file support - EG 21 Nov 2016 - move 3D support to new function - EG 10 jan 2017 - added timeBowl option - TODO : - remove loops - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False t10 = t1 t20 = t2 # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0] + '/' + listFiles[0]) isond0 = fi['isondepth'] # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() latN = isond0.shape[3] levN = isond0.shape[2] basN = isond0.shape[1] varsig = 'ptopsigma' # Declare and open files for writing if os.path.isfile(outDir + '/' + outFile): os.remove(outDir + '/' + outFile) outFile_f = cdm.open(outDir + '/' + outFile, 'w') # Testing mme with less models #listFiles=listFiles[0:4] #timN = isond0.shape[0] timN = t2 - t1 runN = len(listFiles) print ' Number of members:', len(listFiles) valmask = isond0.missing_value[0] varList = [ 'isondepth', 'isonpers', 'isonso', 'isonthetao', 'isonthick', 'isonvol' ] varFill = [0., 0., valmask, valmask, 0., 0.] # init arrays (2D rho/lat) percent = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 0. #minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000. varbowl = npy.ma.ones([runN, timN, basN, latN], dtype='float32') * 1. #varList = ['isondepth'] #print ' !!! ### Testing one variable ###' #varList = ['isonthetao'] # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # loop on variables for iv, var in enumerate(varList): # Array inits (2D rho/lat 3D rho/lat/lon) #shapeR = [basN,levN,latN] isonvar = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * valmask print('isonvar shape: ', isonvar.shape) vardiff, varbowl2D = [ npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') for _ in range(2) ] varstd, varToE1, varToE2 = [ npy.ma.ones([runN, basN, levN, latN], dtype='float32') * valmask for _ in range(3) ] varones = npy.ma.ones([runN, timN, basN, levN, latN], dtype='float32') * 1. print ' Variable ', iv, var # loop over files to fill up array for i, file in enumerate(listFiles): ft = cdm.open(inDir[0] + '/' + file) model = file.split('.')[1] timeax = ft.getAxis('time') file1d = replace(inDir[0] + '/' + file, '2D', '1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:', file1d, 'missing (if mme, run 1D first)' sys.exit(1) tmax = timeax.shape[0] if i == 0: tmax0 = tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax - t20 t2 = tmax else: if tmax != tmax0: print 'wrong time axis: exiting...' return # read array # loop over time/density for memory management for it in range(timN): t1r = t1 + it t2r = t1r + 1 isonRead = ft(var, time=slice(t1r, t2r)) if varFill[iv] != valmask: isonvar[i, it, ...] = isonRead.filled(varFill[iv]) else: isonvar[i, it, ...] = isonRead # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data, valmask).mask percent[i, ...] = npy.float32(npy.equal(maskvar, 0)) if mme: # if mme then just accumulate Bowl, Agree fields varst = var + 'Agree' vardiff[i, ...] = ft(varst, time=slice(t1, t2)) varb = var + 'Bowl' varbowl2D[i, ...] = ft(varb, time=slice(t1, t2)) else: # Compute difference with average of first initN years varinit = cdu.averager(isonvar[i, peri1:peri2, ...], axis=0) for t in range(timN): vardiff[i, t, ...] = isonvar[i, t, ...] - varinit vardiff[i, ...].mask = isonvar[i, ...].mask # Read bowl and truncate 2D field above bowl if iv == 0: bowlRead = f1d(varsig, time=slice(t1, t2)) varbowl[i, ...] = bowlRead # Compute Stddev varstd[i, ...] = npy.ma.std(isonvar[i, ...], axis=0) # Compute ToE if ToeType == 'histnat': # Read mean and Std dev from histnat if i == 0: filehn = glob.glob(inDir[1] + '/cmip5.' + model + '.*zon2D*')[0] #filehn = replace(outFile,'historical','historicalNat') fthn = cdm.open(filehn) varmeanhn = fthn(var) varst = var + 'Std' varmaxstd = fthn(varst) toemult = 1. signal = npy.reshape(isonvar[i, ...] - varmeanhn, (timN, basN * levN * latN)) noise = npy.reshape(varmaxstd, (basN * levN * latN)) varToE1[i, ...] = npy.reshape(findToE(signal, noise, toemult), (basN, levN, latN)) toemult = 2. varToE2[i, ...] = npy.reshape(findToE(signal, noise, toemult), (basN, levN, latN)) ft.close() f1d.close() # <-- end of loop on files # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent, axis=0)) * 100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable( percenta, axes=[time, axesList[1], axesList[2], axesList[3]], id='isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable( vardiffsgSum, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones, vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable( isonVarAve, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Only keep points with rhon > bowl-delta_rho delta_rho = 0. if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = cdm.createVariable( isonVarBowl, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = cdm.createVariable( isonVarStd, axes=[time, axesList[1], axesList[2], axesList[3]], id='foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask if iv == 0: # Read mulitmodel sigma on bowl and average in time file1d = replace(outDir + '/' + outFile, '2D', '1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:', file1d, 'missing (if mme, run 1D first)' sys.exit(1) bowlRead = f1d(varsig, time=slice(t1, t2)) f1d.close() siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): #if ib == 2: # print il, siglimit[ib,il] if siglimit[ib, il] < valmask / 1000.: # if mme bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il])) isonVarBowl[:, ib, 0:index[0], il].mask = True isonVarStd[:, ib, 0:index[0], il].mask = True vardiffsgSum[:, ib, 0:index[0], il].mask = True else: # mask all points isonVarBowl[:, ib, :, il].mask = True isonVarStd[:, ib, :, il].mask = True vardiffsgSum[:, ib, :, il].mask = True else: isonVarBowl = isonVarAve * 1. # start from variable isonVarStd = isonVarAve * 1. # start from variable if iv == 0: siglimit = cdu.averager(varbowl, axis=0) # average accross members # Average bowl in time if timeBowl == 'mean': siglimit = cdu.averager(siglimit, axis=0) - delta_rho # or take largest sigma over time else: siglimit = npy.ma.max(siglimit, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): if siglimit[ib, il] < valmask / 1000.: # if bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib, il]) )[:, 0] #Add [:,0] for python Yona #import code #code.interact(banner='index', local=dict(locals(), **globals())) isonVarBowl[:, ib, 0:index[0], il].mask = True vardiffsgSum[:, ib, 0:index[0], il].mask = True else: # mask all points vardiffsgSum[:, ib, :, il].mask = True isonVarBowl = maskVal(isonVarBowl, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask if varFill[iv] == valmask: isonVarStd = maskVal(isonVarStd, valmask) # Write isonave = cdm.createVariable( isonVarAve, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable( vardiffsgSum, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units isonavebowl = cdm.createVariable( isonVarBowl, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units if not mme: isonmaxstd = cdm.createVariable( isonVarStd, axes=[axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units outFile_f.write(isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) outFile_f.write(isonavebowl.astype('float32')) if not mme: outFile_f.write(isonmaxstd.astype('float32')) if ToeType == 'histnat': isontoe1 = cdm.createVariable( varToE1, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE1') isontoe1.long_name = 'ToE 1 for ' + isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable( varToE2, axes=[ensembleAxis, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ToE2') isontoe2.long_name = 'ToE 2 for ' + isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32')) outFile_f.write(isontoe2.astype('float32')) if mme: isonvarstd = cdm.createVariable( isonVarStd, axes=[time, axesList[1], axesList[2], axesList[3]], id=isonRead.id + 'ModStd') isonvarstd.long_name = isonRead.long_name + ' intermodel std' isonvarstd.units = isonRead.units outFile_f.write(isonvarstd.astype('float32')) # <--- end of loop on variables outFile_f.close() fi.close()
def monsoon_wang_runner(args): # args = P.parse_args(sys.argv[1:]) modpath = genutil.StringConstructor(args.test_data_path) modpath.variable = args.modvar outpathdata = args.results_dir if isinstance(args.modnames, str): mods = eval(args.modnames) else: mods = args.modnames json_filename = args.outnamejson if json_filename == "CMIP_MME": json_filename = "/MPI_" + args.mip + "_" + args.experiment # VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES var = args.modvar thr = args.threshold sig_digits = ".3f" # Get flag for CMEC output cmec = args.cmec ######################################### # PMP monthly default PR obs cdms2.axis.longitude_aliases.append("longitude_prclim_mpd") cdms2.axis.latitude_aliases.append("latitude_prclim_mpd") fobs = cdms2.open(args.reference_data_path) dobs_orig = fobs(args.obsvar) fobs.close() obsgrid = dobs_orig.getGrid() ######################################## # FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX annrange_obs, mpi_obs = mpd(dobs_orig) ######################################### # SETUP WHERE TO OUTPUT RESULTING DATA (netcdf) nout = os.path.join(outpathdata, "_".join([args.experiment, args.mip, "wang-monsoon"])) try: os.makedirs(nout) except BaseException: pass # SETUP WHERE TO OUTPUT RESULTS (json) jout = outpathdata try: os.makedirs(nout) except BaseException: pass gmods = [] # "Got" these MODS for i, mod in enumerate(mods): modpath.model = mod for k in modpath.keys(): try: val = getattr(args, k) except Exception: continue if not isinstance(val, (list, tuple)): setattr(modpath, k, val) else: setattr(modpath, k, val[i]) l1 = modpath() if os.path.isfile(l1) is True: gmods.append(mod) if len(gmods) == 0: raise RuntimeError("No model file found!") ######################################### egg_pth = resources.resource_path() globals = {} locals = {} exec( compile( open(os.path.join(egg_pth, "default_regions.py")).read(), os.path.join(egg_pth, "default_regions.py"), "exec", ), globals, locals, ) regions_specs = locals["regions_specs"] doms = ["AllMW", "AllM", "NAMM", "SAMM", "NAFM", "SAFM", "ASM", "AUSM"] mpi_stats_dic = {} for i, mod in enumerate(gmods): modpath.model = mod for k in modpath.keys(): try: val = getattr(args, k) except Exception: continue if not isinstance(val, (list, tuple)): setattr(modpath, k, val) else: setattr(modpath, k, val[i]) modelFile = modpath() mpi_stats_dic[mod] = {} print( "******************************************************************************************" ) print(modelFile) f = cdms2.open(modelFile) d_orig = f(var) annrange_mod, mpi_mod = mpd(d_orig) annrange_mod = annrange_mod.regrid(obsgrid, regridTool="regrid2", regridMethod="conserve", mkCyclic=True) mpi_mod = mpi_mod.regrid(obsgrid, regridTool="regrid2", regridMethod="conserve", mkCyclic=True) for dom in doms: mpi_stats_dic[mod][dom] = {} reg_sel = regions_specs[dom]["domain"] mpi_obs_reg = mpi_obs(reg_sel) mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis="xy")) mpi_mod_reg = mpi_mod(reg_sel) cor = float( statistics.correlation(mpi_mod_reg, mpi_obs_reg, axis="xy")) rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis="xy")) rmsn = rms / mpi_obs_reg_sd # DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS annrange_mod_dom = annrange_mod(reg_sel) annrange_obs_dom = annrange_obs(reg_sel) # SKILL SCORES # HIT/(HIT + MISSED + FALSE ALARMS) hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores( annrange_mod_dom, annrange_obs_dom, thr) # POPULATE DICTIONARY FOR JSON FILES mpi_stats_dic[mod][dom] = {} mpi_stats_dic[mod][dom]["cor"] = format(cor, sig_digits) mpi_stats_dic[mod][dom]["rmsn"] = format(rmsn, sig_digits) mpi_stats_dic[mod][dom]["threat_score"] = format(score, sig_digits) # SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM fm = os.path.join(nout, "_".join([mod, dom, "wang-monsoon.nc"])) g = cdms2.open(fm, "w") g.write(annrange_mod_dom) g.write(hitmap, dtype=numpy.int32) g.write(missmap, dtype=numpy.int32) g.write(falarmmap, dtype=numpy.int32) g.close() f.close() # OUTPUT METRICS TO JSON FILE OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename) disclaimer = open(os.path.join(egg_pth, "disclaimer.txt")).read() metrics_dictionary = collections.OrderedDict() metrics_dictionary["DISCLAIMER"] = disclaimer metrics_dictionary["REFERENCE"] = ( "The statistics in this file are based on" + " Wang, B., Kim, HJ., Kikuchi, K. et al. " + "Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0") metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict() OUT.var = var OUT.write( metrics_dictionary, json_structure=["model", "domain", "statistic"], indent=4, separators=(",", ": "), ) if cmec: print("Writing cmec file") OUT.write_cmec(indent=4, separators=(",", ": "))
def monsoon_wang_runner(args): # args = P.parse_args(sys.argv[1:]) modpath = args.modpath outpathjsons = args.outpathjsons outpathdata = args.results_dir if isinstance(args.modnames, str): mods = eval(args.modnames) else: mods = args.modnames json_filename = args.jsonname if json_filename == 'CMIP_MME': json_filename = '/MPI_' + args.mip + '_' + args.experiment if args.mip == 'CMIP5' and args.experiment == 'historical' and mods is None: mods = [ 'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM', 'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5', 'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM', 'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2', 'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M', 'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM', 'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P', 'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME'] # VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES var = args.modvar thr = args.threshold sig_digits = '.3f' ######################################### # PMP monthly default PR obs cdms2.axis.longitude_aliases.append("longitude_prclim_mpd") cdms2.axis.latitude_aliases.append("latitude_prclim_mpd") fobs = cdms2.open(args.reference_data_path) dobs_orig = fobs(args.obsvar) fobs.close() obsgrid = dobs_orig.getGrid() ######################################## # FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX annrange_obs, mpi_obs = mpd(dobs_orig) ######################################### # SETUP WHERE TO OUTPUT RESULTING DATA (netcdf) nout = os.path.join(outpathdata, "_".join( [args.experiment, args.mip, 'wang-monsoon'])) try: os.makedirs(nout) except BaseException: pass # SETUP WHERE TO OUTPUT RESULTS (json) jout = outpathjsons try: os.makedirs(nout) except BaseException: pass modpathall = modpath.replace('MODS', '*') lst = glob.glob(modpathall) # CONFIRM DATA FOR MODS IS AVAIL AND REMOVE THOSE IT IS NOT gmods = [] # "Got" these MODS for mod in mods: for l in lst: l1 = modpath.replace('MODS', mod) if os.path.isfile(l1) is True: if mod not in gmods: gmods.append(mod) if args.experiment == 'historical' and mods is None: gmods = [ 'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM', 'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5', 'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM', 'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2', 'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M', 'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM', 'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P', 'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME'] ######################################### try: egg_pth = pkg_resources.resource_filename( pkg_resources.Requirement.parse("pcmdi_metrics"), "share/pmp") except Exception: # python 2 seems to fail when ran in home directory of source? egg_pth = os.path.join(os.getcwd(), "share", "pmp") globals = {} locals = {} exec(compile(open(os.path.join(egg_pth, "default_regions.py")).read(), os.path.join(egg_pth, "default_regions.py"), 'exec'), globals, locals) regions_specs = locals["regions_specs"] doms = ['AllMW', 'AllM', 'NAMM', 'SAMM', 'NAFM', 'SAFM', 'ASM', 'AUSM'] mpi_stats_dic = {} for mod in gmods: modelFile = modpath.replace('MODS', mod) mpi_stats_dic[mod] = {} print("******************************************************************************************") print(modelFile) f = cdms2.open(modelFile) d_orig = f(var) annrange_mod, mpi_mod = mpd(d_orig) annrange_mod = annrange_mod.regrid(obsgrid) mpi_mod = mpi_mod.regrid(obsgrid) for dom in doms: mpi_stats_dic[mod][dom] = {} reg_sel = regions_specs[dom]['domain'] mpi_obs_reg = mpi_obs(reg_sel) mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy')) mpi_mod_reg = mpi_mod(reg_sel) cor = float( statistics.correlation( mpi_mod_reg, mpi_obs_reg, axis='xy')) rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy')) rmsn = rms / mpi_obs_reg_sd # DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS annrange_mod_dom = annrange_mod(reg_sel) annrange_obs_dom = annrange_obs(reg_sel) # SKILL SCORES # HIT/(HIT + MISSED + FALSE ALARMS) hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores( annrange_mod_dom, annrange_obs_dom, thr) # POPULATE DICTIONARY FOR JSON FILES mpi_stats_dic[mod][dom] = {} mpi_stats_dic[mod][dom]['cor'] = format(cor, sig_digits) mpi_stats_dic[mod][dom]['rmsn'] = format(rmsn, sig_digits) mpi_stats_dic[mod][dom]['threat_score'] = format(score, sig_digits) # SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM fm = os.path.join(nout, '_'.join([mod, dom, 'wang-monsoon.nc'])) g = cdms2.open(fm, 'w') g.write(annrange_mod_dom) g.write(hitmap, dtype=numpy.int32) g.write(missmap, dtype=numpy.int32) g.write(falarmmap, dtype=numpy.int32) g.close() f.close() # OUTPUT METRICS TO JSON FILE OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename) disclaimer = open( os.path.join( egg_pth, "disclaimer.txt")).read() metrics_dictionary = collections.OrderedDict() metrics_dictionary["DISCLAIMER"] = disclaimer metrics_dictionary["REFERENCE"] = "The statistics in this file are based on" +\ " Wang, B., Kim, HJ., Kikuchi, K. et al. " +\ "Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0" metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict() OUT.var = var OUT.write( metrics_dictionary, json_structure=["model", "domain", "statistic"], indent=4, separators=( ',', ': '))
print(modelFile) f = cdms2.open(modelFile) d_orig = f(var) annrange_mod, mpi_mod = mpd(d_orig) annrange_mod = annrange_mod.regrid(obsgrid) mpi_mod = mpi_mod.regrid(obsgrid) for dom in doms: mpi_stats_dic[mod][dom] = {} reg_sel = regions_specs[dom]['domain'] mpi_obs_reg = mpi_obs(reg_sel) mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy')) mpi_mod_reg = mpi_mod(reg_sel) cor = float(statistics.correlation(mpi_mod_reg, mpi_obs_reg, axis='xy')) rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy')) rmsn = rms / mpi_obs_reg_sd # DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS annrange_mod_dom = annrange_mod(reg_sel) annrange_obs_dom = annrange_obs(reg_sel) # SKILL SCORES # HIT/(HIT + MISSED + FALSE ALARMS) hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores( annrange_mod_dom, annrange_obs_dom, thr)
def monsoon_wang_runner(args): # args = P.parse_args(sys.argv[1:]) modpath = args.modpath outpathjsons = args.outpathjsons outpathdata = args.results_dir if isinstance(args.modnames, str): mods = eval(args.modnames) else: mods = args.modnames json_filename = args.jsonname if json_filename == 'CMIP_MME': json_filename = '/MPI_' + args.mip + '_' + args.experiment if args.mip == 'CMIP5' and args.experiment == 'historical' and mods is None: mods = [ 'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM', 'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5', 'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM', 'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2', 'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M', 'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM', 'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P', 'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME' ] # VAR IS FIXED TO BE PRECIP FOR CALCULATING MONSOON PRECIPITATION INDICES var = args.modvar thr = args.threshold sig_digits = '.3f' ######################################### # PMP monthly default PR obs cdms2.axis.longitude_aliases.append("longitude_prclim_mpd") cdms2.axis.latitude_aliases.append("latitude_prclim_mpd") fobs = cdms2.open(args.reference_data_path) dobs_orig = fobs(args.obsvar) fobs.close() obsgrid = dobs_orig.getGrid() ######################################## # FCN TO COMPUTE GLOBAL ANNUAL RANGE AND MONSOON PRECIP INDEX annrange_obs, mpi_obs = mpd(dobs_orig) ######################################### # SETUP WHERE TO OUTPUT RESULTING DATA (netcdf) nout = os.path.join(outpathdata, "_".join([args.experiment, args.mip, 'wang-monsoon'])) try: os.makedirs(nout) except BaseException: pass # SETUP WHERE TO OUTPUT RESULTS (json) jout = outpathjsons try: os.makedirs(nout) except BaseException: pass modpathall = modpath.replace('MODS', '*') lst = glob.glob(modpathall) # CONFIRM DATA FOR MODS IS AVAIL AND REMOVE THOSE IT IS NOT gmods = [] # "Got" these MODS for mod in mods: for l in lst: l1 = modpath.replace('MODS', mod) if os.path.isfile(l1) is True: if mod not in gmods: gmods.append(mod) if args.experiment == 'historical' and mods is None: gmods = [ 'ACCESS1-0', 'ACCESS1-3', 'bcc-csm1-1', 'bcc-csm1-1-m', 'BNU-ESM', 'CanCM4', 'CanESM2', 'CCSM4', 'CESM1-BGC', 'CESM1-CAM5', 'CESM1-FASTCHEM', 'CESM1-WACCM', 'CMCC-CESM', 'CMCC-CM', 'CMCC-CMS', 'CNRM-CM5-2', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'FGOALS-g2', 'FIO-ESM', 'GFDL-CM2p1', 'GFDL-CM3', 'GFDL-ESM2G', 'GFDL-ESM2M', 'GISS-E2-H', 'GISS-E2-H-CC', 'GISS-E2-R', 'GISS-E2-R-CC', 'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'inmcm4', 'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC4h', 'MIROC5', 'MIROC-ESM', 'MIROC-ESM-CHEM', 'MPI-ESM-LR', 'MPI-ESM-MR', 'MPI-ESM-P', 'MRI-CGCM3', 'MRI-ESM1', 'NorESM1-M', 'NorESM1-ME' ] ######################################### try: egg_pth = pkg_resources.resource_filename( pkg_resources.Requirement.parse("pcmdi_metrics"), "share/pmp") except Exception: # python 2 seems to fail when ran in home directory of source? egg_pth = os.path.join(os.getcwd(), "share", "pmp") globals = {} locals = {} exec( compile( open(os.path.join(egg_pth, "default_regions.py")).read(), os.path.join(egg_pth, "default_regions.py"), 'exec'), globals, locals) regions_specs = locals["regions_specs"] doms = ['AllMW', 'AllM', 'NAMM', 'SAMM', 'NAFM', 'SAFM', 'ASM', 'AUSM'] mpi_stats_dic = {} for mod in gmods: modelFile = modpath.replace('MODS', mod) mpi_stats_dic[mod] = {} print( "******************************************************************************************" ) print(modelFile) f = cdms2.open(modelFile) d_orig = f(var) annrange_mod, mpi_mod = mpd(d_orig) annrange_mod = annrange_mod.regrid(obsgrid) mpi_mod = mpi_mod.regrid(obsgrid) for dom in doms: mpi_stats_dic[mod][dom] = {} reg_sel = regions_specs[dom]['domain'] mpi_obs_reg = mpi_obs(reg_sel) mpi_obs_reg_sd = float(statistics.std(mpi_obs_reg, axis='xy')) mpi_mod_reg = mpi_mod(reg_sel) cor = float( statistics.correlation(mpi_mod_reg, mpi_obs_reg, axis='xy')) rms = float(statistics.rms(mpi_mod_reg, mpi_obs_reg, axis='xy')) rmsn = rms / mpi_obs_reg_sd # DOMAIN SELECTED FROM GLOBAL ANNUAL RANGE FOR MODS AND OBS annrange_mod_dom = annrange_mod(reg_sel) annrange_obs_dom = annrange_obs(reg_sel) # SKILL SCORES # HIT/(HIT + MISSED + FALSE ALARMS) hit, missed, falarm, score, hitmap, missmap, falarmmap = mpi_skill_scores( annrange_mod_dom, annrange_obs_dom, thr) # POPULATE DICTIONARY FOR JSON FILES mpi_stats_dic[mod][dom] = {} mpi_stats_dic[mod][dom]['cor'] = format(cor, sig_digits) mpi_stats_dic[mod][dom]['rmsn'] = format(rmsn, sig_digits) mpi_stats_dic[mod][dom]['threat_score'] = format(score, sig_digits) # SAVE ANNRANGE AND HIT MISS AND FALSE ALARM FOR EACH MOD DOM fm = os.path.join(nout, '_'.join([mod, dom, 'wang-monsoon.nc'])) g = cdms2.open(fm, 'w') g.write(annrange_mod_dom) g.write(hitmap, dtype=numpy.int32) g.write(missmap, dtype=numpy.int32) g.write(falarmmap, dtype=numpy.int32) g.close() f.close() # OUTPUT METRICS TO JSON FILE OUT = pcmdi_metrics.io.base.Base(os.path.abspath(jout), json_filename) disclaimer = open(os.path.join(egg_pth, "disclaimer.txt")).read() metrics_dictionary = collections.OrderedDict() metrics_dictionary["DISCLAIMER"] = disclaimer metrics_dictionary["REFERENCE"] = "The statistics in this file are based on" +\ " Wang, B., Kim, HJ., Kikuchi, K. et al. " +\ "Clim Dyn (2011) 37: 941. doi:10.1007/s00382-010-0877-0" metrics_dictionary["RESULTS"] = mpi_stats_dic # collections.OrderedDict() OUT.var = var OUT.write(metrics_dictionary, json_structure=["model", "domain", "statistic"], indent=4, separators=(',', ': '))
def getARData(slab, quslab, qvslab, anoslab, quano, qvano, areas, mask_list, axis_list, timestr, param_dict, shift_lon, isplot, outputdir): '''Fetch AR related data Args: slab (cdms.TransientVariable): (n * m) 2D array of IVT, in kg/m/s. quslab (cdms.TransientVariable): (n * m) 2D array of u-flux, in kg/m/s. qvslab (cdms.TransientVariable): (n * m) 2D array of v-flux, in kg/m/s. anoslab (cdms.TransientVariable): (n * m) 2D array of IVT anomalies, in kg/m/s. quano (cdms.TransientVariable): (n * m) 2D array of u-flux anomalies, in kg/m/s. qvano (cdms.TransientVariable): (n * m) 2D array of v-flux anomalies, in kg/m/s. areas (cdms.TransientVariable): (n * m) 2D grid cell area slab, in km^2. mask_list (list): list of 2D binary masks, each with the same shape as <anoslab> etc., and with 1s denoting the location of a found AR. axis_list (list): list of AR axis coordinates. Each coordinate is defined as a Nx2 ndarray storing (y, x) indices of the axis (indices defined in the matrix of corresponding mask in <masks>.) timestr (str): string of time snap. param_dict (dict): parameter dict defined in Global preamble. shift_lon (float): starting longitude of data domain, defined in Global preamble. isplot (bool): if True, create plot of AR axis, flux orientation and cross-sectional flux for each AR. outputdir (str): folder to save plots. If None, don't save. If <isplot> is False, not relevant. Returns: labels (cdms.TransientVariable): (n * m) 2D int map showing all ARs at current time. Each AR is labeled by an int label, starting from 1. Background is filled with 0s. angles (cdms.TransientVariable): (n * m) 2D map showing orientation differences between AR axes and fluxes, for all ARs. In degrees. crossfluxes (cdms.TransientVariable): (n * m) 2D map showing cross- sectional fluxes in all ARs. In kg/m/s. anocrossflux (cdms.TransientVariable): similar as <crossfluxes> but for anomalous fluxes (corresponding to <anoslab>). df (pandas.DataFrame): AR record table. Each row is an AR, see code below for columns. ''' max_isoq = param_dict['max_isoq'] min_length = param_dict['min_length'] min_length_hard = param_dict['min_length_hard'] rdp_thres = param_dict['rdp_thres'] min_area = param_dict['min_area'] lonax = slab.getLongitude() # NOTE: max > 360 latax = slab.getLatitude() # prepare outputs labels = MV.zeros(slab.shape) angles = MV.zeros(slab.shape) crossfluxes = MV.zeros(slab.shape) results = {} #-----------------Loop through ARs----------------- for ii in range(len(mask_list)): maskii = mask_list[ii] # region properties, in pixel units rpii = measure.regionprops(maskii, intensity_image=np.array(slab))[0] # get centroid centroidy, centroidx = rpii.weighted_centroid centroidy = latax[int(centroidy)] centroidx = lonax[int(centroidx)] # get axis coordinate array skelii = axis_list[ii] latsii = latax[skelii[:, 0]] lonsii = lonax[skelii[:, 1]] axisii = np.c_[latsii, lonsii] # segment axis using rdp axis_rdpii = np.array(rdp.rdpGC(axisii.tolist(), rdp_thres)) # lat,lon # area areaii = (maskii * areas).sum() # km^2 # compute length lens = funcs.greatCircle(axis_rdpii[:-1, 0], axis_rdpii[:-1, 1], axis_rdpii[1:, 0], axis_rdpii[1:, 1]) / 1e3 lenii = lens.sum() #km # skip if too small and too short if areaii < min_area or lenii < min_length_hard: continue # mean width widthii = areaii / lenii # km # mask contour contii = funcs.getBinContour(maskii, lonax, latax) # isoperimetric quotient isoquoii = 4 * np.pi * rpii.area / rpii.perimeter**2 # length/width ratio ratioii = lenii / widthii # mean strength slabii = MV.masked_where(maskii == 0, slab) strengthii = cdutil.averager(slabii, axis='xy', weights=['generate', 'generate']) # strength std strengthstdii = float(stats.std(slabii, axis='xy')) # anomaly strength anoslabii = MV.masked_where(maskii == 0, anoslab) anostrengthii = cdutil.averager(anoslabii, axis='xy', weights=['generate', 'generate']) # max strength max_strengthii = float(MV.max(slabii)) # compute angles and cross-section flux of total flux cropmask, cropidx = cropMask(maskii) cropskelii = skelii - np.array([cropidx[0].min(), cropidx[1].min()]) cropu = applyCropIdx(quslab, cropidx) cropv = applyCropIdx(qvslab, cropidx) anglesii, anglesmeanii, crossfluxii, seg_thetasii = crossSectionFlux( cropmask, cropu, cropv, axis_rdpii) # create plots if isplot: pass #plotARCrosssectionFlux(cropmask, cropu, cropv, cropskelii, axis_rdpii, #'%s AR-%d' %(timestr, ii+1), shift_lon, anglesii, anglesmeanii, #crossfluxii, seg_thetasii, outputdir) # insert crop back to the big map anglesii = insertCropSlab(maskii.shape, anglesii, cropidx, slab.getAxisList()) anglesii = MV.where(maskii == 1, anglesii, 0) crossfluxii = insertCropSlab(maskii.shape, crossfluxii, cropidx, slab.getAxisList()) crossfluxii = MV.where(maskii == 1, crossfluxii, 0) # mean meridional flux cropv = applyCropIdx(qvslab, cropidx) cropv = MV.masked_where(cropmask == 0, cropv) qvmeanii=cdutil.averager(cropv,axis='xy',weights=['generate',\ 'generate']) # is candidate a strict AR is_relaxedii = False if isoquoii > max_isoq or ratioii < 2: is_relaxedii = True if lenii < min_length: is_relaxedii = True if qvmeanii <= 0: is_relaxedii = True labels = labels + maskii * (ii + 1) angles = angles + anglesii crossfluxes = crossfluxes + crossfluxii results[ii + 1] = { 'id': ii + 1, 'time': timestr, 'contour_y': contii.vertices[:, 1], 'contour_x': contii.vertices[:, 0], 'centroid_y': centroidy, 'centroid_x': centroidx, 'axis_y': axisii[:, 0], 'axis_x': axisii[:, 1], 'axis_rdp_y': axis_rdpii[:, 0], 'axis_rdp_x': axis_rdpii[:, 1], 'area': areaii, 'length': lenii, 'width': widthii, 'iso_quotient': isoquoii, 'LW_ratio': ratioii, 'strength': strengthii, 'strength_ano': anostrengthii, 'strength_std': strengthstdii, 'max_strength': max_strengthii, 'mean_angle': float(anglesmeanii), 'is_relaxed': is_relaxedii, 'qv_mean': qvmeanii } labels.setAxisList(slab.getAxisList()) angles.setAxisList(slab.getAxisList()) crossfluxes.setAxisList(slab.getAxisList()) labels.id = 'labels' labels.long_name = 'AR labels' labels.standard_name = labels.long_name labels.title = labels.long_name labels.units = '' angles.id = 'angles' angles.long_name = 'AR moisture flux orientation difference' angles.standard_name = angles.long_name angles.title = angles.long_name angles.units = 'degree' crossfluxes.id = 'ivt_cross' crossfluxes.long_name = 'AR total cross sectional moisture flux' crossfluxes.standard_name = crossfluxes.long_name crossfluxes.title = crossfluxes.long_name crossfluxes.units = getattr(slab, 'units', '') keys = [ 'id', 'time', 'contour_y', 'contour_x', 'centroid_y', 'centroid_x', 'axis_y', 'axis_x', 'axis_rdp_y', 'axis_rdp_x', 'area', 'length', 'width', 'iso_quotient', 'LW_ratio', 'strength', 'strength_ano', 'strength_std', 'max_strength', 'mean_angle', 'is_relaxed', 'qv_mean' ] df = pd.DataFrame(results).T if len(df) > 0: df = df[keys] return labels, angles, crossfluxes, df
def mmeAveMsk2D(listFiles, years, inDir, outDir, outFile, timeInt, mme, timeBowl, ToeType, debug=True): ''' The mmeAveMsk2D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 25 13:56:20 CET 2014 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - timeBowl - either time 'mean' or time 'max' bowl used to mask out bowl - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 25 Nov 2014 - Initial function write - EG 27 Nov 2014 - Rewrite with loop on variables - EG 06 Dec 2014 - Added agreement on difference with init period - save as <var>Agree - EG 07 Dec 2014 - Read bowl to remove points above bowl - save as <var>Bowl - EG 19 Apr 2016 - ToE computation (just for 2D files) - EG 07 Oct 2016 - add 3D file support - EG 21 Nov 2016 - move 3D support to new function - EG 10 jan 2017 - added timeBowl option - TODO : - remove loops - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] if t2 <= 0: useLastYears = True t2 = -t2 else: useLastYears = False t10 = t1 t20 = t2 # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0]+'/'+listFiles[0]) isond0 = fi['isondepth'] ; # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() latN = isond0.shape[3] levN = isond0.shape[2] basN = isond0.shape[1] varsig='ptopsigma' # Declare and open files for writing if os.path.isfile(outDir+'/'+outFile): os.remove(outDir+'/'+outFile) outFile_f = cdm.open(outDir+'/'+outFile,'w') # Testing mme with less models #listFiles=listFiles[0:4] #timN = isond0.shape[0] timN = t2-t1 runN = len(listFiles) print ' Number of members:',len(listFiles) valmask = isond0.missing_value[0] varList = ['isondepth','isonpers','isonso','isonthetao','isonthick','isonvol'] varFill = [0.,0.,valmask,valmask,0.,0.] # init arrays (2D rho/lat) percent = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*0. #minbowl = npy.ma.ones([basN,latN], dtype='float32')*1000. varbowl = npy.ma.ones([runN,timN,basN,latN], dtype='float32')*1. #varList = ['isondepth'] #print ' !!! ### Testing one variable ###' #varList = ['isonthetao'] # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' time.designateTime() # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # loop on variables for iv,var in enumerate(varList): # Array inits (2D rho/lat 3D rho/lat/lon) #shapeR = [basN,levN,latN] isonvar = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*valmask vardiff,varbowl2D = [npy.ma.ones(npy.ma.shape(isonvar)) for _ in range(2)] varstd,varToE1,varToE2 = [npy.ma.ones([runN,basN,levN,latN], dtype='float32')*valmask for _ in range(3)] varones = npy.ma.ones([runN,timN,basN,levN,latN], dtype='float32')*1. print ' Variable ',iv, var # loop over files to fill up array for i,file in enumerate(listFiles): ft = cdm.open(inDir[0]+'/'+file) model = file.split('.')[1] timeax = ft.getAxis('time') file1d = replace(inDir[0]+'/'+file,'2D','1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:',file1d,'missing (if mme, run 1D first)' sys.exit(1) tmax = timeax.shape[0] if i == 0: tmax0 = tmax #adapt [t1,t2] time bounds to piControl last NN years if useLastYears: t1 = tmax-t20 t2 = tmax else: if tmax != tmax0: print 'wrong time axis: exiting...' return # read array # loop over time/density for memory management for it in range(timN): t1r = t1 + it t2r = t1r + 1 isonRead = ft(var,time = slice(t1r,t2r)) if varFill[iv] != valmask: isonvar[i,it,...] = isonRead.filled(varFill[iv]) else: isonvar[i,it,...] = isonRead # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data,valmask).mask percent[i,...] = npy.float32(npy.equal(maskvar,0)) if mme: # if mme then just accumulate Bowl, Agree fields varst = var+'Agree' vardiff[i,...] = ft(varst,time = slice(t1,t2)) varb = var+'Bowl' varbowl2D[i,...] = ft(varb,time = slice(t1,t2)) else: # Compute difference with average of first initN years varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) for t in range(timN): vardiff[i,t,...] = isonvar[i,t,...] - varinit vardiff[i,...].mask = isonvar[i,...].mask # Read bowl and truncate 2D field above bowl if iv == 0: bowlRead = f1d(varsig,time = slice(t1,t2)) varbowl[i,...] = bowlRead # Compute Stddev varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0) # Compute ToE if ToeType == 'histnat': # Read mean and Std dev from histnat if i == 0: filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] #filehn = replace(outFile,'historical','historicalNat') fthn = cdm.open(filehn) varmeanhn = fthn(var) varst = var+'Std' varmaxstd = fthn(varst) toemult = 1. signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) noise = npy.reshape(varmaxstd,(basN*levN*latN)) varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) toemult = 2. varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) ft.close() f1d.close() # <-- end of loop on files # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent,axis=0))*100. percenta = mv.masked_less(percenta, 50) percentw = cdm.createVariable(percenta, axes = [time,axesList[1],axesList[2],axesList[3]], id = 'isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32')) # Sign of difference if mme: vardiffsgSum = cdu.averager(vardiff, axis=0) vardiffsgSum = cdm.createVariable(vardiffsgSum , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') vardiffsgSum = maskVal(vardiffsgSum, valmask) vardiffsgSum.mask = percentw.mask else: vardiffsg = npy.copysign(varones,vardiff) # average signs vardiffsgSum = cdu.averager(vardiffsg, axis=0) vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) vardiffsgSum.mask = percentw.mask vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = cdm.createVariable(isonVarAve , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask # Only keep points with rhon > bowl-delta_rho delta_rho = 0. if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = cdm.createVariable(isonVarBowl , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = 'foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask if iv == 0: # Read mulitmodel sigma on bowl and average in time file1d = replace(outDir+'/'+outFile,'2D','1D') if os.path.isfile(file1d): f1d = cdm.open(file1d) else: print 'ERROR:',file1d,'missing (if mme, run 1D first)' sys.exit(1) bowlRead = f1d(varsig,time = slice(t1,t2)) f1d.close() siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): #if ib == 2: # print il, siglimit[ib,il] if siglimit[ib,il] < valmask/1000.: # if mme bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) isonVarBowl [:,ib,0:index[0],il].mask = True isonVarStd [:,ib,0:index[0],il].mask = True vardiffsgSum[:,ib,0:index[0],il].mask = True else: # mask all points isonVarBowl [:,ib,:,il].mask = True isonVarStd [:,ib,:,il].mask = True vardiffsgSum[:,ib,:,il].mask = True else: isonVarBowl = isonVarAve*1. # start from variable isonVarStd = isonVarAve*1. # start from variable if iv == 0: siglimit = cdu.averager(varbowl, axis=0) # average accross members # Average bowl in time if timeBowl == 'mean': siglimit = cdu.averager(siglimit, axis=0) - delta_rho # or take largest sigma over time else: siglimit = npy.ma.max(siglimit, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 for il in range(latN): for ib in range(basN): if siglimit[ib,il] < valmask/1000.: # if bowl density defined, mask above bowl index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) isonVarBowl[:,ib,0:index[0],il].mask = True vardiffsgSum[:,ib,0:index[0],il].mask = True else: # mask all points vardiffsgSum[:,ib,:,il].mask = True isonVarBowl = maskVal(isonVarBowl, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask if varFill[iv] == valmask: isonVarStd = maskVal(isonVarStd, valmask) # Write isonave = cdm.createVariable(isonVarAve, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id) isonave.long_name = isonRead.long_name isonave.units = isonRead.units isonavediff = cdm.createVariable(vardiffsgSum, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Agree') isonavediff.long_name = isonRead.long_name isonavediff.units = isonRead.units isonavebowl = cdm.createVariable(isonVarBowl, axes = [time,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units if not mme: isonmaxstd = cdm.createVariable(isonVarStd, axes = [axesList[1],axesList[2],axesList[3]], id = isonRead.id+'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units outFile_f.write( isonave.astype('float32')) outFile_f.write(isonavediff.astype('float32')) outFile_f.write(isonavebowl.astype('float32')) if not mme: outFile_f.write( isonmaxstd.astype('float32')) if ToeType == 'histnat': isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1') isontoe1.long_name = 'ToE 1 for '+isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2') isontoe2.long_name = 'ToE 2 for '+isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32')) outFile_f.write(isontoe2.astype('float32')) if mme: isonvarstd = cdm.createVariable(isonVarStd , axes =[time,axesList[1],axesList[2],axesList[3]] , id = isonRead.id+'ModStd') isonvarstd.long_name = isonRead.long_name+' intermodel std' isonvarstd.units = isonRead.units outFile_f.write(isonvarstd.astype('float32')) # <--- end of loop on variables outFile_f.close() fi.close()
def mmeAveMsk3D(listFiles, years, inDir, outDir, outFile, timeInt, mme, ToeType, debug=True): ''' The mmeAveMsk3D() function averages rhon/lat density bined files with differing masks It ouputs - the MME - a percentage of non-masked bins - the sign agreement of period2-period1 differences - ToE per run and for MME Author: Eric Guilyardi : [email protected] Created on Tue Nov 21 2016 Inputs: ------- - listFiles(str) - the list of files to be averaged - years(t1,t2) - years for slice read - inDir[](str) - input directory where files are stored (add histnat as inDir[1] for ToE) - outDir(str) - output directory - outFile(str) - output file - timeInt(2xindices) - indices of init period to compare with (e.g. [1,20]) - mme(bool) - multi-model mean (will read in single model ensemble stats) - ToeType(str) - ToE type ('F': none, 'histnat') -> requires running first mm+mme without ToE to compute Stddev - debug <optional> - boolean value Notes: ----- - EG 21 Nov 2016 - Initial function write - TODO : - add computation of ToE per model (toe 1 and toe 2) see ticket #50 - add isonhtc (see ticket #48) ''' # CDMS initialisation - netCDF compression comp = 1 # 0 for no compression cdm.setNetcdfShuffleFlag(comp) cdm.setNetcdfDeflateFlag(comp) cdm.setNetcdfDeflateLevelFlag(comp) cdm.setAutoBounds('on') # Numpy initialisation npy.set_printoptions(precision=2) if debug: debug = True else: debug = False # File dim and grid inits t1 = years[0] t2 = years[1] # Bound of period average to remove peri1 = timeInt[0] peri2 = timeInt[1] fi = cdm.open(inDir[0]+'/'+listFiles[0]) # Switch if only variables below the bowl are present/treated nobowl = True if nobowl: isond0 = fi['isondepthgBowl'] ; # Create variable handle else: isond0 = fi['isondepthg'] ; # Create variable handle # Get grid objects axesList = isond0.getAxisList() sigmaGrd = isond0.getLevel() #time = isond0.getTime() lonN = isond0.shape[3] latN = isond0.shape[2] levN = isond0.shape[1] varsig='ptopsigmaxy' # Limit number of models to 3 for testing of mme #if mme: # listFiles = listFiles[0:2] # print ' !!! ### Testing 3 models ###', listFiles # Declare and open files for writing if os.path.isfile(outDir+'/'+outFile): os.remove(outDir+'/'+outFile) outFile_f = cdm.open(outDir+'/'+outFile,'w') #timN = isond0.shape[0] timN = t2-t1 runN = len(listFiles) print ' Number of members:',len(listFiles) valmask = isond0.missing_value varList = ['isondepthg','persistmxy','sog','thetaog','isonthickg'] varFill = [valmask,valmask,valmask,valmask,valmask] percent = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*0. varbowl = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*1. #varList = ['isondepthg'] #print ' !!! ### Testing one variable ###', varList # init sigma axis sigma = cdm.createAxis(npy.float32(range(1))) sigma.id = axesList[1].id sigma.units = axesList[1].units sigma.designateTime() # init time axis time = cdm.createAxis(npy.float32(range(timN))) time.id = 'time' time.units = 'years since 1861' # init ensemble axis ensembleAxis = cdm.createAxis(npy.float32(range(runN))) ensembleAxis.id = 'members' ensembleAxis.units = 'N' # Output axis sigmaList = [sigma,axesList[2],axesList[3]] ; # sigma, lat, lon sigmaTimeList = [sigma,time,axesList[2],axesList[3]] ; # sigma, time, lat, lon # init arrays isonvar = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask varbowl2D = npy.ma.ones([runN,timN,latN,lonN], dtype='float32')*valmask varstd,varToE1,varToE2 = [npy.ma.ones([runN,latN,lonN], dtype='float32')*valmask for _ in range(3)] # Loop on density levels (for memory management, becomes UNLIMITED axis and requires a ncpq to reorder dimensions) delta_ib = 1 print ' Sigma index:' for ib in range(levN): ib1 = ib + delta_ib print ib, tim0 = timc.clock() # loop on variables for iv,var in enumerate(varList): if nobowl: varb = var+'Bowl' else: varb = var if ib == 0: print ' Variable ',iv, varb # loop over files to fill up array for i,file in enumerate(listFiles): tim01 = timc.clock() ft = cdm.open(inDir[0]+'/'+file) model = file.split('.')[1] timeax = ft.getAxis('time') if i == 0: tmax0 = timeax.shape[0] tmax = timeax.shape[0] if tmax != tmax0: print 'wrong time axis: exiting...' return # read array isonRead = ft(varb,time = slice(t1,t2), lev = slice(ib,ib1)).squeeze() if varFill[iv] != valmask: isonvar[i,...] = isonRead.filled(varFill[iv]) else: isonvar[i,...] = isonRead tim02 = timc.clock() # compute percentage of non-masked points accros MME if iv == 0: maskvar = mv.masked_values(isonRead.data,valmask).mask percent[i,...] = npy.float32(npy.equal(maskvar,0)) tim03 = timc.clock() if mme: # if mme then just accumulate Bowl, Agree and Std fields #varst = var+'Agree' #vardiff[i,...] = ft(varst,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() isonRead = ft(varb,time = slice(t1,t2),lev = slice(ib,ib1)).squeeze() varbowl2D[i,...] = isonRead else: # Compute difference with average of first initN years #varinit = cdu.averager(isonvar[i,peri1:peri2,...],axis=0) #for t in range(timN): # vardiff[i,t,...] = isonvar[i,t,...] - varinit #vardiff[i,...].mask = isonvar[i,...].mask # Read bowl to truncate field above bowl if ib == 0 and iv == 0: varbowl[i,...] = ft(varsig,time = slice(t1,t2)) #varbowl[i,...] = bowlRead # Compute Stddev varstd[i,...] = npy.ma.std(isonvar[i,...], axis=0) # Compute ToE if ToeType == 'histnat': toto=1 # TODO # Read mean and Std dev from histnat # if i == 0: # filehn = glob.glob(inDir[1]+'/cmip5.'+model+'.*zon2D*')[0] # #filehn = replace(outFile,'historical','historicalNat') # fthn = cdm.open(filehn) # varmeanhn = fthn(var) # varst = var+'Std' # varmaxstd = fthn(varst) # toemult = 1. # signal = npy.reshape(isonvar[i,...]-varmeanhn,(timN,basN*levN*latN)) # noise = npy.reshape(varmaxstd,(basN*levN*latN)) # varToE1[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) # toemult = 2. # varToE2[i,...] = npy.reshape(findToE(signal, noise, toemult),(basN,levN,latN)) tim04 = timc.clock() ft.close() #print 'ib, section 1 timing',ib, tim02-tim01,tim03-tim02,tim04-tim03 # <-- end of loop on files (i) tim1 = timc.clock() # Compute percentage of bin presence # Only keep points where percent > 50% if iv == 0: percenta = (cdu.averager(percent,axis=0))*100. percenta = mv.masked_less(percenta, 50) percenta = npy.reshape(percenta,[delta_ib,timN,latN,lonN]) percentw = cdm.createVariable(percenta, axes = sigmaTimeList, id = 'isonpercent') percentw._FillValue = valmask percentw.long_name = 'percentage of MME bin' percentw.units = '%' outFile_f.write(percentw.astype('float32'), extend = 1, index = ib) # Sign of difference #if mme: # vardiffsgSum = cdu.averager(vardiff, axis=0) # vardiffsgSum = cdm.createVariable(vardiffsgSum , axes = sigmaTimeList , id = 'foo') # vardiffsgSum = maskVal(vardiffsgSum, valmask) # vardiffsgSum.mask = percentw.mask #else: # vardiffsg = npy.copysign(varones,vardiff) # # average signs # vardiffsgSum = cdu.averager(vardiffsg, axis=0) # vardiffsgSum = mv.masked_greater(vardiffsgSum, 10000.) # vardiffsgSum.mask = percentw.mask # vardiffsgSum._FillValue = valmask # average variable accross members isonVarAve = cdu.averager(isonvar, axis=0) isonVarAve = npy.reshape(isonVarAve,[delta_ib,timN,latN,lonN]) isonVarAve = cdm.createVariable(isonVarAve , axes = sigmaTimeList , id = 'foo') # mask if varFill[iv] == valmask: isonVarAve = maskVal(isonVarAve, valmask) isonVarAve.mask = percentw.mask tim2 = timc.clock() # Only keep points with rhon > bowl-delta_rho delta_rho = 0. # mme case if mme: # start from average of <var>Agree isonVarBowl = cdu.averager(varbowl2D, axis=0) isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN]) isonVarBowl = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = 'foo') isonVarBowl = maskVal(isonVarBowl, valmask) isonVarBowl.mask = percentw.mask # Compute intermodel stddev isonVarStd = statistics.std(varbowl2D, axis=0) isonVarStd = npy.reshape(isonVarStd,[delta_ib,timN,latN,lonN]) isonVarStd = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = 'foo') isonVarStd = maskVal(isonVarStd, valmask) isonVarStd.mask = percentw.mask # Write isonvarbowlw = cdm.createVariable(isonVarBowl , axes = sigmaTimeList , id = isonRead.id) isonvarbowlw.long_name = isonRead.long_name isonvarbowlw.units = isonRead.units isonvarstdw = cdm.createVariable(isonVarStd , axes = sigmaTimeList , id = isonRead.id+'Std') isonvarstdw.long_name = isonRead.long_name+' intermodel std' isonvarstdw.units = isonRead.units outFile_f.write(isonvarbowlw.astype('float32'), extend = 1, index = ib) outFile_f.write(isonvarstdw.astype('float32'), extend = 1, index = ib) #if ib == 0 and iv == 0: # # TODO review # # Read multimodel sigma on bowl and average in time # file1d = replace(outDir+'/'+outFile,'2D','1D') # if os.path.isfile(file1d): # f1d = cdm.open(file1d) # else: # print 'ERROR:',file1d,'missing (if mme, run 2D first)' # sys.exit(1) # bowlRead = f1d(varsig,time = slice(t1,t2),lev = slice(ib,ib1)) # f1d.close() # siglimit = cdu.averager(bowlRead, axis=0) - delta_rho # TODO: remove loop by building global array with 1/0 #if sw2d == 1: # for il in range(latN): # for ib in range(basN): # #if ib == 2: # # print il, siglimit[ib,il] # if siglimit[ib,il] < valmask/1000.: # # if mme bowl density defined, mask above bowl # index = (npy.argwhere(sigmaGrd[:] >= siglimit[ib,il])) # isonVarBowl [:,ib,0:index[0],il].mask = True # isonVarStd [:,ib,0:index[0],il].mask = True # vardiffsgSum[:,ib,0:index[0],il].mask = True # else: # # mask all points # isonVarBowl [:,ib,:,il].mask = True # isonVarStd [:,ib,:,il].mask = True # vardiffsgSum[:,ib,:,il].mask = True # mm case else: isonVarBowl = isonVarAve*1. # start from variable #isonVarStd = isonVarAve*1. # start from variable if ib == 0 and iv == 0: # build bowl position siglimit = cdu.averager(varbowl, axis=0) # average accross members siglimit = npy.reshape(siglimit,[timN*latN*lonN]) - delta_rho if iv == 0: sigarr = siglimit*1. sigarr[:] = sigmaGrd[ib] # test i = 60 j = 60 ij = j*lonN+i isonVarBowl = npy.reshape(isonVarBowl,[timN*latN*lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN*latN*lonN]) isonVarBowl.mask = npy.where(sigarr < siglimit, True, isonVarBowl.mask) #vardiffsgSum.mask = npy.where(sigarr < siglimit, True, vardiffsgSum.mask) isonVarBowl = npy.reshape(isonVarBowl,[timN,latN,lonN]) #vardiffsgSum = npy.reshape(vardiffsgSum,[timN,latN,lonN]) isonVarBowl = maskVal(isonVarBowl, valmask) #vardiffsgSum = maskVal(vardiffsgSum, valmask) # Find max of Std dev of all members isonVarStd = npy.ma.max(varstd, axis=0) # mask isonVarStd = maskVal(isonVarStd, valmask) # Write #isonave = cdm.createVariable(isonVarAve, axes = sigmaTimeList, id = isonRead.id) #isonave.long_name = isonRead.long_name #isonave.units = isonRead.units #vardiffsgSum = npy.reshape(vardiffsgSum,[delta_ib,timN,latN,lonN]) #isonavediff = cdm.createVariable(vardiffsgSum, axes = sigmaTimeList, id = isonRead.id+'Agree') #isonavediff.long_name = isonRead.long_name #isonavediff.units = isonRead.units isonVarBowl = npy.reshape(isonVarBowl,[delta_ib,timN,latN,lonN]) isonavebowl = cdm.createVariable(isonVarBowl, axes = sigmaTimeList, id = isonRead.id+'Bowl') isonavebowl.long_name = isonRead.long_name isonavebowl.units = isonRead.units isonVarStd = npy.reshape(isonVarStd,[delta_ib,latN,lonN]) isonmaxstd = cdm.createVariable(isonVarStd, axes = sigmaList, id = isonRead.id+'Std') isonmaxstd.long_name = isonRead.long_name isonmaxstd.units = isonRead.units #outFile_f.write( isonave.astype('float32'), extend = 1, index = ib) #outFile_f.write(isonavediff.astype('float32'), extend = 1, index = ib) outFile_f.write(isonavebowl.astype('float32'), extend = 1, index = ib) outFile_f.write(isonmaxstd.astype('float32'), extend = 1, index = ib) tim3 = timc.clock() if ToeType == 'histnat': isontoe1 = cdm.createVariable(varToE1, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE1') isontoe1.long_name = 'ToE 1 for '+isonRead.long_name isontoe1.units = 'Year' isontoe2 = cdm.createVariable(varToE2, axes = [ensembleAxis,axesList[1],axesList[2],axesList[3]], id = isonRead.id+'ToE2') isontoe2.long_name = 'ToE 2 for '+isonRead.long_name isontoe2.units = 'Year' outFile_f.write(isontoe1.astype('float32'), extend = 1, index = ib) outFile_f.write(isontoe2.astype('float32'), extend = 1, index = ib) tim4 = timc.clock() # <--- end of loop on variables #print 'ib, timing',ib, tim01-tim0,tim1-tim01,tim2-tim1,tim3-tim2,tim4-tim3 # <--- end of loop on density print ' ' outFile_f.close() fi.close()