allnatdf=pd.DataFrame(allnatdt,index=years) # =============== OBS: basedir='/HOME/rkm/work/BCs/' subdir='HadISST/' hadfile=basedir + subdir + 'hadisst1.1_bc_128_64_1870_2013m03_' + field + '_1870010100-2013030100.nc' hadsel='1950-01-01,2012-12-31' hadyrs=np.arange(1950,2013) if sea=='DJF': hadyrs=hadyrs[:-1] hadfld=cnc.getNCvar(hadfile,field,timesel=hadsel,seas=season) haddf = pd.Series(hadfld,index=hadyrs) pasthad = haddf.loc[subyrs] preshad = haddf.loc[subyrs2] (hadtstat,hadpv) = cutl.ttest_ind(preshad,pasthad) if dopct: diffhad= (preshad.mean(axis=0) - pasthad.mean(axis=0)) / pasthad.mean(axis=0) *100 else: diffhad=preshad.mean(axis=0) - pasthad.mean(axis=0) subdir='NSIDC/' nsidcfile = basedir + subdir + 'nsidc_bt_128x64_1978m11_2011m12_' + field + '_1978111600-2011121612.nc' nsidcsel='1979-01-01,2011-12-31' nsidcyrs=np.arange(1979,2012) if sea=='DJF': nsidcyrs=nsidcyrs[:-1] nsidcfld=cnc.getNCvar(nsidcfile,field,timesel=nsidcsel,seas=season) nsidcdf=pd.Series(nsidcfld,index=nsidcyrs) pastnsidc = nsidcdf.loc[subyrs] presnsidc = nsidcdf.loc[subyrs2[:-1]]
def calc_nc_agreement_map(ncdt, axs, lin='one',conv=1, subtime=None, magtype='abs'): """ lin = 'one' is ice and co2 (ICEcold, CO2hi) = 'two' is ice2 and co22 (ICEwarm, CO2lo) = 'sub' subtract lin 'one' from lin 'two' (shows how warm clim/low ice different from cold/high) magtype: if 'abs' then magnitude is np.abs(ice) + np.abs(co2) else it is ice+co2 returns: mag=|ice|+|co2|, and agreement (mag with sign) """ pico = ncdt['prei2xco2iceb'] pipi = ncdt['preipreiice'] copi = ncdt['2xco2preiice'] coco = ncdt['2xco22xco2ice'] if lin=='one': _,icepv = cutl.ttest_ind(pico, pipi,axis=0,effdof=False) ice = (np.mean(pico,axis=0) - np.mean(pipi,axis=0))*conv #icep = ncfldzmdt['pi2xco2ipulse'] - ncfldzmdt['preipreiice'] _,co2pv = cutl.ttest_ind(copi, pipi,axis=0,effdof=False) co2 = (np.mean(copi,axis=0) - np.mean(pipi,axis=0))*conv _,combopv = cutl.ttest_ind(pico-pipi+(copi-pipi), coco-pipi) ctl = np.mean(pipi,axis=0)*conv suff='' suff1='cold'; suff2='hi' elif lin=='two': _,icepv = cutl.ttest_ind(coco, copi,axis=0,effdof=False) ice = (np.mean(coco,axis=0) - np.mean(copi,axis=0))*conv _,co2pv = cutl.ttest_ind(coco, pico,axis=0,effdof=False) co2 = (np.mean(coco,axis=0) - np.mean(pico,axis=0))*conv _,combopv = cutl.ttest_ind(coco-copi+(coco-pico), coco-pipi) ctl = np.mean(coco,axis=0)*conv suff='2' suff1='warm'; suff2='lo' elif lin=='sub': #two - one: (coco-copi) - (pico-pipi) _,icepv = cutl.ttest_ind(coco-copi, pico-pipi,axis=0,effdof=False) ice = (np.mean(coco-copi,axis=0) - np.mean(pico-pipi,axis=0))*conv #two - one: (coco-pico) - (copi-pipi) _,co2pv = cutl.ttest_ind(coco-pico, copi-pipi,axis=0,effdof=False) co2 = (np.mean(coco-pico,axis=0) - np.mean(copi-pipi,axis=0))*conv suff='2-1' suff1='warm-cold'; suff2='lo-hi' _,fullpv = cutl.ttest_ind(coco, pipi,axis=0,effdof=False) full = (np.mean(coco,axis=0) - np.mean(pipi,axis=0))*conv if magtype == 'abs': mag = np.abs(ice) + np.abs(co2) else: mag = ice + co2 # could also try: ice + np.abs(co2) import copy as copy magsign=copy.copy(mag) magsign[np.sign(ice) != np.sign(co2)] = -1*mag[np.sign(ice) != np.sign(co2)] return mag,magsign
def loaddata(fields, simulations, ncfields=None,model='CanAM4',timeper='001-121',timefreq=None, levsel=None,meantype=None,filetype='diff',region=None,alsomask=None,rettype='dict'): """ loaddata(fields, simulations,ncfields=None,model='CanAM4',timeper='001-121',timefreq=None, levsel=None, meantype=None,filetype='diff',region=None) fields: tuple of field names [@@update. only one field now but still need tuple] simulations: tuple of simulation names (diff names='E1'...'ENS','NSIDC' etc) ncfields: tuple of ncfield names (var name in file itself). Default to upper case of field model: for now only 'CanAM4' is implemented timeper: time period of the data (used for filename). default '001-121' timefreq: time frequency TO RETURN. default all data 'monthly'|'seasonal'|'climo'|'ANN'|'DJF'|'JJA'|'NDJ'|'MAM'|'SON'| 1,2,3,4,5,6,7,8,9,10,11,12 levsel: select level in Pa (e.g. 50000 for 500hPa). default all levels meantype: 'time','zonal' @@for now. default None, but recommended to choose one if loading multiple variables and multiple simulations at once. It is assumed that time is the first dimension. filetype: 'diff','ctl','pert','pval' Default is diff where both ctl and pert are read in and differenced. region: any of the regions in constants -> region dict. default None. alsomask: if specified as 'land' or 'ocean', then calc_regmean() will mask before computing regional avg (ie. will NOT include it in average). Only used if region!=None. Default None. returns: nested dictionary@@ FIELDS->SIMULATIONS->TIMEFREQ Load requested fields from requested CanAM4 simulations into dictionaries (dataframes?). Function automatically skips first year of simulation and gets a timeseries (or climo if requested) of the rest (assumed through year 121). 3D data and 'turb' not yet implemented! @@ """ if model!='CanAM4': print 'model not supported!' return -1 print '@@ probably should invert the order such that it is field, season, sim?' #bp=con.get_basepath() #basepath=bp['basepath'] + model + '/'; subdir=bp['subdir'] timesel='0002-01-01,0121-12-31' seabool=False # set to True if requested time freq is one season or climo monbool=False # set to True if requested time freq is one month if timefreq=='monthly': # get all months tf = con.get_mon() elif timefreq=='seasonal': # get all 4 seasons tf = 'DJF','MAM','JJA','SON' elif timefreq in range(1,13): # choose an individual month tf=timefreq monbool=True elif timefreq in ('climo','ANN','DJF','JJA','NDJ','MAM','SON','ND','JF','SO'): tf=timefreq seabool=True print tf # @@ # @@@@@ add handling of sia! #datadict = dict.fromkeys(fields,{}) #for fii,field in enumerate(fields): if 1: # GET RID OF FIELD dim too 5/8/2015 fii=0; field=fields[0] if ncfields==None: ncfield=field.upper() else: ncfield=ncfields[fii] print field,ncfield #@@ # assume simulations entered are of the form: E1, R3, ENSE etc. Then # filetype input arg tells which simulation to get (or both) simdict = dict.fromkeys(simulations,{}) for sim in simulations: print sim # @@ #timdict = dict.fromkeys(tf) # construct filename here @@ fnamec,fnamep = con.build_filepathpair(sim,field) #fname = basepath+sim+subdir+sim+'_'+field+'_'+timeper+'_ts.nc' print fnamec tfkey = tf #for tfkey in tf: # print tfkey # @@ #print 'too many levels in the dict...get rid of seasonal keys and just do one@@@ 5/1/2015' # @@ get the data with cnc.getNCvar() here ncparams = {} if monbool: ncparams = {'monsel': tfkey} elif seabool: ncparams = {'seas': tfkey} if levsel!=None: ncparams['levsel'] = levsel if meantype=='zonal': ncparams['calc'] = 'zm' if filetype=='diff' or filetype=='pval': if field in ('turb','net'): fnamec,fnamep = con.build_filepathpair(sim,'hfl') fnamecb,fnamepb = con.build_filepathpair(sim,'hfs') ctl = cnc.getNCvar(fnamec,'HFL',timesel=timesel,**ncparams) +\ cnc.getNCvar(fnamecb,'HFS',timesel=timesel, **ncparams) pert = cnc.getNCvar(fnamep,'HFL',timesel=timesel,**ncparams) +\ cnc.getNCvar(fnamepb,'HFS',timesel=timesel,**ncparams) if field=='net': fnamecc,fnamepc = con.build_filepathpair(sim,'flg') ctl = ctl - cnc.getNCvar(fnamecc,'FLG',timesel=timesel,**ncparams) pert = pert - cnc.getNCvar(fnamepc,'FLG',timesel=timesel,**ncparams) else: pert = cnc.getNCvar(fnamep,ncfield,timesel=timesel,**ncparams) ctl = cnc.getNCvar(fnamec,ncfield,timesel=timesel,**ncparams) fld = pert - ctl elif filetype=='ctl': if field in ('turb','net'): fnamec,fnamep = con.build_filepathpair(sim,'hfl') fnamecb,fnamepb = con.build_filepathpair(sim,'hfs') fld = cnc.getNCvar(fnamec,'HFL',timesel=timesel,**ncparams) +\ cnc.getNCvar(fnamecb,'HFS',timesel=timesel, **ncparams) if field=='net': fnamecc,fnamepc = con.build_filepathpair(sim,'flg') fld = fld - cnc.getNCvar(fnamecc,'FLG',timesel=timesel,**ncparams) else: fld = cnc.getNCvar(fnamec,ncfield,timesel=timesel,**ncparams) elif filetype=='pert': if field in ('turb','net'): fnamec,fnamep = con.build_filepathpair(sim,'hfl') fnamecb,fnamepb = con.build_filepathpair(sim,'hfs') fld = cnc.getNCvar(fnamep,'HFL',timesel=timesel,**ncparams) +\ cnc.getNCvar(fnamepb,'HFS',timesel=timesel,**ncparams) if field=='net': fnamecc,fnamepc = con.build_filepathpair(sim,'flg') fld = fld - cnc.getNCvar(fnamepc,'FLG',timesel=timesel,**ncparams) else: fld = cnc.getNCvar(fnamep,ncfield,timesel=timesel,**ncparams) else: print "filetype not supported! ['diff'|'ctl'|'pert'|'pval']" return -1 if region != None: lat=cnc.getNCvar(fnamec,'lat'); lon=cnc.getNCvar(fnamec,'lon') if filetype=='pval': pert = cutl.calc_regmean(pert,lat,lon,region,alsomask=alsomask) ctl = cutl.calc_regmean(ctl,lat,lon,region,alsomask=alsomask) (tstat,pval) = cutl.ttest_ind(pert,ctl) fld=pval else: fld = cutl.calc_regmean(fld,lat,lon,region,alsomask=alsomask) if meantype=='time': if filetype=='pval': # this is an error. filetype supercedes meantype so time avg won't be done print 'filetype=pval and meantype-time. Ignore meantype and return pvals @@' timdict=fld else: #fldstd = np.std(fld,axis=0) fld = np.mean(fld,axis=0) #timdict[tfkey] = fld #,fldstd timdict=fld else: #timdict[tfkey] = fld timdict=fld simdict[sim] = timdict #datadict[field]=simdict # can I set attributes to a dictionary? @@ like for nfields, nsims, ntimes? #return datadict if rettype=='ndarray': # convert the dict to an array: # get the last sim data to initialize ndarray initshape=simdict[sim].shape initshape=(len(simulations),) + initshape tmp=np.zeros(initshape) for sii,skey in enumerate(simulations): tmp[sii,:] = simdict[skey] return tmp else: return simdict
def plot_nc_linearity_maps(ncdt, axs, lin='one',cmin='',cmax='',cmind='',cmaxd='',cmin2='',cmax2='', conv=1, ptype='nh',suppcb=False,cmap='blue2red_w20',subtime=None, screen=False,vert=False,levlim=None,suppttl=False,addsig=False,sigtype='cont', latlim=None,vertptype=None,addclimcont=False,ctlconts=None, nosum=False,nofull=False,nolin=False,scaledlin=False,ncicedt=None,sigreverse=False, northof=0,fsz=None): """ cmin/cmax: for ice cmin2/cmax2: for co2 & sum & full, if given. otherwise same as ice cmind/cmaxd: for lin combo subtraction from full lin = 'one' is ice and co2 (ICEcold, CO2hi) = 'two' is ice2 and co22 (ICEwarm, CO2lo) = 'sub' subtract lin 'one' from lin 'two' (shows how warm clim/low ice different from cold/high) nosum: include ice+co2 panel or not? nofull: include Full panel or not? nolin: include non-linearity panel or not? scaledlin: is the non-linearity panel to be scaled or not? ncicedt: must be set if scaledlin=True! sigreverse: if True, plot hatching/contours where NOT significant on every map but non-lin. (only if addsig=True) northof: compute pattern correlation b/w SUM and FULL north of this latitude. fsz: fontsize for title returns: plot handles (ph,ph2[,phd]) (ice,co2|sum|full[,difference/nonlin]) """ # @@ subtime not implemented yet. assume average over full time #tmplen = ncdt['2xco2preiice'].shape[0]-1 fmt='%2.1f' # clabel format #if ctlconts!=None: # if (ctlconts < 1.).any() and (ctlconts > -1.).any(): # #if np.logical_and((ctlconts<1).any(),(ctlconts>-1).any()): # print ctlconts # fmt='%2.1f' pico = ncdt['prei2xco2iceb'] pipi = ncdt['preipreiice'] copi = ncdt['2xco2preiice'] coco = ncdt['2xco22xco2ice'] lat = ncdt['lat'] if lin=='one': _,icepv = cutl.ttest_ind(pico, pipi,axis=0,effdof=False) ice = (np.mean(pico,axis=0) - np.mean(pipi,axis=0))*conv #icep = ncfldzmdt['pi2xco2ipulse'] - ncfldzmdt['preipreiice'] _,co2pv = cutl.ttest_ind(copi, pipi,axis=0,effdof=False) co2 = (np.mean(copi,axis=0) - np.mean(pipi,axis=0))*conv _,combopv = cutl.ttest_ind(pico-pipi+(copi-pipi), coco-pipi) ctl = np.mean(pipi,axis=0)*conv suff='' suff1='cold'; suff2='hi' elif lin=='two': _,icepv = cutl.ttest_ind(coco, copi,axis=0,effdof=False) ice = (np.mean(coco,axis=0) - np.mean(copi,axis=0))*conv _,co2pv = cutl.ttest_ind(coco, pico,axis=0,effdof=False) co2 = (np.mean(coco,axis=0) - np.mean(pico,axis=0))*conv _,combopv = cutl.ttest_ind(coco-copi+(coco-pico), coco-pipi) ctl = np.mean(coco,axis=0)*conv suff='2' suff1='warm'; suff2='lo' elif lin=='sub': #two - one: (coco-copi) - (pico-pipi) _,icepv = cutl.ttest_ind(coco-copi, pico-pipi,axis=0,effdof=False) ice = (np.mean(coco-copi,axis=0) - np.mean(pico-pipi,axis=0))*conv #two - one: (coco-pico) - (copi-pipi) _,co2pv = cutl.ttest_ind(coco-pico, copi-pipi,axis=0,effdof=False) co2 = (np.mean(coco-pico,axis=0) - np.mean(copi-pipi,axis=0))*conv suff='2-1' suff1='warm-cold'; suff2='lo-hi' _,fullpv = cutl.ttest_ind(coco, pipi,axis=0,effdof=False) full = (np.mean(coco,axis=0) - np.mean(pipi,axis=0))*conv # compute pattern correlation b/w SUM and FULL prval,ppval = calc_linpatterncorr(ncdt,lin=lin,northof=northof,vert=vert) pvarexp = (prval**2)*100 # pparams for ice! # pparams2 for co2![sum|full] if vert: lev = ncdt['lev'] pparams = {'cmin': cmin, 'cmax': cmax, 'suppcb': suppcb, 'screen': screen, 'cmap':cmap,'levlim':levlim, 'latlim':latlim,'ptype':vertptype} if cmin2 != '': pparams2 = {'cmin': cmin2, 'cmax': cmax2, 'suppcb': suppcb, 'screen': screen, 'cmap':cmap,'levlim':levlim, 'latlim':latlim,'ptype':vertptype} else: pparams2 = {'cmin': cmin, 'cmax': cmax, 'suppcb': suppcb, 'screen': screen, 'cmap':cmap,'levlim':levlim, 'latlim':latlim,'ptype':vertptype} else: lon = ncdt['lon'] pparams = {'ptype': ptype, 'cmin': cmin, 'cmax': cmax, 'suppcb': suppcb, 'cmap':cmap} if cmin2 != '': pparams2 = {'ptype': ptype, 'cmin': cmin2, 'cmax': cmax2, 'suppcb': suppcb, 'cmap':cmap} else: pparams2 = {'ptype': ptype, 'cmin': cmin, 'cmax': cmax, 'suppcb': suppcb, 'cmap':cmap} aii=0 ax=axs[aii] if suppttl: ttl='' else: ttl='ICE'+suff1 if vert: ph = cplt.vert_plot(ice,lev,lat,axis=ax,title=ttl,**pparams) if addsig: cplt.addtsig(ax,icepv,lat,lev/100.,sigtype=sigtype,reverse=sigreverse) if addclimcont: cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,ph = cplt.kemmap(ice,lat,lon,axis=ax,title=ttl,**pparams) if addsig: cplt.addtsigm(bm,icepv,lat,lon,sigtype=sigtype,reverse=sigreverse) if fsz!=None: ax.set_title(ttl,fontsize=fsz) aii+=1 ax=axs[aii] if suppttl: ttl='' else: ttl='CO2'+suff2 if vert: ph2 = cplt.vert_plot(co2,lev,lat,axis=ax,title=ttl,suppylab=True,**pparams2) if addsig: cplt.addtsig(ax,co2pv,lat,lev/100.,sigtype=sigtype,reverse=sigreverse) if addclimcont: cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,ph2 = cplt.kemmap(co2,lat,lon,axis=ax,title=ttl,**pparams2) if addsig: cplt.addtsigm(bm,co2pv,lat,lon,sigtype=sigtype,reverse=sigreverse) if fsz!=None: ax.set_title(ttl,fontsize=fsz) aii+=1 if not nosum: ax=axs[aii] if suppttl: ttl='' else: ttl='Sum' #ice'+suff1+'+co2'+suff2 if vert: ph2 = cplt.vert_plot(ice+co2,lev,lat,axis=ax,title=ttl,suppylab=True,**pparams2) if addclimcont: cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,ph2 = cplt.kemmap(ice+co2,lat,lon,axis=ax,title=ttl,**pparams2) if fsz!=None: ax.set_title(ttl,fontsize=fsz) aii+=1 if not nofull: ax=axs[aii] if suppttl: ttl='' else: ttl='Full' if vert: ph2 = cplt.vert_plot(full,lev,lat,axis=ax,title=ttl,suppylab=True,**pparams2) if addsig: cplt.addtsig(ax,fullpv,lat,lev/100.,sigtype=sigtype,reverse=sigreverse) if addclimcont: cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,ph2 = cplt.kemmap(full, lat,lon,axis=ax,title=ttl,**pparams2) if addsig: cplt.addtsigm(bm,fullpv,lat,lon,sigtype=sigtype,reverse=sigreverse) ax.annotate('$%.0f$'%(pvarexp) + '%', xy=(0.78,0.98),xycoords='axes fraction', fontsize=16) if fsz!=None: ax.set_title(ttl,fontsize=fsz) aii+=1 if not nolin: ax=axs[aii] # the linearity test # @@@ add scaledlin here @@@ # e.g. c1*ICEcold + c2*CO2hi - Full # c1*(pico-pipi) + c2*(copi-pipi) - (coco-pipi) # solve c1, c2 with sea ice s.t. above eqn would give 0 # @@@ Question: how to scale? how to get c1,c2? Use difference from target sea ice? if scaledlin: # compute target offset offs = compute_targoffset(ncicedt,dosia=True) # want to be a fraction so div by 100. # and for ice comparisons, we don't reach target so we need to bump it up (add 1) c1 = offs['ice'+suff1]/100. + 1 # for co2 comparison, not sure how to scale, b/c want to remove ice effect, which isn't a scaling? c2 = offs['co2'+suff2]/100. if suppttl: ttl='' else: ttl='Full-Sum' #ttl='(ice'+suff +'+co2'+suff+')-full' if cmind=='': # just divide the other clims by 2 if vert: phd=cplt.vert_plot(full-(ice+co2),lev,lat,axis=ax,title=ttl, cmin=cmin/2,cmax=cmax/2,suppcb=suppcb,screen=screen,cmap=cmap, suppylab=True,levlim=levlim,latlim=latlim,ptype=vertptype) if addsig: cplt.addtsig(ax,combopv,lat,lev,sigtype=sigtype) if addclimcont: # want in the linearity fig?? @@@@ cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,phd=cplt.kemmap(full-(ice+co2),lat,lon,axis=ax,title=ttl,ptype=ptype, cmin=cmin/2,cmax=cmax/2,suppcb=suppcb,cmap=cmap) if addsig: cplt.addtsigm(bm,combopv,lat,lon,sigtype=sigtype) else: if vert: phd=cplt.vert_plot(full-(ice+co2),lev,lat,axis=ax,title=ttl, cmin=cmind,cmax=cmaxd,suppcb=suppcb,screen=screen,cmap=cmap, suppylab=True,levlim=levlim,latlim=latlim,ptype=vertptype) if addsig: cplt.addtsig(ax,combopv,lat,lev,sigtype=sigtype) if addclimcont: # want in the linearity fig?? @@@@ cplt.add_contoursvert(ax,ctl,lat,lev,verb=True,clab=True,conts=ctlconts,fmt=fmt) else: bm,phd=cplt.kemmap(full-(ice+co2),lat,lon,axis=ax,title=ttl,ptype=ptype, cmin=cmind,cmax=cmaxd,suppcb=suppcb,cmap=cmap) if addsig: cplt.addtsigm(bm,combopv,lat,lon,sigtype=sigtype) if fsz!=None: ax.set_title(ttl,fontsize=fsz) return ph,ph2,phd else: return ph,ph2