def filegetter(filedir=None,savedir=None,filetype='.mpl',**kwargs): """ takes a top-level directory and finds all relevant files returns a list of filenames to iterate through inputs: filedir = top-level directory. If emplty, interactive window is used filetype = file extension to look for (.mpl or .h5), defaults to .mpl """ altitudes = kwargs.get('altitudes',None) starttime = kwargs.get('starttime',datetime.datetime(1970,1,1,0)) endtime = kwargs.get('endtime',datetime.datetime.now()) timestep = kwargs.get('timestep',None) verbose = kwargs.get('verbose',False) SNRmask=kwargs.get('SNRmask',True) SNRthreshold=kwargs.get('SNRthreshold',1.0) bg_alt=kwargs.get('bg_alt',None) datatype=kwargs.get('datatype','NRB') NRBmask=kwargs.get('NRBmask',True) NRBthreshold=kwargs.get('NRBthreshold',5.0) NRBmin=kwargs.get('NRBmin',0.5) minalt=kwargs.get('minalt',0.150) dolayers=kwargs.get('dolayers',True) docorrection=kwargs.get('docorrection',False) winsize=kwargs.get('winsize',5) topickle=kwargs.get('topickle',True) picklefilename=kwargs.get('picklefilename','test.p') #findalllayers kwargs molthresh=kwargs.get('molthresh',1.0) layernoisethresh=kwargs.get('layernoisethresh',1.0) bg_alt=kwargs.get('bg_alt',None) datatype=kwargs.get('datatype','NRB') winsize=kwargs.get('winsize',5) wavelet=kwargs.get('wavelet',signal.ricker) noisethresh=kwargs.get('noisethresh',0.4) cloudthresh=kwargs.get('cloudthresh',(1.0,0.20)) CWTwidth=kwargs.get('CWTwidth',2) minwidth=kwargs.get('minwidth',4) layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5)) PBLwavelet=kwargs.get('PBLwavelet',mproc.dog) PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(5,15)) PBLwidth=kwargs.get('PBLwidth',7) sigma0=kwargs.get('sigma0',0.1) depolsigma0=kwargs.get('depolsigma0',0.05) waterthresh=kwargs.get('waterthresh',0.10) icethresh=kwargs.get('icethresh',0.35) smokethresh=kwargs.get('smokethresh',0.05) dustthresh=kwargs.get('dustthresh',0.20) maxaeroalt=kwargs.get('maxaeroalt',10.0) #correction kwargs refalt=kwargs.get('refalt',None) calrange=kwargs.get('calrange',None) method=kwargs.get('method','klett2') lrat=kwargs.get('lrat',None) mode=kwargs.get('mode','copol') doprogress=kwargs.get('doproc',True) recalc=kwargs.get('recalc',False) if filedir is None: filedir = mtools.set_dir('Sepect top-level folder to search') if savedir is None: savedir = mtools.set_dir('Sepect top-level folder to save to') outlist=[] picklelist=[] n=0 for root, folder, filelist in os.walk(filedir): for filename in filelist: if filename.endswith(filetype): if doprogress: progress(n,len(filelist)) n+=1 tempMPL=mtools.MPL() if filetype=='.mpl': if starttime<=mfile.MPLtodatetime(filename)<=endtime: fileinfo=os.stat(os.path.join(root,filename)) if fileinfo.st_size==0: continue else: savefilename='{0}_proc.h5'.format(filename.split('.')[0]) if recalc: tempMPL.fromMPL(os.path.join(root,filename)) procflag=False elif savefilename in os.listdir(savedir): procflag=True tempMPL.fromHDF(os.path.join(savedir,savefilename)) else: tempMPL.fromMPL(os.path.join(root,filename)) procflag=False if verbose: print filename else: continue elif filetype=='.h5': procflag=True if starttime<=mfile.H5todatetime(filename)<=endtime: fileinfo=os.stat(os.path.join(root,filename)) if fileinfo.st_size==0: continue else: tempMPL.fromHDF(os.path.join(root,filename)) if verbose: print filename else: continue if procflag and not recalc: if topickle: picklelist.append(tempMPL) outlist.append(os.path.join(savedir,savefilename)) else: if timestep: tempMPL = tempMPL.time_resample(timestep=timestep,verbose=verbose) if altitudes.any(): tempMPL = tempMPL.alt_resample(altitudes,verbose=verbose) tempMPL.calc_all() if dolayers: layerkwargs={'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype, 'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh, 'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh, 'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange, 'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0, 'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh, 'dustthresh':dustthresh,'sigma0':sigma0,'depolsigma0':depolsigma0,'maxaeroalt':maxaeroalt} layerdict=mproc.findalllayers(mplin=tempMPL,**layerkwargs) tempMPL=mproc.scenemaker(layerdict) if docorrection: corkwargs={'refalt':refalt,'calrange':calrange,'method':method,'lrat':lrat,'mode':mode} tempMPL=mproc.basiccorrection(tempMPL,**corkwargs) if NRBmask: tempMPL=mtools.NRB_mask_all(tempMPL,NRBthreshold=NRBthreshold, NRBmin=NRBmin,minalt=minalt, winsize=winsize) if SNRmask: tempMPL=mtools.SNR_mask_all(tempMPL,SNRthreshold=SNRthreshold, bg_alt=bg_alt,datatype=datatype) tempMPL.save_to_HDF(os.path.join(savedir,savefilename)) if topickle: picklelist.append(tempMPL) outlist.append(os.path.join(savedir,savefilename)) if topickle: pickle.dump(picklelist,open(picklefilename,'wb')) return outlist
def quickplot(filename, datadir = None, savefigs = False): # MPLdat = mtools.MPL() # # MPLdat.fromHDF(filename[0]) NRBcopol = pan.read_hdf(filename[0],'LNC') NRBdepol = pan.read_hdf(filename[0],'MPL') if savefigs: if datadir is None: datadir = mtools.set_dir("Select Folder for Figures to be depositied") os.chdir(datadir) try: os.chdir('Figures') except WindowsError: os.makedirs('Figures') os.chdir('Figures') #create figure and plot image of depolarization ratios fsize = 18 #baseline font size ar = 2.0 #aspect ratio figheight = 12 #inches plt.rc('font', family='serif', size=fsize) fig = plt.figure() fig.clf() h_set = range(1,25) h_set = map(str,h_set) # NRBcopol = MPLdat.NRB[0] # # NRBdepol = MPLdat.depolrat[0] # datetime = NRBcopol.index alt = NRBcopol.columns print 'Generating Figure' ax1 = fig.add_subplot(2,1,1) im1 = mplot.depol_plot(fig, ax1, ar, datetime,alt[::-1],NRBcopol.T[::-1], (0,.5), fsize = fsize) cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6) cbar1.set_ticks(np.arange(0,0.6,0.1)) cbar1.set_ticklabels(np.arange(0,0.6,0.1)) cbar1.ax.tick_params(labelsize = fsize) # cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6) # cbar1.ax.tick_params(labelsize = fsize) # cbar1.ax.set_ylabel('$[km^{-1}sr^{-1}]$') mplot.dateticks(ax1, datetime, fsize = fsize, tcolor = 'w') ax1.set_xticklabels([]) t1 = ax1.set_title('CORALNet Depol Ratio', fontsize = fsize+10) t1.set_y(1.03) ax2 = fig.add_subplot(2,1,2) im2 = mplot.depol_plot(fig, ax2, ar, datetime,alt[::-1],NRBdepol.T[::-1], (0,0.5), fsize = fsize) cbar2 = fig.colorbar(im2, orientation = 'vertical', aspect = 6) cbar2.set_ticks(np.arange(0,0.6,0.1)) cbar2.set_ticklabels(np.arange(0,0.6,0.1)) cbar2.ax.tick_params(labelsize = fsize) #set axis ranges and tickmarks based on data ranges mplot.dateticks(ax2, datetime, fsize = fsize) ax2.set_xlabel('Time [Local]',fontsize = fsize+4) fig.autofmt_xdate() t2 = ax2.set_title('miniMPL Depol Ratio',fontsize = fsize+10) t2.set_y(1.03) plt.show() if savefigs: ##plt.savefig(savetitle,dpi = 100, edgecolor = 'b', bbox_inches = 'tight') fig.set_size_inches(figheight*ar,figheight) plt.savefig(filename+'NRB.png') print 'Done'
def PBLanalyze(**kwargs): """ Program used to find and hsitogram PBL heights and capping layers """ filedir=kwargs.get('filedir',[]) location=kwargs.get('location',(48.9167,-125.5333)) altrange = kwargs.get('altrange',np.arange(0.150,15.030,0.030)) timestep = kwargs.get('timestep','60S') molthresh=kwargs.get('molthresh',1.0) layernoisethresh=kwargs.get('layernoisethresh',1.0) bg_alt=kwargs.get('bg_alt',None) datatype=kwargs.get('datatype','NRB') winsize=kwargs.get('winsize',5) wavelet=kwargs.get('wavelet',signal.ricker) noisethresh=kwargs.get('noisethresh',0.4) cloudthresh=kwargs.get('cloudthresh',(1.0,0.20)) CWTwidth=kwargs.get('CWTwidth',2) minwidth=kwargs.get('minwidth',4) layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5)) PBLwavelet=kwargs.get('PBLwavelet',mproc.dog) PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10)) PBLwidth=kwargs.get('PBLwidth',7) sigma0=kwargs.get('sigma0',0.01) waterthresh=kwargs.get('waterthresh',0.10) icethresh=kwargs.get('icethresh',0.35) smokethresh=kwargs.get('smokethresh',0.10) dustthresh=kwargs.get('dustthresh',0.20) verbose=kwargs.get('verbose',False) if not filedir: filedir=mtools.set_dir('Select file directory to process') olddir=os.getcwd() os.chdir(filedir) rawfiles = glob.glob('*.mpl') rawfiles.sort() [path,startfile] = os.path.split(rawfiles[0]) [path,endfile] = os.path.split(rawfiles[-1]) startdate = kwargs.get('startdate',mfile.MPLtodatetime(startfile)) enddate = kwargs.get('enddate',mfile.MPLtodatetime(endfile)) mpllist=[] datelist=[] PBL=pan.DataFrame() caplayer=pan.DataFrame() clearcap=pan.DataFrame() layerkwargs={'altrange':altrange,'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype, 'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh, 'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh, 'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange, 'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0, 'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh, 'dustthresh':dustthresh} filedates=[mfile.MPLtodatetime(os.path.split(f)[1]) for f in rawfiles] filteredfiles=[(r,t) for r,t in zip(rawfiles,filedates) if (t>=startdate) and (t<=enddate)] oldday=filedates[0] tempephem,tempperiods=get_ephemera(location,oldday) tempmpl=[] tempdate=[] ephemera=[] periods=[] for f,d in filteredfiles: newday=d if newday.date()==oldday.date(): tempmpl.append(f) tempdate.append(d) else: mpllist.append(tempmpl) datelist.append(tempdate) ephemera.append(tempephem) periods.append(tempperiods) tempmpl=[f] tempdate=[d] tempephem,tempperiods=get_ephemera(location,newday) if f==filteredfiles[-1][0]: mpllist.append(tempmpl) datelist.append(tempdate) ephemera.append(tempephem) periods.append(tempperiods) oldday=newday repnum=1 for mplgrp,datgrp,e,p in zip(mpllist,datelist,ephemera,periods): if verbose: print 'Processing Group {0} of {1}'.format(repnum,len(mplgrp)) repnum+=1 for n in range(len(e)-1): tempmpl=[m for m,t in zip(mplgrp,datgrp) if (t>=e[n])and(t<e[n+1])] if len(tempmpl)==0: continue else: templayerdict=get_layers(tempmpl,**layerkwargs) tempcap,tempclear,tempPBL=find_capper(templayerdict,p[n]) PBL=PBL.append(tempPBL) caplayer=caplayer.append(tempcap) clearcap=clearcap.append(tempclear) os.chdir(olddir) return PBL,caplayer,clearcap
def processall(**kwargs): newdir=kwargs.get('newdir',None) chunksize=kwargs.get('chunksize','days') startdate=kwargs.get('startdate',datetime.datetime(2009,01,01)) enddate=kwargs.get('enddate',datetime.datetime.now()) interactive=kwargs.get('interactive',False) #general kwargs altrange = kwargs.get('altrange',np.arange(150,15000,30)) timestep = kwargs.get('timestep','60S') doplot = kwargs.get('doplot',False) dolayers = kwargs.get('dolayers',False) docorrection = kwargs.get('docorrection',False) dolayerplot = kwargs.get('dolayerplot',False) docorplot = kwargs.get('docorplot',False) saveplot = kwargs.get('saveplot',False) savetype = kwargs.get('savetype','standard') showplot = kwargs.get('showplot',False) procsavepath = kwargs.get('procsavepath','.\Processed') plotsavepath = kwargs.get('plotsavepath','.\Figures') verbose = kwargs.get('verbose',False) #NRBmask kwargs NRBmask = kwargs.get('NRBmask',True) NRBmasktype = kwargs.get('NRBmasktype','profile') NRBthresh=kwargs.get('NRBthresh',3.0) NRBmin=kwargs.get('NRBmin',0.5) NRBminalt=kwargs.get('NRBminalt',0.150) NRBnumprofs=kwargs.get('NRBnumprofs',1) NRBwinsize=kwargs.get('NRBwinsize',3) NRBinplace=kwargs.get('NRBinplace',False) #findalllayers kwargs molthresh=kwargs.get('molthresh',1.0) noisethresh=kwargs.get('noisethresh',1.0) depolnoisethresh=kwargs.get('depolnoisethresh',1.0) bg_alt=kwargs.get('bg_alt',None) datatype=kwargs.get('datatype','NRB') winsize=kwargs.get('winsize',5) wavelet=kwargs.get('wavelet',signal.ricker) noisethresh=kwargs.get('noisethresh',0.4) cloudthresh=kwargs.get('cloudthresh',(1.0,0.20)) CWTwidth=kwargs.get('CWTwidth',2) minwidth=kwargs.get('minwidth',4) layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5)) doPBL=kwargs.get('doPBL',True) PBLwavelet=kwargs.get('PBLwavelet',mproc.dog) PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10)) PBLwidth=kwargs.get('PBLwidth',5) PBLnumprofs=kwargs.get('PBLnumprofs',1) PBLwinsize=kwargs.get('PBLwinsize',5) sigma0=kwargs.get('sigma0',None) depolsigma0=kwargs.get('depolsigma0',None) waterthresh=kwargs.get('waterthresh',0.10) icethresh=kwargs.get('icethresh',0.25) smokethresh=kwargs.get('smokethresh',0.05) dustthresh=kwargs.get('dustthresh',0.15) maxaeroalt=kwargs.get('maxaeroalt',10.0) #scene kwargs PBLrat=kwargs.get('PBLrat',30.0) molrat=kwargs.get('molrat',0.0) moldepol=kwargs.get('moldepol',0.0035) #correction kwargs refalt=kwargs.get('refalt',None) calrange=kwargs.get('calrange',None) method=kwargs.get('method','klett2') lrat=kwargs.get('lrat',None) mode=kwargs.get('mode','copol') cor_numprofs=kwargs.get('cor_numprofs',1) cor_winsize=kwargs.get('cor_winsize',1) #plot kwargs NRB_limits = kwargs.get('NRB_limits',(0.0,1.0,0.2)) depol_limits = kwargs.get('depol_limits',(0.0,0.5,0.1)) back_limits = kwargs.get('back_limits',(0.0,1e-7,2e-8)) ext_limits = kwargs.get('ext_limits',(0.0,2e-6,4e-7)) hours = kwargs.get('hours',['06','12','18']) fsize = kwargs.get('fsize',18) #baseline font size SNRmask = kwargs.get('SNRmask',False) SNRthresh=kwargs.get('SNRthresh',1.0) SNRtype=kwargs.get('SNRtype','NRB') interpolate=kwargs.get('interpolate',None) #other methods include none, bilinear, gaussian and hermite olddir = os.getcwd() if not showplot: plt.ioff() if newdir is None: newdir = mtools.set_dir('Select directory to process') os.chdir(newdir) if interactive: rawfiles = mtools.get_files('Select MPL files', filetype = ('.mpl', '*.mpl')) else: rawfiles = glob.glob('*.mpl') rawfiles.sort() olddate=MPLtodatetime(rawfiles[0]) tempfilelist=[rawfiles[0]] MPLlist=[] prockwargs={'altrange':altrange,'timestep':timestep,'doplot':doplot,'dolayers':dolayers, 'docorrection':docorrection,'cor_numprofs':cor_numprofs,'cor_winsize':cor_winsize, 'dolayerplot':dolayerplot,'docorplot':docorplot,'saveplot':saveplot, 'savetype':savetype,'showplot':showplot,'procsavepath':procsavepath, 'plotsavepath':plotsavepath,'verbose':verbose,'NRBmask':NRBmask,'NRBmasktype':NRBmasktype, 'NRBthresh':NRBthresh,'NRBmin':NRBmin,'NRBminalt':NRBminalt,'NRBnumprofs':NRBnumprofs,'NRBwinsize':NRBwinsize, 'NRBinplace':NRBinplace,'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,'molthresh':molthresh, 'bg_alt':bg_alt,'datatype':datatype, 'winsize':winsize,'wavelet':wavelet,'noisethresh':noisethresh,'depolnoisethresh':depolnoisethresh, 'cloudthresh':cloudthresh,'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange, 'doPBL':doPBL,'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth, 'PBLnumprofs':PBLnumprofs,'PBLwinsize':PBLwinsize,'maxaeroalt':maxaeroalt, 'sigma0':sigma0,'depolsigma0':depolsigma0,'waterthresh':waterthresh,'icethresh':icethresh, 'smokethresh':smokethresh,'dustthresh':dustthresh,'refalt':refalt,'PBLrat':PBLrat,'molrat':molrat,'moldepol':moldepol, 'method':method,'lrat':lrat,'mode':mode,'NRB_limits':NRB_limits,'depol_limits':depol_limits, 'back_limits':back_limits,'ext_limits':ext_limits,'hours':hours,'fsize':fsize,'interpolate':interpolate} for f in rawfiles: newdate=MPLtodatetime(f) if newdate>=startdate and newdate<=enddate: if chunksize=='days': if olddate.date()==newdate.date(): tempfilelist.append(f) if f==rawfiles[-1]: if verbose: print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day) print len(tempfilelist) prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) olddate=newdate else: if verbose: print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day) print len(tempfilelist) prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) tempfilelist=[f] olddate=newdate elif chunksize=='weeks': if newdate-olddate<=datetime.timedelta(7): tempfilelist.append(f) if f==rawfiles[-1]: prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) olddate=newdate else: if verbose: print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day) prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) tempfilelist.append(f) olddate=newdate elif chunksize=='months': if olddate.month==newdate.month: tempfilelist.append(f) if f==rawfiles[-1]: prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) olddate=newdate else: if verbose: print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day) prockwargs['rawfiles']=tempfilelist mpltemp=fileproc(**prockwargs) MPLlist.append(mpltemp) tempfilelist.append(f) olddate=newdate os.chdir(olddir) return MPLlist