def filegetter(filedir=None,savedir=None,filetype='.mpl',**kwargs):
"""
takes a top-level directory and finds all relevant files
returns a list of filenames to iterate through
inputs:
filedir = top-level directory. If emplty, interactive window is used
filetype = file extension to look for (.mpl or .h5), defaults to .mpl
"""
altitudes = kwargs.get('altitudes',None)
starttime = kwargs.get('starttime',datetime.datetime(1970,1,1,0))
endtime = kwargs.get('endtime',datetime.datetime.now())
timestep = kwargs.get('timestep',None)
verbose = kwargs.get('verbose',False)
SNRmask=kwargs.get('SNRmask',True)
SNRthreshold=kwargs.get('SNRthreshold',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
NRBmask=kwargs.get('NRBmask',True)
NRBthreshold=kwargs.get('NRBthreshold',5.0)
NRBmin=kwargs.get('NRBmin',0.5)
minalt=kwargs.get('minalt',0.150)
dolayers=kwargs.get('dolayers',True)
docorrection=kwargs.get('docorrection',False)
winsize=kwargs.get('winsize',5)
topickle=kwargs.get('topickle',True)
picklefilename=kwargs.get('picklefilename','test.p')
#findalllayers kwargs
molthresh=kwargs.get('molthresh',1.0)
layernoisethresh=kwargs.get('layernoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(5,15))
PBLwidth=kwargs.get('PBLwidth',7)
sigma0=kwargs.get('sigma0',0.1)
depolsigma0=kwargs.get('depolsigma0',0.05)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.35)
smokethresh=kwargs.get('smokethresh',0.05)
dustthresh=kwargs.get('dustthresh',0.20)
maxaeroalt=kwargs.get('maxaeroalt',10.0)
#correction kwargs
refalt=kwargs.get('refalt',None)
calrange=kwargs.get('calrange',None)
method=kwargs.get('method','klett2')
lrat=kwargs.get('lrat',None)
mode=kwargs.get('mode','copol')
doprogress=kwargs.get('doproc',True)
recalc=kwargs.get('recalc',False)
if filedir is None:
filedir = mtools.set_dir('Sepect top-level folder to search')
if savedir is None:
savedir = mtools.set_dir('Sepect top-level folder to save to')
outlist=[]
picklelist=[]
n=0
for root, folder, filelist in os.walk(filedir):
for filename in filelist:
if filename.endswith(filetype):
if doprogress:
progress(n,len(filelist))
n+=1
tempMPL=mtools.MPL()
if filetype=='.mpl':
if starttime<=mfile.MPLtodatetime(filename)<=endtime:
fileinfo=os.stat(os.path.join(root,filename))
if fileinfo.st_size==0:
continue
else:
savefilename='{0}_proc.h5'.format(filename.split('.')[0])
if recalc:
tempMPL.fromMPL(os.path.join(root,filename))
procflag=False
elif savefilename in os.listdir(savedir):
procflag=True
tempMPL.fromHDF(os.path.join(savedir,savefilename))
else:
tempMPL.fromMPL(os.path.join(root,filename))
procflag=False
if verbose:
print filename
else:
continue
elif filetype=='.h5':
procflag=True
if starttime<=mfile.H5todatetime(filename)<=endtime:
fileinfo=os.stat(os.path.join(root,filename))
if fileinfo.st_size==0:
continue
else:
tempMPL.fromHDF(os.path.join(root,filename))
if verbose:
print filename
else:
continue
if procflag and not recalc:
if topickle:
picklelist.append(tempMPL)
outlist.append(os.path.join(savedir,savefilename))
else:
if timestep:
tempMPL = tempMPL.time_resample(timestep=timestep,verbose=verbose)
if altitudes.any():
tempMPL = tempMPL.alt_resample(altitudes,verbose=verbose)
tempMPL.calc_all()
if dolayers:
layerkwargs={'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype,
'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh,
'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh,
'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh,'sigma0':sigma0,'depolsigma0':depolsigma0,'maxaeroalt':maxaeroalt}
layerdict=mproc.findalllayers(mplin=tempMPL,**layerkwargs)
tempMPL=mproc.scenemaker(layerdict)
if docorrection:
corkwargs={'refalt':refalt,'calrange':calrange,'method':method,'lrat':lrat,'mode':mode}
tempMPL=mproc.basiccorrection(tempMPL,**corkwargs)
if NRBmask:
tempMPL=mtools.NRB_mask_all(tempMPL,NRBthreshold=NRBthreshold,
NRBmin=NRBmin,minalt=minalt,
winsize=winsize)
if SNRmask:
tempMPL=mtools.SNR_mask_all(tempMPL,SNRthreshold=SNRthreshold,
bg_alt=bg_alt,datatype=datatype)
tempMPL.save_to_HDF(os.path.join(savedir,savefilename))
if topickle:
picklelist.append(tempMPL)
outlist.append(os.path.join(savedir,savefilename))
if topickle:
pickle.dump(picklelist,open(picklefilename,'wb'))
return outlist
def quickplot(filename, datadir = None, savefigs = False):
# MPLdat = mtools.MPL()
#
# MPLdat.fromHDF(filename[0])
NRBcopol = pan.read_hdf(filename[0],'LNC')
NRBdepol = pan.read_hdf(filename[0],'MPL')
if savefigs:
if datadir is None:
datadir = mtools.set_dir("Select Folder for Figures to be depositied")
os.chdir(datadir)
try:
os.chdir('Figures')
except WindowsError:
os.makedirs('Figures')
os.chdir('Figures')
#create figure and plot image of depolarization ratios
fsize = 18 #baseline font size
ar = 2.0 #aspect ratio
figheight = 12 #inches
plt.rc('font', family='serif', size=fsize)
fig = plt.figure()
fig.clf()
h_set = range(1,25)
h_set = map(str,h_set)
# NRBcopol = MPLdat.NRB[0]
#
# NRBdepol = MPLdat.depolrat[0]
#
datetime = NRBcopol.index
alt = NRBcopol.columns
print 'Generating Figure'
ax1 = fig.add_subplot(2,1,1)
im1 = mplot.depol_plot(fig, ax1, ar, datetime,alt[::-1],NRBcopol.T[::-1], (0,.5), fsize = fsize)
cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6)
cbar1.set_ticks(np.arange(0,0.6,0.1))
cbar1.set_ticklabels(np.arange(0,0.6,0.1))
cbar1.ax.tick_params(labelsize = fsize)
# cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6)
# cbar1.ax.tick_params(labelsize = fsize)
# cbar1.ax.set_ylabel('$[km^{-1}sr^{-1}]$')
mplot.dateticks(ax1, datetime, fsize = fsize, tcolor = 'w')
ax1.set_xticklabels([])
t1 = ax1.set_title('CORALNet Depol Ratio', fontsize = fsize+10)
t1.set_y(1.03)
ax2 = fig.add_subplot(2,1,2)
im2 = mplot.depol_plot(fig, ax2, ar, datetime,alt[::-1],NRBdepol.T[::-1], (0,0.5), fsize = fsize)
cbar2 = fig.colorbar(im2, orientation = 'vertical', aspect = 6)
cbar2.set_ticks(np.arange(0,0.6,0.1))
cbar2.set_ticklabels(np.arange(0,0.6,0.1))
cbar2.ax.tick_params(labelsize = fsize)
#set axis ranges and tickmarks based on data ranges
mplot.dateticks(ax2, datetime, fsize = fsize)
ax2.set_xlabel('Time [Local]',fontsize = fsize+4)
fig.autofmt_xdate()
t2 = ax2.set_title('miniMPL Depol Ratio',fontsize = fsize+10)
t2.set_y(1.03)
plt.show()
if savefigs:
##plt.savefig(savetitle,dpi = 100, edgecolor = 'b', bbox_inches = 'tight')
fig.set_size_inches(figheight*ar,figheight)
plt.savefig(filename+'NRB.png')
print 'Done'
def PBLanalyze(**kwargs):
"""
Program used to find and hsitogram PBL heights and capping layers
"""
filedir=kwargs.get('filedir',[])
location=kwargs.get('location',(48.9167,-125.5333))
altrange = kwargs.get('altrange',np.arange(0.150,15.030,0.030))
timestep = kwargs.get('timestep','60S')
molthresh=kwargs.get('molthresh',1.0)
layernoisethresh=kwargs.get('layernoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10))
PBLwidth=kwargs.get('PBLwidth',7)
sigma0=kwargs.get('sigma0',0.01)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.35)
smokethresh=kwargs.get('smokethresh',0.10)
dustthresh=kwargs.get('dustthresh',0.20)
verbose=kwargs.get('verbose',False)
if not filedir:
filedir=mtools.set_dir('Select file directory to process')
olddir=os.getcwd()
os.chdir(filedir)
rawfiles = glob.glob('*.mpl')
rawfiles.sort()
[path,startfile] = os.path.split(rawfiles[0])
[path,endfile] = os.path.split(rawfiles[-1])
startdate = kwargs.get('startdate',mfile.MPLtodatetime(startfile))
enddate = kwargs.get('enddate',mfile.MPLtodatetime(endfile))
mpllist=[]
datelist=[]
PBL=pan.DataFrame()
caplayer=pan.DataFrame()
clearcap=pan.DataFrame()
layerkwargs={'altrange':altrange,'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype,
'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh,
'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh,
'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh}
filedates=[mfile.MPLtodatetime(os.path.split(f)[1]) for f in rawfiles]
filteredfiles=[(r,t) for r,t in zip(rawfiles,filedates) if (t>=startdate) and (t<=enddate)]
oldday=filedates[0]
tempephem,tempperiods=get_ephemera(location,oldday)
tempmpl=[]
tempdate=[]
ephemera=[]
periods=[]
for f,d in filteredfiles:
newday=d
if newday.date()==oldday.date():
tempmpl.append(f)
tempdate.append(d)
else:
mpllist.append(tempmpl)
datelist.append(tempdate)
ephemera.append(tempephem)
periods.append(tempperiods)
tempmpl=[f]
tempdate=[d]
tempephem,tempperiods=get_ephemera(location,newday)
if f==filteredfiles[-1][0]:
mpllist.append(tempmpl)
datelist.append(tempdate)
ephemera.append(tempephem)
periods.append(tempperiods)
oldday=newday
repnum=1
for mplgrp,datgrp,e,p in zip(mpllist,datelist,ephemera,periods):
if verbose:
print 'Processing Group {0} of {1}'.format(repnum,len(mplgrp))
repnum+=1
for n in range(len(e)-1):
tempmpl=[m for m,t in zip(mplgrp,datgrp) if (t>=e[n])and(t<e[n+1])]
if len(tempmpl)==0:
continue
else:
templayerdict=get_layers(tempmpl,**layerkwargs)
tempcap,tempclear,tempPBL=find_capper(templayerdict,p[n])
PBL=PBL.append(tempPBL)
caplayer=caplayer.append(tempcap)
clearcap=clearcap.append(tempclear)
os.chdir(olddir)
return PBL,caplayer,clearcap
def processall(**kwargs):
newdir=kwargs.get('newdir',None)
chunksize=kwargs.get('chunksize','days')
startdate=kwargs.get('startdate',datetime.datetime(2009,01,01))
enddate=kwargs.get('enddate',datetime.datetime.now())
interactive=kwargs.get('interactive',False)
#general kwargs
altrange = kwargs.get('altrange',np.arange(150,15000,30))
timestep = kwargs.get('timestep','60S')
doplot = kwargs.get('doplot',False)
dolayers = kwargs.get('dolayers',False)
docorrection = kwargs.get('docorrection',False)
dolayerplot = kwargs.get('dolayerplot',False)
docorplot = kwargs.get('docorplot',False)
saveplot = kwargs.get('saveplot',False)
savetype = kwargs.get('savetype','standard')
showplot = kwargs.get('showplot',False)
procsavepath = kwargs.get('procsavepath','.\Processed')
plotsavepath = kwargs.get('plotsavepath','.\Figures')
verbose = kwargs.get('verbose',False)
#NRBmask kwargs
NRBmask = kwargs.get('NRBmask',True)
NRBmasktype = kwargs.get('NRBmasktype','profile')
NRBthresh=kwargs.get('NRBthresh',3.0)
NRBmin=kwargs.get('NRBmin',0.5)
NRBminalt=kwargs.get('NRBminalt',0.150)
NRBnumprofs=kwargs.get('NRBnumprofs',1)
NRBwinsize=kwargs.get('NRBwinsize',3)
NRBinplace=kwargs.get('NRBinplace',False)
#findalllayers kwargs
molthresh=kwargs.get('molthresh',1.0)
noisethresh=kwargs.get('noisethresh',1.0)
depolnoisethresh=kwargs.get('depolnoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
doPBL=kwargs.get('doPBL',True)
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10))
PBLwidth=kwargs.get('PBLwidth',5)
PBLnumprofs=kwargs.get('PBLnumprofs',1)
PBLwinsize=kwargs.get('PBLwinsize',5)
sigma0=kwargs.get('sigma0',None)
depolsigma0=kwargs.get('depolsigma0',None)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.25)
smokethresh=kwargs.get('smokethresh',0.05)
dustthresh=kwargs.get('dustthresh',0.15)
maxaeroalt=kwargs.get('maxaeroalt',10.0)
#scene kwargs
PBLrat=kwargs.get('PBLrat',30.0)
molrat=kwargs.get('molrat',0.0)
moldepol=kwargs.get('moldepol',0.0035)
#correction kwargs
refalt=kwargs.get('refalt',None)
calrange=kwargs.get('calrange',None)
method=kwargs.get('method','klett2')
lrat=kwargs.get('lrat',None)
mode=kwargs.get('mode','copol')
cor_numprofs=kwargs.get('cor_numprofs',1)
cor_winsize=kwargs.get('cor_winsize',1)
#plot kwargs
NRB_limits = kwargs.get('NRB_limits',(0.0,1.0,0.2))
depol_limits = kwargs.get('depol_limits',(0.0,0.5,0.1))
back_limits = kwargs.get('back_limits',(0.0,1e-7,2e-8))
ext_limits = kwargs.get('ext_limits',(0.0,2e-6,4e-7))
hours = kwargs.get('hours',['06','12','18'])
fsize = kwargs.get('fsize',18) #baseline font size
SNRmask = kwargs.get('SNRmask',False)
SNRthresh=kwargs.get('SNRthresh',1.0)
SNRtype=kwargs.get('SNRtype','NRB')
interpolate=kwargs.get('interpolate',None) #other methods include none, bilinear, gaussian and hermite
olddir = os.getcwd()
if not showplot:
plt.ioff()
if newdir is None:
newdir = mtools.set_dir('Select directory to process')
os.chdir(newdir)
if interactive:
rawfiles = mtools.get_files('Select MPL files', filetype = ('.mpl', '*.mpl'))
else:
rawfiles = glob.glob('*.mpl')
rawfiles.sort()
olddate=MPLtodatetime(rawfiles[0])
tempfilelist=[rawfiles[0]]
MPLlist=[]
prockwargs={'altrange':altrange,'timestep':timestep,'doplot':doplot,'dolayers':dolayers,
'docorrection':docorrection,'cor_numprofs':cor_numprofs,'cor_winsize':cor_winsize,
'dolayerplot':dolayerplot,'docorplot':docorplot,'saveplot':saveplot,
'savetype':savetype,'showplot':showplot,'procsavepath':procsavepath,
'plotsavepath':plotsavepath,'verbose':verbose,'NRBmask':NRBmask,'NRBmasktype':NRBmasktype,
'NRBthresh':NRBthresh,'NRBmin':NRBmin,'NRBminalt':NRBminalt,'NRBnumprofs':NRBnumprofs,'NRBwinsize':NRBwinsize,
'NRBinplace':NRBinplace,'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,'molthresh':molthresh,
'bg_alt':bg_alt,'datatype':datatype,
'winsize':winsize,'wavelet':wavelet,'noisethresh':noisethresh,'depolnoisethresh':depolnoisethresh,
'cloudthresh':cloudthresh,'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'doPBL':doPBL,'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,
'PBLnumprofs':PBLnumprofs,'PBLwinsize':PBLwinsize,'maxaeroalt':maxaeroalt,
'sigma0':sigma0,'depolsigma0':depolsigma0,'waterthresh':waterthresh,'icethresh':icethresh,
'smokethresh':smokethresh,'dustthresh':dustthresh,'refalt':refalt,'PBLrat':PBLrat,'molrat':molrat,'moldepol':moldepol,
'method':method,'lrat':lrat,'mode':mode,'NRB_limits':NRB_limits,'depol_limits':depol_limits,
'back_limits':back_limits,'ext_limits':ext_limits,'hours':hours,'fsize':fsize,'interpolate':interpolate}
for f in rawfiles:
newdate=MPLtodatetime(f)
if newdate>=startdate and newdate<=enddate:
if chunksize=='days':
if olddate.date()==newdate.date():
tempfilelist.append(f)
if f==rawfiles[-1]:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
print len(tempfilelist)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
print len(tempfilelist)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist=[f]
olddate=newdate
elif chunksize=='weeks':
if newdate-olddate<=datetime.timedelta(7):
tempfilelist.append(f)
if f==rawfiles[-1]:
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist.append(f)
olddate=newdate
elif chunksize=='months':
if olddate.month==newdate.month:
tempfilelist.append(f)
if f==rawfiles[-1]:
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist.append(f)
olddate=newdate
os.chdir(olddir)
return MPLlist
