def getmplfiles(newdir,interactive=True, verbose=False):
olddir = os.getcwd()
os.chdir(newdir)
if interactive:
rawfiles = mtools.get_files('Select MPL files', filetype = ('.mpl', '*.mpl'))
else:
rawfiles = glob.glob('*.mpl')
rawfiles.sort()
if verbose:
print rawfiles
os.chdir(olddir)
return rawfiles
import MPLtools as mtools
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import cm
import MPLhisttools as mhist
import json
olddir = os.getcwd()
#os.chdir('C:\SigmaMPL\DATA')
os.chdir('E:\Smoke2015\Processed')
filepath = mtools.get_files('Select MPL file', filetype = ('.h5', '*.h5'))
depollist = []
depoldates = []
jsondat={}
for f in filepath:
MPLfile = mtools.MPL()
MPLfile.fromHDF(f)
altrange = np.arange(0.150,3.03,0.030)
print 'Resampling {0}'.format(f)
MPLfile.alt_resample(altrange)
depolraw=MPLfile.depolrat[0]
#----------------------------------------------------------------------------
#Uses tools created in MPL_tools to open all files in a folder and resample
#them to a regular spacing in altitude/date the concatenates them into one
#pandas dataframe and plots it using LNC_plot
#July 05, 2012
#----------------------------------------------------------------------------
os.chdir('C:\Program Files (x86)\SigmaMPL\DATA')
#newdir = mtools.set_dir('Select Event Folder')
#
#os.chdir(newdir)
#
#files = os.listdir(newdir)
files = mtools.get_files('Select MPL files', filetype = ('.mpl', '*.mpl'))
rawfiles = []
#set altitude range and date step sizes
altrange = np.arange(150,20000,10)#meters
timestep = '60S' #seconds
#check to see if each file has been processed before and separate processed
#files into a new list
for f in files:
if '.mpl' in f:
rawfiles.append(f)
#open, altitude resample, and concatenate data and mask files
def filegetter(filedir=None,savedir=None,filetype='.mpl',**kwargs):
"""
takes a top-level directory and finds all relevant files
returns a list of filenames to iterate through
inputs:
filedir = top-level directory. If emplty, interactive window is used
filetype = file extension to look for (.mpl or .h5), defaults to .mpl
"""
altitudes = kwargs.get('altitudes',None)
starttime = kwargs.get('starttime',datetime.datetime(1970,1,1,0))
endtime = kwargs.get('endtime',datetime.datetime.now())
timestep = kwargs.get('timestep',None)
verbose = kwargs.get('verbose',False)
SNRmask=kwargs.get('SNRmask',True)
SNRthreshold=kwargs.get('SNRthreshold',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
NRBmask=kwargs.get('NRBmask',True)
NRBthreshold=kwargs.get('NRBthreshold',5.0)
NRBmin=kwargs.get('NRBmin',0.5)
minalt=kwargs.get('minalt',0.150)
dolayers=kwargs.get('dolayers',True)
docorrection=kwargs.get('docorrection',False)
winsize=kwargs.get('winsize',5)
topickle=kwargs.get('topickle',True)
picklefilename=kwargs.get('picklefilename','test.p')
#findalllayers kwargs
molthresh=kwargs.get('molthresh',1.0)
layernoisethresh=kwargs.get('layernoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(5,15))
PBLwidth=kwargs.get('PBLwidth',7)
sigma0=kwargs.get('sigma0',0.1)
depolsigma0=kwargs.get('depolsigma0',0.05)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.35)
smokethresh=kwargs.get('smokethresh',0.05)
dustthresh=kwargs.get('dustthresh',0.20)
maxaeroalt=kwargs.get('maxaeroalt',10.0)
#correction kwargs
refalt=kwargs.get('refalt',None)
calrange=kwargs.get('calrange',None)
method=kwargs.get('method','klett2')
lrat=kwargs.get('lrat',None)
mode=kwargs.get('mode','copol')
doprogress=kwargs.get('doproc',True)
recalc=kwargs.get('recalc',False)
if filedir is None:
filedir = mtools.set_dir('Sepect top-level folder to search')
if savedir is None:
savedir = mtools.set_dir('Sepect top-level folder to save to')
outlist=[]
picklelist=[]
n=0
for root, folder, filelist in os.walk(filedir):
for filename in filelist:
if filename.endswith(filetype):
if doprogress:
progress(n,len(filelist))
n+=1
tempMPL=mtools.MPL()
if filetype=='.mpl':
if starttime<=mfile.MPLtodatetime(filename)<=endtime:
fileinfo=os.stat(os.path.join(root,filename))
if fileinfo.st_size==0:
continue
else:
savefilename='{0}_proc.h5'.format(filename.split('.')[0])
if recalc:
tempMPL.fromMPL(os.path.join(root,filename))
procflag=False
elif savefilename in os.listdir(savedir):
procflag=True
tempMPL.fromHDF(os.path.join(savedir,savefilename))
else:
tempMPL.fromMPL(os.path.join(root,filename))
procflag=False
if verbose:
print filename
else:
continue
elif filetype=='.h5':
procflag=True
if starttime<=mfile.H5todatetime(filename)<=endtime:
fileinfo=os.stat(os.path.join(root,filename))
if fileinfo.st_size==0:
continue
else:
tempMPL.fromHDF(os.path.join(root,filename))
if verbose:
print filename
else:
continue
if procflag and not recalc:
if topickle:
picklelist.append(tempMPL)
outlist.append(os.path.join(savedir,savefilename))
else:
if timestep:
tempMPL = tempMPL.time_resample(timestep=timestep,verbose=verbose)
if altitudes.any():
tempMPL = tempMPL.alt_resample(altitudes,verbose=verbose)
tempMPL.calc_all()
if dolayers:
layerkwargs={'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype,
'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh,
'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh,
'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh,'sigma0':sigma0,'depolsigma0':depolsigma0,'maxaeroalt':maxaeroalt}
layerdict=mproc.findalllayers(mplin=tempMPL,**layerkwargs)
tempMPL=mproc.scenemaker(layerdict)
if docorrection:
corkwargs={'refalt':refalt,'calrange':calrange,'method':method,'lrat':lrat,'mode':mode}
tempMPL=mproc.basiccorrection(tempMPL,**corkwargs)
if NRBmask:
tempMPL=mtools.NRB_mask_all(tempMPL,NRBthreshold=NRBthreshold,
NRBmin=NRBmin,minalt=minalt,
winsize=winsize)
if SNRmask:
tempMPL=mtools.SNR_mask_all(tempMPL,SNRthreshold=SNRthreshold,
bg_alt=bg_alt,datatype=datatype)
tempMPL.save_to_HDF(os.path.join(savedir,savefilename))
if topickle:
picklelist.append(tempMPL)
outlist.append(os.path.join(savedir,savefilename))
if topickle:
pickle.dump(picklelist,open(picklefilename,'wb'))
return outlist
# allfiles=pickle.load(sfile)
datatypes={'NRB':['NRBhistdat_sigma.h5',(0.0,5.0),'NRBhistplot_sigma.png'],
'Depolrat':['Depolhistdat_sigma.h5',(0,1.0),'Depolhistplot_sigma.png'],
'SNR':['SNRhistdat_sigma.h5',(0,10.0),'SNRhistplot_sigma.png']}
alldict={}
n=0
for dtype,dspecs in datatypes.iteritems():
df=dataextractor(datatype=dtype,loadfiletype='pickle',loadfilename=picklefile,
toHDF=True,HDFfile=os.path.join(topdir,dspecs[0]))
dfplot=df[df>0]
dfplot.fillna(value=-99999,inplace=True)
alldict[dtype]=histplot1D(df,numbins=100,binrange=dspecs[1],xlabel=dtype,
fignum=n,cumulative=False,normalize=True,
saveplot=True,plotfilename=os.path.join(topdir,dspecs[2]))
n+=1
os.chdir(topdir)
mplfile=mtools.get_files('blah',filetype = ('.h5','*.h5'))
MPLtest=mtools.MPL()
MPLtest.fromHDF(mplfile[0])
MPLtest.calc_all()
NRB=MPLtest.NRB[0]
NRB.fillna(-99999,inplace=True)
depol=MPLtest.depolrat[0]
depol.fillna(-99999,inplace=True)
hist1,NRBedges,altedges=althistplot(NRB,numdatbins=500,numaltbins=len(NRB.columns),databinrange=[0.0,10.0],doplot=True)
saveplots=True
showplots=True
dolayerplot=True
docorplot=True
# hours=['02','04','06','08','10','12','14','168','18','20','22']
# hours=['03','06','09','12','15','18','21']
hours=['06','12','18']
NRB_limits=(0.0,0.5,0.1)
depol_limits=(0.0,0.5,0.1)
back_limits=(0.0,2e-3,5e-4)
ext_limits=(0.0,1e-1,2e-2)
interpolate='none'
os.chdir('K:\Smoke2015\Processed')
filepath = mtools.get_files('Select Processed MPL file(s)', filetype = ('.h5', '*.h5'))
kwargs={'alt_resample':alt_resample,'altrange':altrange,
'time_resample':time_resample,'timestep':timestep,'starttime':starttime,'endtime':endtime,
'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,
'saveplots':saveplots,'showplots':showplots,
'dolayerplot':dolayerplot,'docorplot':docorplot,
'NRBmask':NRBmask,'NRBthresh':NRBthresh,'NRBmin':NRBmin,
'NRBminalt':NRBminalt,'NRBnumprofs':NRBnumprofs,
'NRB_limits':NRB_limits,'depol_limits':depol_limits,
'back_limits':back_limits,'ext_limits':ext_limits,
'interpolate':interpolate,'hours':hours}
plotall(filepath[0],**kwargs)
def PBLanalyze(**kwargs):
"""
Program used to find and hsitogram PBL heights and capping layers
"""
filedir=kwargs.get('filedir',[])
location=kwargs.get('location',(48.9167,-125.5333))
altrange = kwargs.get('altrange',np.arange(0.150,15.030,0.030))
timestep = kwargs.get('timestep','60S')
molthresh=kwargs.get('molthresh',1.0)
layernoisethresh=kwargs.get('layernoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10))
PBLwidth=kwargs.get('PBLwidth',7)
sigma0=kwargs.get('sigma0',0.01)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.35)
smokethresh=kwargs.get('smokethresh',0.10)
dustthresh=kwargs.get('dustthresh',0.20)
verbose=kwargs.get('verbose',False)
if not filedir:
filedir=mtools.set_dir('Select file directory to process')
olddir=os.getcwd()
os.chdir(filedir)
rawfiles = glob.glob('*.mpl')
rawfiles.sort()
[path,startfile] = os.path.split(rawfiles[0])
[path,endfile] = os.path.split(rawfiles[-1])
startdate = kwargs.get('startdate',mfile.MPLtodatetime(startfile))
enddate = kwargs.get('enddate',mfile.MPLtodatetime(endfile))
mpllist=[]
datelist=[]
PBL=pan.DataFrame()
caplayer=pan.DataFrame()
clearcap=pan.DataFrame()
layerkwargs={'altrange':altrange,'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype,
'molthresh':molthresh,'winsize':winsize,'layernoisethresh':layernoisethresh,
'wavelet':wavelet,'noisethresh':noisethresh,'cloudthresh':cloudthresh,
'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,'sigma0':sigma0,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh}
filedates=[mfile.MPLtodatetime(os.path.split(f)[1]) for f in rawfiles]
filteredfiles=[(r,t) for r,t in zip(rawfiles,filedates) if (t>=startdate) and (t<=enddate)]
oldday=filedates[0]
tempephem,tempperiods=get_ephemera(location,oldday)
tempmpl=[]
tempdate=[]
ephemera=[]
periods=[]
for f,d in filteredfiles:
newday=d
if newday.date()==oldday.date():
tempmpl.append(f)
tempdate.append(d)
else:
mpllist.append(tempmpl)
datelist.append(tempdate)
ephemera.append(tempephem)
periods.append(tempperiods)
tempmpl=[f]
tempdate=[d]
tempephem,tempperiods=get_ephemera(location,newday)
if f==filteredfiles[-1][0]:
mpllist.append(tempmpl)
datelist.append(tempdate)
ephemera.append(tempephem)
periods.append(tempperiods)
oldday=newday
repnum=1
for mplgrp,datgrp,e,p in zip(mpllist,datelist,ephemera,periods):
if verbose:
print 'Processing Group {0} of {1}'.format(repnum,len(mplgrp))
repnum+=1
for n in range(len(e)-1):
tempmpl=[m for m,t in zip(mplgrp,datgrp) if (t>=e[n])and(t<e[n+1])]
if len(tempmpl)==0:
continue
else:
templayerdict=get_layers(tempmpl,**layerkwargs)
tempcap,tempclear,tempPBL=find_capper(templayerdict,p[n])
PBL=PBL.append(tempPBL)
caplayer=caplayer.append(tempcap)
clearcap=clearcap.append(tempclear)
os.chdir(olddir)
return PBL,caplayer,clearcap
edge_above=e
#if upper edge is found, add indices of (lower,center,upper) to layers
layers.append((edge_below,peakloc,edge_above))
break
#Step 5: Return list of tuples where each element is ()
return layers
#open a test case from an actual .mpl file
os.chdir('C:\Users\dashamstyr\Dropbox\Lidar Files\MPL Data\DATA\Whistler-0330\Processed')
mpltest = mtools.MPL()
mpltest.fromHDF('201403300000-201403302300_proc.h5')
testprof = mpltest.data[0].iloc[80,:]
depolprof = mpltest.data[0].iloc[101,:]
real_z=testprof.index.values
z = np.arange(100,15000,3)
z = z+0.0
wave = 532.0 #nm
lrat_p = 30.0
lrat_m = 8.0*np.pi/3.0
P_mol = ltools.molprof(z,wave)
def quickplot(filename, datadir = None, savefigs = False):
# MPLdat = mtools.MPL()
#
# MPLdat.fromHDF(filename[0])
NRBcopol = pan.read_hdf(filename[0],'LNC')
NRBdepol = pan.read_hdf(filename[0],'MPL')
if savefigs:
if datadir is None:
datadir = mtools.set_dir("Select Folder for Figures to be depositied")
os.chdir(datadir)
try:
os.chdir('Figures')
except WindowsError:
os.makedirs('Figures')
os.chdir('Figures')
#create figure and plot image of depolarization ratios
fsize = 18 #baseline font size
ar = 2.0 #aspect ratio
figheight = 12 #inches
plt.rc('font', family='serif', size=fsize)
fig = plt.figure()
fig.clf()
h_set = range(1,25)
h_set = map(str,h_set)
# NRBcopol = MPLdat.NRB[0]
#
# NRBdepol = MPLdat.depolrat[0]
#
datetime = NRBcopol.index
alt = NRBcopol.columns
print 'Generating Figure'
ax1 = fig.add_subplot(2,1,1)
im1 = mplot.depol_plot(fig, ax1, ar, datetime,alt[::-1],NRBcopol.T[::-1], (0,.5), fsize = fsize)
cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6)
cbar1.set_ticks(np.arange(0,0.6,0.1))
cbar1.set_ticklabels(np.arange(0,0.6,0.1))
cbar1.ax.tick_params(labelsize = fsize)
# cbar1 = fig.colorbar(im1, orientation = 'vertical', aspect = 6)
# cbar1.ax.tick_params(labelsize = fsize)
# cbar1.ax.set_ylabel('$[km^{-1}sr^{-1}]$')
mplot.dateticks(ax1, datetime, fsize = fsize, tcolor = 'w')
ax1.set_xticklabels([])
t1 = ax1.set_title('CORALNet Depol Ratio', fontsize = fsize+10)
t1.set_y(1.03)
ax2 = fig.add_subplot(2,1,2)
im2 = mplot.depol_plot(fig, ax2, ar, datetime,alt[::-1],NRBdepol.T[::-1], (0,0.5), fsize = fsize)
cbar2 = fig.colorbar(im2, orientation = 'vertical', aspect = 6)
cbar2.set_ticks(np.arange(0,0.6,0.1))
cbar2.set_ticklabels(np.arange(0,0.6,0.1))
cbar2.ax.tick_params(labelsize = fsize)
#set axis ranges and tickmarks based on data ranges
mplot.dateticks(ax2, datetime, fsize = fsize)
ax2.set_xlabel('Time [Local]',fontsize = fsize+4)
fig.autofmt_xdate()
t2 = ax2.set_title('miniMPL Depol Ratio',fontsize = fsize+10)
t2.set_y(1.03)
plt.show()
if savefigs:
##plt.savefig(savetitle,dpi = 100, edgecolor = 'b', bbox_inches = 'tight')
fig.set_size_inches(figheight*ar,figheight)
plt.savefig(filename+'NRB.png')
print 'Done'
def processall(**kwargs):
newdir=kwargs.get('newdir',None)
chunksize=kwargs.get('chunksize','days')
startdate=kwargs.get('startdate',datetime.datetime(2009,01,01))
enddate=kwargs.get('enddate',datetime.datetime.now())
interactive=kwargs.get('interactive',False)
#general kwargs
altrange = kwargs.get('altrange',np.arange(150,15000,30))
timestep = kwargs.get('timestep','60S')
doplot = kwargs.get('doplot',False)
dolayers = kwargs.get('dolayers',False)
docorrection = kwargs.get('docorrection',False)
dolayerplot = kwargs.get('dolayerplot',False)
docorplot = kwargs.get('docorplot',False)
saveplot = kwargs.get('saveplot',False)
savetype = kwargs.get('savetype','standard')
showplot = kwargs.get('showplot',False)
procsavepath = kwargs.get('procsavepath','.\Processed')
plotsavepath = kwargs.get('plotsavepath','.\Figures')
verbose = kwargs.get('verbose',False)
#NRBmask kwargs
NRBmask = kwargs.get('NRBmask',True)
NRBmasktype = kwargs.get('NRBmasktype','profile')
NRBthresh=kwargs.get('NRBthresh',3.0)
NRBmin=kwargs.get('NRBmin',0.5)
NRBminalt=kwargs.get('NRBminalt',0.150)
NRBnumprofs=kwargs.get('NRBnumprofs',1)
NRBwinsize=kwargs.get('NRBwinsize',3)
NRBinplace=kwargs.get('NRBinplace',False)
#findalllayers kwargs
molthresh=kwargs.get('molthresh',1.0)
noisethresh=kwargs.get('noisethresh',1.0)
depolnoisethresh=kwargs.get('depolnoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
noisethresh=kwargs.get('noisethresh',0.4)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
doPBL=kwargs.get('doPBL',True)
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10))
PBLwidth=kwargs.get('PBLwidth',5)
PBLnumprofs=kwargs.get('PBLnumprofs',1)
PBLwinsize=kwargs.get('PBLwinsize',5)
sigma0=kwargs.get('sigma0',None)
depolsigma0=kwargs.get('depolsigma0',None)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.25)
smokethresh=kwargs.get('smokethresh',0.05)
dustthresh=kwargs.get('dustthresh',0.15)
maxaeroalt=kwargs.get('maxaeroalt',10.0)
#scene kwargs
PBLrat=kwargs.get('PBLrat',30.0)
molrat=kwargs.get('molrat',0.0)
moldepol=kwargs.get('moldepol',0.0035)
#correction kwargs
refalt=kwargs.get('refalt',None)
calrange=kwargs.get('calrange',None)
method=kwargs.get('method','klett2')
lrat=kwargs.get('lrat',None)
mode=kwargs.get('mode','copol')
cor_numprofs=kwargs.get('cor_numprofs',1)
cor_winsize=kwargs.get('cor_winsize',1)
#plot kwargs
NRB_limits = kwargs.get('NRB_limits',(0.0,1.0,0.2))
depol_limits = kwargs.get('depol_limits',(0.0,0.5,0.1))
back_limits = kwargs.get('back_limits',(0.0,1e-7,2e-8))
ext_limits = kwargs.get('ext_limits',(0.0,2e-6,4e-7))
hours = kwargs.get('hours',['06','12','18'])
fsize = kwargs.get('fsize',18) #baseline font size
SNRmask = kwargs.get('SNRmask',False)
SNRthresh=kwargs.get('SNRthresh',1.0)
SNRtype=kwargs.get('SNRtype','NRB')
interpolate=kwargs.get('interpolate',None) #other methods include none, bilinear, gaussian and hermite
olddir = os.getcwd()
if not showplot:
plt.ioff()
if newdir is None:
newdir = mtools.set_dir('Select directory to process')
os.chdir(newdir)
if interactive:
rawfiles = mtools.get_files('Select MPL files', filetype = ('.mpl', '*.mpl'))
else:
rawfiles = glob.glob('*.mpl')
rawfiles.sort()
olddate=MPLtodatetime(rawfiles[0])
tempfilelist=[rawfiles[0]]
MPLlist=[]
prockwargs={'altrange':altrange,'timestep':timestep,'doplot':doplot,'dolayers':dolayers,
'docorrection':docorrection,'cor_numprofs':cor_numprofs,'cor_winsize':cor_winsize,
'dolayerplot':dolayerplot,'docorplot':docorplot,'saveplot':saveplot,
'savetype':savetype,'showplot':showplot,'procsavepath':procsavepath,
'plotsavepath':plotsavepath,'verbose':verbose,'NRBmask':NRBmask,'NRBmasktype':NRBmasktype,
'NRBthresh':NRBthresh,'NRBmin':NRBmin,'NRBminalt':NRBminalt,'NRBnumprofs':NRBnumprofs,'NRBwinsize':NRBwinsize,
'NRBinplace':NRBinplace,'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,'molthresh':molthresh,
'bg_alt':bg_alt,'datatype':datatype,
'winsize':winsize,'wavelet':wavelet,'noisethresh':noisethresh,'depolnoisethresh':depolnoisethresh,
'cloudthresh':cloudthresh,'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,
'doPBL':doPBL,'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,
'PBLnumprofs':PBLnumprofs,'PBLwinsize':PBLwinsize,'maxaeroalt':maxaeroalt,
'sigma0':sigma0,'depolsigma0':depolsigma0,'waterthresh':waterthresh,'icethresh':icethresh,
'smokethresh':smokethresh,'dustthresh':dustthresh,'refalt':refalt,'PBLrat':PBLrat,'molrat':molrat,'moldepol':moldepol,
'method':method,'lrat':lrat,'mode':mode,'NRB_limits':NRB_limits,'depol_limits':depol_limits,
'back_limits':back_limits,'ext_limits':ext_limits,'hours':hours,'fsize':fsize,'interpolate':interpolate}
for f in rawfiles:
newdate=MPLtodatetime(f)
if newdate>=startdate and newdate<=enddate:
if chunksize=='days':
if olddate.date()==newdate.date():
tempfilelist.append(f)
if f==rawfiles[-1]:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
print len(tempfilelist)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
print len(tempfilelist)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist=[f]
olddate=newdate
elif chunksize=='weeks':
if newdate-olddate<=datetime.timedelta(7):
tempfilelist.append(f)
if f==rawfiles[-1]:
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist.append(f)
olddate=newdate
elif chunksize=='months':
if olddate.month==newdate.month:
tempfilelist.append(f)
if f==rawfiles[-1]:
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
olddate=newdate
else:
if verbose:
print "Processing file group ending on {0}-{1}-{2}".format(olddate.year,olddate.month,olddate.day)
prockwargs['rawfiles']=tempfilelist
mpltemp=fileproc(**prockwargs)
MPLlist.append(mpltemp)
tempfilelist.append(f)
olddate=newdate
os.chdir(olddir)
return MPLlist
def fileproc(**kwargs):
"""
----------------------------------------------------------------------------
Uses tools created in MPL_tools to open all files in a folder and resample
them to a regular spacing in altitude/date the concatenates them into one
pandas dataframe and saves it as a
created: July 05, 2012
updated: May 09,2014
----------------------------------------------------------------------------
inputs:
**kwargs:
altrange = a list of altitudes to process(becomes pandas columns)
starttime = time point for first profile
endtime = time point for last profile
timestep = time between profiles
interactive = boolean determining if user will select files or if all in folder will be processed
doplot = boolean determining if plot will be made from processed file
dolayers = boolean determining if the
saveplot = boolean determining if plot will be saved
verbose = boolean determining if messages will be displayed
"""
#general kwargs
recalc=kwargs.get('recalc',True)
rawfiles = kwargs.get('rawfiles',None)
altrange = kwargs.get('altrange',np.arange(150,15000,30))
timestep = kwargs.get('timestep','60S')
saveproc = kwargs.get('saveproc',True)
doplot = kwargs.get('doplot',False)
dolayers = kwargs.get('dolayers',False)
docorrection = kwargs.get('docorrection',False)
dolayerplot = kwargs.get('dolayerplot',False)
docorplot = kwargs.get('docorplot',False)
saveplot = kwargs.get('saveplot',False)
savetype = kwargs.get('savetype','standard')
showplot = kwargs.get('showplot',False)
procsavepath = kwargs.get('procsavepath','.\Processed')
plotsavepath = kwargs.get('plotsavepath','.\Figures')
verbose = kwargs.get('verbose',False)
#NRBmask kwargs
NRBmask = kwargs.get('NRBmask',True)
NRBmasktype = kwargs.get('NRBmasktype','profile')
NRBthresh=kwargs.get('NRBthresh',3.0)
NRBmin=kwargs.get('NRBmin',0.5)
NRBminalt=kwargs.get('NRBminalt',0.150)
NRBnumprofs=kwargs.get('NRBnumprofs',1)
NRBwinsize=kwargs.get('NRBwinsize',3)
NRBinplace=kwargs.get('NRBinplace',False)
#findalllayers kwargs
molthresh=kwargs.get('molthresh',1.0)
noisethresh=kwargs.get('noisethresh',1.0)
depolnoisethresh=kwargs.get('depolnoisethresh',1.0)
bg_alt=kwargs.get('bg_alt',None)
datatype=kwargs.get('datatype','NRB')
winsize=kwargs.get('winsize',5)
wavelet=kwargs.get('wavelet',signal.ricker)
cloudthresh=kwargs.get('cloudthresh',(1.0,0.20))
CWTwidth=kwargs.get('CWTwidth',2)
minwidth=kwargs.get('minwidth',4)
layerCWTrange=kwargs.get('layerCWTrange',np.arange(2,5))
doPBL=kwargs.get('doPBL',True)
PBLwavelet=kwargs.get('PBLwavelet',mproc.dog)
PBLCWTrange=kwargs.get('PBLCWTrange',np.arange(2,10))
PBLwidth=kwargs.get('PBLwidth',5)
PBLnumprofs=kwargs.get('PBLnumprofs',1)
PBLwinsize=kwargs.get('PBLwinsize',5)
sigma0=kwargs.get('sigma0',None)
depolsigma0=kwargs.get('depolsigma0',None)
waterthresh=kwargs.get('waterthresh',0.10)
icethresh=kwargs.get('icethresh',0.25)
smokethresh=kwargs.get('smokethresh',0.05)
dustthresh=kwargs.get('dustthresh',0.15)
maxaeroalt=kwargs.get('maxaeroalt',10.0)
#scene kwargs
PBLrat=kwargs.get('PBLrat',30.0)
molrat=kwargs.get('molrat',0.0)
moldepol=kwargs.get('moldepol',0.0035)
#correction kwargs
refalt=kwargs.get('refalt',None)
calrange=kwargs.get('calrange',None)
method=kwargs.get('method','klett2')
lrat=kwargs.get('lrat',None)
mode=kwargs.get('mode','copol')
cor_numprofs=kwargs.get('cor_numprofs',1)
cor_winsize=kwargs.get('cor_winsize',1)
#plot kwargs
NRB_limits = kwargs.get('NRB_limits',(0.0,1.0,0.2))
depol_limits = kwargs.get('depol_limits',(0.0,0.5,0.1))
back_limits = kwargs.get('back_limits',(0.0,1e-7,2e-8))
ext_limits = kwargs.get('ext_limits',(0.0,2e-6,4e-7))
hours = kwargs.get('hours',['06','12','18'])
fsize = kwargs.get('fsize',18) #baseline font size
SNRmask = kwargs.get('SNRmask',False)
SNRthresh=kwargs.get('SNRthresh',1.0)
SNRtype=kwargs.get('SNRtype','NRB')
interpolate=kwargs.get('interpolate',None) #other methods include none, bilinear, gaussian and hermite
#starttime and endtime are defined later
olddir=os.getcwd()
if recalc:
if rawfiles is None:
rawfiles = mtools.get_files('Select files to process',filetype = ('.mpl','*.mpl'))
[path,startfile] = os.path.split(rawfiles[0])
[path,endfile] = os.path.split(rawfiles[-1])
starttime = kwargs.get('starttime',MPLtodatetime(startfile))
endtime = kwargs.get('endtime',MPLtodatetime(endfile))
for r in rawfiles:
[path,tempname] = os.path.split(r)
if starttime <= MPLtodatetime(tempname) <= endtime:
MPLdat_temp = mtools.MPL()
MPLdat_temp.fromMPL(r)
MPLdat_temp.alt_resample(altrange,verbose=verbose)
try:
MPLdat_event.append(MPLdat_temp)
except NameError:
MPLdat_event = MPLdat_temp
#sort by index to make certain data is in order then set date ranges to match
MPLdat_event.header.sort_index()
for n in range(MPLdat_event.header['numchans'][0]):
data = MPLdat_event.data[n]
data = data.sort_index()
MPLdat_event.time_resample(timestep,verbose=verbose)
MPLdat_event.calc_all()
if dolayers:
layerkwargs={'timestep':timestep,'bg_alt':bg_alt,'datatype':datatype,
'molthresh':molthresh,'winsize':winsize,'noisethresh':noisethresh,
'depolnoisethresh':depolnoisethresh,'wavelet':wavelet,'cloudthresh':cloudthresh,
'CWTwidth':CWTwidth,'minwidth':minwidth,'layerCWTrange':layerCWTrange,'doPBL':doPBL,
'PBLwavelet':PBLwavelet,'PBLCWTrange':PBLCWTrange,'PBLwidth':PBLwidth,
'PBLnumprofs':PBLnumprofs,'PBLwinsize':PBLwinsize,'sigma0':sigma0,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh,'sigma0':sigma0,'depolsigma0':depolsigma0,'maxaeroalt':maxaeroalt}
layerdict=mproc.findalllayers(mplin=MPLdat_event,**layerkwargs)
scenekwargs={'PBLrat':PBLrat,'molrat':molrat,'moldepol':moldepol,'cloudthresh':cloudthresh,
'waterthresh':waterthresh,'icethresh':icethresh,'smokethresh':smokethresh,
'dustthresh':dustthresh,'maxaeroalt':maxaeroalt}
MPLdat_event=mproc.scenemaker(layerdict,**scenekwargs)
if docorrection:
corkwargs={'refalt':refalt,'calrange':calrange,'method':method,'lrat':lrat,'mode':mode,
'numprofs':cor_numprofs,'winsize':cor_winsize}
MPLdat_event=mproc.basiccorrection(MPLdat_event,**corkwargs)
if saveproc:
if savetype=='standard':
if len(rawfiles) == 1:
d_filename = '{0}_proc.h5'.format(startfile.split('.')[0])
else:
d_filename = '{0}-{1}_proc.h5'.format(startfile.split('.')[0],endfile.split('.')[0])
savepath=os.path.join(procsavepath,d_filename)
try:
MPLdat_event.save_to_HDF(savepath)
except IOError:
os.mkdir(procsavepath)
MPLdat_event.save_to_HDF(savepath)
elif savetype=='IDL':
if len(rawfiles) == 1:
d_filename = '{0}_IDL.h5'.format(startfile.split('.')[0])
else:
d_filename = '{0}-{1}_IDL.h5'.format(startfile.split('.')[0],endfile.split('.')[0])
savepath=os.path.join(procsavepath,d_filename)
try:
MPLdat_event.save_to_IDL(procsavepath)
except IOError:
os.mkdir(savepath)
MPLdat_event.save_to_IDL(savepath)
else:
if rawfiles is None:
rawfiles = mtools.get_files('Select files to process',filetype = ('.h5','*.h5'))
d_filename=rawfiles[0]
MPLdat_event=mtools.MPL()
MPLdat_event.fromHDF(d_filename)
MPLdat_event.alt_resample(altrange,verbose=verbose)
if NRBmask:
NRBmaskkwargs={'NRBmasktype':NRBmasktype,'NRBthreshold':NRBthresh,'NRBmin':NRBmin,
'minalt':NRBminalt,'numprofs':NRBnumprofs,'winsize':NRBwinsize,
'inplace':NRBinplace}
MPLdat_event=mtools.NRB_mask_all(MPLdat_event,**NRBmaskkwargs)
if doplot:
plotfilename = '{0}.png'.format(d_filename.split('.')[0])
plotkwargs={'altrange':altrange,'topplot_limits':NRB_limits,'bottomplot_limits':depol_limits,
'hours':hours,'fsize':fsize,'savefilename':plotfilename,'savefilepath':plotsavepath,
'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,'interpolate':interpolate,
'showplot':showplot,'saveplot':saveplot}
mplot.doubleplot(MPLdat_event,plotfilename=plotfilename,**plotkwargs)
if docorplot:
coefplotfilename = '{0}-coefficients.png'.format(d_filename.split('.')[0])
coefplotkwargs={'altrange':altrange,'toptype':'backscatter','bottomtype':'extinction',
'topplot_limits':back_limits,'bottomplot_limits':ext_limits,
'hours':hours,'fsize':fsize,'savefilename':coefplotfilename,
'savefilepath':plotsavepath,#'SNRmask':SNRmask,'SNRthresh':SNRthresh,
'interpolate':interpolate,'SNRtype':SNRtype,'showplot':showplot,'saveplot':saveplot}
mplot.doubleplot(MPLdat_event,plotfilename=coefplotfilename,**coefplotkwargs)
if dolayerplot:
layerplotfilename = '{0}-layers.png'.format(d_filename.split('.')[0])
layerplotkwargs={'altrange':altrange,'hours':hours,'fontsize':fsize,'plotfilename':layerplotfilename,
#'SNRmask':SNRmask,'SNRthresh':SNRthresh,'SNRtype':SNRtype,
'plotfilepath':plotsavepath,'showplot':showplot,'saveplot':saveplot}
mplot.colormask_plot(MPLdat_event,**layerplotkwargs)
# if os.path.isdir(procsavepath):
# os.chdir(procsavepath)
# else:
# os.makedirs(procsavepath)
# os.chdir(procsavepath)
#
# if verbose:
# print 'Saving '+d_filename
#
# if savetype=='standard':
# MPLdat_event.save_to_HDF(d_filename)
# elif savetype=='IDL':
# MPLdat_event.save_to_IDL(d_filename)
if verbose:
print 'Done'
os.chdir(olddir)
return MPLdat_event
def main():
olddir = os.getcwd()
#os.chdir('C:\SigmaMPL\DATA')
os.chdir('C:\SigmaMPL\DATA\Processed')
filepath = mtools.get_files('Select MPL file', filetype = ('.h5', '*.h5'))
MPLfile = mtools.MPL()
MPLfile.fromHDF(filepath[0])
altrange = np.arange(150,1000,30)
MPLfile.alt_resample(altrange)
copol = MPLfile.NRB[0]
crosspol = MPLfile.NRB[1]
copolvals = np.hstack(copol.values).astype('float32')
depolMPL = crosspol.values/copol.values
depolvals = depolMPL/(depolMPL+1)
numbins = 100
depolmin = 0.25
depolmax = 0.45
copolmin = 0.0
copolmax = 3e-2
copolhist=h2d.fullhist(copolvals,numbins,copolmin,copolmax,-9999.,-8888.)
depolhist=h2d.fullhist(np.hstack(depolvals),numbins,depolmin,depolmax,-9999.,-8888.)
altOut = h2d.althist(depolvals,altrange,numbins,(depolmin,depolmax))
copolOut=h2d.hist2D(copolhist['fullbins'],depolhist['fullbins'],copolhist['numBins'],depolhist['numBins'])
altcounts=altOut['coverage']
copolcounts = copolOut['coverage']
altcounts[altcounts < 1] = 1
copolcounts[copolcounts < 1] = 1
altlogcounts=np.log10(altcounts)
copollogcounts=np.log10(copolcounts)
try:
os.chdir('../HistPlots')
except WindowsError:
os.makedirs('../HistPlots')
os.chdir('../HistPlots')
startdate = copol.index[0].strftime("%Y-%m-%d")
enddate = copol.index[-1].strftime("%Y-%m-%d")
starttime = copol.index[0].strftime("%H")
endtime = copol.index[-1].strftime("%H")
if startdate == enddate:
if starttime == endtime:
savetime = startdate+'_'+starttime
else:
savetime = startdate+'_'+starttime+'-'+endtime
else:
savetime = startdate+'-'+enddate
fsize = 21 #baseline font size
ar = 1.1 #aspect ratio
figheight = 12 #inches
plt.rc('font', family='serif', size=fsize)
fignum = 0
#fignum+=1
#fig=plt.figure(fignum)
#fig.clf()
#the_axis=fig.add_subplot(111)
#the_axis.plot(depolvals,copolvals,'b+')
#the_axis.set_xlabel('depolvals')
#the_axis.set_ylabel('copolvals')
#the_axis.set_title('raw scatterplot')
#fig.savefig('{0}_{1}-{2}m-copoldepolraw.png'.format(savetime,altrange[0],altrange[-1]))
#fig.canvas.draw()
cmap=cm.bone
cmap.set_over('r')
cmap.set_under('b')
fignum+=1
fig=plt.figure(fignum)
fig.clf()
axis=fig.add_subplot(111)
im=axis.pcolormesh(depolhist['centers'],altrange,altlogcounts.T, cmap = cmap)
cb=plt.colorbar(im,extend='both')
title="2-D Histogram: Altitude vs. Depol. Ratio"
colorbar="log10(counts)"
the_label=cb.ax.set_ylabel(colorbar,rotation=270)
altticks(axis,altrange,numticks = 5, fsize = fsize, tcolor='w')
for line in axis.xaxis.get_ticklines():
line.set_markersize(10)
line.set_markeredgewidth(3)
axis.set_xlabel('Volume Depolarization Ratio')
axis.set_ylabel('Altitude [m]')
# t1 = axis.set_title(title)
fig.set_size_inches(figheight*ar,figheight)
fig.tight_layout()
t1.set_y(1.02)
# fig.canvas.draw()
fig.savefig('{0}_{1}-{2}m-althist.png'.format(savetime,altrange[0],altrange[-1]))
fignum+=1
fig=plt.figure(fignum)
fig.clf()
axis=fig.add_subplot(111)
im=axis.pcolormesh(depolhist['centers'],copolhist['centers'],copollogcounts, cmap = cmap)
axis.yaxis.set_major_locator(MaxNLocator(prune='lower'))
cb=plt.colorbar(im,extend='both')
title="2-D Histogram: Backscatter vs. Depol. Ratio"
colorbar="log10(counts)"
the_label=cb.ax.set_ylabel(colorbar,rotation=270)
for line in axis.yaxis.get_ticklines():
line.set_color('w')
line.set_markersize(10)
line.set_markeredgewidth(3)
for line in axis.xaxis.get_ticklines():
line.set_color('w')
line.set_markersize(10)
line.set_markeredgewidth(3)
axis.set_xlabel('Volume Depolarization Ratio')
axis.set_ylabel('Attenuated Backscatter $[km^{-1} sr^{-1}]$')
# t2 = axis.set_title(title)
fig.set_size_inches(figheight*ar,figheight)
fig.tight_layout()
t2.set_y(1.02)
# fig.canvas.draw()
fig.savefig('{0}_{1}-{2}m-copoldepol.png'.format(savetime,altrange[0],altrange[-1]))
#fignum+=1
#fig = plt.figure(fignum)
#fig.clf()
#axis = fig.add_subplot(111)
#hist = axis.bar(np.arange(len(depolhist['fullbins'])),depolhist['fullbins'], 0.35)
#axis.set_xlabel('Volume Depolaization Ratio')
#fig.savefig('{0}_{1}-{2}m-1Ddepolhist.png'.format(savetime,altrange[0],altrange[-1]))
#fig.canvas.draw()
#plt.show()
os.chdir(olddir)
