#!/usr/bin/env python

# coding: utf-8

# # Intro

#

# Purpose:

#

# Steps through the cloud retrieval of Sp_params with ORACLES 2017 data

# Checks each parameter to verify fit and appropriateness

#

# Output:

#

# figures and save files...

#

# Dependencies:

#

# - Sp_parameters.py : for Sp class definition, and for defining the functions used to build parameters

# - run_kisq_retrieval.py : for the retrieval functions

# - load_utils.py : for loading modis files

# - matplotlib

# - mpltools

# - numpy

#

#

# Needed Files:

#

# - starzen.mat file from ORACLES 2017

# - lut file from ORACLES 2017 (v3_ORACLES_lut.mat)

# # Load the modules and set up the paths

# In[1]:

get_ipython().magic(u'config InlineBackend.rc = {}')

import matplotlib

import os

matplotlib.rc_file(os.path.join(os.getcwd(),'file.rc'))

import matplotlib.pyplot as plt

import matplotlib.colors as colors

from mpltools import color

get_ipython().magic(u'matplotlib notebook')

import numpy as np, h5py

import scipy.io as sio

import os

import Sp_parameters as Sp

from path_utils import getpath

import hdf5storage as h5

# In[2]:

# set the basic directory path

fp=getpath('ORACLES')

fp

# # Load the files

# ## Load the lut file and do the first processing

# In[3]:

# load the idl save file containing the modeled radiances

s=h5.loadmat(fp+'model/v3_ORACLES_lut.mat')

print s.keys()

# In[117]:

s['rad'].shape

# In[5]:

s['sza']

# In[6]:

# select only one sza to analyze

# Choose sza 61

isza = 5

s['sza'][isza]

# In[7]:

# Run the Sp calcs

lut0 = {}

lut0['tau'] = s['tau']

lut0['ref'] = s['ref']

lut0['zout'] = s['zout']

lut0['sza'] = s['sza']

lut0['phase'] = s['phase']

lut0['irr_dn_diff'] = s['irr_dn_diff'][:,:,:,:,:,isza]

lut0['irr_dn'] = s['irr_dn'][:,:,:,:,:,isza]

lut0['irr_up'] = s['irr_up'][:,:,:,:,:,isza]

lut0['wvl'] = [s['wvl']]

lut0['rad'] = s['rad'][:,:,:,:,:,isza]

lut = Sp.Sp(lut0,verbose=False)

# In[9]:

lut.params(liq_only=True,ice_only=False)

lut.param_hires(start_ref=2,end_ref=30,start_tau=0.1,end_tau=60)

# In[162]:

lut.sp_hires()

lut.norm = lut.normsp(lut.sp,iws=lut.isubwvl)

# ## Load the starzen measurement file and run through the parameters

# In[10]:

# load the matlab file containing the measured TCAP radiances

m = sio.loadmat(fp+'starzen_2017/4STAR_20170809starzen.mat')

sm = sio.idl.AttrDict(m)

print sm.keys()

print 'Measured radiance Shape: ', sm.rad.shape

# In[12]:

m.keys()

# In[88]:

reload(Sp)

# In[73]:

meas = Sp.Sp(m)

meas.params()

# # Now plot the measurements and luts

# ## Plot all measurements

# In[85]:

pz = Sp.plt_zenrad(meas,good_only=True)

# In[86]:

pn = Sp.plt_norm_zenrad(meas,good_only=True)

# In[87]:

pc = Sp.curtain_zenrad(meas)

# In[90]:

pcn = Sp.curtain_zenrad(meas,utc=False)

# In[89]:

pnc = Sp.plt_curtain_norm_zenrad(meas)

# In[91]:

pncn = Sp.plt_curtain_norm_zenrad(meas,utc=False)

# ## Now plot the measurements for only the selected sample airmass

# In[95]:

airmass = 1./np.cos(s['sza']*np.pi/180.0)

meas.airmass = 1.0/np.cos(meas.sza*np.pi/180.0)

idx = Sp.find_closest(airmass,meas.airmass)[:,0]

# In[96]:

len(idx)

# In[97]:

idx.shape

# In[99]:

#Create new 'good' filter

meas.good = np.where((idx==isza) & (meas.utc>0.0) &(m['Md'][meas.iset][:,0]==8))[0]

len(meas.good)

# In[103]:

p = Sp.plt_zenrad(meas)

plt.ylim(-0.01,0.2)

plt.grid()

# In[104]:

p = Sp.plt_norm_zenrad(meas)

plt.grid()

# ## Now plot a few LUT spectra

# In[200]:

pl = Sp.plt_lut_zenrad(lut,range_variable='tau',norm=False,lims=[0,60],cmapname='gist_earth')

# In[190]:

pl = Sp.plt_lut_zenrad(lut,range_variable='tau',norm=True,lims=[0,60])

# In[191]:

pl = Sp.plt_lut_zenrad(lut,range_variable='ref',norm=False,other_index=10,lims=[2,30])

# In[192]:

pl = Sp.plt_lut_zenrad(lut,range_variable='ref',norm=True,other_index=10,lims=[2,30])

# ## Plot the lut on top of the measurements

# In[203]:

p = Sp.plt_zenrad(meas)

p = Sp.plt_lut_zenrad(lut,range_variable='tau',norm=False,other_index=2,lims=[2,35],cmapname='gist_earth',fig=p)

plt.ylim(-0.01,0.2)

plt.grid()

# In[207]:

p = Sp.plt_norm_zenrad(meas)

p = Sp.plt_lut_zenrad(lut,range_variable='tau',norm=True,other_index=2,lims=[2,35],cmapname='spring',fig=p)

plt.title('Model and measured radiances')

plt.grid()

# #### Plotting functions defined

# In[9]:

# set up plotting of a few of the zenith radiance spectra

def pltzen(fig=None,ax=None, tit='Zenith spectra'):

return fig,ax

def norm(fig=None,ax=None,dolegend=True):

return fig,ax

def dashlen(dashlength,dashseperation,fig=plt.gcf()):

return tuple([f(i) for i in range(numdash)])

def plot_line_gradients(ax,s,names,cmap,iphase,irefs,itau,iwvls,pos,normalize=False):

return [alow,ahigh]

def plot_greys(fig=None,ax=None):

plt.axvspan(1565,1634,color=cl); #eta15

# ### Plotting iterations

# In[9]:

fig,ax=pltzen()

# Next figure with modeled spectra

# In[10]:

# now go through and add the different modeled spectra

fig,ax=pltzen()

lines = [('Liquid Cloud Model, COD=0.5','Reds',0,[0,13],1,[420,0.01]),

('Ice Cloud Model, COD=0.5','Greens',1,[13,34],1,[380,0.02]),

('Liquid Cloud Model, COD=10','RdPu',0,[0,13],9,[700,0.16]),

('Ice Cloud Model, COD=10','Blues',1,[13,34],9,[750,0.15])]

for names,cmap,iphase,irefs,itau,pos in lines:

[alow,ahigh] = plot_line_gradients(ax,s,names,cmap,iphase,irefs,itau,iwvls,pos)

lbl=["Small R$_{eff}$ (Ice=" + str(s.ref[13]) + " $\mu m$, Liquid=" + str(s.ref[0]) + " $\mu m$)",

"Large R$_{eff}$ (Ice=" + str(s.ref[34]) + " $\mu m$, Liquid=" + str(s.ref[13]) + " $\mu m$)"]

plt.legend([alow[0],ahigh[0]],

lbl,

frameon=False,loc=7,prop={'size':10})

ax.text(600,0.19,'4STAR Measurement from TCAP')

pltzen(fig,ax)

# ### Next figure with normalized spectra and areas of parameters

# In[13]:

fig,ax = norm(dolegend=False)

plt.axvspan(350,1700,color='#FFFFFF')

ax.text(600,0.19/0.22,'4STAR Measurement')

plt.savefig(fp+'plots/zen_spectra_nomodel.png',dpi=600,transparent=True)

# In[11]:

fig,ax=norm()

lines = [('Liquid Cloud Model, $\\tau$=0.5','Reds',0,[0,13],1,[420,0.01]),

('Ice Cloud Model, $\\tau$=0.5','Greens',1,[13,34],1,[380,0.02]),

('Liquid Cloud Model, $\\tau$=10','RdPu',0,[0,13],9,[700,0.16]),

('Ice Cloud Model, $\\tau$=10','Blues',1,[13,34],9,[750,0.15])]

for names,cmap,iphase,irefs,itau,pos in lines:

[alow,ahigh] = plot_line_gradients(ax,s,names,cmap,iphase,irefs,itau,iwvls,pos,normalize=True)

plt.legend([alow[0],ahigh[0]],lbl,

frameon=False,loc=7,prop={'size':10})

ax.text(600,0.19/0.22,'4STAR Measurement')

norm(fig,ax)

plt.axvspan(350,1700,color='#FFFFFF')

plot_greys()

plt.savefig(fp+'plots/zen_spectra_model.png',dpi=600,transparent=True)

#plt.savefig(fp+'plots/zen_spectra_model.eps')

# Plot the same but with squeezed axes

# In[17]:

fp

# In[16]:

fig,ax=norm()

fig.set_figheight(5)

fig.set_figwidth(6)

lines = [('Liquid Cloud Model, $\\tau$=0.5','Reds',0,[0,13],1,[420,0.01]),

('Ice Cloud Model, $\\tau$=0.5','Greens',1,[13,34],1,[380,0.02]),

('Liquid Cloud Model, $\\tau$=10','RdPu',0,[0,13],9,[700,0.16]),

('Ice Cloud Model, $\\tau$=10','Blues',1,[13,34],9,[750,0.15])]

for names,cmap,iphase,irefs,itau,pos in lines:

[alow,ahigh] = plot_line_gradients(ax,s,names,cmap,iphase,irefs,itau,iwvls,pos,normalize=True)

plt.legend([alow[0],ahigh[0]],lbl,

frameon=False,loc=7,prop={'size':10})

ax.text(600,0.19/0.22,'4STAR Measurement')

norm(fig,ax)

plt.axvspan(350,1700,color='#FFFFFF')

plot_greys()

plt.savefig(fp+'plots/zen_spectra_model_squeeze.png',dpi=600,transparent=True)

# ### Now calculate the parameters for the measured spectra

# In[12]:

map(lambda x:x*x,[-1,1,24])

# In[13]:

reload(Sp)

if 'meas' in locals():

del meas

import gc; gc.collect()

# In[14]:

# first convert measurements to Sp class, with inherent parameters defined

meas = Sp.Sp(m)

meas.params()

# Plot the parameters for the specified time

# In[15]:

fig2,ax2 = plt.subplots(5,3,sharex=True,figsize=(15,8))

ax2 = ax2.ravel()

for i in range(meas.npar-1):

ax2[i].plot(meas.utc,Sp.smooth(meas.par[:,i],3))

ax2[i].set_title('Parameter '+str(i))

ax2[i].grid()

ax2[i].set_xlim([17,19])

if i > 11:

ax2[i].set_xlabel('UTC [h]')

fig2.tight_layout()

plt.show()

# ### Prepare the LUT for the modeled spectra

# In[ ]:

reload(Sp)

if 'lut' in locals():

del lut

import gc; gc.collect()

# In[16]:

lut = Sp.Sp(s)

lut.params()

lut.param_hires()

# In[17]:

import gc; gc.collect()

import sys

print sys.version

# In[18]:

lut.sp_hires()

# In[19]:

print lut.tau.shape

print lut.ref.shape

print lut.sp.ndim

print lut.par.size

print lut.par.shape

# In[20]:

print lut.ref

print lut.sp[0,400,0,23,10]

print lut.sp[1,400,0,:,10]

print lut.par.shape

# Now plot the resulting lut of parameters

# In[21]:

fig3,ax3 = plt.subplots(5,3,sharex=True,figsize=(15,8))

ax3 = ax3.ravel()

for i in range(lut.npar-1):

color.cycle_cmap(len(lut.ref[lut.ref<30]),cmap=plt.cm.RdBu,ax=ax3[i])

for j in xrange(len(lut.ref)):

ax3[i].plot(lut.tau,lut.par[0,j,:,i])

ax3[i].set_title('Parameter '+str(i))

ax3[i].grid()

ax3[i].set_xlim([0,100])

if i > 11:

ax3[i].set_xlabel('Tau')

fig3.tight_layout()

plt.suptitle('Liquid')

plt.subplots_adjust(top=0.93,right=0.93)

cbar_ax = fig3.add_axes([0.95,0.10,0.02,0.8])

scalarmap = plt.cm.ScalarMappable(cmap=plt.cm.RdBu,norm=plt.Normalize(vmin=0,vmax=1))

scalarmap.set_array(lut.ref[lut.ref<30])

cba = plt.colorbar(scalarmap,ticks=np.linspace(0,1,6),cax=cbar_ax)

cba.ax.set_ylabel('R$_{ef}$ [$\\mu$m]')

cba.ax.set_yticklabels(np.linspace(lut.ref[0],29,6));

plt.show()

# In[22]:

fig4,ax4 = plt.subplots(5,3,sharex=True,figsize=(15,8))

ax4 = ax4.ravel()

for i in range(lut.npar-1):

color.cycle_cmap(len(lut.tau),cmap=plt.cm.gist_ncar,ax=ax4[i])

for j in xrange(len(lut.tau)):

ax4[i].plot(lut.ref,lut.par[0,:,j,i])

ax4[i].set_title('Parameter '+str(i))

ax4[i].grid()

ax4[i].set_xlim([0,30])

if i > 11:

ax4[i].set_xlabel('R$_{ef}$ [$\mu$m]')

fig4.tight_layout()

plt.suptitle('Liquid')

plt.subplots_adjust(top=0.93,right=0.93)

cbar_ax = fig4.add_axes([0.95,0.10,0.02,0.8])

scalarmap = plt.cm.ScalarMappable(cmap=plt.cm.gist_ncar,norm=plt.Normalize(vmin=0,vmax=1))

scalarmap.set_array(lut.tau)

cba = plt.colorbar(scalarmap,ticks=np.linspace(0,1,6),cax=cbar_ax)

cba.ax.set_ylabel('$\\tau$')

labels = ['%5.1f' %F for F in np.linspace(lut.tau[0],lut.tau[-1],6)]

cba.ax.set_yticklabels(labels);

plt.show()

# In[23]:

fig5,ax5 = plt.subplots(5,3,sharex=True,figsize=(15,8))

ax5 = ax5.ravel()

for i in range(lut.npar-1):

color.cycle_cmap(len(lut.ref),cmap=plt.cm.RdBu,ax=ax5[i])

for j in xrange(len(lut.ref)):

ax5[i].plot(lut.tau,lut.par[1,j,:,i])

ax5[i].set_title('Parameter '+str(i))

ax5[i].grid()

ax5[i].set_xlim([0,100])

if i > 11:

ax5[i].set_xlabel('Tau')

fig5.tight_layout()

plt.suptitle('Ice')

plt.subplots_adjust(top=0.93,right=0.93)

cbar_ax = fig5.add_axes([0.95,0.10,0.02,0.8])

scalarmap = plt.cm.ScalarMappable(cmap=plt.cm.RdBu,norm=plt.Normalize(vmin=0,vmax=1))

scalarmap.set_array(lut.ref)

cba = plt.colorbar(scalarmap,ticks=np.linspace(0,1,6),cax=cbar_ax)

cba.ax.set_ylabel('R$_{ef}$ [$\\mu$m]')

cba.ax.set_yticklabels(np.linspace(lut.ref[0],lut.ref[-1],6));

plt.show()

# In[24]:

fig6,ax6 = plt.subplots(5,3,sharex=True,figsize=(15,8))

ax6 = ax6.ravel()

for i in range(lut.npar-1):

color.cycle_cmap(len(lut.tau),cmap=plt.cm.gist_ncar,ax=ax6[i])

for j in xrange(len(lut.tau)):

ax6[i].plot(lut.ref,lut.par[1,:,j,i])

ax6[i].set_title('Parameter '+str(i))

ax6[i].grid()

ax6[i].set_xlim([0,50])

if i > 11:

ax6[i].set_xlabel('R$_{ef}$ [$\mu$m]')

fig6.tight_layout()

plt.suptitle('Ice')

plt.subplots_adjust(top=0.93,right=0.93)

cbar_ax = fig6.add_axes([0.95,0.10,0.02,0.8])

scalarmap = plt.cm.ScalarMappable(cmap=plt.cm.gist_ncar,norm=plt.Normalize(vmin=0,vmax=1))

scalarmap.set_array(lut.tau)

cba = plt.colorbar(scalarmap,ticks=np.linspace(0,1,6),cax=cbar_ax)

cba.ax.set_ylabel('$\\tau$')

labels = ['%5.1f' %F for F in np.linspace(lut.tau[0],lut.tau[-1],6)]

cba.ax.set_yticklabels(labels);

plt.show()

# Now run through a few spectra for double checking the input to make sure everythin matches

# In[25]:

print lut.sp.shape

print lut.tau[80]

# In[26]:

print meas.par.shape

# In[27]:

print meas.par[meas.good[200],13]

# In[28]:

plt.figure()

color.cycle_cmap(31,cmap=plt.cm.gist_ncar)

for i in xrange(30):

plt.plot(lut.wvl,lut.sp[0,:,0,i,80])

plt.title('Liquid water cloud with varying R$_{ef}$ at $\\tau$ of '+str(lut.tau[80]))

plt.xlabel('Wavelength [nm]')

plt.ylabel('Radiance [Wm$^{-2}$sr$^{-1}$nm$^{-1}$]')

scalarmap = plt.cm.ScalarMappable(cmap=plt.cm.gist_ncar

,norm=plt.Normalize(vmin=0,vmax=1))

scalarmap.set_array(lut.ref[range(30)])

cba = plt.colorbar(scalarmap,ticks=np.linspace(0,1,6))

cba.ax.set_ylabel('R$_{ef}$ [$\\mu$m]')

cba.ax.set_yticklabels(np.linspace(lut.ref[0],lut.ref[30],6));

# In[29]:

all_zeros = not np.any(lut.sp[0,:,0,40,39])

print all_zeros

print np.max(lut.sp[0,:,0,20,80]), np.min(lut.sp[0,:,0,20,80])

print np.any(lut.sp[0,:,0,20,70])

print lut.ref[28]

plt.figure()

for i in xrange(75,85):

plt.plot(lut.wvl,lut.sp[0,:,0,28,i], label="tau:"+str(lut.tau[i]))

plt.legend()

plt.title('Liquid water cloud with R$_{ef}$ = '+str(lut.ref[28])+' at varying $\\tau$ between '+str(lut.tau[75])+' to '+str(lut.tau[85]))

plt.xlabel('Wavelength [nm]')

plt.ylabel('Radiance [Wm$^{-2}$sr$^{-1}$nm$^{-1}$]')

# In[30]:

plt.figure()

plt.plot(lut.wvl,lut.sp[0,:,0,28,70])

plt.title('Liquid water cloud with R$_{ef}$ = '+str(lut.ref[28])+' at $\\tau$ '+str(lut.tau[70]))

plt.xlabel('Wavelength [nm]')

plt.ylabel('Radiance [Wm$^{-2}$sr$^{-1}$nm$^{-1}$]')

# In[31]:

print np.any(lut.sp[0,:,0,28,70])

print lut.ref[3]

# In[32]:

ro = (range(1,20),range(4,50))

print ro[1]

print ro[0]

# In[33]:

from scipy import interpolate

print np.shape([lut.tau[69],lut.tau[71]])

print np.shape([lut.sp[0,:,0,28,69],lut.sp[0,:,0,28,69]])

fs = interpolate.interp1d([lut.tau[69],lut.tau[71]],[lut.sp[0,:,0,28,69],lut.sp[0,:,0,28,69]],axis=0)

sss = fs(lut.tau[70])

# In[34]:

print np.shape(sss)

plt.figure()

plt.plot(lut.wvl,sss)

plt.plot(lut.wvl,lut.sp[0,:,0,28,70],'r')

print type(sss)

plt.title('Liquid water cloud with R$_{ef}$ = '+str(lut.ref[28])+' interpolated through $\\tau$='+str(lut.tau[70]))

plt.xlabel('Wavelength [nm]')

plt.ylabel('Radiance [Wm$^{-2}$sr$^{-1}$nm$^{-1}$]')

# ## Now run through the retrieval scheme

# In[35]:

import run_kisq_retrieval as rk

reload(rk)

import Sp_parameters as Sp

reload(Sp)

#del lut

#del meas

# In[36]:

print max(meas.good)

# In[37]:

(tau,ref,phase,ki) = rk.run_retrieval(meas,lut)

# In[38]:

del lut

# In[39]:

print meas.utc.shape

print len(meas.good), max(meas.good)

# In[40]:

reload(Sp)

from Sp_parameters import smooth

# In[41]:

fig,ax = plt.subplots(4,sharex=True)

ax[0].set_title('Retrieval results time trace')

ax[0].plot(meas.utc,tau,'rx')

ax[0].plot(meas.utc[meas.good[:,0]],smooth(tau[meas.good[:,0],0],20),'k')

ax[0].set_ylabel('$\\tau$')

ax[1].plot(meas.utc,ref,'g+')

ax[1].set_ylabel('R$_{ef}$ [$\\mu$m]')

ax[1].plot(meas.utc[meas.good[:,0]],smooth(ref[meas.good[:,0],0],20),'k')

ax[2].plot(meas.utc,phase,'k.')

ax[2].set_ylabel('Phase')

ax[2].set_ylim([-0.5,1.5])

ax[2].set_yticks([0,1])

ax[2].set_yticklabels(['liq','ice'])

ax[3].plot(meas.utc,ki)

ax[3].set_ylabel('$\\chi^{2}$')

ax[3].set_xlabel('UTC [Hours]')

ax[3].set_xlim([17,19.05])

plt.savefig(fp+'plots/TCAP_retri_results.png',dpi=600)

#plt.savefig(fp+'plots/TCAP_retri_results.eps')

# Now save the smoothed values

# In[42]:

print tau.shape

# In[43]:

tau[meas.good[:,0],0] = smooth(tau[meas.good[:,0],0],20)

ref[meas.good[:,0],0] = smooth(ref[meas.good[:,0],0],20)

# ### Now load the results from MODIS to compare

# Check the day of year

# In[44]:

import datetime

doy = datetime.datetime(2013,2,19)

print doy.timetuple().tm_yday

dd = datetime.datetime(2014,9,19)

print dd.timetuple().tm_yday

# Now import the hdf files of MODIS

# In[45]:

from mpl_toolkits.basemap import Basemap,cm

# In[46]:

myd06_file = fp+'MODIS\\MYD06_L2.A2013050.1725.006.2014260074007.hdf'

myd03_file = fp+'MODIS\\MYD03.A2013050.1725.006.2013051163424.hdf'

print os.path.isfile(myd03_file) #check if it exists

print os.path.isfile(myd06_file)

# In[47]:

import load_utils as lm

reload(lm)

if 'modis' in locals():

del modis, modis_dicts

import gc; gc.collect()

# Load the geolocated data:

# In[48]:

print 'after import'

import gc; gc.collect()

modis,modis_dicts = lm.load_modis(myd03_file,myd06_file)

# In[49]:

print modis_dicts['qa']['description']

# In[50]:

modis['qa'].shape

# In[51]:

bb = modis['qa'][100,100,:]

print bb

bb.dtype

print bb.astype('ubyte')

def bin(x):

return ''.join(x & (1 << i) and '1' or '0' for i in range(7,-1,-1))

bin(bb[0])

# In[52]:

bin8 = lambda x : ''.join(reversed( [str((x >> i) & 1) for i in range(8)] ) )

# In[53]:

mqa = modis['qa']

# In[54]:

for i in mqa:

print i

# In[55]:

if bin8(bb[1])[-4]:

print 'yes'

bin8(bb[1])[-4]

# Now plot the resulting imagery

# In[56]:

#set up a easy plotting function

def tcap_map(ax=plt.gca()):