def read_data_matched(fname='matched_data_live.nc', mask=True, **kwargs):
''' read in file produced by write_matched_data() '''
from netCDF4 import Dataset
from numpy import zeros
from libclass import var
from libgeo import planck_inv
from lblrtm_utils import AIRS_ancillary
from numpy.ma import masked_where
cdf = Dataset(fname, 'r')
nw = len(cdf.dimensions['wavenumber'])
nt = len(cdf.dimensions['time'])
data = Matched_data(size=nt)
wvn = cdf.variables['wavenumber'][:]
cld = cdf.variables['cloud']
dis = cdf.variables['lbldis']
air = cdf.variables['airs']
lat = cdf.variables['latitude']
lon = cdf.variables['longitude']
tim = cdf.variables['time']
arg = {'attrn': ['wavenumber'], 'attrv': [wvn]}
mask_l2 = AIRS_ancillary().l2_ignore
for n in xrange(nt):
lbldis = dis[n, :]
airs = air[n, :]
#_blank out bad channels
if mask:
lbldis = masked_where(mask_l2, lbldis)
airs = masked_where(mask_l2, airs)
lbldis = masked_where(lbldis == -9999, lbldis)
airs = masked_where(airs == -9999, airs)
lbldis = var(lbldis, **arg)
airs = var(airs, **arg)
lbldis_tb = var(planck_inv(lbldis, wvn, domain='wavenumber'), **arg)
airs_tb = var(planck_inv(airs, wvn, domain='wavenumber'), **arg)
data[n] = (cld[n], lbldis, airs, lbldis_tb, airs_tb, lat[n], lon[n],
tim[n])
return data
def check_ctp(fov, wave_obs, apriori=[], dynamic_CTP=False, **kwargs):
'''
setting dyanmic_CTP to True replaces the
apriori cloud top pressure with one from a separate
source. Currently, only supporting dual regression
heights from Scanning-HIS
'''
from libgeo import p2z, planck_inv
from lblrtm_utils import microwindow_average
if not dynamic_CTP:
return
#_loop over each layer
for layer in apriori:
#_skip aerosol layers
if layer['type'] != 'cloud':
continue
#_giving source priority by list order
for src in dynamic_CTP:
if src == 'SHIS':
#_calc cloud top height
CTP = getattr(fov, '{0}_CTP'.format(src))
CTZ = p2z(CTP) / 1e3
#_get brightness temperatures for a few windows
obs_rads = getattr(fov, '{0}_radiances'.format(src))
r, w = microwindow_average(obs_rads, wave_obs, -26)
bt = planck_inv(r, w * 100, domain='wavenumber')[-3:].mean()
#_if apprioriate layer and there is a value, use SHIS-DR
if CTZ > 0:
layer['z_top'] = CTZ
layer['z'] = CTZ - 0.1
dbg(('WRS0', CTZ))
#_check if 'ice' cloud
if bt < 273.15: #_maybe lower
#_add ice ssp to list and change cloud database to it
ice = '/data/wsessions/lbldis_inputs/' \
+ 'ssp_db.mie_ice.gamma_sigma_0p100'
kwargs['ssp_db_files'].append(ice)
layer['dbnum'] = len(kwargs['ssp_db_files'])
#_other source options here elif 'cloudsat' or whatever
else:
continue
#_once complete for one source, skip rest
break
def calc_bias(experiment='hs3',
dv=None,
dir_out=DIR_PLOT,
fmt='/data/wsessions/LBL-RTM/{3}_{0:s}.{1:s}_fov{2:06d}',
**kwargs):
''' read in all the output from run_clear_skies and get average bias '''
from clear_sky_fovs_hand_filtered import clear_sky as cs
from hs3_utils import Flight_segment as F
import os
from lblrtm_utils import read_lbldis_out, microwindow_average
from numpy import append
from libgeo import planck_inv
#_open a csv to put all the values into
fname = 'shis-lbl.{0}.csv'.format(experiment)
fname = os.path.join(dir_out, fname)
f = open(fname, 'w')
f.write('SHIS - LBL\n')
first = True
for file_seg, opts in cs.iteritems():
#_read in flight
flight = F(file_seg=file_seg)
for i in opts['fidx']:
#_construct directory name
dir_lblrtm = fmt.format(flight.dtg0, flight.dtg1, i, experiment)
fname = os.path.join(dir_lblrtm, 'lbldis_output.clear.cdf')
print fname
#_read in data
clear = read_lbldis_out(fname)
rad_shs = flight[i].SHIS_radiances
wvn_shs = flight.SHIS_wavenumber
rad_lbl = clear.radiances
wvn_lbl = clear.wavenumber
#_if run on microwindows, only average the obs
if dv < 0:
rad_shs, wvn = microwindow_average(rad_shs, wvn_shs, dv)
else:
rad_shs, wvn = microwindow_average(rad_shs, wvn_shs, -7)
rad_lbl, wvn = microwindow_average(rad_lbl, wvn_lbl, -7)
tb_shs = planck_inv(rad_shs, wvn * 100, domain='wavenumber')
tb_lbl = planck_inv(rad_lbl, wvn * 100, domain='wavenumber')
#_write wavenumbers first time
if first:
fmt0 = ','.join(
['{{{0:d}:10.5f}}'.format(j) for j in range(wvn.size)])
f.write(fmt0.format(*wvn) + '\n')
#_put into one thinger
rads = append(tb_shs, tb_lbl).tolist()
fmt0 = ','.join(
['{{{0:d}:10.5f}}'.format(j) for j in range(wvn.size * 2)])
fmt0 += '\n'
f.write(fmt0.format(*rads))
first = False
#_write
f.close()
print fname
return fname
def write_data_live(dir_gdas=os.path.join(os.environ['PRODUCTS'], 'gdas'),
ngdas=99999,
**kwargs):
from numpy import ndarray, recarray
from lblrtm_utils import read_lbldis_out, read_airs, convolve2airs
from lblrtm_utils import airs_channels
from hs3_utils import Flight_data, FLIGHTS #_read_shis
from glob import glob
from libgeo import planck_inv
'''
reads in all gdas files in dir_gdas, then tries to find
LBL-DIS output directories that match the timestamp/naming
returns: all dat noise in a recarray
'''
#_get list of directories to read in
dirs_lbldis, files_gdas = get_lbldis_directories()
nrun = len(dirs_lbldis)
wave = airs_channels()[:].copy()
nair = len(wave)
#_testing purposes
files_gdas = files_gdas[:ngdas]
#_initialize output file
cdf = initialize_cdf(nrun, nair)
vvv = cdf.variables
vvv['wavenumber'][:] = wave
#_conversion args
args = (wave, )
kwargs.update({'domain': 'wavenumber'})
n = 0
for file_gdas in files_gdas:
airs = read_airs(file_gdas)
#_redundancy! This limits the case to this gdas file
dirs_lbldis = glob(file_gdas[:-4].replace('gdas', 'LBL-RTM') + '/0*')
dirs_lbldis.sort()
for dir_lbldis in dirs_lbldis:
#_read lblrtm output
fname = os.path.join(dir_lbldis, 'lbldis_out.cdf')
lbldis = read_lbldis_out(fname)
f0 = lbldis.wavenumber.min()
f1 = lbldis.wavenumber.max()
lblcon = convolve2airs(lbldis, f0=f0, f1=f1)
#_get time index of collocated scene and where to start/end
idx = abs(airs.time - lblcon.time).argmin()
#_get differences (with wnum, T, lat, lon, cloud, time attached)
vvv['cloud'][n] = lblcon.cloud
vvv['latitude'][n] = lblcon.latitude
vvv['longitude'][n] = lblcon.longitude
vvv['time'][n] = lblcon.time
vvv['rads_lbldis'][n, :] = lblcon.radiances
vvv['rads_airs'][n, :] = airs[idx].spectrum.radiances
vvv['tb_lbldis'][n, :] = planck_inv(lblcon.radiances, *args,
**kwargs)
vvv['tb_airs'][n, :] = planck_inv(airs[idx].spectrum.radiances,
*args, **kwargs)
print nrun, n
n += 1
cdf.close()
def fig_00(pname='delta_bt.png', dir_plot='.', dv=-26, **kwargs):
'''
Spectral signatures of habits by wavenumber
x == wavenumber
y == change in radiance from clear sky?
'''
import matplotlib.pyplot as plt
import os
from lblrtm_utils import read_lbldis_out as read_lbldis
from libtools import shrink_ticks
from libgeo import planck_inv
from lblrtm_utils import read_microwindows_header as mw
from numpy import abs
dir_in = '/data/wsessions/LBL-RTM_simulated/std_tropical'
habits = { #'quartz' : 'blue',
'kaolinite': 'red',
'gypsum': 'green'
}
fig, axis = plt.subplots(1)
lines = []
names = []
#_read in all data and plot
for h, c in habits.iteritems():
#_gen filename
fname = os.path.join(dir_in, '{0}.cdf'.format(h))
fn_in = os.path.join(dir_in, 'lbldis_input.{0}'.format(h))
#_read in output file
data = read_lbldis(fname, fname_input=fn_in)
#_convert to brightness temperatures
wavs = data[0].wavenumber
bt0 = planck_inv(data[0].radiances, wavs * 100, domain='wavenumber')
bt1 = planck_inv(data[1].radiances, wavs * 100, domain='wavenumber')
d_bt = bt0 - bt1
line = axis.plot(wavs,
d_bt,
c=c,
linewidth=0.4,
label='changes by {0}'.format(h),
zorder=1)
#_plot clear sky bt
atwn = axis.twinx()
line = atwn.plot(wavs, bt0, 'k-', linewidth=0.2, label='clear sky')
#_plot microwindow numbers
chans = mw()[abs(dv + 1)]
for n, x in chans:
v = (n + x) / 2.
axis.plot([v, v], [0, 20], 'k--', linewidth=0.2)
axis.set_xlim(600, 1300)
axis.set_ylim(0, 20)
axis.set_ylabel("$\Delta B_{\\nu}T$ $(K)$", size='small')
axis.set_xlabel("wavenumber $(cm^{-1})$", size='small')
atwn.set_ylabel("$B_{\\nu}T$ $(K)$", size='small', rotation=-90)
axis.legend(loc=2, frameon=False, fontsize=8)
atwn.legend(loc=1, frameon=False, fontsize=8)
#_save
[shrink_ticks(a) for a in [axis, atwn]]
pname = os.path.join(dir_plot, pname)
dbg(pname)
plt.savefig(pname)