def eof_anal(data, show=False, label='', **kwargs):
'''
eof/pca to synthetic dataset
'''
import numpy as np
import matplotlib.pyplot as plt
from numpy.random import randn
import matplotlib.cm as cm
from numpy.linalg import svd, eig
from numpy import dot, product, arange
import libtools as lt
#_if this is a 3d dataset, blorgh
if data.ndim > 2:
shape = data.shape
data = data.reshape( shape[0], product( shape[1:]) )
multi = True
else:
multi = False
#_create fake data
nt, nx = data.shape
x, t = arange(nx), arange(nt)
#_get covariance matrix of data
covar = dot( data.T, data ) / (nt-1.)
eigV, eigL, pca, percent = eof(data)
#_create fake data
nt, nx = data.shape
x, t = np.arange(nx), np.arange(nt)
#_apply eof analysis (s is the sqrt of eigenvalues)
#_ using SVG
## l, s, r = svd(dataN, full_matrices=False)
## eigV = r.T.copy()
## eigL = s**2 / (nt-1.)
## pca = np.dot(l, np.diag(eigL))
## #_using EIG
## eigL, eigV = eig( covar ) #_get eigValues and eigVectors
## idx = np.argsort(eigL)[::-1] #_get eigenvalues in order
## eigL = eigL[idx] #
## eigV = eigV[:,idx] #_and eigenvectors
## pca = np.dot( dataN, eigV ) #_calculate principal comps
#_calc variance explained by each EOF
percent = eigL / eigL.sum() * 100.
lamberr = eigL * np.sqrt(2./(nt-1))
#_initialize draw object
fig = plt.figure( )
arg = {
'extend': 'both',
'cmap' : cm.RdBu_r,
'levels': np.linspace(-1,1,11) }
shape = (5,2)
#_make unimportant eigenvectors plot lightly
lw = np.ones((percent.size)) * 0.5
lw[percent < 10.] = 0.2
if not multi:
#_create hovmuller diagram of synthetic data
ax0 = plt.subplot2grid( shape, (0,0), rowspan=2 )
ap0 = ax0.contourf( x, t, data, **arg )
ax0.set_title( label ); ax0.set_ylabel('t')
plt.colorbar( ap0 )
#_create hovmuller diagram of synthetic data
ax1 = plt.subplot2grid( shape, (0,1), rowspan=2 )
ap1 = ax1.contourf( x, t, dataN, **arg )
ax1.set_title( 'with white noise' )
plt.colorbar( ap1 )
#_plot the eigenspectrum with error bars (100 indep samples)
ax2 = plt.subplot2grid( shape, (2,0), colspan=2 )
## ax2.errorbar( x+0.01, percent, yerr=lamberr, fmt='o', markersize=1.3)
ax2.errorbar( x+0.01, eigL, yerr=lamberr, fmt='o', markersize=1.3)
ax2.set_xlim(-1,nx); ##ax2.set_ylim(0,40)
ax2.set_title('eigenspectrum scree plot')
#_plot the EOFs scaled by the sqrt of the eigenvalues
eof0 = dot( eigV[:,0], np.sqrt(eigL[0]) )
eof1 = dot( eigV[:,1], np.sqrt(eigL[1]) )
eof2 = dot( eigV[:,2], np.sqrt(eigL[2]) )
ax3 = plt.subplot2grid( shape, (3,0), colspan=2 )
ax3.plot( x, eof0, '-k', linewidth=lw[0] )
ax3.plot( x, eof1, '-r', linewidth=lw[1] )
ax3.plot( x, eof2, '-b', linewidth=lw[2] )
ax3.set_title('first three eofs : black=1,red=2,blue=3')
ax3.set_xlim(0,len(x)-1)
#_plot first three pcs
ax4 = plt.subplot2grid( shape, (4,0), colspan=2 )
pc0 = pca[:,0] / np.sqrt(eigL[0])
pc1 = pca[:,1] / np.sqrt(eigL[1])
pc2 = pca[:,2] / np.sqrt(eigL[2])
ax4.plot( t, pc0, '-k', linewidth=lw[0] )
ax4.plot( t, pc1, '-r', linewidth=lw[1] )
ax4.plot( t, pc2, '-b', linewidth=lw[2] )
ax4.set_title('first three pcs : black=1,red=2,blue=3')
## ax4.set_ylim(-4,4)
ax4.set_xlim(0,len(t)-1)
elif multi:
pass
for ax in fig.axes: lt.shrink_ticks(ax,size=8)
plt.tight_layout()
if show: plt.show()
else: plt.savefig('problem_'+label+'_.png')
def plot_bias(fname=None,
out_label='',
experiment='hs3',
dir_out='.',
dir_plot='.',
**kwargs):
''' read in output from calc bias, plot '''
import numpy as np
import matplotlib.pyplot as plt
from libtools import shrink_ticks
#_
if fname is None:
fname = 'shis-lbl.{0}.csv'.format(experiment)
fname = os.path.join(dir_out, fname)
#_open file
f = open(fname, 'r')
#_get wavenumbers
f.readline()
wvn = f.readline().split(',')
wvn = [float(a) for a in wvn]
data = []
for line in f.readlines():
data.append([float(a) for a in line.split(',')])
data = np.array(data)
nfov, nchan2 = data.shape
#_shis is [:,0,:], lbldis is [:,1,:]
data = data.reshape(nfov, 2, nchan2 / 2)
diff = data[:, 1, :] - data[:, 0, :]
bias_mu = diff.mean(0)
bias_sd = diff.std(0)
h, w = plt.figaspect(6)
fig = plt.figure(figsize=(w, h))
ax = fig.add_subplot(111)
#fig, ax = plt.subplots(1)
with open('/home/wsessions/bias.py', 'w') as f:
for i, wave in enumerate(wvn):
f.write("{0:7.3f}\t:\t{1:8.5f}\n".format(wave, bias_mu[i]))
arg = (experiment, diff.shape[0])
ax.set_title('Clear Sky Bias (LBLDIS minus S-HIS)')
# ax.set_title('LBLDIS minus S-HIS, experiment={0}, N={1:d}'.format(*arg))
ax.scatter(wvn, bias_mu)
ax.errorbar(wvn, bias_mu, yerr=bias_sd, linestyle='None')
ax.set_ylim(-3, 3)
ax.set_xlim([800, 1200])
ax.grid(True)
ax.set_xlabel('wavenumber')
ax.set_ylabel('$\Delta B_{\\nu}T$ (K)')
shrink_ticks(ax)
pname = 'bias.{0}.png'.format(out_label)
pname = os.path.join(dir_plot, pname)
print pname
plt.savefig(pname)
#_close up file
f.close()
def fig_00(pname='delta_bt.png', dir_plot='.', dv=-26, **kwargs):
'''
Spectral signatures of habits by wavenumber
x == wavenumber
y == change in radiance from clear sky?
'''
import matplotlib.pyplot as plt
import os
from lblrtm_utils import read_lbldis_out as read_lbldis
from libtools import shrink_ticks
from libgeo import planck_inv
from lblrtm_utils import read_microwindows_header as mw
from numpy import abs
dir_in = '/data/wsessions/LBL-RTM_simulated/std_tropical'
habits = { #'quartz' : 'blue',
'kaolinite': 'red',
'gypsum': 'green'
}
fig, axis = plt.subplots(1)
lines = []
names = []
#_read in all data and plot
for h, c in habits.iteritems():
#_gen filename
fname = os.path.join(dir_in, '{0}.cdf'.format(h))
fn_in = os.path.join(dir_in, 'lbldis_input.{0}'.format(h))
#_read in output file
data = read_lbldis(fname, fname_input=fn_in)
#_convert to brightness temperatures
wavs = data[0].wavenumber
bt0 = planck_inv(data[0].radiances, wavs * 100, domain='wavenumber')
bt1 = planck_inv(data[1].radiances, wavs * 100, domain='wavenumber')
d_bt = bt0 - bt1
line = axis.plot(wavs,
d_bt,
c=c,
linewidth=0.4,
label='changes by {0}'.format(h),
zorder=1)
#_plot clear sky bt
atwn = axis.twinx()
line = atwn.plot(wavs, bt0, 'k-', linewidth=0.2, label='clear sky')
#_plot microwindow numbers
chans = mw()[abs(dv + 1)]
for n, x in chans:
v = (n + x) / 2.
axis.plot([v, v], [0, 20], 'k--', linewidth=0.2)
axis.set_xlim(600, 1300)
axis.set_ylim(0, 20)
axis.set_ylabel("$\Delta B_{\\nu}T$ $(K)$", size='small')
axis.set_xlabel("wavenumber $(cm^{-1})$", size='small')
atwn.set_ylabel("$B_{\\nu}T$ $(K)$", size='small', rotation=-90)
axis.legend(loc=2, frameon=False, fontsize=8)
atwn.legend(loc=1, frameon=False, fontsize=8)
#_save
[shrink_ticks(a) for a in [axis, atwn]]
pname = os.path.join(dir_plot, pname)
dbg(pname)
plt.savefig(pname)
def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']), start_dtg=None,
dir_plot='.', end_dtg=None, type_coa='a', **kwargs):
'''
start_dtg str{14}, Starting date-time-group for subsetting
end_dtg str{14}, Can you solve the mystery?
'''
from glob import glob
from hs3_utils import Flight_segment
import matplotlib.pyplot as plt
from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p
from numpy import arange, array, ceil, linspace
from numpy.ma import mean
from libcmap import rgbcmap
from flight_namelists import experiments
mkdir_p(dir_plot)
#_get segments
segments = experiments['HS3']
#_convert bounding limits
if start_dtg is not None:
ep0 = dtg2epoch(start_dtg, full=True)
ep1 = dtg2epoch(end_dtg, full=True)
else:
ep0 = '19000101000000'
ep1 = '99999999999999'
#_image setup
smooth = False
label = 'nosmooth'
files = glob('{0}/nc/*nc'.format(dir_cpl))
files.sort()
nrow = 6
ncol = 2
npag = ceil(len(files) / (nrow/ncol))
calipso = rgbcmap('calipso')
dtgs = segments.keys()
dtgs.sort()
#_loop over all cpl files (netcdf)
# for i, fname in enumerate(files):
for i, dtg in enumerate(dtgs):
#_start new page
if not i % (nrow*ncol):
fig = plt.figure()
fig.suptitle('CPL 532 Backscatter', size='small')
#_calc column index
#_calc row index
ax_backscatter = fig.add_subplot(nrow, ncol, i%(nrow*ncol)+1)
ax = ax_backscatter.twinx()
#_read in cpl data
# cpl = read_cpl(f, None)
cpl = Flight_segment(dtg=dtg)
#_check if file is within limits
if cpl.CPL_epoch.min() > ep1 and cpl.CPL_epoch.max() < ep0:
continue
#_get values of just aerosl
aod = calc_aod(cpl, type_coa)
bck = calc_backscatter(cpl)
if smooth:
aod = smooth_aod(aod)
#_generate list of times
time = [epoch2iso(e) for e in cpl.CPL_epoch]
#_get number of fovs
nfov = aod.size
nt = nfov / 2
ax.set_xticks(arange(aod.size)[nt:-nt:nt])
ax.set_xticklabels(time[nt:-nt:nt])
ax.xaxis.set_visible(False)
props = dict(boxstyle='round', facecolor='white', alpha=.5)
# ax.text(nt/4., 3, f, color='k', size='x-small', bbox=props)
f = '{0} - {1}'.format(time[0], time[-1])
ax.text(nt/4., .75, f, color='k', size='xx-small', bbox=props)
ax.set_ylim(0, 1)
#_plotbackscatter
cb=ax_backscatter.pcolormesh(bck,vmin=-4e-7,vmax=1e-4,cmap=calipso,
zorder=0)
ax.plot(aod, linewidth=0.2, zorder=1, color=(1,1,1,.5))
if i % ncol:
# ax.set_ylabel('aod', size='xx-small')
ax_backscatter.yaxis.set_visible(False)
elif not i % ncol:
ax.yaxis.set_visible(False)
xlim=ax_backscatter.xaxis.get_data_interval()
ax_backscatter.set_xlim(xlim)
ax_backscatter.set_yticks(linspace(170,900,9))
ax_backscatter.set_yticklabels(linspace(0,20,9))
ax_backscatter.set_ylim(170,900)
shrink_ticks(ax)
shrink_ticks(ax_backscatter)
#_I think this is bailing
if (i % (nrow*ncol)) == ((nrow*ncol) - 1):
page_num = i / (nrow*ncol)
pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num,
label, type_coa)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04, 0.5, 'altitude (km)', va='center',
rotation='vertical', size='x-small')
fig.text(0.96, 0.5, '{0}OD'.format(type_coa.upper()), va='center',
rotation=270, size='x-small')
fig.savefig(pname)
plt.close()
#_after everything is done
else:
page_num = i / (nrow*ncol)
pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num,
label, type_coa)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04, 0.5, 'altitude (km)', va='center',
rotation='vertical', size='x-small')
fig.text(0.96, 0.5, 'AOD', va='center',
rotation=270, size='x-small')
fig.savefig(pname)
plt.close()
def plot_simulated_histogram(out_label='',
plot_var='tau',
dir_hs3=os.path.join(os.environ['PRODUCTS'],
'hs3'),
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.retrieved'.format(out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 2.0),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_produce for each state variable
for sv in state_vars:
#_get residual
res = truth[sv] - retr[sv]
#_initialize figure and axes
fig = plt.figure()
axes = []
figtext = []
ymax, xmax, xmin = 0, 0, 0
nbin = 100
axes.append(plt.subplot2grid((1, 1), (0, 0)))
#_plot histogram of residuals
(n, bins, patches) = axes[0].hist(res,
bins=nbin,
facecolor='b',
edgecolor='b')
xmax = max((xmax, bins.max()))
xmin = min((xmin, bins.min()))
ymax = max((ymax, n.max()))
axes[0].plot([0, 0], [0, n.max()], 'k--')
axes[0].set_xlabel('diff in tau, {0}'.format(sv), size='x-small')
axes[0].set_title('retrieval residuals'.format(out_label),
size='xx-small')
axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small')
#_max symmetric
xbnd = max((abs(xmin), abs(xmax)))
#_shrink...ticks...
[ax.set_xlim(-xbnd, xbnd) for ax in axes]
[ax.set_ylim(top=100) for ax in axes]
[shrink_ticks(ax) for ax in axes]
#_create image name
pname = 'hist_{0}_{1}.png'.format(out_label, sv)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
def plot_simulated_retrieval(out_label='',
plot_var='tau',
fsim=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'],
'hs3'),
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 9999),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_produce for each state variable
for sv in state_vars:
#_get residual
res = truth[sv] - retr[sv]
#_get indices for ranges
for label, opts in ranges.iteritems():
min_val, max_val = opts[sv]
# idx0 = truth[sv] >= min_val
# idx1 = truth[sv] < max_val
# idx = append(idx0[:,None], idx1[:,None], axis=1).all(axis=1)
idx = (truth[sv] >= min_val) * (truth[sv] < max_val)
ranges[label]['idx'] = idx
#_initialize figure and axes
fig = plt.figure()
axes = []
axis = plt.subplot2grid((2, 4), (1, 0), colspan=len(ranges.keys()))
figtext = []
ymax, xmax, xmin = 0, 0, 0
nbin = 20
for i, label in enumerate(order): #ranges):
min_val, max_val = ranges[label][sv]
idx = ranges[label]['idx']
col = ranges[label]['color']
if idx.sum() == 0:
continue
axes.append(plt.subplot2grid((2, 4), (0, i)))
#_plot histogram of residuals
(n, bins, patches) = axes[i].hist(res[idx],
bins=nbin,
facecolor=col,
edgecolor=col)
xmax = max((xmax, bins.max()))
xmin = min((xmin, bins.min()))
ymax = max((ymax, n.max()))
#_add label
figtext.append('{0}\nmin {1:5.1f}\nmax {2:5.1f}'.format(
label, min_val, max_val))
axes[i].set_xlabel('difference in {0}'.format(sv), size='x-small')
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
for value in set(truth[sv][idx]):
x_truth.append(value)
y_fract.append(fract[truth[sv][idx] == value])
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
## axis.plot(truth[sv][idx], fract, marker='x', color=col)
## axis.scatter(truth[sv][idx], uncrt[sv][idx], marker='x', color=col)
axis.set_xticks(list(set(truth[sv])))
axes[0].set_title('{0} (experiment label)'.format(out_label),
size='xx-small')
#_various things for uncertainty
axis.set_ylim(bottom=0)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('posterior uncercertainty', size='x-small')
axis.set_xlabel('{0}'.format(sv), size='x-small')
axis.grid(False)
axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small')
#_max symmetric
xbnd = max((abs(xmin), abs(xmax)))
for i, label in enumerate(order):
axes[i].text(0, ymax - ymax / 5., figtext[i], size='xx-small')
#_shrink...ticks...
[ax.set_xlim(-xbnd, xbnd) for ax in axes]
[ax.set_ylim(top=ymax) for ax in axes]
[shrink_ticks(ax) for ax in axes]
shrink_ticks(axis)
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.png'.format(sim_experiment, out_label,
sv)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
def fig_02(sv='tau',
fnames=[],
delta_ts=[0],
dir_plot='.',
dir_hs3='.',
by_var=False,
threshold_aod=0.1,
**kwargs):
'''
plot histograms of all the surf temperature tests (alpha it up)
delta_ts array of departures from the true surface temp
fnames list, names of input files (SIMULATED)
'''
from oe_sim import read_simulated_retrieval
import matplotlib.pyplot as plt
from numpy import array, append, linspace
from libtools import shrink_ticks, colors_spag
fig, axis = plt.subplots(1)
nshift = len(delta_ts)
#_loop over shift in T
for j, shift_temp in enumerate(delta_ts):
#_ignore non-shifted
if shift_temp == 0:
continue
#_define out_label
## kwargs.update({'out_label' : 'SFC_TEMP_{0:4.2f}'.format(shift_temp)})
out_label = 'SFC_TEMP_{0:4.2f}'.format(shift_temp)
#_loop over input files
for i, fname in enumerate(fnames):
lab = re.search('SIMULATED.(.*?).nc$', fname).group(1)
lab = '{0}.{1}'.format(lab, out_label)
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.retrieved'.format(lab)
fname = os.path.join(dir_hs3, fname)
if not i: #_read in data
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
else: #_append data together
t, r, u = read_simulated_retrieval(fname, **kwargs)
for sv in truth:
truth[sv] = append(truth[sv], t[sv])
for sv in retr:
retr[sv] = append(retr[sv], r[sv])
for sv in uncrt:
uncrt[sv] = append(uncrt[sv], u[sv])
#_calculate residuals
res = truth[sv] - retr[sv]
#_setup plot
args = {'bins': 100, 'alpha': 0.5, 'range': (-1, 1)}
#_Make and image separating out heights
#_make whisker plot
#_make histogram
if by_var != False:
vars = list(set(truth[by_var]))
nvar = len(vars)
#_only do this crap first time through
if j == 0:
cols = colors_spag(nshift * nvar)
for k, v in enumerate(vars):
idx = truth[by_var] == v
#_calculate percentage within thresh?
P = (abs(res[idx]) <= threshold_aod).sum() / float(idx.sum())
P *= 100
fmt = '$\Delta$T={0:>5.2f}, {4}={1:>4.2f},{2:>6.1f}%' +\
'$\pm${3:>5.2f}'
args.update({
'facecolor':
cols[j * nvar + k],
'edgecolor':
cols[j * nvar + k],
'label':
fmt.format(shift_temp, v, P, threshold_aod, by_var)
})
(n, bins, patches) = axis.hist(res[idx], **args)
#_make histogram
else:
if j == 0:
#_only do this crap first time through
cols = colors_spag(nshift)
k = 0
#_calculate percentage within thresh?
P = (abs(res) <= threshold_aod).sum() / float(res.size)
P *= 100
fmt = '$\Delta$T={0:>5.2f},{1:>6.1f}%$\pm${2:>5.2f}'
args.update({
'facecolor': cols[j],
'edgecolor': cols[j],
'label': fmt.format(shift_temp, P, threshold_aod)
})
(n, bins, patches) = axis.hist(res, **args)
#_add legend
rem = 1 if (j + k) % 26 else 0
ncol = (j + k) / 26 + rem
dbg((ncol, rem, j + k))
lgd = axis.legend(fontsize=8,
loc=2,
bbox_to_anchor=(1.05, 1.),
borderaxespad=0.,
frameon=False)
#_highlight within TOLERANCE level (90% within 0.01?)
#_particulars
axis.set_xlabel('Residuals (True OD - S-HIS OE Retrieval)', size='small')
axis.set_xlim(-1, 1)
axis.set_xticks(linspace(-1, 1, 21))
shrink_ticks(axis)
#_savefig
tf = type(by_var) == str
pname = 'surf_temp_tolerance{0}.png'.format(tf * '_{0}'.format(by_var))
pname = os.path.join(dir_plot, pname)
dbg(pname)
plt.savefig(pname, bbox_extra_artists=(lgd, ), bbox_inches='tight')
def fig_01_check(var,
out_label='',
plot_var='tau',
fsim=None,
dir_lblrtm=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'),
**kwargs):
'''
Uncertainty by optical depth
First does plot of all median uncertainties? Or by specific ref/z
Plot uncertainty broken down by ref/z with box/whisker plots.
Plot them in total.
plot_simulated_by_var
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
dbg('running...')
#_read in retrieval data
if type(fsim) != list:
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
else:
for fname in fsim:
res = re.search('SIMULATED.(.*?).nc', fname)
sim_experiment = res.group(1)
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
t, r, u = read_simulated_retrieval(fname, **kwargs)
try:
[append(truth[sv], t[sv]) for sv in t.keys()]
[append(retr[sv], r[sv]) for sv in r.keys()]
[append(uncrt[sv], u[sv]) for sv in u.keys()]
except UnboundLocalError:
truth, retr, uncrt = t, r, u
sim_experiment = 'NOISE'
#_what are we working with?
state_vars = retr.keys() # truth.keys()
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_get potential settings for value
values = list(set(truth[var]))
nval = len(values)
values.sort()
nrow = 6
npag = nval / nrow #_do we need this?
page = 0
#_produce for each state variable
for sv in state_vars:
for j, value in enumerate(values):
dbg((sv, var, value))
#_initialize new figure
if not j % nrow:
page += 1
fig = plt.figure()
k = 0
#_get indices of limit
idx = truth[var] == value
#_get residual
res = truth[sv][idx] - retr[sv][idx]
#_initialize figure and axes
axis = fig.add_subplot(nrow, 1, k + 1)
ymax, xmax, xmin = 0, 0, 0
nbin = 20
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
for value_truth in set(truth[sv]):
x_truth.append(value_truth)
y_fract.append(fract[truth[sv][idx] == value_truth])
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
axis.set_xticks(list(set(truth[sv])))
#_various things for uncertainty
axis.set_ylim(bottom=0, top=2)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small')
axis.grid(False)
#_shrink...ticks...
shrink_ticks(axis)
#_save when page full
if k == (nrow - 1) or j == nval - 1:
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format(
sim_experiment, out_label, sv, var, page)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
k += 1
def fig_01(var,
out_label='',
plot_var='tau',
fsim=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'),
dir_plot='.',
files=[],
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
import re
from oe_sim import read_simulated_retrieval
if len(files) == 0:
print type(fsim)
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
else:
for i, f in enumerate(files):
res = re.search('SIMULATED.(.*?).nc', f)
sim_experiment = res.group(1)
f = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
f = os.path.join(dir_hs3, f)
try:
if i == 0: #_initialize
truth, retr, uncrt = read_simulated_retrieval(f, **kwargs)
else: #_append
t, r, u = read_simulated_retrieval(f, **kwargs)
for sv in truth:
truth[sv] = append(truth[sv], t[sv])
for sv in retr:
retr[sv] = append(retr[sv], r[sv])
for sv in uncrt:
uncrt[sv] = append(uncrt[sv], u[sv])
except IOError:
dbg(('Did you complete this retrieval yet?', f))
os._exit(23)
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1).split('.')[0]
out_label = 'summary'
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg('')
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 9999),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = '{1}.{0}'.format(out_label, sim_experiment)
ppath = os.path.join(dir_plot, ppath)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_get potential settings for value
values = list(set(truth[var]))
nval = len(values)
values.sort()
''' make loop over pages, 6 per page '''
nrow = 6
npag = nval / nrow #_do we need this?
page = 0
#_produce for each state variable
for sv in state_vars:
for j, value in enumerate(values):
dbg((sv, var, value))
#_initialize new figure
if not j % nrow:
page += 1
fig = plt.figure()
k = 0
fig.suptitle('Fractional Uncertainty')
#_get indices of limit
idx = truth[var] == value
#_get residual
res = truth[sv][idx] - retr[sv][idx]
#_initialize figure and axes
axis = fig.add_subplot(nrow, 1, k + 1)
ymax, xmax, xmin = 0, 0, 0
nbin = 20
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
#_pull out values across all over vars, at truth value
for value_truth in set(truth[sv]):
x_truth.append(value_truth)
y_fract.append(fract[truth[sv][idx] == value_truth])
#_pull out number of retrievals going into these
y_bar = [len(y) for y in y_fract]
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
axis.set_xticks(list(set(truth[sv])))
abar = axis.twinx()
bar_width = (max(x_truth) - min(x_truth)) / float(len(x_truth))\
- 0.05
abar.bar(x_truth - bar_width / 2.,
y_bar,
bar_width,
alpha=0.24,
color='brown')
#_get how high we can go and still be 95 % certain of the AOD
## m95 = truth[sv][certainty_garbage(res)]
#_various things for uncertainty
# axis.text(truth[sv].max()-0.2, 1.6, 'n={0}'.format(len(y_fract))
axis.set_ylim(bottom=0, top=2)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('{0} = {1}'.format(var, value), size='xx-small')
axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small')
## axis.set_xlabel('{0}, max_95 {1:5.2f}'.format(sv,m95), size='x-small')
axis.grid(False)
#_shrink...ticks...
shrink_ticks(axis)
shrink_ticks(abar)
#_save when page full
if k == (nrow - 1) or j == nval - 1:
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format(
sim_experiment, out_label, sv, var, page)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
k += 1
def plot_period(data, e0, e1, plot_oe=True, plot_caliop=False,
cmfile='/home/wsessions/lib/cmaps/calipso-backscatter.cmap', **kwargs):
''' plot all the crep in data '''
#_read in retrieval data for period
## obs = read_oe(airs.AIRS_epoch, **kwargs)
import matplotlib.pyplot as plt
from libcmap import rgbcmap
from libtools import epoch2iso as e2i
from numpy import array, vstack, arange, recarray, append
from libtools import shrink_ticks
from numpy.ma import masked_where as mw
import ccplot_cmap as ccc
nplot = sum([plot_oe, plot_caliop])
fig, ax = plt.subplots(nplot)
try:
ax_oe, ax_cal = ax
except:
ax_oe = ax
dbg(e2i([e0, e1]))
#_plot retrievals from optimal estimation
#_AIRS_____#
def fix_oe_crap(d, **kwargs):
''' get the oe values out of stupid dictionaries '''
example = array([f is not None for f in d.values])
example = d[example.argmin()]
ssp_dbs = example.values.ssp_db
names = [n.split('.')[-2] for n in ssp_dbs]
#_build recarray to store these
nssp = len(ssp_dbs)
nt = d.size
dtype = [('tau','f4'), ('lat','f4'), ('lon','f4'), ('epoch', 'f8'),
('name','a20'), ('habit', 'a20'), ('ref_wvnum', 'f4'),
('z_top', 'f4'), ('z', 'f4'), ('ref', 'f4')]
data = recarray((nssp*nt), dtype)
#_loop over layers and pull all this crap out
for i, vals in enumerate(d.values):
for ii, l in enumerate(vals.layers):
idx = ii + nssp*i
habit = vals.ssp_db[l['dbnum']].split('.')[-2]
arg = (l['tau'][0], d.lat[i], d.lon[i], d.epoch[i], d.name[i],
habit, l['ref_wvnum'], l['z_top'], l['z'], l['ref'])
data[idx] = arg
data = data[data.tau != -9999]
return data
#_pull out oe
oe = data[data.name == 'oe']
idx = array([d is not None for d in oe.values])
oe = oe[idx]
oe = oe[oe.epoch.argsort()]
oe = oe[(oe.epoch >= e0) * (oe.epoch <= e1)]
#_pull out optical depths and put into dictionary by specie
oe = fix_oe_crap(oe, **kwargs)
ssps = set(oe.habit)
for habit in set(oe.habit):
if habit == 'mie_wat':
continue
#_pull out this ssp habit
oe_habit = oe[oe.habit == habit]
oe_habit = oe_habit[oe_habit.epoch.argsort()]
#_
x = oe_habit.epoch
y = oe_habit.tau
nt = x.size
max_oe_epoch, min_oe_epoch = x.max(), x.min()
ax_oe.plot(x, y, label=habit, linewidth=0.5)
xticks = append(x[::nt/5], x[-1])
if xticks[-1] - xticks[-2] < nt/20:
xticks[-2] = xticks[-1]
xticklabels = [tmp[11:19] for tmp in e2i(xticks)]
ax_oe.set_xticks(xticks)
ax_oe.set_xticklabels(xticklabels)
shrink_ticks(ax_oe)
#_crop plotting area.
ax_oe.set_xlim(x.min(), x.max())
ax_oe.set_ylim(0, 3.)
#_drop box, make smaller
ax_oe.legend()
##########
#_CALIOP_#
if plot_caliop:
import numpy as np
#_ pull out caliop data
caliop = data[data.name == 'caliop']
idx_c = (caliop.epoch >= e0) * (caliop.epoch <= e1)
caliop = caliop[(caliop.epoch >= min_oe_epoch) * (caliop.epoch <= max_oe_epoch)]
## caliop = caliop[(caliop.epoch >= e0) * (caliop.epoch <= e1)]
caliop = caliop[caliop.epoch.argsort()]
x = caliop.epoch
#_put into ONE BIG OL 2d array
caliop = vstack(caliop.values)
caliop = mw(caliop == -9999, caliop)
calipo = np.ma.masked_invalid(caliop)
## caliop[caliop < 0] = 0.
## caliop[caliop > 1.5] = 1.5
#_coords
y = arange(caliop.shape[1])
dbg(('REMOVE TRUNCATION'))
#_load up backscatter colormaps
cmap, norm, ticks = ccc.loadcolormap(cmfile, 'caliop')
## cb = ax_cal.imshow(caliop.T, cmap=cmap, norm=norm, interpolation='nearest')
im = ax_cal.pcolormesh(x, y, caliop.T, cmap=cmap, norm=norm)#, vmin=0, vmax=1.5)
ax_cal.set_xlabel('{0:e}, {1:e}'.format(caliop.max(), caliop.min()))
#_label bottom
xticks = append(x[::x.size/5], x[-1])
if xticks[-1] - xticks[-2] < 100:
xticks[-2] = xticks[-1]
xticklabels = [tmp[11:19] for tmp in e2i(xticks)]
ax_cal.set_xticks(xticks)
ax_cal.set_xticklabels(xticklabels)
ax_cal.invert_yaxis()
ax_cal.get_yaxis().set_visible(False)
ax_cal.set_xlim(x.min(), x.max())
shrink_ticks(ax_cal)
## cb = fig.colorbar(im, ax=axes, cax=cbaxes, orientation="vertical",
## extend="both", ticks=ticks, norm=norm,
## format=SciFormatter())
## cb.set_label(name)
pname = 'oe_{0}_{1}.png'.format(e2i(e0), e2i(e1))
pname = pname.replace(':', '-')
dbg(pname)
plt.savefig(pname)
def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']),
start_dtg=None,
dir_plot='.',
end_dtg=None,
**kwargs):
'''
start_dtg str{14}, Starting date-time-group for subsetting
end_dtg str{14}, Can you solve the mystery?
'''
from glob import glob
from hs3_utils import read_cpl
import matplotlib.pyplot as plt
from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p
from numpy import arange, array, ceil, linspace
from numpy.ma import mean
from libcmap import rgbcmap
mkdir_p(dir_plot)
#_convert bounding limits
if start_dtg is not None:
ep0 = dtg2epoch(start_dtg, full=True)
ep1 = dtg2epoch(end_dtg, full=True)
else:
ep0 = '19000101000000'
ep1 = '99999999999999'
#_image setup
smooth = False
label = 'nosmooth'
files = glob('{0}/nc/*nc'.format(dir_cpl))
files.sort()
nrow = 6
npag = ceil(len(files) / nrow)
calipso = rgbcmap('calipso')
#_loop over all cpl files (netcdf)
i = 0
for q, fname in enumerate(files):
f = fname.split('/')[-1]
#_read in cpl data
cpl = read_cpl(f, None)
#_check if file is within limits
if cpl.epoch.min() > ep1 or cpl.epoch.max() < ep0:
continue
#_start new page
if not i % nrow:
fig = plt.figure()
ax_backscatter = fig.add_subplot(nrow, 1, i % nrow + 1)
ax = ax_backscatter.twinx()
print 'MADE IT'
#_get values of just aerosl
aod = calc_aod(cpl)
bck = calc_backscatter(cpl)
if smooth:
aod = smooth_aod(aod)
#_generate list of times
time = [epoch2iso(e) for e in cpl.epoch]
#_get number of fovs
nfov = aod.size
nt = nfov / 5
ax.set_xticks(arange(aod.size)[nt:-nt:nt])
ax.set_xticklabels(time[nt:-nt:nt])
props = dict(boxstyle='round', facecolor='white', alpha=.5)
ax.text(nt / 4., 3, f, color='k', size='x-small', bbox=props)
ax.set_ylim(0, 4)
#_plotbackscatter
cb = ax_backscatter.pcolormesh(bck,
vmin=-4e-7,
vmax=1e-4,
cmap=calipso,
zorder=0)
ax.plot(aod, linewidth=0.2, zorder=1, color=(1, 1, 1, .5))
ax.set_ylabel('aod', size='xx-small')
xlim = ax_backscatter.xaxis.get_data_interval()
ax_backscatter.set_xlim(xlim)
## ax_backscatter.set_ylim(0,900)
ax_backscatter.set_yticks(linspace(170, 900, 11))
ax_backscatter.set_yticklabels(linspace(0, 20, 11))
ax_backscatter.set_ylim(170, 900)
shrink_ticks(ax)
shrink_ticks(ax_backscatter)
i += 1
if (i % nrow) == (nrow - 1):
page_num = i / nrow
pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04,
0.5,
'altitude (km)',
va='center',
rotation='vertical',
size='x-small')
fig.savefig(pname)
plt.close()
else:
page_num = i / nrow
pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04,
0.5,
'altitude (km)',
va='center',
rotation='vertical',
size='x-small')
fig.savefig(pname)
plt.close()