def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']), start_dtg=None,
dir_plot='.', end_dtg=None, type_coa='a', **kwargs):
'''
start_dtg str{14}, Starting date-time-group for subsetting
end_dtg str{14}, Can you solve the mystery?
'''
from glob import glob
from hs3_utils import Flight_segment
import matplotlib.pyplot as plt
from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p
from numpy import arange, array, ceil, linspace
from numpy.ma import mean
from libcmap import rgbcmap
from flight_namelists import experiments
mkdir_p(dir_plot)
#_get segments
segments = experiments['HS3']
#_convert bounding limits
if start_dtg is not None:
ep0 = dtg2epoch(start_dtg, full=True)
ep1 = dtg2epoch(end_dtg, full=True)
else:
ep0 = '19000101000000'
ep1 = '99999999999999'
#_image setup
smooth = False
label = 'nosmooth'
files = glob('{0}/nc/*nc'.format(dir_cpl))
files.sort()
nrow = 6
ncol = 2
npag = ceil(len(files) / (nrow/ncol))
calipso = rgbcmap('calipso')
dtgs = segments.keys()
dtgs.sort()
#_loop over all cpl files (netcdf)
# for i, fname in enumerate(files):
for i, dtg in enumerate(dtgs):
#_start new page
if not i % (nrow*ncol):
fig = plt.figure()
fig.suptitle('CPL 532 Backscatter', size='small')
#_calc column index
#_calc row index
ax_backscatter = fig.add_subplot(nrow, ncol, i%(nrow*ncol)+1)
ax = ax_backscatter.twinx()
#_read in cpl data
# cpl = read_cpl(f, None)
cpl = Flight_segment(dtg=dtg)
#_check if file is within limits
if cpl.CPL_epoch.min() > ep1 and cpl.CPL_epoch.max() < ep0:
continue
#_get values of just aerosl
aod = calc_aod(cpl, type_coa)
bck = calc_backscatter(cpl)
if smooth:
aod = smooth_aod(aod)
#_generate list of times
time = [epoch2iso(e) for e in cpl.CPL_epoch]
#_get number of fovs
nfov = aod.size
nt = nfov / 2
ax.set_xticks(arange(aod.size)[nt:-nt:nt])
ax.set_xticklabels(time[nt:-nt:nt])
ax.xaxis.set_visible(False)
props = dict(boxstyle='round', facecolor='white', alpha=.5)
# ax.text(nt/4., 3, f, color='k', size='x-small', bbox=props)
f = '{0} - {1}'.format(time[0], time[-1])
ax.text(nt/4., .75, f, color='k', size='xx-small', bbox=props)
ax.set_ylim(0, 1)
#_plotbackscatter
cb=ax_backscatter.pcolormesh(bck,vmin=-4e-7,vmax=1e-4,cmap=calipso,
zorder=0)
ax.plot(aod, linewidth=0.2, zorder=1, color=(1,1,1,.5))
if i % ncol:
# ax.set_ylabel('aod', size='xx-small')
ax_backscatter.yaxis.set_visible(False)
elif not i % ncol:
ax.yaxis.set_visible(False)
xlim=ax_backscatter.xaxis.get_data_interval()
ax_backscatter.set_xlim(xlim)
ax_backscatter.set_yticks(linspace(170,900,9))
ax_backscatter.set_yticklabels(linspace(0,20,9))
ax_backscatter.set_ylim(170,900)
shrink_ticks(ax)
shrink_ticks(ax_backscatter)
#_I think this is bailing
if (i % (nrow*ncol)) == ((nrow*ncol) - 1):
page_num = i / (nrow*ncol)
pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num,
label, type_coa)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04, 0.5, 'altitude (km)', va='center',
rotation='vertical', size='x-small')
fig.text(0.96, 0.5, '{0}OD'.format(type_coa.upper()), va='center',
rotation=270, size='x-small')
fig.savefig(pname)
plt.close()
#_after everything is done
else:
page_num = i / (nrow*ncol)
pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num,
label, type_coa)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04, 0.5, 'altitude (km)', va='center',
rotation='vertical', size='x-small')
fig.text(0.96, 0.5, 'AOD', va='center',
rotation=270, size='x-small')
fig.savefig(pname)
plt.close()
def plot_sat( recs, field='aod', path='.', **kwargs ): ''' rec : np.recarray, plots all sensor data and differenc plots and whatever else ed wanted ''' import matplotlib.pyplot as plt import matplotlib.cm as cm from matplotlib.colors import LogNorm import libnva as ln import libcmap as lc reg_dict = lm.regions() plt_dict = lm.fields() #_pull out metadata for title dtg = lt.unique( recs.dtg, unique=True ) sensor = lt.unique( recs.sensor, unique=True ) long_name = lt.unique( recs.long_name, unique=True ) wavelength = lt.unique( recs.wavelength, unique=True ) level = lt.unique( recs.level, unique=True ) title = ' '.join(( lt.human_date(dtg), sensor, long_name )) plt.title( title, size='small', ha='left', va='bottom',position=(0.,1.)) #_plot specifics corn = reg_dict['global']['corn'] grid = reg_dict['global']['grid'] delt = reg_dict['global']['delta'] #_setup colors icap = lc.rgbcmap( 'icap', N=256 ) #_get specifics for field field = field.lower() levs = [0.1,0.2,0.4,0.8] norm = LogNorm( levs[0], levs[-1] ) #_setup output name dir_out = '/'.join(( path, 'sat_01', 'global', dtg )) file = '_'.join(( dtg, 'total_aod', str(wavelength), sensor.lower(), 'l'+str(level)+'.png')) file = '/'.join(( dir_out, file )) lt.mkdir_p( dir_out ) #_generate map to plot over m = ln.draw_map( grid, corn, 'cyl', delt, fill='white', **kwargs ) #_put into 2d array tau2d = obsnew_field( recs, **kwargs ) #_put data onto grid d, x, y = ln.map_data( tau2d, m ) #_plot data CS = m.pcolormesh( x, y, d, cmap=cm.jet, norm=norm, rasterized=True ) #_generate colorbar bar = plt.colorbar( CS, orientation='horizontal', shrink=0.8, aspect=40, pad=0.05, ticks=levs ) bar.set_ticklabels( levs ) dbg(file) plt.savefig(file) plt.close()
def plot_simulated_retrieval(out_label='',
plot_var='tau',
fsim=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'],
'hs3'),
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 9999),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_produce for each state variable
for sv in state_vars:
#_get residual
res = truth[sv] - retr[sv]
#_get indices for ranges
for label, opts in ranges.iteritems():
min_val, max_val = opts[sv]
# idx0 = truth[sv] >= min_val
# idx1 = truth[sv] < max_val
# idx = append(idx0[:,None], idx1[:,None], axis=1).all(axis=1)
idx = (truth[sv] >= min_val) * (truth[sv] < max_val)
ranges[label]['idx'] = idx
#_initialize figure and axes
fig = plt.figure()
axes = []
axis = plt.subplot2grid((2, 4), (1, 0), colspan=len(ranges.keys()))
figtext = []
ymax, xmax, xmin = 0, 0, 0
nbin = 20
for i, label in enumerate(order): #ranges):
min_val, max_val = ranges[label][sv]
idx = ranges[label]['idx']
col = ranges[label]['color']
if idx.sum() == 0:
continue
axes.append(plt.subplot2grid((2, 4), (0, i)))
#_plot histogram of residuals
(n, bins, patches) = axes[i].hist(res[idx],
bins=nbin,
facecolor=col,
edgecolor=col)
xmax = max((xmax, bins.max()))
xmin = min((xmin, bins.min()))
ymax = max((ymax, n.max()))
#_add label
figtext.append('{0}\nmin {1:5.1f}\nmax {2:5.1f}'.format(
label, min_val, max_val))
axes[i].set_xlabel('difference in {0}'.format(sv), size='x-small')
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
for value in set(truth[sv][idx]):
x_truth.append(value)
y_fract.append(fract[truth[sv][idx] == value])
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
## axis.plot(truth[sv][idx], fract, marker='x', color=col)
## axis.scatter(truth[sv][idx], uncrt[sv][idx], marker='x', color=col)
axis.set_xticks(list(set(truth[sv])))
axes[0].set_title('{0} (experiment label)'.format(out_label),
size='xx-small')
#_various things for uncertainty
axis.set_ylim(bottom=0)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('posterior uncercertainty', size='x-small')
axis.set_xlabel('{0}'.format(sv), size='x-small')
axis.grid(False)
axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small')
#_max symmetric
xbnd = max((abs(xmin), abs(xmax)))
for i, label in enumerate(order):
axes[i].text(0, ymax - ymax / 5., figtext[i], size='xx-small')
#_shrink...ticks...
[ax.set_xlim(-xbnd, xbnd) for ax in axes]
[ax.set_ylim(top=ymax) for ax in axes]
[shrink_ticks(ax) for ax in axes]
shrink_ticks(axis)
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.png'.format(sim_experiment, out_label,
sv)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
def main(fidx,
dir_lblrtm_fmt='',
dir_lblrtm=None,
dir_plot='.',
persistence=False,
hostname=False,
file_col=None,
experiment='HS3',
surf_temp='GDAS',
rerun=False,
**kwargs):
'''
completely relies upon the kwargs pickle
written by optimal_estimation.real_case()
fidx int, Field of view number associated with
flight segment file in $PRODUCTS/hs3
'''
import matplotlib.pyplot as plt
from optimal_estimation import optimal_estimation as oe
from numpy import diag, matrix
from lblrtm_utils import microwindow_average, microwindow_stdev
from lblrtm_utils import get_surf_temp
from libtools import mkdir_p, epoch2dtg, dbg
from shutil import copytree, rmtree, copy2
from glob import glob
from oe_utils import read_airs
from time import sleep
from hs3_utils import Flight_segment as F
from os import unlink
try:
file_col, hostname = remote_copy(file_col)
except:
pass
#_read in radiance data for profile
flight = F(**kwargs)
fov = flight[fidx]
wave = flight.SHIS_wavenumber
rads = fov.SHIS_radiances
errs = flight.SHIS_HBB_NESR
#_initialize plot
# fig, ax = plt.subplots(1, **{'figsize' : (4,1)})
fig, ax = plt.subplots(1, figsize=(4, 1))
fig.suptitle(kwargs.get('desc'), size='xx-small')
out_label = kwargs.get('out_label')
pname = 'oe_{0}_{1}_f{2:04d}.png'.format(out_label, flight.dtg0, fidx)
pname = os.path.join(dir_plot, 'OE', pname)
mkdir_p(os.path.join(dir_plot, 'OE'))
if os.path.exists(pname):
os.unlink(pname)
#_build covariance matrix
#_when using microwindows, average observation data
dv = kwargs.get('dv')
if dv < 0:
std, d = microwindow_stdev(rads, wave, dv)
nesr, d = microwindow_average(errs, wave, dv, error=True)
y, wave_obs = microwindow_average(rads, wave, dv)
#_std deviation within microwindows applied to ref
# blackbody
cov = matrix(diag(nesr) + diag(std))
else:
cov = matrix(diag(errs))
#_allow for arbitrary lblrtm directory, get location of LBL-RTM output
dtgfov = (flight.dtg0, flight.dtg1, fidx, experiment, flight.dtg0[2:])
dir_lblrtm = dir_lblrtm_fmt.format(*dtgfov)
dir_lblrtm_arc = dir_lblrtm_fmt.format(*dtgfov) if dir_lblrtm is None \
else dir_lblrtm
#_update kwargs based upon dynamic ATP/CTP
surf_temp = get_surf_temp(fov,
surf_temp_src=surf_temp,
dir_lblrtm=dir_lblrtm_arc,
**kwargs)
check_ctp(fov, wave, **kwargs)
check_atp(fov, **kwargs)
#_if already run and not desired to redo, kill
lbldis_output = kwargs.get('lbldis_output')
file_out = os.path.join(dir_lblrtm_arc, lbldis_output)
#_check last field of view for persistence
if type(persistence) == dict and fidx != 0:
print 'DONT USE THIS IS PROBABLY WRONG DIRECTORY'
dtgfov_tmp = (experiment, dtg)
dir_lblrtm_arc_tmp = dir_lblrtm_fmt.format(*dtgfov_tmp)
print 'CHECKING PERSIST', kwargs['apriori']
#_put this in a try clause to allow for failure
try:
check_persistence(dir_lblrtm_arc_tmp, persistence, **kwargs)
except IOError:
pass
print 'CHECKING OUTSIST', kwargs['apriori']
#_copy the archive LBLRTM directory locally and work form there
dir_lblrtm = os.path.join(DIR_WORK,
'/'.join(dir_lblrtm_arc.split('/')[-2:]))
if not os.path.exists(dir_lblrtm + '/TAPE5'):
killcount = 0
while killcount < 10:
try:
mkdir_p(dir_lblrtm)
[copy2(cp, dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*')]
break
except:
#_if fails, try ten times
sleep(10)
killcount += 1
if killcount >= 10:
exit()
#_update keyword arguments for this fov
args = (rads, wave, cov)
kwargs.update({
'fig': fig,
'ax': ax,
'pname': pname,
'nproc': 1, #_just in case I forgot to tone it down
'dir_lblrtm_arc': dir_lblrtm_arc,
'dir_lblrtm': dir_lblrtm,
'surf_temp': surf_temp,
})
#_pull_the_trigger_#
#__________________#
dbg('LAUNCHING...')
oe(*args, **kwargs)
#_archive crap
for out_file in glob(dir_lblrtm + '/*' + out_label + '*'):
killcount = 0
out = os.path.join(dir_lblrtm_arc, out_file.split('/')[-1])
#_remove old output if rerunning
if rerun and os.path.exists(out):
unlink(out)
while not os.path.exists(out):
try:
dbg(('archiving', out_file, '->', out))
copy2(out_file, dir_lblrtm_arc)
except:
dbg(('failed to copy', out_file, '->', out))
sleep(10)
#_plot up results with flight data
kwargs.update({'dir_plot': os.path.join(dir_plot, 'DASHBOARD')})
flight.plot(fidx=fidx, rerun=rerun, **kwargs)
#_remove temporary directory
rmtree(dir_lblrtm)
def fig_01_check(var,
out_label='',
plot_var='tau',
fsim=None,
dir_lblrtm=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'),
**kwargs):
'''
Uncertainty by optical depth
First does plot of all median uncertainties? Or by specific ref/z
Plot uncertainty broken down by ref/z with box/whisker plots.
Plot them in total.
plot_simulated_by_var
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
dbg('running...')
#_read in retrieval data
if type(fsim) != list:
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
else:
for fname in fsim:
res = re.search('SIMULATED.(.*?).nc', fname)
sim_experiment = res.group(1)
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
t, r, u = read_simulated_retrieval(fname, **kwargs)
try:
[append(truth[sv], t[sv]) for sv in t.keys()]
[append(retr[sv], r[sv]) for sv in r.keys()]
[append(uncrt[sv], u[sv]) for sv in u.keys()]
except UnboundLocalError:
truth, retr, uncrt = t, r, u
sim_experiment = 'NOISE'
#_what are we working with?
state_vars = retr.keys() # truth.keys()
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_get potential settings for value
values = list(set(truth[var]))
nval = len(values)
values.sort()
nrow = 6
npag = nval / nrow #_do we need this?
page = 0
#_produce for each state variable
for sv in state_vars:
for j, value in enumerate(values):
dbg((sv, var, value))
#_initialize new figure
if not j % nrow:
page += 1
fig = plt.figure()
k = 0
#_get indices of limit
idx = truth[var] == value
#_get residual
res = truth[sv][idx] - retr[sv][idx]
#_initialize figure and axes
axis = fig.add_subplot(nrow, 1, k + 1)
ymax, xmax, xmin = 0, 0, 0
nbin = 20
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
for value_truth in set(truth[sv]):
x_truth.append(value_truth)
y_fract.append(fract[truth[sv][idx] == value_truth])
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
axis.set_xticks(list(set(truth[sv])))
#_various things for uncertainty
axis.set_ylim(bottom=0, top=2)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small')
axis.grid(False)
#_shrink...ticks...
shrink_ticks(axis)
#_save when page full
if k == (nrow - 1) or j == nval - 1:
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format(
sim_experiment, out_label, sv, var, page)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
k += 1
def plot_simulated_histogram(out_label='',
plot_var='tau',
dir_hs3=os.path.join(os.environ['PRODUCTS'],
'hs3'),
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.retrieved'.format(out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 2.0),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = 'PLOTS_{0}'.format(out_label)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_produce for each state variable
for sv in state_vars:
#_get residual
res = truth[sv] - retr[sv]
#_initialize figure and axes
fig = plt.figure()
axes = []
figtext = []
ymax, xmax, xmin = 0, 0, 0
nbin = 100
axes.append(plt.subplot2grid((1, 1), (0, 0)))
#_plot histogram of residuals
(n, bins, patches) = axes[0].hist(res,
bins=nbin,
facecolor='b',
edgecolor='b')
xmax = max((xmax, bins.max()))
xmin = min((xmin, bins.min()))
ymax = max((ymax, n.max()))
axes[0].plot([0, 0], [0, n.max()], 'k--')
axes[0].set_xlabel('diff in tau, {0}'.format(sv), size='x-small')
axes[0].set_title('retrieval residuals'.format(out_label),
size='xx-small')
axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small')
#_max symmetric
xbnd = max((abs(xmin), abs(xmax)))
#_shrink...ticks...
[ax.set_xlim(-xbnd, xbnd) for ax in axes]
[ax.set_ylim(top=100) for ax in axes]
[shrink_ticks(ax) for ax in axes]
#_create image name
pname = 'hist_{0}_{1}.png'.format(out_label, sv)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
def fig_01(var,
out_label='',
plot_var='tau',
fsim=None,
dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'),
dir_plot='.',
files=[],
**kwargs):
'''
produce some plots for this god damn stuff
out_label str, experiment is not used so that it can
denote specific input files as opposed
to output. in_label should have been used.
'''
import matplotlib.pyplot as plt
from libtools import mkdir_p, shrink_ticks
from numpy import array, append
import re
from oe_sim import read_simulated_retrieval
if len(files) == 0:
print type(fsim)
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1)
#_read in retrieval data
fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
fname = os.path.join(dir_hs3, fname)
try:
truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs)
except IOError:
dbg(('Did you complete this retrieval yet?', out_label))
os._exit(23)
else:
for i, f in enumerate(files):
res = re.search('SIMULATED.(.*?).nc', f)
sim_experiment = res.group(1)
f = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format(
sim_experiment, out_label)
f = os.path.join(dir_hs3, f)
try:
if i == 0: #_initialize
truth, retr, uncrt = read_simulated_retrieval(f, **kwargs)
else: #_append
t, r, u = read_simulated_retrieval(f, **kwargs)
for sv in truth:
truth[sv] = append(truth[sv], t[sv])
for sv in retr:
retr[sv] = append(retr[sv], r[sv])
for sv in uncrt:
uncrt[sv] = append(uncrt[sv], u[sv])
except IOError:
dbg(('Did you complete this retrieval yet?', f))
os._exit(23)
res = re.search('SIMULATED.(.*?).nc', fsim)
sim_experiment = res.group(1).split('.')[0]
out_label = 'summary'
#_what are we working with?
state_vars = retr.keys() # truth.keys()
dbg('')
dbg(state_vars)
#_kick back if not available
if plot_var not in state_vars:
dbg(('Variable not retrieved:', plot_var))
return
#_color dictionary
order = ['low', 'mid', 'high', 'very_high']
ranges = {
'low': {
'color': 'blue',
'tau': (-9999., 0.5),
'ref': (0.5, 0.675),
},
'mid': {
'color': 'green',
'tau': (0.5, 1.0),
'ref': (0.625, 0.75),
},
'high': {
'color': 'orange',
'tau': (1.0, 1.5),
'ref': (0.75, 0.875),
},
'very_high': {
'color': 'red',
'tau': (1.5, 9999),
'ref': (0.875, 1.0),
},
}
#_output directory for plots
ppath = '{1}.{0}'.format(out_label, sim_experiment)
ppath = os.path.join(dir_plot, ppath)
if not os.path.exists(ppath):
mkdir_p(ppath)
#_get potential settings for value
values = list(set(truth[var]))
nval = len(values)
values.sort()
''' make loop over pages, 6 per page '''
nrow = 6
npag = nval / nrow #_do we need this?
page = 0
#_produce for each state variable
for sv in state_vars:
for j, value in enumerate(values):
dbg((sv, var, value))
#_initialize new figure
if not j % nrow:
page += 1
fig = plt.figure()
k = 0
fig.suptitle('Fractional Uncertainty')
#_get indices of limit
idx = truth[var] == value
#_get residual
res = truth[sv][idx] - retr[sv][idx]
#_initialize figure and axes
axis = fig.add_subplot(nrow, 1, k + 1)
ymax, xmax, xmin = 0, 0, 0
nbin = 20
#_plot uncertainty by truth
fract = uncrt[sv][idx] / truth[sv][idx]
x_truth, y_fract = [], []
#_pull out values across all over vars, at truth value
for value_truth in set(truth[sv]):
x_truth.append(value_truth)
y_fract.append(fract[truth[sv][idx] == value_truth])
#_pull out number of retrievals going into these
y_bar = [len(y) for y in y_fract]
## axis.scatter(truth[sv][idx], fract, marker='x', color=col)
axis.boxplot(y_fract, positions=x_truth, widths=0.05)
axis.set_xticks(list(set(truth[sv])))
abar = axis.twinx()
bar_width = (max(x_truth) - min(x_truth)) / float(len(x_truth))\
- 0.05
abar.bar(x_truth - bar_width / 2.,
y_bar,
bar_width,
alpha=0.24,
color='brown')
#_get how high we can go and still be 95 % certain of the AOD
## m95 = truth[sv][certainty_garbage(res)]
#_various things for uncertainty
# axis.text(truth[sv].max()-0.2, 1.6, 'n={0}'.format(len(y_fract))
axis.set_ylim(bottom=0, top=2)
axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1)
axis.set_ylabel('{0} = {1}'.format(var, value), size='xx-small')
axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small')
## axis.set_xlabel('{0}, max_95 {1:5.2f}'.format(sv,m95), size='x-small')
axis.grid(False)
#_shrink...ticks...
shrink_ticks(axis)
shrink_ticks(abar)
#_save when page full
if k == (nrow - 1) or j == nval - 1:
#_create image name
pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format(
sim_experiment, out_label, sv, var, page)
pname = os.path.join(ppath, pname)
plt.savefig(pname)
plt.close()
dbg(pname)
k += 1
def main(fidx,
dir_lblrtm_fmt='',
dir_lblrtm=None,
dir_plot='.',
persistence=False,
hostname=False,
file_col=None,
rerun=False,
experiment='HS3',
surf_temp='GDAS',
**kwargs):
'''
completely relies upon the kwargs pickle
written by optimal_estimation.real_case()
fidx int, Field of view number associated with
flight segment file in $PRODUCTS/hs3
'''
import matplotlib.pyplot as plt
from optimal_estimation import optimal_estimation as oe
from numpy import diag, matrix
from lblrtm_utils import microwindow_average, microwindow_stdev
from lblrtm_utils import get_surf_temp
from libtools import mkdir_p, epoch2dtg, dbg
from shutil import copytree, rmtree, copy2
from glob import glob
from oe_utils import read_airs
from time import sleep
try:
file_col, hostname = remote_copy(file_col)
except:
pass
#_read in radiance data for profile
idx_x, idx_a, file_airs = get_colloc_crap(fidx, file_col)
obs, wave = read_airs(file_airs, idx_x, idx_a, **kwargs) #_WRS
if len(obs) > 1:
raise RuntimeError, "Nope"
obs = obs[0]
#_update surface temperature value to be used
if surf_temp == 'GDAS':
from oe_utils import read_profile_gdas
gdas = read_profile_gdas(file_airs, idx_x, idx_a, **kwargs)
surf_temp = gdas.GDAS_sfc_temperature[0]
kwargs.update({'surf_temp': surf_temp})
#_pull out radiances
rads = getattr(obs, '{0}_radiances'.format('AIRS')) #_WRS
errs = getattr(obs, '{0}_nen'.format('AIRS')) #_WRS
#_get current location
dtg = epoch2dtg(obs.AIRS_epoch, full=True) #_WRS
#_initialize plot
# fig, ax = plt.subplots(1, **{'figsize' : (4,1)})
fig, ax = plt.subplots(1, figsize=(4, 1))
fig.suptitle(kwargs.get('desc'), size='xx-small')
out_label = kwargs.get('out_label')
pname = 'oe_{0}_{1}_f{2:04d}.png'.format(out_label, dtg, fidx)
pname = os.path.join(dir_plot, 'OE', pname)
mkdir_p(os.path.join(dir_plot, 'OE'))
if os.path.exists(pname):
os.unlink(pname)
#_build covariance matrix
#_when using microwindows, average observation data
dv = kwargs.get('dv')
if dv < 0:
std, d = microwindow_stdev(rads, wave, dv)
nesr, d = microwindow_average(errs, wave, dv, error=True)
y, wave_obs = microwindow_average(rads, wave, dv)
#_std deviation within microwindows applied to ref
# blackbody
cov = matrix(diag(nesr) + diag(std))
else:
cov = matrix(diag(errs))
#_allow for arbitrary lblrtm directory, get location of LBL-RTM output
dtgfov = (experiment, dtg)
dir_lblrtm = dir_lblrtm_fmt.format(*dtgfov)
dir_lblrtm_arc = dir_lblrtm_fmt.format(*dtgfov) if dir_lblrtm is None \
else dir_lblrtm
#_if already run and not desired to redo, kill
lbldis_output = kwargs.get('lbldis_output')
file_out = os.path.join(dir_lblrtm_arc, lbldis_output)
if not rerun and os.path.exists(file_out):
dbg(('Scene already processed', file_out))
return
#_check last field of view for persistence
if type(persistence) == dict and fidx != 0:
print 'DONT USE THIS IS PROBABLY WRONG DIRECTORY'
dtgfov_tmp = (experiment, dtg)
dir_lblrtm_arc_tmp = dir_lblrtm_fmt.format(*dtgfov_tmp)
print 'CHECKING PERSIST', kwargs['apriori']
#_put this in a try clause to allow for failure
try:
check_persistence(dir_lblrtm_arc_tmp, persistence, **kwargs)
except IOError:
pass
print 'CHECKING OUTSIST', kwargs['apriori']
#_copy the archive LBLRTM directory locally and work form there
dir_lblrtm = os.path.join(DIR_WORK,
'/'.join(dir_lblrtm_arc.split('/')[-2:]))
if not os.path.exists(dir_lblrtm + '/TAPE5'):
killcount = 0
while killcount < 10:
try:
mkdir_p(dir_lblrtm)
## [ copy2(dir_lblrtm_arc + '/*', dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*') ]
[copy2(cp, dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*')]
## copytree(dir_lblrtm_arc + '/*', dir_lblrtm)
break
except:
#_if fails, try ten times
sleep(10)
killcount += 1
if killcount >= 10:
exit()
#_update keyword arguments for this fov
args = (rads, wave, cov)
kwargs.update({
'fig': fig,
'ax': ax,
'pname': pname,
'nproc': 1, #_just in case I forgot to tone it down
'dir_lblrtm_arc': dir_lblrtm_arc,
'dir_lblrtm': dir_lblrtm
})
#_pull_the_trigger_#
#__________________#
dbg('LAUNCHING...')
oe(*args, **kwargs)
#_plot up results with flight data
kwargs.update({'dir_plot': os.path.join(dir_plot, 'DASHBOARD')})
dbg('REPLACE PLOTTING PROCEDURE, FILE_COL TOO SHORT, MAKE SEPARATE')
## flight.plot(fidx=fidx, **kwargs)
for out_file in glob(dir_lblrtm + '/*' + out_label + '*'):
killcount = 0
out = os.path.join(dir_lblrtm_arc, out_file.split('/')[-1])
while not os.path.exists(out):
try:
dbg(('archiving', out_file, '->', out))
copy2(out_file, dir_lblrtm_arc)
except:
dbg(('failed to copy', out_file, '->', out))
sleep(10)
#_remove temporary directory
rmtree(dir_lblrtm)
def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']),
start_dtg=None,
dir_plot='.',
end_dtg=None,
**kwargs):
'''
start_dtg str{14}, Starting date-time-group for subsetting
end_dtg str{14}, Can you solve the mystery?
'''
from glob import glob
from hs3_utils import read_cpl
import matplotlib.pyplot as plt
from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p
from numpy import arange, array, ceil, linspace
from numpy.ma import mean
from libcmap import rgbcmap
mkdir_p(dir_plot)
#_convert bounding limits
if start_dtg is not None:
ep0 = dtg2epoch(start_dtg, full=True)
ep1 = dtg2epoch(end_dtg, full=True)
else:
ep0 = '19000101000000'
ep1 = '99999999999999'
#_image setup
smooth = False
label = 'nosmooth'
files = glob('{0}/nc/*nc'.format(dir_cpl))
files.sort()
nrow = 6
npag = ceil(len(files) / nrow)
calipso = rgbcmap('calipso')
#_loop over all cpl files (netcdf)
i = 0
for q, fname in enumerate(files):
f = fname.split('/')[-1]
#_read in cpl data
cpl = read_cpl(f, None)
#_check if file is within limits
if cpl.epoch.min() > ep1 or cpl.epoch.max() < ep0:
continue
#_start new page
if not i % nrow:
fig = plt.figure()
ax_backscatter = fig.add_subplot(nrow, 1, i % nrow + 1)
ax = ax_backscatter.twinx()
print 'MADE IT'
#_get values of just aerosl
aod = calc_aod(cpl)
bck = calc_backscatter(cpl)
if smooth:
aod = smooth_aod(aod)
#_generate list of times
time = [epoch2iso(e) for e in cpl.epoch]
#_get number of fovs
nfov = aod.size
nt = nfov / 5
ax.set_xticks(arange(aod.size)[nt:-nt:nt])
ax.set_xticklabels(time[nt:-nt:nt])
props = dict(boxstyle='round', facecolor='white', alpha=.5)
ax.text(nt / 4., 3, f, color='k', size='x-small', bbox=props)
ax.set_ylim(0, 4)
#_plotbackscatter
cb = ax_backscatter.pcolormesh(bck,
vmin=-4e-7,
vmax=1e-4,
cmap=calipso,
zorder=0)
ax.plot(aod, linewidth=0.2, zorder=1, color=(1, 1, 1, .5))
ax.set_ylabel('aod', size='xx-small')
xlim = ax_backscatter.xaxis.get_data_interval()
ax_backscatter.set_xlim(xlim)
## ax_backscatter.set_ylim(0,900)
ax_backscatter.set_yticks(linspace(170, 900, 11))
ax_backscatter.set_yticklabels(linspace(0, 20, 11))
ax_backscatter.set_ylim(170, 900)
shrink_ticks(ax)
shrink_ticks(ax_backscatter)
i += 1
if (i % nrow) == (nrow - 1):
page_num = i / nrow
pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04,
0.5,
'altitude (km)',
va='center',
rotation='vertical',
size='x-small')
fig.savefig(pname)
plt.close()
else:
page_num = i / nrow
pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label)
pname = os.path.join(dir_plot, pname)
print pname
#_make a common y label
fig.text(0.04,
0.5,
'altitude (km)',
va='center',
rotation='vertical',
size='x-small')
fig.savefig(pname)
plt.close()
def cronlog(dtg, path='/'.join((dir_prod, 'status')), job={}):
'''
Looks into $PRODUCT/status directory and uses True/False files
to figure out the current progress of runs, downloads, and plotting
dtg : str*10, DTG for log file
path : str, Path to output directory
job : dict, keys == jobnames, values are status
'''
from time import strftime
from glob import glob
import re
dbg(('passed', dtg, job))
#_get current time
current_time = strftime('%Y-%m-%d_%H:%M:%S')
#_set log filename
dir_out = '/'.join((path, dtg[:6]))
lt.mkdir_p(dir_out)
file = '/'.join((dir_out, dtg + '_status'))
#_setup format for log writes
fmt = '%-20s %-10s %-22s %-22s\n'
#_Initialize log file if it does not exist
if not os.path.exists(file):
with filelock(file, mode='w') as f:
#_write header
header = ('job', 'status', 'start-time', 'end-time')
f.write(fmt % header)
#_write model statuses
for member in lm.current_icap():
line = (member, 'UNKNOWN', 'UNKNOWN', 'UNKNOWN')
dbg(line, l=9)
f.write(fmt % line)
#_add e-naaps to file
line = ('ENAAPS-NAV', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN')
dbg(line, l=9)
f.write(fmt % line)
#_add plotting
line = ('PLOTTING', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN')
dbg(line, l=9)
f.write(fmt % line)
#### #_look for status files
#### status_regex = re.compile( '(\w+)_\d{10}_status_(\w+)' )
#### glob_search = '/'.join(( dir_prod, 'status', dtg[:6], '*'+dtg+'*' ))
#### for statusfile in glob( glob_search ):
#### s = status_regex.search( statusfile )
####
#### #_if the file matches, get status
#### if s:
#### result = s.group(1)
#### jobname = s.group(2)
#### if result == 'True':
#### pass
#### ## #_LEAVE THIS COMMENTED OUT
#### ## #_leave False files to trigger backfill()
#### ## os.unlink( statusfile )
#### status = 'COMPLETE'
#### else:
#### status = 'FAILED'
####
#### #_update job dictionary
#### if jobname not in job:
#### job.update({ jobname : status })
####
dbg(('writing', dtg, job))
#_loop over job dictionary and update main log file
for jobname, status in job.iteritems():
#_read in old log
with filelock(file, 'r') as f:
lines = f.readlines()
#_write updated log
with filelock(file, 'w') as f:
for line in lines:
values = line.split()
#_if reporting on this job, update entry
if values[0] == jobname:
#_check to see what is being
# reported, then set the appropriate
# value. Times could be gathered
# from the filestats of the status
# files, but that's for another day
if status == 'STARTED':
start = current_time
else:
start = values[2]
if status == 'COMPLETE':
end = current_time
elif status == 'FAILED':
end = current_time
else:
end = 'UNKNOWN'
line = (jobname, status, start, end)
f.write(fmt % line)
#_otherwise, don't change
else:
f.write(line)
def plot_availability(dtgs,
dtge,
fcst_finc=24,
models=lm.current_icap(),
variables=['dust_aod'],
path='.',
**kwargs):
'''
For a given period, creates a bar chart to easily reference when
model forecasts are present or missing.
Only check dust_aod for now
'''
import matplotlib.pyplot as plt
lw = 30
#_Create list of forecast DTGs to check
dtg_list = []
dtg = dtgs
while dtg <= dtge:
dtg_list.append(dtg)
dtg = lt.newdtg(dtg, fcst_finc)
nt = len(dtg_list)
x = np.arange(nt)
#_Create dictionary of model keys that correspond to a y values
false_mask = [False] * nt
y = {}
models.insert(0, 'ICAP')
for model in models:
n = models.index(model)
y[model] = np.ma.masked_array([n] * nt)
y[model].mask = false_mask
h, w = plt.figaspect(10)
fig = plt.figure(figsize=(w, h))
#_Loop over dtgs and try to read forecasts
for dtg in dtg_list:
n = dtg_list.index(dtg)
try: #_read in forecast
fcst = read_icapfcst(dtg)
except: #_if the ICAP read function fails, assume all missing
continue
#_can expand to others later, but for now focus on dust
for variable in variables:
#_grab a single record to represent period
fhr = 72 #_Use 72 hours as the testing sample
rec = ln.subset(fcst, variable=variable, fhr=fhr, unique=True)
icap_mask = False
#_Check for each model
for member in rec.values.member:
#_Assumes member index is 0
m = rec.values.member.tolist().index(member)
col = mod_dict[member]['color']
mask = rec.values.mask[m]
#_Test full masked array
idx = np.where(mask == True)[0]
if len(idx) < (mask.size / 4):
plt.hlines([m + 1], [n - .55], [n + .55],
lw=lw,
colors=col)
continue
#_If any models fail, mask ICAP
icap_mask = True
else:
#_After all members done, plot ICAP
col = mod_dict['ICAP']['color']
if not icap_mask:
plt.hlines([0], [n - .55], [n + .55], lw=lw, colors='red')
x_max = 0
lab_y = []
for member in models:
#_Some Y-labeling specifics
if member == 'NGAC':
lab_y.append('NGAC\n(dust only)')
elif member == 'NMMB':
lab_y.append('NMMB/BSC-CTM\n(dust and seasalt only)')
else:
lab_y.append(member)
#_Limit size of plot area
plt.xlim(0, nt - 1)
ax = plt.subplot(111)
mjl = plt.MultipleLocator(nt / 5)
ax.xaxis.set_major_locator(mjl)
mjl = plt.MultipleLocator(90)
ax.yaxis.set_major_locator(mjl)
ax.grid(True, zorder=-1)
#_make list of dtgs for x axis
dtg_labels = dtg_list[0:nt:nt / 5]
tick_lab = lt.human_date(dtg_labels)
#_make list of x values to put the dtg labels
tick_loc = []
[tick_loc.append(dtg_list.index(p)) for p in dtg_labels]
plt.xticks(tick_loc, tick_lab, size='x-small')
#_Setup y axis
tick_loc = range(0, len(models), 1)
plt.yticks(tick_loc, lab_y, size='x-small')
plt.ylim(-0.5, len(models) - 0.5)
#_Write plot title
plt.title('ICAP Forecast Availablity at NRL-MRY for ' + dtgs + '-' + dtge)
#_Save file
lt.mkdir_p(path)
file_out = path + '/' + dtgs + '.availability.png'
dbg(file_out)
plt.savefig(file_out)
plt.close()
