def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']), start_dtg=None, dir_plot='.', end_dtg=None, type_coa='a', **kwargs): ''' start_dtg str{14}, Starting date-time-group for subsetting end_dtg str{14}, Can you solve the mystery? ''' from glob import glob from hs3_utils import Flight_segment import matplotlib.pyplot as plt from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p from numpy import arange, array, ceil, linspace from numpy.ma import mean from libcmap import rgbcmap from flight_namelists import experiments mkdir_p(dir_plot) #_get segments segments = experiments['HS3'] #_convert bounding limits if start_dtg is not None: ep0 = dtg2epoch(start_dtg, full=True) ep1 = dtg2epoch(end_dtg, full=True) else: ep0 = '19000101000000' ep1 = '99999999999999' #_image setup smooth = False label = 'nosmooth' files = glob('{0}/nc/*nc'.format(dir_cpl)) files.sort() nrow = 6 ncol = 2 npag = ceil(len(files) / (nrow/ncol)) calipso = rgbcmap('calipso') dtgs = segments.keys() dtgs.sort() #_loop over all cpl files (netcdf) # for i, fname in enumerate(files): for i, dtg in enumerate(dtgs): #_start new page if not i % (nrow*ncol): fig = plt.figure() fig.suptitle('CPL 532 Backscatter', size='small') #_calc column index #_calc row index ax_backscatter = fig.add_subplot(nrow, ncol, i%(nrow*ncol)+1) ax = ax_backscatter.twinx() #_read in cpl data # cpl = read_cpl(f, None) cpl = Flight_segment(dtg=dtg) #_check if file is within limits if cpl.CPL_epoch.min() > ep1 and cpl.CPL_epoch.max() < ep0: continue #_get values of just aerosl aod = calc_aod(cpl, type_coa) bck = calc_backscatter(cpl) if smooth: aod = smooth_aod(aod) #_generate list of times time = [epoch2iso(e) for e in cpl.CPL_epoch] #_get number of fovs nfov = aod.size nt = nfov / 2 ax.set_xticks(arange(aod.size)[nt:-nt:nt]) ax.set_xticklabels(time[nt:-nt:nt]) ax.xaxis.set_visible(False) props = dict(boxstyle='round', facecolor='white', alpha=.5) # ax.text(nt/4., 3, f, color='k', size='x-small', bbox=props) f = '{0} - {1}'.format(time[0], time[-1]) ax.text(nt/4., .75, f, color='k', size='xx-small', bbox=props) ax.set_ylim(0, 1) #_plotbackscatter cb=ax_backscatter.pcolormesh(bck,vmin=-4e-7,vmax=1e-4,cmap=calipso, zorder=0) ax.plot(aod, linewidth=0.2, zorder=1, color=(1,1,1,.5)) if i % ncol: # ax.set_ylabel('aod', size='xx-small') ax_backscatter.yaxis.set_visible(False) elif not i % ncol: ax.yaxis.set_visible(False) xlim=ax_backscatter.xaxis.get_data_interval() ax_backscatter.set_xlim(xlim) ax_backscatter.set_yticks(linspace(170,900,9)) ax_backscatter.set_yticklabels(linspace(0,20,9)) ax_backscatter.set_ylim(170,900) shrink_ticks(ax) shrink_ticks(ax_backscatter) #_I think this is bailing if (i % (nrow*ncol)) == ((nrow*ncol) - 1): page_num = i / (nrow*ncol) pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num, label, type_coa) pname = os.path.join(dir_plot, pname) print pname #_make a common y label fig.text(0.04, 0.5, 'altitude (km)', va='center', rotation='vertical', size='x-small') fig.text(0.96, 0.5, '{0}OD'.format(type_coa.upper()), va='center', rotation=270, size='x-small') fig.savefig(pname) plt.close() #_after everything is done else: page_num = i / (nrow*ncol) pname = 'page_{1}_{0:02d}_{2}OD_segments.png'.format(page_num, label, type_coa) pname = os.path.join(dir_plot, pname) print pname #_make a common y label fig.text(0.04, 0.5, 'altitude (km)', va='center', rotation='vertical', size='x-small') fig.text(0.96, 0.5, 'AOD', va='center', rotation=270, size='x-small') fig.savefig(pname) plt.close()
def plot_sat( recs, field='aod', path='.', **kwargs ): ''' rec : np.recarray, plots all sensor data and differenc plots and whatever else ed wanted ''' import matplotlib.pyplot as plt import matplotlib.cm as cm from matplotlib.colors import LogNorm import libnva as ln import libcmap as lc reg_dict = lm.regions() plt_dict = lm.fields() #_pull out metadata for title dtg = lt.unique( recs.dtg, unique=True ) sensor = lt.unique( recs.sensor, unique=True ) long_name = lt.unique( recs.long_name, unique=True ) wavelength = lt.unique( recs.wavelength, unique=True ) level = lt.unique( recs.level, unique=True ) title = ' '.join(( lt.human_date(dtg), sensor, long_name )) plt.title( title, size='small', ha='left', va='bottom',position=(0.,1.)) #_plot specifics corn = reg_dict['global']['corn'] grid = reg_dict['global']['grid'] delt = reg_dict['global']['delta'] #_setup colors icap = lc.rgbcmap( 'icap', N=256 ) #_get specifics for field field = field.lower() levs = [0.1,0.2,0.4,0.8] norm = LogNorm( levs[0], levs[-1] ) #_setup output name dir_out = '/'.join(( path, 'sat_01', 'global', dtg )) file = '_'.join(( dtg, 'total_aod', str(wavelength), sensor.lower(), 'l'+str(level)+'.png')) file = '/'.join(( dir_out, file )) lt.mkdir_p( dir_out ) #_generate map to plot over m = ln.draw_map( grid, corn, 'cyl', delt, fill='white', **kwargs ) #_put into 2d array tau2d = obsnew_field( recs, **kwargs ) #_put data onto grid d, x, y = ln.map_data( tau2d, m ) #_plot data CS = m.pcolormesh( x, y, d, cmap=cm.jet, norm=norm, rasterized=True ) #_generate colorbar bar = plt.colorbar( CS, orientation='horizontal', shrink=0.8, aspect=40, pad=0.05, ticks=levs ) bar.set_ticklabels( levs ) dbg(file) plt.savefig(file) plt.close()
def plot_simulated_retrieval(out_label='', plot_var='tau', fsim=None, dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'), **kwargs): ''' produce some plots for this god damn stuff out_label str, experiment is not used so that it can denote specific input files as opposed to output. in_label should have been used. ''' import matplotlib.pyplot as plt from libtools import mkdir_p, shrink_ticks from numpy import array, append res = re.search('SIMULATED.(.*?).nc', fsim) sim_experiment = res.group(1) #_read in retrieval data fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format( sim_experiment, out_label) fname = os.path.join(dir_hs3, fname) try: truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs) except IOError: dbg(('Did you complete this retrieval yet?', out_label)) os._exit(23) #_what are we working with? state_vars = retr.keys() # truth.keys() dbg(state_vars) #_kick back if not available if plot_var not in state_vars: dbg(('Variable not retrieved:', plot_var)) return #_color dictionary order = ['low', 'mid', 'high', 'very_high'] ranges = { 'low': { 'color': 'blue', 'tau': (-9999., 0.5), 'ref': (0.5, 0.675), }, 'mid': { 'color': 'green', 'tau': (0.5, 1.0), 'ref': (0.625, 0.75), }, 'high': { 'color': 'orange', 'tau': (1.0, 1.5), 'ref': (0.75, 0.875), }, 'very_high': { 'color': 'red', 'tau': (1.5, 9999), 'ref': (0.875, 1.0), }, } #_output directory for plots ppath = 'PLOTS_{0}'.format(out_label) if not os.path.exists(ppath): mkdir_p(ppath) #_produce for each state variable for sv in state_vars: #_get residual res = truth[sv] - retr[sv] #_get indices for ranges for label, opts in ranges.iteritems(): min_val, max_val = opts[sv] # idx0 = truth[sv] >= min_val # idx1 = truth[sv] < max_val # idx = append(idx0[:,None], idx1[:,None], axis=1).all(axis=1) idx = (truth[sv] >= min_val) * (truth[sv] < max_val) ranges[label]['idx'] = idx #_initialize figure and axes fig = plt.figure() axes = [] axis = plt.subplot2grid((2, 4), (1, 0), colspan=len(ranges.keys())) figtext = [] ymax, xmax, xmin = 0, 0, 0 nbin = 20 for i, label in enumerate(order): #ranges): min_val, max_val = ranges[label][sv] idx = ranges[label]['idx'] col = ranges[label]['color'] if idx.sum() == 0: continue axes.append(plt.subplot2grid((2, 4), (0, i))) #_plot histogram of residuals (n, bins, patches) = axes[i].hist(res[idx], bins=nbin, facecolor=col, edgecolor=col) xmax = max((xmax, bins.max())) xmin = min((xmin, bins.min())) ymax = max((ymax, n.max())) #_add label figtext.append('{0}\nmin {1:5.1f}\nmax {2:5.1f}'.format( label, min_val, max_val)) axes[i].set_xlabel('difference in {0}'.format(sv), size='x-small') #_plot uncertainty by truth fract = uncrt[sv][idx] / truth[sv][idx] x_truth, y_fract = [], [] for value in set(truth[sv][idx]): x_truth.append(value) y_fract.append(fract[truth[sv][idx] == value]) ## axis.scatter(truth[sv][idx], fract, marker='x', color=col) axis.boxplot(y_fract, positions=x_truth, widths=0.05) ## axis.plot(truth[sv][idx], fract, marker='x', color=col) ## axis.scatter(truth[sv][idx], uncrt[sv][idx], marker='x', color=col) axis.set_xticks(list(set(truth[sv]))) axes[0].set_title('{0} (experiment label)'.format(out_label), size='xx-small') #_various things for uncertainty axis.set_ylim(bottom=0) axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1) axis.set_ylabel('posterior uncercertainty', size='x-small') axis.set_xlabel('{0}'.format(sv), size='x-small') axis.grid(False) axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small') #_max symmetric xbnd = max((abs(xmin), abs(xmax))) for i, label in enumerate(order): axes[i].text(0, ymax - ymax / 5., figtext[i], size='xx-small') #_shrink...ticks... [ax.set_xlim(-xbnd, xbnd) for ax in axes] [ax.set_ylim(top=ymax) for ax in axes] [shrink_ticks(ax) for ax in axes] shrink_ticks(axis) #_create image name pname = 'hist_uncert.{0}.{1}.{2}.png'.format(sim_experiment, out_label, sv) pname = os.path.join(ppath, pname) plt.savefig(pname) plt.close() dbg(pname)
def main(fidx, dir_lblrtm_fmt='', dir_lblrtm=None, dir_plot='.', persistence=False, hostname=False, file_col=None, experiment='HS3', surf_temp='GDAS', rerun=False, **kwargs): ''' completely relies upon the kwargs pickle written by optimal_estimation.real_case() fidx int, Field of view number associated with flight segment file in $PRODUCTS/hs3 ''' import matplotlib.pyplot as plt from optimal_estimation import optimal_estimation as oe from numpy import diag, matrix from lblrtm_utils import microwindow_average, microwindow_stdev from lblrtm_utils import get_surf_temp from libtools import mkdir_p, epoch2dtg, dbg from shutil import copytree, rmtree, copy2 from glob import glob from oe_utils import read_airs from time import sleep from hs3_utils import Flight_segment as F from os import unlink try: file_col, hostname = remote_copy(file_col) except: pass #_read in radiance data for profile flight = F(**kwargs) fov = flight[fidx] wave = flight.SHIS_wavenumber rads = fov.SHIS_radiances errs = flight.SHIS_HBB_NESR #_initialize plot # fig, ax = plt.subplots(1, **{'figsize' : (4,1)}) fig, ax = plt.subplots(1, figsize=(4, 1)) fig.suptitle(kwargs.get('desc'), size='xx-small') out_label = kwargs.get('out_label') pname = 'oe_{0}_{1}_f{2:04d}.png'.format(out_label, flight.dtg0, fidx) pname = os.path.join(dir_plot, 'OE', pname) mkdir_p(os.path.join(dir_plot, 'OE')) if os.path.exists(pname): os.unlink(pname) #_build covariance matrix #_when using microwindows, average observation data dv = kwargs.get('dv') if dv < 0: std, d = microwindow_stdev(rads, wave, dv) nesr, d = microwindow_average(errs, wave, dv, error=True) y, wave_obs = microwindow_average(rads, wave, dv) #_std deviation within microwindows applied to ref # blackbody cov = matrix(diag(nesr) + diag(std)) else: cov = matrix(diag(errs)) #_allow for arbitrary lblrtm directory, get location of LBL-RTM output dtgfov = (flight.dtg0, flight.dtg1, fidx, experiment, flight.dtg0[2:]) dir_lblrtm = dir_lblrtm_fmt.format(*dtgfov) dir_lblrtm_arc = dir_lblrtm_fmt.format(*dtgfov) if dir_lblrtm is None \ else dir_lblrtm #_update kwargs based upon dynamic ATP/CTP surf_temp = get_surf_temp(fov, surf_temp_src=surf_temp, dir_lblrtm=dir_lblrtm_arc, **kwargs) check_ctp(fov, wave, **kwargs) check_atp(fov, **kwargs) #_if already run and not desired to redo, kill lbldis_output = kwargs.get('lbldis_output') file_out = os.path.join(dir_lblrtm_arc, lbldis_output) #_check last field of view for persistence if type(persistence) == dict and fidx != 0: print 'DONT USE THIS IS PROBABLY WRONG DIRECTORY' dtgfov_tmp = (experiment, dtg) dir_lblrtm_arc_tmp = dir_lblrtm_fmt.format(*dtgfov_tmp) print 'CHECKING PERSIST', kwargs['apriori'] #_put this in a try clause to allow for failure try: check_persistence(dir_lblrtm_arc_tmp, persistence, **kwargs) except IOError: pass print 'CHECKING OUTSIST', kwargs['apriori'] #_copy the archive LBLRTM directory locally and work form there dir_lblrtm = os.path.join(DIR_WORK, '/'.join(dir_lblrtm_arc.split('/')[-2:])) if not os.path.exists(dir_lblrtm + '/TAPE5'): killcount = 0 while killcount < 10: try: mkdir_p(dir_lblrtm) [copy2(cp, dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*')] break except: #_if fails, try ten times sleep(10) killcount += 1 if killcount >= 10: exit() #_update keyword arguments for this fov args = (rads, wave, cov) kwargs.update({ 'fig': fig, 'ax': ax, 'pname': pname, 'nproc': 1, #_just in case I forgot to tone it down 'dir_lblrtm_arc': dir_lblrtm_arc, 'dir_lblrtm': dir_lblrtm, 'surf_temp': surf_temp, }) #_pull_the_trigger_# #__________________# dbg('LAUNCHING...') oe(*args, **kwargs) #_archive crap for out_file in glob(dir_lblrtm + '/*' + out_label + '*'): killcount = 0 out = os.path.join(dir_lblrtm_arc, out_file.split('/')[-1]) #_remove old output if rerunning if rerun and os.path.exists(out): unlink(out) while not os.path.exists(out): try: dbg(('archiving', out_file, '->', out)) copy2(out_file, dir_lblrtm_arc) except: dbg(('failed to copy', out_file, '->', out)) sleep(10) #_plot up results with flight data kwargs.update({'dir_plot': os.path.join(dir_plot, 'DASHBOARD')}) flight.plot(fidx=fidx, rerun=rerun, **kwargs) #_remove temporary directory rmtree(dir_lblrtm)
def fig_01_check(var, out_label='', plot_var='tau', fsim=None, dir_lblrtm=None, dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'), **kwargs): ''' Uncertainty by optical depth First does plot of all median uncertainties? Or by specific ref/z Plot uncertainty broken down by ref/z with box/whisker plots. Plot them in total. plot_simulated_by_var produce some plots for this god damn stuff out_label str, experiment is not used so that it can denote specific input files as opposed to output. in_label should have been used. ''' import matplotlib.pyplot as plt from libtools import mkdir_p, shrink_ticks from numpy import array, append dbg('running...') #_read in retrieval data if type(fsim) != list: res = re.search('SIMULATED.(.*?).nc', fsim) sim_experiment = res.group(1) fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format( sim_experiment, out_label) fname = os.path.join(dir_hs3, fname) try: truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs) except IOError: dbg(('Did you complete this retrieval yet?', out_label)) os._exit(23) else: for fname in fsim: res = re.search('SIMULATED.(.*?).nc', fname) sim_experiment = res.group(1) fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format( sim_experiment, out_label) fname = os.path.join(dir_hs3, fname) t, r, u = read_simulated_retrieval(fname, **kwargs) try: [append(truth[sv], t[sv]) for sv in t.keys()] [append(retr[sv], r[sv]) for sv in r.keys()] [append(uncrt[sv], u[sv]) for sv in u.keys()] except UnboundLocalError: truth, retr, uncrt = t, r, u sim_experiment = 'NOISE' #_what are we working with? state_vars = retr.keys() # truth.keys() #_kick back if not available if plot_var not in state_vars: dbg(('Variable not retrieved:', plot_var)) return #_output directory for plots ppath = 'PLOTS_{0}'.format(out_label) if not os.path.exists(ppath): mkdir_p(ppath) #_get potential settings for value values = list(set(truth[var])) nval = len(values) values.sort() nrow = 6 npag = nval / nrow #_do we need this? page = 0 #_produce for each state variable for sv in state_vars: for j, value in enumerate(values): dbg((sv, var, value)) #_initialize new figure if not j % nrow: page += 1 fig = plt.figure() k = 0 #_get indices of limit idx = truth[var] == value #_get residual res = truth[sv][idx] - retr[sv][idx] #_initialize figure and axes axis = fig.add_subplot(nrow, 1, k + 1) ymax, xmax, xmin = 0, 0, 0 nbin = 20 #_plot uncertainty by truth fract = uncrt[sv][idx] / truth[sv][idx] x_truth, y_fract = [], [] for value_truth in set(truth[sv]): x_truth.append(value_truth) y_fract.append(fract[truth[sv][idx] == value_truth]) ## axis.scatter(truth[sv][idx], fract, marker='x', color=col) axis.boxplot(y_fract, positions=x_truth, widths=0.05) axis.set_xticks(list(set(truth[sv]))) #_various things for uncertainty axis.set_ylim(bottom=0, top=2) axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1) axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small') axis.grid(False) #_shrink...ticks... shrink_ticks(axis) #_save when page full if k == (nrow - 1) or j == nval - 1: #_create image name pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format( sim_experiment, out_label, sv, var, page) pname = os.path.join(ppath, pname) plt.savefig(pname) plt.close() dbg(pname) k += 1
def plot_simulated_histogram(out_label='', plot_var='tau', dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'), **kwargs): ''' produce some plots for this god damn stuff out_label str, experiment is not used so that it can denote specific input files as opposed to output. in_label should have been used. ''' import matplotlib.pyplot as plt from libtools import mkdir_p, shrink_ticks from numpy import array, append #_read in retrieval data fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.retrieved'.format(out_label) fname = os.path.join(dir_hs3, fname) try: truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs) except IOError: dbg(('Did you complete this retrieval yet?', out_label)) os._exit(23) #_what are we working with? state_vars = retr.keys() # truth.keys() dbg(state_vars) #_kick back if not available if plot_var not in state_vars: dbg(('Variable not retrieved:', plot_var)) return #_color dictionary order = ['low', 'mid', 'high', 'very_high'] ranges = { 'low': { 'color': 'blue', 'tau': (-9999., 0.5), 'ref': (0.5, 0.675), }, 'mid': { 'color': 'green', 'tau': (0.5, 1.0), 'ref': (0.625, 0.75), }, 'high': { 'color': 'orange', 'tau': (1.0, 1.5), 'ref': (0.75, 0.875), }, 'very_high': { 'color': 'red', 'tau': (1.5, 2.0), 'ref': (0.875, 1.0), }, } #_output directory for plots ppath = 'PLOTS_{0}'.format(out_label) if not os.path.exists(ppath): mkdir_p(ppath) #_produce for each state variable for sv in state_vars: #_get residual res = truth[sv] - retr[sv] #_initialize figure and axes fig = plt.figure() axes = [] figtext = [] ymax, xmax, xmin = 0, 0, 0 nbin = 100 axes.append(plt.subplot2grid((1, 1), (0, 0))) #_plot histogram of residuals (n, bins, patches) = axes[0].hist(res, bins=nbin, facecolor='b', edgecolor='b') xmax = max((xmax, bins.max())) xmin = min((xmin, bins.min())) ymax = max((ymax, n.max())) axes[0].plot([0, 0], [0, n.max()], 'k--') axes[0].set_xlabel('diff in tau, {0}'.format(sv), size='x-small') axes[0].set_title('retrieval residuals'.format(out_label), size='xx-small') axes[0].set_ylabel('residuals, truth - oe_retrieval', size='x-small') #_max symmetric xbnd = max((abs(xmin), abs(xmax))) #_shrink...ticks... [ax.set_xlim(-xbnd, xbnd) for ax in axes] [ax.set_ylim(top=100) for ax in axes] [shrink_ticks(ax) for ax in axes] #_create image name pname = 'hist_{0}_{1}.png'.format(out_label, sv) pname = os.path.join(ppath, pname) plt.savefig(pname) plt.close() dbg(pname)
def fig_01(var, out_label='', plot_var='tau', fsim=None, dir_hs3=os.path.join(os.environ['PRODUCTS'], 'hs3'), dir_plot='.', files=[], **kwargs): ''' produce some plots for this god damn stuff out_label str, experiment is not used so that it can denote specific input files as opposed to output. in_label should have been used. ''' import matplotlib.pyplot as plt from libtools import mkdir_p, shrink_ticks from numpy import array, append import re from oe_sim import read_simulated_retrieval if len(files) == 0: print type(fsim) res = re.search('SIMULATED.(.*?).nc', fsim) sim_experiment = res.group(1) #_read in retrieval data fname = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format( sim_experiment, out_label) fname = os.path.join(dir_hs3, fname) try: truth, retr, uncrt = read_simulated_retrieval(fname, **kwargs) except IOError: dbg(('Did you complete this retrieval yet?', out_label)) os._exit(23) else: for i, f in enumerate(files): res = re.search('SIMULATED.(.*?).nc', f) sim_experiment = res.group(1) f = 'SHIS.CPL.GDAS.COLLOC.SIMULATED.{0}.{1}.retrieved'.format( sim_experiment, out_label) f = os.path.join(dir_hs3, f) try: if i == 0: #_initialize truth, retr, uncrt = read_simulated_retrieval(f, **kwargs) else: #_append t, r, u = read_simulated_retrieval(f, **kwargs) for sv in truth: truth[sv] = append(truth[sv], t[sv]) for sv in retr: retr[sv] = append(retr[sv], r[sv]) for sv in uncrt: uncrt[sv] = append(uncrt[sv], u[sv]) except IOError: dbg(('Did you complete this retrieval yet?', f)) os._exit(23) res = re.search('SIMULATED.(.*?).nc', fsim) sim_experiment = res.group(1).split('.')[0] out_label = 'summary' #_what are we working with? state_vars = retr.keys() # truth.keys() dbg('') dbg(state_vars) #_kick back if not available if plot_var not in state_vars: dbg(('Variable not retrieved:', plot_var)) return #_color dictionary order = ['low', 'mid', 'high', 'very_high'] ranges = { 'low': { 'color': 'blue', 'tau': (-9999., 0.5), 'ref': (0.5, 0.675), }, 'mid': { 'color': 'green', 'tau': (0.5, 1.0), 'ref': (0.625, 0.75), }, 'high': { 'color': 'orange', 'tau': (1.0, 1.5), 'ref': (0.75, 0.875), }, 'very_high': { 'color': 'red', 'tau': (1.5, 9999), 'ref': (0.875, 1.0), }, } #_output directory for plots ppath = '{1}.{0}'.format(out_label, sim_experiment) ppath = os.path.join(dir_plot, ppath) if not os.path.exists(ppath): mkdir_p(ppath) #_get potential settings for value values = list(set(truth[var])) nval = len(values) values.sort() ''' make loop over pages, 6 per page ''' nrow = 6 npag = nval / nrow #_do we need this? page = 0 #_produce for each state variable for sv in state_vars: for j, value in enumerate(values): dbg((sv, var, value)) #_initialize new figure if not j % nrow: page += 1 fig = plt.figure() k = 0 fig.suptitle('Fractional Uncertainty') #_get indices of limit idx = truth[var] == value #_get residual res = truth[sv][idx] - retr[sv][idx] #_initialize figure and axes axis = fig.add_subplot(nrow, 1, k + 1) ymax, xmax, xmin = 0, 0, 0 nbin = 20 #_plot uncertainty by truth fract = uncrt[sv][idx] / truth[sv][idx] x_truth, y_fract = [], [] #_pull out values across all over vars, at truth value for value_truth in set(truth[sv]): x_truth.append(value_truth) y_fract.append(fract[truth[sv][idx] == value_truth]) #_pull out number of retrievals going into these y_bar = [len(y) for y in y_fract] ## axis.scatter(truth[sv][idx], fract, marker='x', color=col) axis.boxplot(y_fract, positions=x_truth, widths=0.05) axis.set_xticks(list(set(truth[sv]))) abar = axis.twinx() bar_width = (max(x_truth) - min(x_truth)) / float(len(x_truth))\ - 0.05 abar.bar(x_truth - bar_width / 2., y_bar, bar_width, alpha=0.24, color='brown') #_get how high we can go and still be 95 % certain of the AOD ## m95 = truth[sv][certainty_garbage(res)] #_various things for uncertainty # axis.text(truth[sv].max()-0.2, 1.6, 'n={0}'.format(len(y_fract)) axis.set_ylim(bottom=0, top=2) axis.set_xlim(truth[sv].min() - 0.1, truth[sv].max() + 0.1) axis.set_ylabel('{0} = {1}'.format(var, value), size='xx-small') axis.set_ylabel('{0} = {1}'.format(var, value), size='x-small') ## axis.set_xlabel('{0}, max_95 {1:5.2f}'.format(sv,m95), size='x-small') axis.grid(False) #_shrink...ticks... shrink_ticks(axis) shrink_ticks(abar) #_save when page full if k == (nrow - 1) or j == nval - 1: #_create image name pname = 'hist_uncert.{0}.{1}.{2}.by_{3}.p{4:02d}.png'.format( sim_experiment, out_label, sv, var, page) pname = os.path.join(ppath, pname) plt.savefig(pname) plt.close() dbg(pname) k += 1
def main(fidx, dir_lblrtm_fmt='', dir_lblrtm=None, dir_plot='.', persistence=False, hostname=False, file_col=None, rerun=False, experiment='HS3', surf_temp='GDAS', **kwargs): ''' completely relies upon the kwargs pickle written by optimal_estimation.real_case() fidx int, Field of view number associated with flight segment file in $PRODUCTS/hs3 ''' import matplotlib.pyplot as plt from optimal_estimation import optimal_estimation as oe from numpy import diag, matrix from lblrtm_utils import microwindow_average, microwindow_stdev from lblrtm_utils import get_surf_temp from libtools import mkdir_p, epoch2dtg, dbg from shutil import copytree, rmtree, copy2 from glob import glob from oe_utils import read_airs from time import sleep try: file_col, hostname = remote_copy(file_col) except: pass #_read in radiance data for profile idx_x, idx_a, file_airs = get_colloc_crap(fidx, file_col) obs, wave = read_airs(file_airs, idx_x, idx_a, **kwargs) #_WRS if len(obs) > 1: raise RuntimeError, "Nope" obs = obs[0] #_update surface temperature value to be used if surf_temp == 'GDAS': from oe_utils import read_profile_gdas gdas = read_profile_gdas(file_airs, idx_x, idx_a, **kwargs) surf_temp = gdas.GDAS_sfc_temperature[0] kwargs.update({'surf_temp': surf_temp}) #_pull out radiances rads = getattr(obs, '{0}_radiances'.format('AIRS')) #_WRS errs = getattr(obs, '{0}_nen'.format('AIRS')) #_WRS #_get current location dtg = epoch2dtg(obs.AIRS_epoch, full=True) #_WRS #_initialize plot # fig, ax = plt.subplots(1, **{'figsize' : (4,1)}) fig, ax = plt.subplots(1, figsize=(4, 1)) fig.suptitle(kwargs.get('desc'), size='xx-small') out_label = kwargs.get('out_label') pname = 'oe_{0}_{1}_f{2:04d}.png'.format(out_label, dtg, fidx) pname = os.path.join(dir_plot, 'OE', pname) mkdir_p(os.path.join(dir_plot, 'OE')) if os.path.exists(pname): os.unlink(pname) #_build covariance matrix #_when using microwindows, average observation data dv = kwargs.get('dv') if dv < 0: std, d = microwindow_stdev(rads, wave, dv) nesr, d = microwindow_average(errs, wave, dv, error=True) y, wave_obs = microwindow_average(rads, wave, dv) #_std deviation within microwindows applied to ref # blackbody cov = matrix(diag(nesr) + diag(std)) else: cov = matrix(diag(errs)) #_allow for arbitrary lblrtm directory, get location of LBL-RTM output dtgfov = (experiment, dtg) dir_lblrtm = dir_lblrtm_fmt.format(*dtgfov) dir_lblrtm_arc = dir_lblrtm_fmt.format(*dtgfov) if dir_lblrtm is None \ else dir_lblrtm #_if already run and not desired to redo, kill lbldis_output = kwargs.get('lbldis_output') file_out = os.path.join(dir_lblrtm_arc, lbldis_output) if not rerun and os.path.exists(file_out): dbg(('Scene already processed', file_out)) return #_check last field of view for persistence if type(persistence) == dict and fidx != 0: print 'DONT USE THIS IS PROBABLY WRONG DIRECTORY' dtgfov_tmp = (experiment, dtg) dir_lblrtm_arc_tmp = dir_lblrtm_fmt.format(*dtgfov_tmp) print 'CHECKING PERSIST', kwargs['apriori'] #_put this in a try clause to allow for failure try: check_persistence(dir_lblrtm_arc_tmp, persistence, **kwargs) except IOError: pass print 'CHECKING OUTSIST', kwargs['apriori'] #_copy the archive LBLRTM directory locally and work form there dir_lblrtm = os.path.join(DIR_WORK, '/'.join(dir_lblrtm_arc.split('/')[-2:])) if not os.path.exists(dir_lblrtm + '/TAPE5'): killcount = 0 while killcount < 10: try: mkdir_p(dir_lblrtm) ## [ copy2(dir_lblrtm_arc + '/*', dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*') ] [copy2(cp, dir_lblrtm) for cp in glob(dir_lblrtm_arc + '/*')] ## copytree(dir_lblrtm_arc + '/*', dir_lblrtm) break except: #_if fails, try ten times sleep(10) killcount += 1 if killcount >= 10: exit() #_update keyword arguments for this fov args = (rads, wave, cov) kwargs.update({ 'fig': fig, 'ax': ax, 'pname': pname, 'nproc': 1, #_just in case I forgot to tone it down 'dir_lblrtm_arc': dir_lblrtm_arc, 'dir_lblrtm': dir_lblrtm }) #_pull_the_trigger_# #__________________# dbg('LAUNCHING...') oe(*args, **kwargs) #_plot up results with flight data kwargs.update({'dir_plot': os.path.join(dir_plot, 'DASHBOARD')}) dbg('REPLACE PLOTTING PROCEDURE, FILE_COL TOO SHORT, MAKE SEPARATE') ## flight.plot(fidx=fidx, **kwargs) for out_file in glob(dir_lblrtm + '/*' + out_label + '*'): killcount = 0 out = os.path.join(dir_lblrtm_arc, out_file.split('/')[-1]) while not os.path.exists(out): try: dbg(('archiving', out_file, '->', out)) copy2(out_file, dir_lblrtm_arc) except: dbg(('failed to copy', out_file, '->', out)) sleep(10) #_remove temporary directory rmtree(dir_lblrtm)
def main(dir_cpl='{0}/cpl'.format(os.environ['PRODUCTS']), start_dtg=None, dir_plot='.', end_dtg=None, **kwargs): ''' start_dtg str{14}, Starting date-time-group for subsetting end_dtg str{14}, Can you solve the mystery? ''' from glob import glob from hs3_utils import read_cpl import matplotlib.pyplot as plt from libtools import shrink_ticks, epoch2iso, dtg2epoch, mkdir_p from numpy import arange, array, ceil, linspace from numpy.ma import mean from libcmap import rgbcmap mkdir_p(dir_plot) #_convert bounding limits if start_dtg is not None: ep0 = dtg2epoch(start_dtg, full=True) ep1 = dtg2epoch(end_dtg, full=True) else: ep0 = '19000101000000' ep1 = '99999999999999' #_image setup smooth = False label = 'nosmooth' files = glob('{0}/nc/*nc'.format(dir_cpl)) files.sort() nrow = 6 npag = ceil(len(files) / nrow) calipso = rgbcmap('calipso') #_loop over all cpl files (netcdf) i = 0 for q, fname in enumerate(files): f = fname.split('/')[-1] #_read in cpl data cpl = read_cpl(f, None) #_check if file is within limits if cpl.epoch.min() > ep1 or cpl.epoch.max() < ep0: continue #_start new page if not i % nrow: fig = plt.figure() ax_backscatter = fig.add_subplot(nrow, 1, i % nrow + 1) ax = ax_backscatter.twinx() print 'MADE IT' #_get values of just aerosl aod = calc_aod(cpl) bck = calc_backscatter(cpl) if smooth: aod = smooth_aod(aod) #_generate list of times time = [epoch2iso(e) for e in cpl.epoch] #_get number of fovs nfov = aod.size nt = nfov / 5 ax.set_xticks(arange(aod.size)[nt:-nt:nt]) ax.set_xticklabels(time[nt:-nt:nt]) props = dict(boxstyle='round', facecolor='white', alpha=.5) ax.text(nt / 4., 3, f, color='k', size='x-small', bbox=props) ax.set_ylim(0, 4) #_plotbackscatter cb = ax_backscatter.pcolormesh(bck, vmin=-4e-7, vmax=1e-4, cmap=calipso, zorder=0) ax.plot(aod, linewidth=0.2, zorder=1, color=(1, 1, 1, .5)) ax.set_ylabel('aod', size='xx-small') xlim = ax_backscatter.xaxis.get_data_interval() ax_backscatter.set_xlim(xlim) ## ax_backscatter.set_ylim(0,900) ax_backscatter.set_yticks(linspace(170, 900, 11)) ax_backscatter.set_yticklabels(linspace(0, 20, 11)) ax_backscatter.set_ylim(170, 900) shrink_ticks(ax) shrink_ticks(ax_backscatter) i += 1 if (i % nrow) == (nrow - 1): page_num = i / nrow pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label) pname = os.path.join(dir_plot, pname) print pname #_make a common y label fig.text(0.04, 0.5, 'altitude (km)', va='center', rotation='vertical', size='x-small') fig.savefig(pname) plt.close() else: page_num = i / nrow pname = 'page_{1}_{0:02d}_0.png'.format(page_num, label) pname = os.path.join(dir_plot, pname) print pname #_make a common y label fig.text(0.04, 0.5, 'altitude (km)', va='center', rotation='vertical', size='x-small') fig.savefig(pname) plt.close()
def cronlog(dtg, path='/'.join((dir_prod, 'status')), job={}): ''' Looks into $PRODUCT/status directory and uses True/False files to figure out the current progress of runs, downloads, and plotting dtg : str*10, DTG for log file path : str, Path to output directory job : dict, keys == jobnames, values are status ''' from time import strftime from glob import glob import re dbg(('passed', dtg, job)) #_get current time current_time = strftime('%Y-%m-%d_%H:%M:%S') #_set log filename dir_out = '/'.join((path, dtg[:6])) lt.mkdir_p(dir_out) file = '/'.join((dir_out, dtg + '_status')) #_setup format for log writes fmt = '%-20s %-10s %-22s %-22s\n' #_Initialize log file if it does not exist if not os.path.exists(file): with filelock(file, mode='w') as f: #_write header header = ('job', 'status', 'start-time', 'end-time') f.write(fmt % header) #_write model statuses for member in lm.current_icap(): line = (member, 'UNKNOWN', 'UNKNOWN', 'UNKNOWN') dbg(line, l=9) f.write(fmt % line) #_add e-naaps to file line = ('ENAAPS-NAV', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN') dbg(line, l=9) f.write(fmt % line) #_add plotting line = ('PLOTTING', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN') dbg(line, l=9) f.write(fmt % line) #### #_look for status files #### status_regex = re.compile( '(\w+)_\d{10}_status_(\w+)' ) #### glob_search = '/'.join(( dir_prod, 'status', dtg[:6], '*'+dtg+'*' )) #### for statusfile in glob( glob_search ): #### s = status_regex.search( statusfile ) #### #### #_if the file matches, get status #### if s: #### result = s.group(1) #### jobname = s.group(2) #### if result == 'True': #### pass #### ## #_LEAVE THIS COMMENTED OUT #### ## #_leave False files to trigger backfill() #### ## os.unlink( statusfile ) #### status = 'COMPLETE' #### else: #### status = 'FAILED' #### #### #_update job dictionary #### if jobname not in job: #### job.update({ jobname : status }) #### dbg(('writing', dtg, job)) #_loop over job dictionary and update main log file for jobname, status in job.iteritems(): #_read in old log with filelock(file, 'r') as f: lines = f.readlines() #_write updated log with filelock(file, 'w') as f: for line in lines: values = line.split() #_if reporting on this job, update entry if values[0] == jobname: #_check to see what is being # reported, then set the appropriate # value. Times could be gathered # from the filestats of the status # files, but that's for another day if status == 'STARTED': start = current_time else: start = values[2] if status == 'COMPLETE': end = current_time elif status == 'FAILED': end = current_time else: end = 'UNKNOWN' line = (jobname, status, start, end) f.write(fmt % line) #_otherwise, don't change else: f.write(line)
def plot_availability(dtgs, dtge, fcst_finc=24, models=lm.current_icap(), variables=['dust_aod'], path='.', **kwargs): ''' For a given period, creates a bar chart to easily reference when model forecasts are present or missing. Only check dust_aod for now ''' import matplotlib.pyplot as plt lw = 30 #_Create list of forecast DTGs to check dtg_list = [] dtg = dtgs while dtg <= dtge: dtg_list.append(dtg) dtg = lt.newdtg(dtg, fcst_finc) nt = len(dtg_list) x = np.arange(nt) #_Create dictionary of model keys that correspond to a y values false_mask = [False] * nt y = {} models.insert(0, 'ICAP') for model in models: n = models.index(model) y[model] = np.ma.masked_array([n] * nt) y[model].mask = false_mask h, w = plt.figaspect(10) fig = plt.figure(figsize=(w, h)) #_Loop over dtgs and try to read forecasts for dtg in dtg_list: n = dtg_list.index(dtg) try: #_read in forecast fcst = read_icapfcst(dtg) except: #_if the ICAP read function fails, assume all missing continue #_can expand to others later, but for now focus on dust for variable in variables: #_grab a single record to represent period fhr = 72 #_Use 72 hours as the testing sample rec = ln.subset(fcst, variable=variable, fhr=fhr, unique=True) icap_mask = False #_Check for each model for member in rec.values.member: #_Assumes member index is 0 m = rec.values.member.tolist().index(member) col = mod_dict[member]['color'] mask = rec.values.mask[m] #_Test full masked array idx = np.where(mask == True)[0] if len(idx) < (mask.size / 4): plt.hlines([m + 1], [n - .55], [n + .55], lw=lw, colors=col) continue #_If any models fail, mask ICAP icap_mask = True else: #_After all members done, plot ICAP col = mod_dict['ICAP']['color'] if not icap_mask: plt.hlines([0], [n - .55], [n + .55], lw=lw, colors='red') x_max = 0 lab_y = [] for member in models: #_Some Y-labeling specifics if member == 'NGAC': lab_y.append('NGAC\n(dust only)') elif member == 'NMMB': lab_y.append('NMMB/BSC-CTM\n(dust and seasalt only)') else: lab_y.append(member) #_Limit size of plot area plt.xlim(0, nt - 1) ax = plt.subplot(111) mjl = plt.MultipleLocator(nt / 5) ax.xaxis.set_major_locator(mjl) mjl = plt.MultipleLocator(90) ax.yaxis.set_major_locator(mjl) ax.grid(True, zorder=-1) #_make list of dtgs for x axis dtg_labels = dtg_list[0:nt:nt / 5] tick_lab = lt.human_date(dtg_labels) #_make list of x values to put the dtg labels tick_loc = [] [tick_loc.append(dtg_list.index(p)) for p in dtg_labels] plt.xticks(tick_loc, tick_lab, size='x-small') #_Setup y axis tick_loc = range(0, len(models), 1) plt.yticks(tick_loc, lab_y, size='x-small') plt.ylim(-0.5, len(models) - 0.5) #_Write plot title plt.title('ICAP Forecast Availablity at NRL-MRY for ' + dtgs + '-' + dtge) #_Save file lt.mkdir_p(path) file_out = path + '/' + dtgs + '.availability.png' dbg(file_out) plt.savefig(file_out) plt.close()