def write_matched_data(data):
from netCDF4 import Dataset
from libtools import unique, subset
from libnva import sort_rec
from numpy import argsort
raise RuntimeError, 'USE LIVE_WRITE'
cdf = Dataset('matched_data.nc', 'w', format='NETCDF3_CLASSIC')
#_define dimensions
time = unique(data.time)
wnum = unique(data.wavenumber)
nt = len(time)
nw = len(wnum)
cdf.createDimension('wavenumber', nw)
cdf.createDimension('time', nt)
cdf.createVariable('wavenumber', 'f8', ('wavenumber', ))
cdf.createVariable('time', 'f8', ('time', ))
cdf.variables['wavenumber'][:] = wnum
cdf.variables['time'][:] = time
cdf.createVariable('cloud', 'f4', ('time', ))
cdf.createVariable('latitude', 'f8', ('time', ))
cdf.createVariable('longitude', 'f8', ('time', ))
cdf.createVariable('radiances_lbldis',
'f8', ('time', 'wavenumber'),
fill_value=-9999)
cdf.createVariable('radiances_airs',
'f8', ('time', 'wavenumber'),
fill_value=-9999)
cdf.createVariable('tb_lbldis',
'f8', ('time', 'wavenumber'),
fill_value=-9999)
cdf.createVariable('tb_airs',
'f8', ('time', 'wavenumber'),
fill_value=-9999)
for tidx in xrange(nt):
t = time[tidx]
#_pull out this time
data_t = subset(data, time=t)
#_pull out thiiiisss
cdf.variables['cloud'][tidx] = unique(data_t.cloud, unique=True)
cdf.variables['latitude'][tidx] = unique(data_t.latitude, unique=True)
cdf.variables['longitude'][tidx] = unique(data_t.longitude,
unique=True)
#_sort these
idx = argsort(data_t.wavenumber)
cdf.variables['lbldis'][tidx] = data_t.lbldis[idx]
cdf.variables['airs'][tidx] = data_t.airs[idx]
cdf.close()
def obsnew_field( recs, nx=360, ny=180, **kwargs ): ''' Takes recarray of obsnewdata and puts it into a 2d field for plot recs : cl.sat_object Will crash if more than 1 dtg pass ''' dtg = lt.unique( recs.dtg, unique=True ) lat = np.linspace( -89.5, 89.5, ny ) lon = np.linspace( -179.5, 179.5, nx ) #_create dummy array to hold values tau = np.zeros((ny,nx)) - 9999. #_convert lats and lons to indices i, j = lt.ll2ij( recs.lat, recs.lon, lat, lon ) tau[j,i] = recs.tau idx = np.where( tau == -9999. ) tau[idx] = 0. ## tau = np.ma.masked_where( tau == -9999., tau ) ## tau = np.ma.MaskedArray.filled( tau, 0. ) ## tau = np.linspace(0.1,0.8,nx*ny).reshape(ny,nx) tau = cl.var( tau, attrv=(lat,lon) ) return tau
def plot_forker( recs, **kwargs ): ''' forks plots based on dtg ''' dtgs = lt.unique( recs.dtg ) groups = lt.setup_groups( dtgs, **kwargs ) #_loop over proc groups for group in groups: children = [] for dtg in group: pid = os.fork() if pid != 0: children.append(pid) elif pid == 0: #_pull out day day = ln.subset( recs, dtg=dtg ) #_pass day to plotter plot_sat( day, **kwargs ) os._exit(0) for kid in children: os.waitpid(kid,0)
def plot_sat( recs, field='aod', path='.', **kwargs ): ''' rec : np.recarray, plots all sensor data and differenc plots and whatever else ed wanted ''' import matplotlib.pyplot as plt import matplotlib.cm as cm from matplotlib.colors import LogNorm import libnva as ln import libcmap as lc reg_dict = lm.regions() plt_dict = lm.fields() #_pull out metadata for title dtg = lt.unique( recs.dtg, unique=True ) sensor = lt.unique( recs.sensor, unique=True ) long_name = lt.unique( recs.long_name, unique=True ) wavelength = lt.unique( recs.wavelength, unique=True ) level = lt.unique( recs.level, unique=True ) title = ' '.join(( lt.human_date(dtg), sensor, long_name )) plt.title( title, size='small', ha='left', va='bottom',position=(0.,1.)) #_plot specifics corn = reg_dict['global']['corn'] grid = reg_dict['global']['grid'] delt = reg_dict['global']['delta'] #_setup colors icap = lc.rgbcmap( 'icap', N=256 ) #_get specifics for field field = field.lower() levs = [0.1,0.2,0.4,0.8] norm = LogNorm( levs[0], levs[-1] ) #_setup output name dir_out = '/'.join(( path, 'sat_01', 'global', dtg )) file = '_'.join(( dtg, 'total_aod', str(wavelength), sensor.lower(), 'l'+str(level)+'.png')) file = '/'.join(( dir_out, file )) lt.mkdir_p( dir_out ) #_generate map to plot over m = ln.draw_map( grid, corn, 'cyl', delt, fill='white', **kwargs ) #_put into 2d array tau2d = obsnew_field( recs, **kwargs ) #_put data onto grid d, x, y = ln.map_data( tau2d, m ) #_plot data CS = m.pcolormesh( x, y, d, cmap=cm.jet, norm=norm, rasterized=True ) #_generate colorbar bar = plt.colorbar( CS, orientation='horizontal', shrink=0.8, aspect=40, pad=0.05, ticks=levs ) bar.set_ticklabels( levs ) dbg(file) plt.savefig(file) plt.close()
def join_icap(aod,
fhr=120,
fstrt=0,
nt=None,
finc=6,
members=lm.current_icap(),
**kwargs):
'''
Put all icap forecasts on common NAAPS grid for usage with ensemble
statistics
require_all limits the returned values to timesteps when every model
present (at all)
can provide data. So if MACC is in the mix, no 00z.
'''
if 'ICAP' in members: members.remove('ICAP')
dbg(aod.size)
#_Calculate last dtg
species = [s.lower() for s in mod_dict['ICAP']['specs']]
nx = mod_dict['ICAP']['nx']
ny = mod_dict['ICAP']['ny']
lons = np.linspace(-179.5, 179.5, nx)
lats = np.linspace(-89.5, 89.5, ny)
finc = 6
icap = cl.model_object()
vars = lm.aod_vars()
#_Create list of models with ANY data and icap models
#_Loop over species, join models we have for specs
dtg_valds = set(aod.dtg_vald) #_list of unique dtgs
dtg_init = lt.unique(aod.dtg_init)[0]
#_Create array of for missing data
## nan_2darray = np.empty((ny, nx))
## nan_2darray[:] = NaN #_There's gotta ba shorthand for this
nan_2darray = np.zeros((ny, nx)) - 9999.
nens_max = len(members)
for spec in species:
dbg(spec, l=2)
long_name = vars[spec]['long_name']
for dtg_vald in dtg_valds:
#_make recarray for one dtg, spec, but multiple models
aod_sub = ln.subset(aod,
variable=spec,
model=members,
dtg_vald=dtg_vald)
#_regrid models
aod_rgd = np.empty((nens_max, ny, nx))
aod_rgd[:] = -9999.
for e in np.arange(nens_max):
#_get model name and append it to dimension
name = members[e]
#_pull gridded data for specific model
tmp = ln.subset(aod_sub, model=name)
if tmp.size == 1: #_Should have single rec
d = tmp.values[0]
x = d.lon
y = d.lat
#_Regrid model data to icap x/y
aod_rgd[e, :, :] = lt.regrid_field(x, y, d, lons,
lats).transpose()
elif tmp.size == 0: #_Model data missing
aod_rgd[e, :, :] = nan_2darray.copy()
else:
print 'How did this happen?'
return -1
#_Get indices that are non-physical
neg_idx = np.where(aod_rgd < -1e-5)
aod_rgd[neg_idx] = -9999. #_SLOW
aod_rgd = np.ma.masked_where(aod_rgd == -9999., aod_rgd)
#_Convert to masked array and count present models
nens = ln.check_members(aod_rgd)
### miss = ( aod_rgd[:,0,0] == -9999. ).tolist().count(True)
### nens = nens_max - miss
data = cl.var(aod_rgd, attrv=(
members,
lats,
lons,
))
dimsize = data.shape
dimname = (
'member',
'lat',
'lon',
)
vhr = lt.find_runlength(dtg_init, dtg_vald) / 3600
icap.resize(icap.size + 1)
icap[-1] = (data, 'ICAP', dtg_init, dtg_vald, vhr, 'global', spec,
nens, dimname, dimsize, '', long_name)
#_Limit to forecasts every finc hours
idx_fhr = np.where(icap.fhr % finc == 0)[0]
icap = icap[idx_fhr]
return icap
def filter(records,
strict_icap=True,
members=['NAAPS', 'GEOS5', 'MACC', 'MASINGAR', 'NGAC'],
modes=False,
**kwargs):
'''
Builds recarray model_object() containing only ICAP records that
1. Contain all models in members
2. Contain all species of that member as defined in libmeta
records : model_object(), np.recarray() of aod model data
members : list, list of names to require to return
'''
dbg(records.size)
tmp_model = lt.unique(records.model)
if tmp_model != ['ICAP']:
dbg(('filter for icap only', tmp_model), l=3)
return records
if not strict_icap:
dbg(('icap not set to strict, returning'), l=3)
return records
#_Make expected species list for each model
specs = ln.expected_vars()
#_Initialize return object
out = cl.model_object()
#_REDUCE________________________________________________________________
#_CLEANUP_______________________________________________________________
#_Remove records lacking any of the required members
#_Loop over each ICAP record
for rec in records:
#_take slice to check for masked members
mask = rec.values[:, 0, 0].mask
v = rec.variable
#_loop over each model for this record, see if variable
# is both expected and present
desired = []
for model in members:
#_make list of expected species for each model
idx = rec.values.member.tolist().index(model)
### dbg(( rec.values.member, model, idx ))
#_make list of indices to keep
desired.append(idx)
#_see if model is masked, and if so, break loop
# leaving record out
test = mask if type(mask) == np.bool_ else mask[idx]
if test and v in specs[model]:
dbg((rec.dtg_vald, v, 'filtered'), l=1)
dbg((model, 'was the cause'), l=1)
break
#_if it makes it passed all members, add to return array
else:
#_Need to reducse attrv to plug var back into recarry
atn, atv = ln.get_attr(rec)
mem_idx = atn.index('member')
atv[mem_idx] = atv[mem_idx][desired]
#_Update dimsize
vals = rec.values.copy()
vals_out = vals[desired, :, :]
rec.dimsize = vals_out.shape
#_Put back into original record
rec.values = cl.var(vals_out, attrn=atn, attrv=atv)
out = lt.merge((out, rec))
dbg(out.size)
return out