def read_nmmbfcst(dtg,
path=dir_prod + '/NMMB',
require=['dust_aod', 'seasalt_aod']):
'''
Read NMMB netcdf file
aod[dtg][spec] = read_geos5fcst(dtg)
dtg = string, start dtg
finc = integer, step up to fhr
nt = integer, number of timesteps to fhr,
normally diagnosed
fhr = integer, maximum forecast hour from dtg
path = string, directory of MASINGAR files
'''
from netCDF4 import Dataset, Variable
dtg_fcst = dtg
file = '/'.join((path, dtg_fcst[:6], dtg_fcst + '-NMMB_BSC_CTM-ICAP.nc'))
dbg(dtg)
model = 'NMMB'
aod = cl.model_object()
vars = lm.aod_vars()
species = {'dust_aod550': 'dust_aod', 'salt_aod550': 'seasalt_aod'}
#_BSC added seasalt on this date
if dtg < '2012112000': del species['salt_aod550']
if not os.path.isfile(file):
error = file + ' is missing'
dbg(error, l=2)
else:
dbg(file, l=2)
handle = Dataset(file, mode='r', format='NETCDF3_CLASSIC')
x, ny, nx = handle.variables['dust_aod550'].shape
lons = handle.variables['lon'][nx / 2, 1:]
lats = handle.variables['lat'][:, ny / 2]
times = handle.variables['time'][:]
dtg_init = handle.dtg_init #_Brings dtg to 00z instead of 22
nt = len(times)
#_Loop over species dictionary to read in ncdf variables
for spec in species:
spec_loc = species[spec]
long_name = vars[spec_loc]['long_name']
tmp = handle.variables[spec]
for t in np.arange(nt):
s = times[t]
dtg_loop = lt.newdtg(dtg_init, s)
vhr = lt.find_runlength( dtg_init, dtg_loop ) \
/ 3600
#_Setup attributes and add to recarray
attrv = (
lats,
lons,
)
data = cl.var(tmp[t, :, 1:], attrv=attrv)
dimname = vars[spec_loc]['dims']
dimsize = data.shape
aod.resize(aod.size + 1)
aod[-1] = (data, model, dtg_init, dtg_loop, vhr, 'global',
spec_loc, 0, dimname, dimsize, '', long_name)
handle.close()
#_Check for required species
for s in require:
size = ln.subset(aod, variable=s).size
if size == 0:
err = 'Not all ' + model + ' species available'
raise RuntimeError, err
return aod
def read_masingarfcst(dtg,
require=[
'sulfate_aod', 'dust_aod', 'smoke_aod',
'seasalt_aod', 'total_aod'
],
path=dir_prod + '/MASINGAR'):
'''
Read MASINGAR netcdf file
aod[dtg][spec] = read_geos5fcst(dtg)
dtg = string, start dtg
finc = integer, step up to fhr
nt = integer, number of timesteps to fhr,
normally diagnosed
fhr = integer, maximum forecast hour from dtg
path = string, directory of MASINGAR files
'''
from netCDF4 import Dataset, Variable
dtg_fcst = dtg
file = path + '/' + dtg_fcst[:6] + '/' + dtg_fcst \
+ '_aod_masingar.nc'
dbg(dtg)
model = 'MASINGAR'
aod = cl.model_object()
vars = lm.aod_vars()
fhr = 120
specs = [s.lower() for s in mod_dict[model]['specs']]
if os.path.isfile(file) == False:
raise IOError, 'WARNING: ' + file + ' is missing.'
else:
dbg(file, l=2)
handle = Dataset(file, mode='r', format='NETCDF3_CLASSIC')
null, ny, nx = handle.variables['total'].shape
lons = handle.variables['lon'][:]
lats = handle.variables['lat'][:]
times = handle.variables['time'][:]
dtg_init = lt.epoch2dtg(times[0]) #_Brings dtg to 00z instead of 22
dtg_fhr = lt.newdtg(dtg_fcst, fhr)
nt = len(times)
#_Loop over each variable, store in dictionary
for spec in specs: #handle.variables:
if spec == 'lat' or spec == 'lon' or spec == 'time':
continue
long_name = vars[spec]['long_name']
spec_mas = spec.replace('_aod', '')
tmp = handle.variables[spec_mas]
for t in np.arange(nt):
s = times[t]
dtg_loop = lt.epoch2dtg(s)
if dtg_loop > dtg_fhr: break
vhr = lt.find_runlength(dtg_init, dtg_loop) / 3600
attrv = (
lats,
lons,
)
data = cl.var(handle.variables[spec_mas][t, :, :], attrv=attrv)
dimname = vars[spec]['dims']
dimsize = data.shape
aod.resize(aod.size + 1)
aod[-1] = (data, model, dtg_init, dtg_loop, vhr, 'global',
spec, 0, dimname, dimsize, '', long_name)
handle.close()
#_Check for required species
for s in require:
size = ln.subset(aod, variable=s).size
if size == 0:
err = 'Not all ' + model + ' species available'
raise RuntimeError, err
return aod
def plot_availability(dtgs,
dtge,
fcst_finc=24,
models=lm.current_icap(),
variables=['dust_aod'],
path='.',
**kwargs):
'''
For a given period, creates a bar chart to easily reference when
model forecasts are present or missing.
Only check dust_aod for now
'''
import matplotlib.pyplot as plt
lw = 30
#_Create list of forecast DTGs to check
dtg_list = []
dtg = dtgs
while dtg <= dtge:
dtg_list.append(dtg)
dtg = lt.newdtg(dtg, fcst_finc)
nt = len(dtg_list)
x = np.arange(nt)
#_Create dictionary of model keys that correspond to a y values
false_mask = [False] * nt
y = {}
models.insert(0, 'ICAP')
for model in models:
n = models.index(model)
y[model] = np.ma.masked_array([n] * nt)
y[model].mask = false_mask
h, w = plt.figaspect(10)
fig = plt.figure(figsize=(w, h))
#_Loop over dtgs and try to read forecasts
for dtg in dtg_list:
n = dtg_list.index(dtg)
try: #_read in forecast
fcst = read_icapfcst(dtg)
except: #_if the ICAP read function fails, assume all missing
continue
#_can expand to others later, but for now focus on dust
for variable in variables:
#_grab a single record to represent period
fhr = 72 #_Use 72 hours as the testing sample
rec = ln.subset(fcst, variable=variable, fhr=fhr, unique=True)
icap_mask = False
#_Check for each model
for member in rec.values.member:
#_Assumes member index is 0
m = rec.values.member.tolist().index(member)
col = mod_dict[member]['color']
mask = rec.values.mask[m]
#_Test full masked array
idx = np.where(mask == True)[0]
if len(idx) < (mask.size / 4):
plt.hlines([m + 1], [n - .55], [n + .55],
lw=lw,
colors=col)
continue
#_If any models fail, mask ICAP
icap_mask = True
else:
#_After all members done, plot ICAP
col = mod_dict['ICAP']['color']
if not icap_mask:
plt.hlines([0], [n - .55], [n + .55], lw=lw, colors='red')
x_max = 0
lab_y = []
for member in models:
#_Some Y-labeling specifics
if member == 'NGAC':
lab_y.append('NGAC\n(dust only)')
elif member == 'NMMB':
lab_y.append('NMMB/BSC-CTM\n(dust and seasalt only)')
else:
lab_y.append(member)
#_Limit size of plot area
plt.xlim(0, nt - 1)
ax = plt.subplot(111)
mjl = plt.MultipleLocator(nt / 5)
ax.xaxis.set_major_locator(mjl)
mjl = plt.MultipleLocator(90)
ax.yaxis.set_major_locator(mjl)
ax.grid(True, zorder=-1)
#_make list of dtgs for x axis
dtg_labels = dtg_list[0:nt:nt / 5]
tick_lab = lt.human_date(dtg_labels)
#_make list of x values to put the dtg labels
tick_loc = []
[tick_loc.append(dtg_list.index(p)) for p in dtg_labels]
plt.xticks(tick_loc, tick_lab, size='x-small')
#_Setup y axis
tick_loc = range(0, len(models), 1)
plt.yticks(tick_loc, lab_y, size='x-small')
plt.ylim(-0.5, len(models) - 0.5)
#_Write plot title
plt.title('ICAP Forecast Availablity at NRL-MRY for ' + dtgs + '-' + dtge)
#_Save file
lt.mkdir_p(path)
file_out = path + '/' + dtgs + '.availability.png'
dbg(file_out)
plt.savefig(file_out)
plt.close()
def read_geos5fcst(dtg,
require=[
'dust_aod', 'smoke_aod', 'seasalt_aod', 'sulfate_aod',
'total_aod'
],
path=dir_prod + '/GEOS5'):
'''
Read GEOS-5 netcdf file
aod[dtg][spec] = read_geos5fcst(dtg)
dtg = string, start dtg
finc = integer, step up to fhr
nt = integer, number of timesteps to fhr,
normally diagnosed
fhr = integer, maximum forecast hour from dtg
path = string, directory of GEOS-5 files
'''
#_These are the species FROM GMAO.
# Not the list getting PLOTTED. That
# is in the model dictionary
from netCDF4 import Dataset, Variable
dbg(dtg)
aod = cl.model_object()
vars = lm.aod_vars()
model = 'GEOS5'
finc = 6
fhr = 120
species = [
'dust_aod', 'blackcarbon_aod', 'organiccarbon_aod', 'seasalt_aod',
'sulfate_aod'
]
file = path + '/' + dtg[:6] + '/' + dtg + '_aod_geos5.nc'
dbg('Reading ' + file, l=2)
if os.path.isfile(file) == False:
raise IOError, file + ' is missing.'
else:
handle = Dataset(file, mode='r', format='NETCDF3_CLASSIC')
lons = handle.variables['lons'][:]
lats = handle.variables['lats'][:]
times = handle.variables['times'][:]
dtg_init = lt.gsfc_day2dtg(times[0]) #_Brings dtg to 00z, not 22
dtg_fcst = lt.newdtg(dtg_init, fhr)
times = times[::finc]
nt = len(times)
for spec in species: # handle.variables:
spec_geos = spec.replace('_aod', '')
tmp = handle.variables[spec_geos]
long_name = vars[spec]['long_name']
for t in np.arange(nt):
days = times[t]
dtg_loop = lt.gsfc_day2dtg(days)
if dtg_loop > dtg_fcst:
break
vhr = lt.find_runlength(dtg_init, dtg_loop) / 3600
data = cl.var(tmp[t * finc, :, :], attrv=[lats, lons])
dimname = vars[spec]['dims']
dimsize = data.shape
aod.resize(aod.size + 1)
aod[-1] = (data, model, dtg_init, dtg_loop, vhr, 'global',
spec, 0, dimname, dimsize, '', long_name)
#_Replace this with masked_outside
if len(np.where(tmp > 1e3)) > 0:
raise ValueError, 'GEOS-5 data out of range for ' + \
dtg_loop + ' ' + spec
handle.close()
#_If we have bc and oc, sum them to generate smoke specie
spec_smoke = ['blackcarbon_aod', 'organiccarbon_aod']
dtg_pres = aod.dtg_vald[0]
for spec in spec_smoke:
test = ln.subset(aod, variable=spec)
if test.size > 0:
pass
else:
break
else: #-If all species present, sum and generate 'total'i WALTER
long_name = vars['smoke_aod']['long_name']
bc = ln.subset(aod, variable='blackcarbon_aod')
oc = ln.subset(aod, variable='organiccarbon_aod')
#_Get arrays of dtgs available for species
dtg_bc = bc.dtg_vald
dtg_oc = oc.dtg_vald
#_Find where we have both values
dtg_smoke = lt.intersection([dtg_bc, dtg_oc])
for dtg_loop in dtg_smoke:
if dtg_loop > dtg_fcst: break #_KLUUUUUUUUDGE
vhr = lt.find_runlength(dtg_init, dtg_loop) / 3600
bc_loop = ln.subset(bc, dtg_vald=dtg_loop).values[0]
oc_loop = ln.subset(oc, dtg_vald=dtg_loop).values[0]
attrv = [lats, lons]
tmp = np.sum([
bc_loop,
oc_loop,
], axis=0)
data = cl.var(tmp, attrv=attrv)
dimname = vars['smoke_aod']['dims']
dimsize = data.shape
aod.resize(aod.size + 1)
aod[-1] = (data, model, dtg_init, dtg_loop, vhr, 'global',
'smoke_aod', 0, dimname, dimsize, '', long_name)
#_If we have all necessary species, sum total
spec_total = [
'dust_aod', 'seasalt_aod', 'sulfate_aod', 'smoke_aod'
]
for spec in spec_total:
test = ln.subset(aod, variable=spec, dtg_vald=dtg_loop)
if test.size > 0:
pass
else:
dbg('Cannot calc total AOD ' + dtg_loop + ' ' + spec,
l=2)
break
else: #-If bc and oc present, generate 'smoke' specie
long_name = vars['total_aod']['long_name']
aod_loop = ln.subset(aod, dtg_vald=dtg_loop)
dust_loop = ln.subset(aod_loop,
variable=['dust_aod']).values[0]
salt_loop = ln.subset(aod_loop,
variable=['seasalt_aod']).values[0]
sulf_loop = ln.subset(aod_loop,
variable=['sulfate_aod']).values[0]
smoke_loop = ln.subset(aod_loop,
variable=['smoke_aod']).values[0]
tmp = np.sum([dust_loop, salt_loop, sulf_loop, smoke_loop],
axis=0)
attrv = [lats, lons]
data = cl.var(tmp, attrv=attrv)
dimname = vars['total_aod']['dims']
dimsize = data.shape
aod.resize(aod.size + 1)
aod[-1] = (data, model, dtg_init, dtg_loop, vhr, 'global',
'total_aod', 0, dimname, dimsize, '', long_name)
#_Check for required species
for s in require:
size = ln.subset(aod, variable=s).size
if size == 0:
raise RuntimeError, 'Not all GEOS-5 species available'
return aod
def join_icap(aod,
fhr=120,
fstrt=0,
nt=None,
finc=6,
members=lm.current_icap(),
**kwargs):
'''
Put all icap forecasts on common NAAPS grid for usage with ensemble
statistics
require_all limits the returned values to timesteps when every model
present (at all)
can provide data. So if MACC is in the mix, no 00z.
'''
if 'ICAP' in members: members.remove('ICAP')
dbg(aod.size)
#_Calculate last dtg
species = [s.lower() for s in mod_dict['ICAP']['specs']]
nx = mod_dict['ICAP']['nx']
ny = mod_dict['ICAP']['ny']
lons = np.linspace(-179.5, 179.5, nx)
lats = np.linspace(-89.5, 89.5, ny)
finc = 6
icap = cl.model_object()
vars = lm.aod_vars()
#_Create list of models with ANY data and icap models
#_Loop over species, join models we have for specs
dtg_valds = set(aod.dtg_vald) #_list of unique dtgs
dtg_init = lt.unique(aod.dtg_init)[0]
#_Create array of for missing data
## nan_2darray = np.empty((ny, nx))
## nan_2darray[:] = NaN #_There's gotta ba shorthand for this
nan_2darray = np.zeros((ny, nx)) - 9999.
nens_max = len(members)
for spec in species:
dbg(spec, l=2)
long_name = vars[spec]['long_name']
for dtg_vald in dtg_valds:
#_make recarray for one dtg, spec, but multiple models
aod_sub = ln.subset(aod,
variable=spec,
model=members,
dtg_vald=dtg_vald)
#_regrid models
aod_rgd = np.empty((nens_max, ny, nx))
aod_rgd[:] = -9999.
for e in np.arange(nens_max):
#_get model name and append it to dimension
name = members[e]
#_pull gridded data for specific model
tmp = ln.subset(aod_sub, model=name)
if tmp.size == 1: #_Should have single rec
d = tmp.values[0]
x = d.lon
y = d.lat
#_Regrid model data to icap x/y
aod_rgd[e, :, :] = lt.regrid_field(x, y, d, lons,
lats).transpose()
elif tmp.size == 0: #_Model data missing
aod_rgd[e, :, :] = nan_2darray.copy()
else:
print 'How did this happen?'
return -1
#_Get indices that are non-physical
neg_idx = np.where(aod_rgd < -1e-5)
aod_rgd[neg_idx] = -9999. #_SLOW
aod_rgd = np.ma.masked_where(aod_rgd == -9999., aod_rgd)
#_Convert to masked array and count present models
nens = ln.check_members(aod_rgd)
### miss = ( aod_rgd[:,0,0] == -9999. ).tolist().count(True)
### nens = nens_max - miss
data = cl.var(aod_rgd, attrv=(
members,
lats,
lons,
))
dimsize = data.shape
dimname = (
'member',
'lat',
'lon',
)
vhr = lt.find_runlength(dtg_init, dtg_vald) / 3600
icap.resize(icap.size + 1)
icap[-1] = (data, 'ICAP', dtg_init, dtg_vald, vhr, 'global', spec,
nens, dimname, dimsize, '', long_name)
#_Limit to forecasts every finc hours
idx_fhr = np.where(icap.fhr % finc == 0)[0]
icap = icap[idx_fhr]
return icap
def read_icapfcst_raw(dtg,
remove_members=False,
fhr=None,
require=[
'total_aod', 'sulfate_aod', 'dust_aod', 'smoke_aod',
'seasalt_aod'
],
members=lm.current_icap(),
**kwargs):
'''
Attempts to read in all centers' five day forecasts
Will not merge without members in members having require specs
fhr is a list of desired fhr
'''
dbg(members)
aod = cl.model_object()
if 'MACC' in members: #_READ MACC
try:
macc = read_maccfcst(dtg, require=require)
if type(macc) == np.recarray:
aod = lt.merge((aod, macc))
except:
dbg('Error reading MACC', l=2)
if 'GEOS5' in members: #_READ GEOS5
try:
dtg_g = lt.newdtg(dtg, -2)
geos5 = read_geos5fcst(dtg_g, require=require)
if type(geos5) == np.recarray:
aod = lt.merge((aod, geos5))
except:
dbg('Error reading GEOS5', l=2)
if 'MASINGAR' in members: #_READ MASINGAR
try:
masingar = read_masingarfcst(dtg, require=require)
if type(masingar) == np.recarray:
aod = lt.merge((aod, masingar))
except:
dbg('Error reading MASINGAR', l=2)
if 'NGAC' in members: #_READ GFS NGAC-
try:
require_ngac = ['dust_aod']
ngac = read_ngacfcst(dtg, require=require_ngac)
if type(ngac) == np.recarray:
aod = lt.merge((aod, ngac))
except:
dbg('Error reading NGAC', l=2)
if 'NMMB' in members: #_READ GFS NGAC-
try:
require_nmmb = ['dust_aod', 'seasalt_aod']
nmmb = read_nmmbfcst(dtg, require=require_nmmb)
if type(nmmb) == np.recarray:
aod = lt.merge((aod, nmmb))
except:
dbg('Error reading NMMB', l=2)
if 'NAAPS' in members: #_READ NAAPS DETERMINISTIC
try:
nva = ln.read_naapsfcst(dtg, **kwargs)
if type(nva) == np.recarray:
aod = lt.merge((aod, nva))
except:
dbg('Error reading NAAPS', l=2)
if 'ENAAPS-NAV' in members: #_READ NAAPS MET ENSEMBLE
try:
ensemble = ln.read_ensfcst(dtg, model='ENAAPS-NAV')
if type(ensemble) == np.recarray:
aod = lt.merge((aod, ensemble))
except:
dbg('Error reading eNAAPS-NAV MET', l=2)
if 'dart' in members: #_READ NAAPS DART ENSEMBLE
try:
ensemble = read_ensfcst(dtg, model='ENAAPS-DART')
if type(ensemble) == np.recarray:
aod = lt.merge((aod, ensemble))
except:
dbg('Error reading eNAAPS-DART', l=2)
#_If subsetting by fhr
aod = ln.subset(aod, fhr=fhr)
try:
icap = join_icap(aod, dtg, members=members, **kwargs)
aod = lt.merge((aod, icap))
except:
dbg(('error merging ICAP_FCST', dtg))
#_return only ICAP
if remove_members: aod = ln.subset(aod, model='ICAP')
return aod