def get_tempdata_fpath(caseutc, fmt, vrbl, validutc, initutc, mem):
if caseutc is None:
casestr = "cNone"
else:
casestr = utils.string_from_time('dir', caseutc, strlen='day')
if fmt is None:
fmt = "fNone"
if vrbl is None:
vrbl = "vNone"
if validutc is None:
validstr = "uNone"
else:
validstr = utils.string_from_time('dir', validutc, strlen='minute')
if initutc is None:
initstr = "iNone"
else:
initstr = utils.string_from_time('dir', initutc, strlen='hour')
if mem is None:
mem = "mNone"
fname = "temp_{}_{}_{}_{}_{}_{}.npy".format(casestr, initstr, validstr,
vrbl, fmt, mem)
fpath = os.path.join(tempdir_root, fname)
utils.trycreate(fpath, debug=False)
return fpath
def plot_variable(self,va,lv,**kwargs):
self.fig = plt.figure()
m, x, y = self.basemap_setup(nc,**kwargs)
# Scales, colourmaps in here
data = self.get(va,**kwargs)
if not 'scale' in kwargs:
m.contourf(x,y,data)
else:
m.contourf(x,y,data,N.arange(*kwargs['scale']))
if self.C.plot_titles:
title = utils.string_from_time('title',self.t)
plt.title(title)
if self.C.colorbar:
plt.colorbar(orientation='horizontal')
# SAVE FIGURE
datestr = utils.string_from_time('output',self.t)
lv_na = utils.get_level_naming(lv,va)
na = (va,lv_na,datestr)
self.fname = self.create_fname(*na)
# pdb.set_trace()
self.save(self.fig,self.output_root,self.fname)
plt.close()
def get_random_d01(caseutc):
initutc = CASES[caseutc][0]
casestr = utils.string_from_time('dir', caseutc, strlen='day')
initstr = utils.string_from_time('dir', initutc, strlen='hour')
print("Loading random d01 wrfout for", casestr)
m01dir = os.path.join(ensroot, initstr, "m01")
d01_fname = get_wrfout_fname(initutc, 1)
d01_fpath = os.path.join(m01dir, d01_fname)
d01_nc = Dataset(d01_fpath)
return d01_nc
def plot_streamlines(self,va,lv,**kwargs):
fig = plt.figure()
# Scales, colourmaps in here
# Get data
u_all = self.get('U10',**kwargs)[:]
v_all = self.get('V10',**kwargs)[:]
u = self.cut_2D_array(u_all)
v = self.cut_2D_array(v_all)
m, x, y = self.basemap_setup(**kwargs)
"""
# Density depends on which version
# Wanting to match 3 km WRF (which had 2.5 density)
# Should work out dx, dy in __init__ method!
WRF_density = 2.5
WRF_res = 3.0
if self.version == 3:
RUC_res = 13.0
density = WRF_density * RUC_res/WRF_res
"""
density = 2.5
#x = N.array(range(u.shape[0]))
#y = N.array(range(v.shape[1]))
#pdb.set_trace()
#m.streamplot(x[self.x_dim/2,:],y[:,self.y_dim/2],u,v,density=density,linewidth=0.75,color='k')
#m.streamplot(x[y.shape[1]/2,:],y[:,x.shape[0]/2],u,v,density=density,linewidth=0.75,color='k')
#plt.streamplot(x[:,0],y[0,:],u,v)#,density=density,linewidth=0.75,color='k')
m.streamplot(y[0,:],x[:,0],u,v,density=density,linewidth=0.75,color='k')
#m.quiver(x,y,u,v)
if self.C.plot_titles:
title = utils.string_from_time('title',self.t)
plt.title(title)
# SAVE FIGURE
datestr = utils.string_from_time('output',self.t)
lv_na = utils.get_level_naming(va,lv=lv)
na = (va,lv_na,datestr)
self.fname = self.create_fname(*na)
# pdb.set_trace()
self.save(fig,self.output_root,self.fname)
plt.close()
def plot(self,va,lv,times,outdir=False,**kwargs):
for t in times:
fig = plt.figure()
data = self.get(va,lv,t)
m, x, y = self.basemap_setup()
if 'scale' in kwargs:
S = kwargs['scale']
f1 = m.contour(x,y,data,S,colors='k')
else:
f1 = m.contour(x,y,data,colors='k')
if kwargs['title']:
title = utils.string_from_time('title',t)
plt.title(title)
if 'wind_overlay' in kwargs:
jet = kwargs['wind_overlay']
wind = self.get('wind',lv,t)
windplot = m.contourf(x,y,wind,jet,alpha=0.6)
plt.colorbar(windplot)
elif 'W_overlay' in kwargs:
Wscale = kwargs['W_overlay']
W = self.get('W',lv,t)
windplot = m.contourf(x,y,W,alpha=0.6)
plt.colorbar(windplot)
# if self.C.colorbar:
# plt.colorbar(orientation='horizontal')
datestr = utils.string_from_time('output',t)
fname = '_'.join(('ECMWF',va,str(lv),datestr)) + '.png'
print(("Plotting {0} at {1} for {2}".format(
va,lv,datestr)))
plt.clabel(f1, inline=1, fmt='%4u', fontsize=12, colors='k')
utils.trycreate(outdir)
plt.savefig(os.path.join(outdir,fname))
plt.clf()
plt.close()
def return_all_mrms(caseutc, vrbl):
""" Return sorted dictionary of times and files.
"""
fdir, key, dlat = lookup_mrms_metadata(caseutc, vrbl)
casestr = utils.string_from_time('dir', caseutc, strlen='day')
FILES = {}
# Open all netcdf files in this folder, and create list of times
nclist = glob.glob(os.path.join(fdir, "*.netcdf"))
for f in nclist:
utcstr, _ext = os.path.basename(f).split('.')
assert _ext == "netcdf"
fmt = "%Y%m%d-%H%M%S"
utc = datetime.datetime.strptime(utcstr, fmt)
FILES[utc] = f
utcs = sorted(FILES.keys())
# pdb.set_trace()
return FILES, utcs, key, fdir
CASES_narr[datetime.datetime(2017, 5, 4, 0, 0,
0)] = datetime.datetime(2017, 5, 4, 12, 0, 0)
# Load tornado report CSV to plot lat/lon
# Load severe hail reports, or draw a swathe of SVR reports with Python?
# Could plot hail reports as proxy for mesocyclonic activity.
fig, _axes = plt.subplots(ncols=4, nrows=4, figsize=(12, 10))
fig_fname = "case_outline.png"
fig_fpath = os.path.join(outroot, fig_fname)
axes = _axes.flat
# For each case...
for caseutc, plotutc in CASES_narr.items():
casestr = utils.string_from_time('dir', plotutc, strlen='hour')
narr_fname = f'merged_AWIP32.{casestr}.3D'
narr_fpath = os.path.join(narrroot, narr_fname)
# Load NARR data
# G = GribFile(fpath)
G = NARRFile(narr_fpath)
data_925 = G.get("HGT", level="level 925")[0, 0, :, :]
data_500 = G.get("HGT", level="level 500")[0, 0, :, :]
# lats = G.lats
# lons = G.lons
# Plot 500 hPa height, 925 hPa height from NARR
ax = next(axes)
m = create_bmap(50.0, -55.0, 25.0, -130.0, ax=ax)
nneighs = len(neighs)
# DATA = {dom: {dBZ: {score: N.zeros([ninit,4]) for score in scores} for dBZ in dBZs} for dom in thedoms.keys()}
DATA = {s: N.zeros([2, len(dBZs), nneighs, len(fcstmins),len(initutcs)]) for s in scores}
# '-' if score == "FI" else '--'
LS = dict(FI='-',FISS='-.',RES='--',REL='--',UNC='--',FISS2='-.',RES_REL=':',FISS_WoF='-.',eFSS='-')
assert sorted(LS.keys()) == sorted(scores)
all_neighs = []
for dom_pp, initutc, fcstmin, score in itertools.product(domkeys,initutcs,fcstmins,scores):
# if score in ("RES_REL","FISS_WoF"):
# continue
domidx = domkeys.index(dom_pp)
fidx = fcstmins.index(fcstmin)
iidx = initutcs.index(initutc)
dx = thedoms[dom_pp]
initstr = utils.string_from_time('dir',initutc,strlen='hour')
fi_fpath = os.path.join(outroot,initstr,'FI_{}_{}min_{}.png'.format(dom_pp,fcstmin,score))
fdir_fi = os.path.join(npyroot,initstr,"FI")
fpath = generate_fname(vrbl=vrbl,metric='FI',suffix="efss",
fcstmin=fcstmin,dom_pp=dom_pp,ensmember='all',
fdir=fdir_fi,extension='.pickle')
with open(fpath,'rb') as f:
fi = pickle.load(f)
toplot = {}
# dBZs = sorted(fi.keys())
neighs = sorted(fi[dBZs[0]].keys())
all_neighs.append(neighs)
def plot_radar(self,
outdir=False,
fig=False,
ax=False,
fname=False,
Nlim=False,
Elim=False,
Slim=False,
Wlim=False,
cb=True,
drawcounties=False,
save='auto'):
"""
Plot radar data.
Args:
save : (str,bool) - If 'auto', saving will only occur if
fig and ax are not specified.
If True, the figure is saved; if False, not.
"""
# if not fig:
# fig, ax = plt.subplots()
# self.generate_basemap(fig,ax,Nlim,Elim,Slim,Wlim)
#lons, lats = self.m.makegrid(self.xlen,self.ylen)
if isinstance(Nlim, float):
data, lats, lons = self.get_subdomain(Nlim, Elim, Slim, Wlim)
else:
data = self.data
lats = self.lats #flip lats upside down?
lons = self.lons
# x,y = self.m(*N.meshgrid(lons,lats))
# xx,uy = self.m(lons,lats)
# x,y = self.m(*N.meshgrid(lons,lats))
# x,y = self.m(*N.meshgrid(lons,lats[::-1]))
# Custom colorbar
radarcmap = ct.reflect_ncdc(self.clvs)
# radarcmap = ct.ncdc_modified_ISU(self.clvs)
# Convert pixel levels to dBZ
dBZ = self.get_dBZ(data)
# dBZ[dBZ<0] = 0
# def plot2D(self,data,fname,outdir,plottype='contourf',
# save=1,smooth=1,lats=False,lons=False,
# clvs=False,cmap=False,title=False,colorbar=True,
# locations=False):
if not fname:
tstr = utils.string_from_time('output', self.utc)
fname = 'verif_radar_{0}.png'.format(tstr)
F = BirdsEye(fig=fig, ax=ax, proj=self.proj)
if cb:
cb = 'horizontal'
if save is 'auto':
if (fig is not False) and (ax is not False):
save = False
else:
save = True
F.plot2D(
dBZ,
fname,
outdir=outdir,
lats=lats,
lons=lons,
cmap=radarcmap,
clvs=N.arange(5, 90, 5),
cb=cb,
cblabel='Composite reflectivity (dBZ)',
drawcounties=drawcounties,
save=save,
lat_ts=50.0,
)
# PLOTTING COMPUTED SCORES
plot_detscores = 0
plot_crps = 0
plot_object_hist = 0
# OBSERVATION GROUPS
ST4 = ObsGroup(st4dir, 'stageiv')
RADARS = ObsGroup(radardir, 'radar')
OBS = (ST4, RADARS)
init_dict = {}
for caseutc, initutcs in CASES.items():
if caseutc in skiplist:
print("Skipping {} - in skiplist.".format(caseutc))
continue
casestr = utils.string_from_time('dir', caseutc, strlen='day')
for initutc in initutcs:
initdir = utils.string_from_time('dir', initutc, strlen='hour')
ensdir = os.path.join(ensroot, initdir)
npydir = os.path.join(npyroot, initdir)
outdir = os.path.join(outroot, casestr, initdir)
init_dict[initutc] = npydir
if not load_ens_switch:
continue
E = Ensemble(ensdir, initutc, ndoms=2, ctrl=False)
V = Verif(ensemble=E,
obs=OBS,
outdir=outdir,
datadir=npydir,
def gather_commands_one(i):
commands = []
caseutc, initutc, mem, vrbl = i
initstr = utils.string_from_time('dir', initutc, strlen='hour')
casestr = utils.string_from_time('dir', caseutc, strlen='day')
memdir = os.path.join(ensroot, initstr, mem)
# print("Appending interpolation commands for",mem,"forecast grids of",vrbl,"on", initstr)
for validutc in generate_valid_utcs(initutc):
#print(stars,validutc,stars)
### d01_3km (d01 cut to d02)
save_to_fpath = get_extraction_fpaths(vrbl,
"d01_3km",
validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
if not os.path.exists(save_to_fpath):
data, latsA, lonsA = get_data(vrbl=vrbl,
fmt="d01_raw",
validutc=validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
latsB, lonsB = get_data(caseutc, "d02_raw", latlon_only=True)
commands.append(
(data, latsA, lonsA, latsB, lonsB, True, save_to_fpath))
### d02_1km - should be a simple one...
### todo: compare interp to raw data! They should be very similar
save_to_fpath = get_extraction_fpaths(vrbl,
"d02_1km",
validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
if not os.path.exists(save_to_fpath):
data, latsA, lonsA = get_data(vrbl=vrbl,
fmt="d02_raw",
validutc=validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
latsB, lonsB = get_data(caseutc, "d02_raw", latlon_only=True)
commands.append(
(data, latsA, lonsA, latsB, lonsB, False, save_to_fpath))
### d02_3km (d02 interpolated to 3 km)
save_to_fpath = get_extraction_fpaths(vrbl,
"d02_3km",
validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
if not os.path.exists(save_to_fpath):
data, latsA, lonsA = get_data(vrbl=vrbl,
fmt="d02_raw",
validutc=validutc,
caseutc=caseutc,
initutc=initutc,
mem=mem)
latsB, lonsB = get_data(caseutc, "d01_3km", latlon_only=True)
commands.append(
(data, latsA, lonsA, latsB, lonsB, False, save_to_fpath))
# pdb.set_trace()
return commands
def lookup_mrms_metadata(caseutc, vrbl):
""" Return the folder, variable key, grid-spacing of a MRMS dataset.
fname convention is
YYYYMMDD-HHMMSS.netcdf
(Az Shear is per second)
2016, LL Az Shear:
/work/john.lawson/MRMS_data/rot-dz/20160331/processed/nofake/merged/MergedLLShear/00.00/*
LatGridSpacing = 0.01
MergedLLShear, pixel_x, pixel_y, pixel_count
2016, ML Az Shear:
/work/john.lawson/MRMS_data/rot-dz/20160331/processed/nofake/merged/MergedMLShear/00.00/*
LatGridSpacing = 0.01
MergedMLShear, pixel_x, pixel_y, pixel_count
2016, DZ:
/work/john.lawson/MRMS_data/rot-dz/20160331/processed/nofake/merged/MergedReflectivityQCComposite/00.00/*
LatGridSpacing: 0.01
MergedReflectivityQCComposite, ...
2017, LL Az Shear:
/work/john.lawson/MRMS_data/rot-dz/20170501/MergedAzShear_0-2kmAGL/00.00/*
LatGridSpacing = 0.005
MergedAzShear_0-2kmAGL, ...
2017, ML Az Shear:
/work/john.lawson/MRMS_data/rot-dz/20170501/MergedAzShear_2-5kmAGL/00.00/*
LatGridSpacing: 0.005
MergedAzShear_2-5kmAGL, ...
2017 DZ:
/work/john.lawson/MRMS_data/rot-dz/20170501/MergedReflectivityQCComposite/00.50/*
LatGridSpacing: 0.01
MergedReflectivityQCComposite, ...
"""
casestr = utils.string_from_time('dir', caseutc, strlen='day')
if caseutc.year == 2016:
dlat = 0.01
if vrbl == "DZ":
key = "MergedReflectivityQCComposite"
elif vrbl == "AWS02":
key = 'MergedLLShear'
elif vrbl == "AWS25":
key = 'MergedMLShear'
else:
raise Exception
fdir = os.path.join(mrmsdir, casestr, "processed", "nofake", "merged",
key, "00.00")
elif caseutc.year == 2017:
if vrbl == "DZ":
dlat = 0.01
key = 'MergedReflectivityQCComposite'
subdir = "00.50"
elif vrbl == "AWS02":
dlat = 0.005
key = 'MergedAzShear_0-2kmAGL'
subdir = "00.00"
elif vrbl == "AWS25":
dlat = 0.005
key = 'MergedAzShear_2-5kmAGL'
subdir = "00.00"
else:
raise Exception
fdir = os.path.join(mrmsdir, casestr, key, subdir)
else:
raise Exception
return fdir, key, dlat
def get_data(caseutc,
fmt,
vrbl=None,
validutc=None,
initutc=None,
mem=None,
latlon_only=False,
load_latlons=False):
""" Whatever
"""
casestr = utils.string_from_time('dir', caseutc, strlen='day')
if mem:
assert initutc
assert (vrbl in FCST_VRBLS) or (vrbl in other_list)
initstr = utils.string_from_time('dir', initutc, strlen='hour')
memdir = os.path.join(ensroot, initstr, mem)
latf, lonf = get_llf_fpath(casestr, fmt)
if latlon_only:
if load_latlons:
return N.load(latf), N.load(lonf)
else:
return latf, lonf
else:
data_fpath = get_tempdata_fpath(caseutc, fmt, vrbl, validutc, initutc,
mem)
if fmt == "d01_raw":
# Load lat/lon grid
latf, lonf = get_llf_fpath(casestr, "d01_raw")
# Load 3-km wrfout domain as is, on 2D
fname = get_wrfout_fname(initutc, dom=1)
fpath = os.path.join(memdir, fname)
W = WRFOut(fpath)
# if vrbl in ("Wmax","UH02","UH25","RAINNC"):
data = W.get(utc=validutc, vrbl=vrbl)[0, 0, :, :]
elif fmt == "d02_raw":
# Load lat/lon grid
latf, lonf = get_llf_fpath(casestr, "d02_raw")
# Load 1-km wrfout domain as is, on 2D
fname = get_wrfout_fname(initutc, dom=2)
fpath = os.path.join(memdir, fname)
W = WRFOut(fpath)
# if vrbl in ("Wmax","UH02","UH25","RAINNC"):
data = W.get(utc=validutc, vrbl=vrbl)[0, 0, :, :]
elif fmt in ("mrms_aws_raw", "mrms_dz_raw"):
latf, lonf = get_llf_fpath(casestr, fmt)
# Load MRMS data, 2D
FILES, utcs, key, fdir = return_all_mrms(caseutc, vrbl)
t = closest(utcs, validutc, return_val=True)
fpath = os.path.join(fdir, FILES[t])
mrms_nc = Dataset(fpath)
# d01_nc = get_random_d01(caseutc)
data = _unsparsify(
vrbl=key,
mrms_nc=mrms_nc,
) #d01_nc=d01_nc)
with N.errstate(invalid='ignore'):
data[data < -100] = N.nan
data[data > 100] = N.nan
# pdb.set_trace()
elif fmt == "stageiv_raw":
latf, lonf = get_llf_fpath(casestr, "stageiv_raw")
# Load stage iv data
data = ST4.get(utc=validutc)[0, 0, :, :]
# pdb.set_trace()
elif fmt == 'nexrad_raw':
latf, lonf = get_llf_fpath(casestr, "nexrad_raw")
# Load stage iv data
data = RADARS.get(utc=validutc)
else:
print("fmt not valid.")
assert True is False
# This is now done in _save function
# if N.ma.is_masked(data):
# data = data.data
# pdb.set_trace()
_save(arr=data, file=data_fpath)
if load_latlons:
return data_fpath, N.load(latf), N.load(lonf)
else:
return data_fpath, latf, lonf
def plot(
self,
data=None,
outdir=False,
fig=False,
ax=False,
fname=False,
# Nlim=False, Elim=False, Slim=False,Wlim=False,
cb=True,
drawcounties=False,
save='auto',
lats=None,
lons=None,
proj='merc',
):
""" Plot data.
Todo:
* This doesn't feel right. Plotting should be done elsewhere.
Args:
save : (str,bool) - If 'auto', saving will only occur if
fig and ax are not specified.
If True, the figure is saved; if False, not.
"""
# if isinstance(Nlim,float):
# data, lats, lons = self.get_subdomain(Nlim,Elim,Slim,Wlim)
# else:
# if not given, try to use self's
if data == None:
data = self.get()
# data = getattr(self,'data',data)
if lats == None:
lats = self.lats
# lats = getattr(self,'lats',lats)
if lons == None:
lons = self.lons
# lons = getattr(self,'lons',lons)
# Need to implement options for this
cmap = None
clvs = None
if not fname:
tstr = utils.string_from_time('output', self.utc)
fname = 'verif_ST4_{0}.png'.format(tstr)
F = BirdsEye(fig=fig, ax=ax, proj=proj)
if cb:
cb = 'horizontal'
if save == 'auto':
if (fig is not False) and (ax is not False):
save = False
else:
save = True
F.plot2D(
data,
fname,
outdir=outdir,
lats=lats,
lons=lons,
cmap=cmap,
clvs=clvs,
cb=cb,
cblabel='Accumulated precipitation (mm)',
drawcounties=drawcounties,
save=save,
lat_ts=50.0,
)