def _vector2raster(self, d, var, tndx):
"""
Postprocess a vector field into barbs (used for wind) and contains hacks for WRF staggered grids.
:param d: the open netCDF file
:param var: the name of the variable in var_wisdom
:param tndx: the time index
:return: the raster png as a StringIO, and the coordinates
"""
# gather wisdom about the variable
wisdom = get_wisdom(var)
wisdom.update(self.wisdom_update.get(var, {}))
native_unit = wisdom['native_unit']
u_name, v_name = wisdom['components']
lat, lon = wisdom['grid'](d)
# extract variable
uw, vw = get_wisdom(u_name), get_wisdom(v_name)
uw.update(self.wisdom_update.get(u_name, {}))
vw.update(self.wisdom_update.get(v_name, {}))
u = uw['retrieve_as'](d, tndx)
v = vw['retrieve_as'](d, tndx)
if u.shape != lat.shape:
raise PostprocError(
"Variable %s size does not correspond to grid size: var %s grid %s."
% (u_name, u.shape, u_lat.shape))
if v.shape != lat.shape:
raise PostprocError(
"Variable %s size does not correspond to grid size." % v_name)
# look at mins and maxes
fa_min, fa_max = min(np.nanmin(u),
np.nanmin(v)), max(np.nanmax(u), np.nanmax(v))
# determine if we will use the range in the variable or a fixed range
scale = wisdom['scale']
if scale != 'original':
fa_min, fa_max = scale[0], scale[1]
u[u < fa_min] = fa_min
u[u > fa_max] = fa_max
v[v < fa_min] = fa_min
v[v > fa_max] = fa_max
# create the raster & get coordinate bounds, HACK to get better quiver resolution
s = 3
raster_png_data, corner_coords = basemap_barbs_mercator(
u[::s, ::s], v[::s, ::s], lat[::s, ::s], lon[::s, ::s])
return raster_png_data, corner_coords
def _vector2raster(self, d, var, tndx):
"""
Postprocess a vector field into barbs (used for wind) and contains hacks for WRF staggered grids.
:param d: the open netCDF file
:param var: the name of the variable in var_wisdom
:param tndx: the time index
:return: the raster png as a StringIO, and the coordinates
"""
# gather wisdom about the variable
wisdom = get_wisdom(var)
wisdom.update(self.wisdom_update.get(var, {}))
native_unit = wisdom['native_unit']
u_name, v_name = wisdom['components']
lat, lon = wisdom['grid'](d)
# extract variable
uw, vw = get_wisdom(u_name), get_wisdom(v_name)
uw.update(self.wisdom_update.get(u_name, {}))
vw.update(self.wisdom_update.get(v_name, {}))
u = uw['retrieve_as'](d, tndx)
v = vw['retrieve_as'](d, tndx)
if u.shape != lat.shape:
raise PostprocError("Variable %s size does not correspond to grid size: var %s grid %s." % (u_name, u.shape, u_lat.shape))
if v.shape != lat.shape:
raise PostprocError("Variable %s size does not correspond to grid size." % v_name)
# look at mins and maxes
fa_min,fa_max = min(np.nanmin(u), np.nanmin(v)),max(np.nanmax(u), np.nanmax(v))
# determine if we will use the range in the variable or a fixed range
scale = wisdom['scale']
if scale != 'original':
fa_min, fa_max = scale[0], scale[1]
u[u < fa_min] = fa_min
u[u > fa_max] = fa_max
v[v < fa_min] = fa_min
v[v > fa_max] = fa_max
# create the raster & get coordinate bounds, HACK to get better quiver resolution
s = 3
raster_png_data,corner_coords = basemap_barbs_mercator(u[::s,::s],v[::s,::s],lat[::s,::s],lon[::s,::s])
return raster_png_data, corner_coords
def _scalar2raster(self, d, var, tndx):
"""
Convert a single variable into a raster and colorbar.
:param d: the netcdf file
:param var: the variable name
:param tndx: the time index
:return: two StringIO objects, first is the PNG raster, second is PNG colorbar
"""
# gather wisdom about the variable
wisdom = get_wisdom(var).copy()
wisdom.update(self.wisdom_update.get(var, {}))
native_unit = wisdom['native_unit']
cmap_name = wisdom['colormap']
cmap = mpl.cm.get_cmap(cmap_name)
# extract variable
fa = wisdom['retrieve_as'](
d,
tndx) # this calls a lambda defined to read the required 2d field
lat, lon = wisdom['grid'](d)
if lat.shape != fa.shape:
raise PostprocError(
"Variable %s size does not correspond to grid size." % var)
# look at mins and maxes
fa_min, fa_max = np.nanmin(fa), np.nanmax(fa)
# determine if we will use the range in the variable or a fixed range
scale = wisdom['scale']
if scale != 'original':
fa_min, fa_max = scale[0], scale[1]
fa[fa < fa_min] = fa_min
fa[fa > fa_max] = fa_max
# only create the colorbar if requested
cb_png_data = None
if wisdom['colorbar'] is not None:
cb_unit = wisdom['colorbar']
cbu_min, cbu_max = convert_value(native_unit, cb_unit,
fa_min), convert_value(
native_unit, cb_unit, fa_max)
# colorbar + add it to the KMZ as a screen overlay
cb_png_data = make_colorbar([cbu_min, cbu_max], 'vertical', 2,
cmap, wisdom['name'] + ' ' + cb_unit)
# check for 'transparent' color value and replace with nans
if 'transparent_values' in wisdom:
rng = wisdom['transparent_values']
fa = np.ma.masked_array(fa,
np.logical_and(fa >= rng[0], fa <= rng[1]))
# create the raster & get coordinate bounds
raster_png_data, corner_coords = basemap_raster_mercator(
lon, lat, fa, fa_min, fa_max, cmap)
return raster_png_data, corner_coords, cb_png_data
def process_vars_tiff(pp, d, wrfout_path, dom_id, times, vars):
"""
Postprocess a list of scalar or vector fields for a given wrfout file into TIFF files.
:param pp: Postprocess class
:param d: the open netCDF file
:param dom_id: the domain identifier
:param times: list of times to process
:param vars: list of variables to process
"""
logging.info('process_vars_tiff: looking for file %s' % wrfout_path)
# netCDF WRF metadata
projection, geotransform_atm, geotransform_fire = ncwrfmeta(d)
outpath_base = osp.join(pp.output_path, pp.product_name + ("-%02d-" % dom_id))
# build an output file per variable
for var in vars:
logging.info('process_vars_tiff: postprocessing %s' % var)
try:
tiff_path, coords, mf_upd = None, None, {}
wisdom = get_wisdom(var).copy()
wisdom.update(pp.wisdom_update.get(var, {}))
if is_fire_var(var):
geotransform = geotransform_fire
else:
geotransform = geotransform_atm
if is_windvec(var):
u_name, v_name = wisdom['components']
uw, vw = get_wisdom(u_name), get_wisdom(v_name)
uw.update(pp.wisdom_update.get(u_name, {}))
vw.update(pp.wisdom_update.get(v_name, {}))
tiff_path = vector2tiffs(outpath_base, d, (wisdom,uw,vw), projection, geotransform, times, var)
else:
tiff_path = scalar2tiffs(outpath_base, d, wisdom, projection, geotransform, times, var)
for idx,time in enumerate(times):
mf_upd['tiff'] = osp.basename(tiff_path[idx])
ts_esmf = time.replace('_','T')+'Z'
pp._update_manifest(dom_id, ts_esmf, var, mf_upd)
except Exception as e:
logging.warning("Exception %s while postprocessing %s" % (e, var))
logging.warning(traceback.print_exc())
def _scalar2raster(self, d, var, tndx):
"""
Convert a single variable into a raster and colorbar.
:param d: the netcdf file
:param var: the variable name
:param tndx: the time index
:return: two StringIO objects, first is the PNG raster, second is PNG colorbar
"""
# gather wisdom about the variable
wisdom = get_wisdom(var).copy()
wisdom.update(self.wisdom_update.get(var, {}))
native_unit = wisdom['native_unit']
cmap_name = wisdom['colormap']
cmap = mpl.cm.get_cmap(cmap_name)
# extract variable
fa = wisdom['retrieve_as'](d,tndx) # this calls a lambda defined to read the required 2d field
lat, lon = wisdom['grid'](d)
if lat.shape != fa.shape:
raise PostprocError("Variable %s size does not correspond to grid size." % var)
# look at mins and maxes
fa_min,fa_max = np.nanmin(fa),np.nanmax(fa)
# determine if we will use the range in the variable or a fixed range
scale = wisdom['scale']
if scale != 'original':
fa_min, fa_max = scale[0], scale[1]
fa[fa < fa_min] = fa_min
fa[fa > fa_max] = fa_max
# only create the colorbar if requested
cb_png_data = None
if wisdom['colorbar'] is not None:
cb_unit = wisdom['colorbar']
cbu_min,cbu_max = convert_value(native_unit, cb_unit, fa_min), convert_value(native_unit, cb_unit, fa_max)
# colorbar + add it to the KMZ as a screen overlay
cb_png_data = make_colorbar([cbu_min, cbu_max],'vertical',2,cmap,wisdom['name'] + ' ' + cb_unit)
# check for 'transparent' color value and replace with nans
if 'transparent_values' in wisdom:
rng = wisdom['transparent_values']
fa = np.ma.masked_array(fa, np.logical_and(fa >= rng[0], fa <= rng[1]))
# create the raster & get coordinate bounds
raster_png_data,corner_coords = basemap_raster_mercator(lon,lat,fa,fa_min,fa_max,cmap)
return raster_png_data, corner_coords, cb_png_data
def process_file(self, wrfout_path, var_list, skip=1):
"""
Process an entire file, all timestamps and generate images for var_instr.keys().
:param wrfout_path: the wrfout to process
:param var_list: list of variables to process
:param skip: only process every skip-th frame
"""
traceargs()
# open the netCDF dataset
d = nc4.Dataset(wrfout_path)
# extract ESMF string times and identify timestamp of interest
times = [''.join(x) for x in d.variables['Times'][:]]
# build one KMZ per variable
fixed_colorbars = {}
for tndx, ts_esmf in enumerate(times[::skip]):
print('Processing time %s ...' % ts_esmf)
for var in var_list:
try:
outpath_base = os.path.join(self.output_path, self.product_name + '-' + ts_esmf + '-' + var)
if var in ['WINDVEC']:
kmz_path,raster_path,coords = self._vector2kmz(d, var, tndx, outpath_base, cleanup=False)
raster_name = osp.basename(raster_path)
kmz_name = osp.basename(kmz_path)
self._update_manifest(dom_id, ts_esmf, var, { 'raster' : raster_name, 'coords' : coords, 'kml' : kmz_name})
else:
kmz_path,raster_path,cb_path,coords = self._scalar2kmz(d, var, tndx, outpath_base, cleanup=False)
mf_upd = { 'raster' : osp.basename(raster_path), 'coords' : coords, 'kml' : osp.basename(kmz_path) }
if cb_path is not None:
# optimization for display when we know colorbar has fixed scale
# we memoize the colorbar computed at time 0 and use it for all frames
# although other colorbars are generated, they are deleted
scale = self.wisdom_update.get('scale', get_wisdom(var)['scale'])
if type(scale) == list and np.isfinite(scale[0]) and np.isfinite(scale[1]):
logging.info("Using fixed colorbar strategy for variable " + var)
if tndx == 0:
fixed_colorbars[var] = osp.basename(cb_path)
else:
os.remove(cb_path)
mf_upd['colorbar'] = fixed_colorbars[var]
else:
mf_upd['colorbar'] = osp.basename(cb_path)
self._update_manifest(dom_id, ts_esmf, var, mf_upd)
except Exception as e:
logging.warning("Exception %s while postprocessing %s for time %s" % (e.message, var, ts_esmf))
logging.warning(traceback.print_exc())
def __init__(self, output_path, prod_name, tslist, num_doms):
"""
Initialize timeseries with output parameters.
:param output_path: path where timeseries files are stored
:param prod_name: name of manifest json file and prefix of all output files
:param tslist: dictionary with time series information
:param num_doms: number of domains
"""
logging.info("Timeseries: output_path=%s prod_name=%s" %
(output_path, prod_name))
self.output_path = output_path
self.product_name = prod_name
self.stations = tslist['stations'].copy()
self.variables = {
var: get_wisdom(var).copy()
for var in tslist['vars']
}
logging.info("Timeseries: stations=%s" %
[st['name'] for st in self.stations.values()])
logging.info("Timeseries: variables=%s" % list(self.variables.keys()))
self.num_doms = num_doms
# initialize the CSV files for each station
self.initialize_stations()
def process_file(self, wrfout_path, var_list, skip=1):
"""
Process an entire file, all timestamps and generate images for var_instr.keys().
:param wrfout_path: the wrfout to process
:param var_list: list of variables to process
:param skip: only process every skip-th frame
"""
traceargs()
# open the netCDF dataset
d = nc4.Dataset(wrfout_path)
# extract ESMF string times and identify timestamp of interest
times = [''.join(x) for x in d.variables['Times'][:]]
# build one KMZ per variable
fixed_colorbars = {}
for tndx, ts_esmf in enumerate(times[::skip]):
print('Processing time %s ...' % ts_esmf)
for var in var_list:
try:
outpath_base = os.path.join(
self.output_path,
self.product_name + '-' + ts_esmf + '-' + var)
if is_windvec(var):
kmz_path, raster_path, coords = self._vector2kmz(
d, var, tndx, ts_esmf, outpath_base, cleanup=False)
raster_name = osp.basename(raster_path)
kmz_name = osp.basename(kmz_path)
self._update_manifest(
dom_id, ts_esmf, var, {
'raster': raster_name,
'coords': coords,
'kml': kmz_name
})
else:
kmz_path, raster_path, cb_path, coords = self._scalar2kmz(
d, var, tndx, ts_esmf, outpath_base, cleanup=False)
mf_upd = {
'raster': osp.basename(raster_path),
'coords': coords,
'kml': osp.basename(kmz_path)
}
if cb_path is not None:
# optimization for display when we know colorbar has fixed scale
# we memoize the colorbar computed at time 0 and use it for all frames
# although other colorbars are generated, they are deleted
scale = self.wisdom_update.get(
'scale',
get_wisdom(var)['scale'])
if type(scale) == list and np.isfinite(
scale[0]) and np.isfinite(scale[1]):
logging.info(
"Using fixed colorbar strategy for variable "
+ var)
if tndx == 0:
fixed_colorbars[var] = osp.basename(
cb_path)
else:
os.remove(cb_path)
mf_upd['colorbar'] = fixed_colorbars[var]
else:
mf_upd['colorbar'] = osp.basename(cb_path)
self._update_manifest(dom_id, ts_esmf, var, mf_upd)
except Exception as e:
logging.warning(
"Exception %s while postprocessing %s for time %s" %
(e.message, var, ts_esmf))
logging.warning(traceback.print_exc())
def _scalar2raster(self, d, var, tndx):
"""
Convert a single variable into a raster and colorbar.
:param d: the netcdf file
:param var: the variable name
:param tndx: the time index
:return: two StringIO objects, first is the PNG raster, second is PNG colorbar
"""
# gather wisdom about the variable
wisdom = get_wisdom(var).copy()
wisdom.update(self.wisdom_update.get(var, {}))
native_unit = wisdom['native_unit']
cmap_name = wisdom['colormap']
cmap = mpl.cm.get_cmap(cmap_name)
# extract variable
fa = wisdom['retrieve_as'](
d,
tndx) # this calls a lambda defined to read the required 2d field
lat, lon = wisdom['grid'](d)
if lat.shape != fa.shape:
raise PostprocError(
"Variable %s size does not correspond to grid size." % var)
# check for 'transparent' color value and mask
if 'transparent_values' in wisdom:
rng = wisdom['transparent_values']
logging.info(
'_scalar2raster: variable %s min %s max %s masking transparent from %s to %s'
% (var, np.nanmin(fa), np.nanmax(fa), rng[0], rng[1]))
fa = np.ma.masked_array(fa,
np.logical_and(fa >= rng[0], fa <= rng[1]))
else:
fa = np.ma.masked_array(fa)
# create the raster & get coordinate bounds
# look at mins and maxes
# fa_min,fa_max = np.nanmin(fa),np.nanmax(fa)
# look at mins and maxes, transparent don't count
fa_min, fa_max = np.nanmin(fa), np.nanmax(fa)
# determine if we will use the range in the variable or a fixed range
scale = wisdom['scale']
if scale != 'original':
m = fa.mask.copy(
) # save mask, resetting values below will destroy the mask
fa_min, fa_max = scale[0], scale[1]
fa[fa < fa_min] = fa_min
fa[fa > fa_max] = fa_max
fa.mask = m # restore the mask
# only create the colorbar if requested
cb_png_data = None
if wisdom['colorbar'] is not None:
cb_unit = wisdom['colorbar']
cbu_min, cbu_max = convert_value(native_unit, cb_unit,
fa_min), convert_value(
native_unit, cb_unit, fa_max)
# colorbar + add it to the KMZ as a screen overlay
legend = wisdom['name'] + ' ' + cb_unit
logging.info(
'_scalar2raster: variable %s colorbar from %s to %s %s' %
(var, cbu_min, cbu_max, legend))
cb_png_data = make_colorbar([cbu_min, cbu_max], 'vertical', 2,
cmap, legend)
# replace masked values by nans just in case
fa.data[fa.mask] = np.nan
fa.fill_value = np.nan
logging.info(
'_scalar2raster: variable %s elements %s count %s not masked %s min %s max %s'
% (var, fa.size, fa.count(), np.count_nonzero(fa.mask == False),
np.nanmin(fa), np.nanmax(fa)))
raster_png_data, corner_coords = basemap_raster_mercator(
lon, lat, fa, fa_min, fa_max, cmap)
return raster_png_data, corner_coords, cb_png_data