Exemple #1
0
    def _vector2raster(self, d, var, tndx):
        """
        Postprocess a vector field into barbs (used for wind) and contains hacks for WRF staggered grids.

        :param d: the open netCDF file
        :param var: the name of the variable in var_wisdom
        :param tndx: the time index
        :return: the raster png as a StringIO, and the coordinates
        """
        # gather wisdom about the variable
        wisdom = get_wisdom(var)
        wisdom.update(self.wisdom_update.get(var, {}))
        native_unit = wisdom['native_unit']
        u_name, v_name = wisdom['components']
        lat, lon = wisdom['grid'](d)

        # extract variable
        uw, vw = get_wisdom(u_name), get_wisdom(v_name)
        uw.update(self.wisdom_update.get(u_name, {}))
        vw.update(self.wisdom_update.get(v_name, {}))
        u = uw['retrieve_as'](d, tndx)
        v = vw['retrieve_as'](d, tndx)

        if u.shape != lat.shape:
            raise PostprocError(
                "Variable %s size does not correspond to grid size: var %s grid %s."
                % (u_name, u.shape, u_lat.shape))

        if v.shape != lat.shape:
            raise PostprocError(
                "Variable %s size does not correspond to grid size." % v_name)

        # look at mins and maxes
        fa_min, fa_max = min(np.nanmin(u),
                             np.nanmin(v)), max(np.nanmax(u), np.nanmax(v))

        # determine if we will use the range in the variable or a fixed range
        scale = wisdom['scale']
        if scale != 'original':
            fa_min, fa_max = scale[0], scale[1]
            u[u < fa_min] = fa_min
            u[u > fa_max] = fa_max
            v[v < fa_min] = fa_min
            v[v > fa_max] = fa_max

        # create the raster & get coordinate bounds, HACK to get better quiver resolution
        s = 3
        raster_png_data, corner_coords = basemap_barbs_mercator(
            u[::s, ::s], v[::s, ::s], lat[::s, ::s], lon[::s, ::s])

        return raster_png_data, corner_coords
Exemple #2
0
    def _vector2raster(self, d, var, tndx):
        """
        Postprocess a vector field into barbs (used for wind) and contains hacks for WRF staggered grids.

        :param d: the open netCDF file
        :param var: the name of the variable in var_wisdom
        :param tndx: the time index
        :return: the raster png as a StringIO, and the coordinates
        """
        # gather wisdom about the variable
        wisdom = get_wisdom(var)
        wisdom.update(self.wisdom_update.get(var, {}))
        native_unit = wisdom['native_unit']
        u_name, v_name = wisdom['components']
        lat, lon = wisdom['grid'](d)

        # extract variable
        uw, vw = get_wisdom(u_name), get_wisdom(v_name)
        uw.update(self.wisdom_update.get(u_name, {}))
        vw.update(self.wisdom_update.get(v_name, {}))
        u = uw['retrieve_as'](d, tndx)
        v = vw['retrieve_as'](d, tndx)

        if u.shape != lat.shape:
            raise PostprocError("Variable %s size does not correspond to grid size: var %s grid %s." % (u_name, u.shape, u_lat.shape))
        
        if v.shape != lat.shape:
            raise PostprocError("Variable %s size does not correspond to grid size." % v_name)

        # look at mins and maxes
        fa_min,fa_max = min(np.nanmin(u), np.nanmin(v)),max(np.nanmax(u), np.nanmax(v))

        # determine if we will use the range in the variable or a fixed range
        scale = wisdom['scale']
        if scale != 'original':
            fa_min, fa_max = scale[0], scale[1]
            u[u < fa_min] = fa_min
            u[u > fa_max] = fa_max
            v[v < fa_min] = fa_min
            v[v > fa_max] = fa_max

        # create the raster & get coordinate bounds, HACK to get better quiver resolution
        s = 3
        raster_png_data,corner_coords = basemap_barbs_mercator(u[::s,::s],v[::s,::s],lat[::s,::s],lon[::s,::s])

        return raster_png_data, corner_coords
Exemple #3
0
    def _scalar2raster(self, d, var, tndx):
        """
        Convert a single variable into a raster and colorbar.

        :param d: the netcdf file
        :param var: the variable name
        :param tndx: the time index
        :return: two StringIO objects, first is the PNG raster, second is PNG colorbar
        """
        # gather wisdom about the variable
        wisdom = get_wisdom(var).copy()
        wisdom.update(self.wisdom_update.get(var, {}))
        native_unit = wisdom['native_unit']
        cmap_name = wisdom['colormap']
        cmap = mpl.cm.get_cmap(cmap_name)

        # extract variable
        fa = wisdom['retrieve_as'](
            d,
            tndx)  # this calls a lambda defined to read the required 2d field
        lat, lon = wisdom['grid'](d)

        if lat.shape != fa.shape:
            raise PostprocError(
                "Variable %s size does not correspond to grid size." % var)

        # look at mins and maxes
        fa_min, fa_max = np.nanmin(fa), np.nanmax(fa)

        # determine if we will use the range in the variable or a fixed range
        scale = wisdom['scale']
        if scale != 'original':
            fa_min, fa_max = scale[0], scale[1]
            fa[fa < fa_min] = fa_min
            fa[fa > fa_max] = fa_max

        # only create the colorbar if requested
        cb_png_data = None
        if wisdom['colorbar'] is not None:
            cb_unit = wisdom['colorbar']
            cbu_min, cbu_max = convert_value(native_unit, cb_unit,
                                             fa_min), convert_value(
                                                 native_unit, cb_unit, fa_max)
            #  colorbar + add it to the KMZ as a screen overlay
            cb_png_data = make_colorbar([cbu_min, cbu_max], 'vertical', 2,
                                        cmap, wisdom['name'] + ' ' + cb_unit)

        # check for 'transparent' color value and replace with nans
        if 'transparent_values' in wisdom:
            rng = wisdom['transparent_values']
            fa = np.ma.masked_array(fa,
                                    np.logical_and(fa >= rng[0], fa <= rng[1]))

        # create the raster & get coordinate bounds
        raster_png_data, corner_coords = basemap_raster_mercator(
            lon, lat, fa, fa_min, fa_max, cmap)

        return raster_png_data, corner_coords, cb_png_data
def process_vars_tiff(pp, d, wrfout_path, dom_id, times, vars):
    """
    Postprocess a list of scalar or vector fields for a given wrfout file into TIFF files.

    :param pp: Postprocess class
    :param d: the open netCDF file
    :param dom_id: the domain identifier
    :param times: list of times to process
    :param vars: list of variables to process
    """

    logging.info('process_vars_tiff: looking for file %s' % wrfout_path)
    # netCDF WRF metadata
    projection, geotransform_atm, geotransform_fire = ncwrfmeta(d)

    outpath_base = osp.join(pp.output_path, pp.product_name + ("-%02d-" % dom_id))
    # build an output file per variable
    for var in vars:
        logging.info('process_vars_tiff: postprocessing %s' % var)
        try:
            tiff_path, coords, mf_upd = None, None, {}
            wisdom = get_wisdom(var).copy()
            wisdom.update(pp.wisdom_update.get(var, {}))
            if is_fire_var(var):
                geotransform = geotransform_fire
            else:
                geotransform = geotransform_atm

            if is_windvec(var):
                u_name, v_name = wisdom['components']
                uw, vw = get_wisdom(u_name), get_wisdom(v_name)
                uw.update(pp.wisdom_update.get(u_name, {}))
                vw.update(pp.wisdom_update.get(v_name, {}))
                tiff_path = vector2tiffs(outpath_base, d, (wisdom,uw,vw), projection, geotransform, times, var)
            else:
                tiff_path = scalar2tiffs(outpath_base, d, wisdom, projection, geotransform, times, var)

            for idx,time in enumerate(times):
                mf_upd['tiff'] = osp.basename(tiff_path[idx])
                ts_esmf = time.replace('_','T')+'Z'
                pp._update_manifest(dom_id, ts_esmf, var, mf_upd)

        except Exception as e:
            logging.warning("Exception %s while postprocessing %s" % (e, var))
            logging.warning(traceback.print_exc())
Exemple #5
0
    def _scalar2raster(self, d, var, tndx):
        """
        Convert a single variable into a raster and colorbar.

        :param d: the netcdf file
        :param var: the variable name
        :param tndx: the time index
        :return: two StringIO objects, first is the PNG raster, second is PNG colorbar
        """
        # gather wisdom about the variable
        wisdom = get_wisdom(var).copy()
        wisdom.update(self.wisdom_update.get(var, {}))
        native_unit = wisdom['native_unit']
        cmap_name = wisdom['colormap']
        cmap = mpl.cm.get_cmap(cmap_name)

        # extract variable
        fa = wisdom['retrieve_as'](d,tndx) # this calls a lambda defined to read the required 2d field
        lat, lon = wisdom['grid'](d)

        if lat.shape != fa.shape:
            raise PostprocError("Variable %s size does not correspond to grid size." % var)

        # look at mins and maxes
        fa_min,fa_max = np.nanmin(fa),np.nanmax(fa)

        # determine if we will use the range in the variable or a fixed range
        scale = wisdom['scale']
        if scale != 'original':
            fa_min, fa_max = scale[0], scale[1]
            fa[fa < fa_min] = fa_min
            fa[fa > fa_max] = fa_max

        # only create the colorbar if requested
        cb_png_data = None
        if wisdom['colorbar'] is not None:
            cb_unit = wisdom['colorbar']
            cbu_min,cbu_max = convert_value(native_unit, cb_unit, fa_min), convert_value(native_unit, cb_unit, fa_max)
            #  colorbar + add it to the KMZ as a screen overlay
            cb_png_data = make_colorbar([cbu_min, cbu_max],'vertical',2,cmap,wisdom['name'] + ' ' + cb_unit)

        # check for 'transparent' color value and replace with nans
        if 'transparent_values' in wisdom:
            rng = wisdom['transparent_values']
            fa = np.ma.masked_array(fa, np.logical_and(fa >= rng[0], fa <= rng[1]))

        # create the raster & get coordinate bounds
        raster_png_data,corner_coords = basemap_raster_mercator(lon,lat,fa,fa_min,fa_max,cmap)

        return raster_png_data, corner_coords, cb_png_data
Exemple #6
0
    def process_file(self, wrfout_path, var_list, skip=1):
        """
        Process an entire file, all timestamps and generate images for var_instr.keys().

        :param wrfout_path: the wrfout to process
        :param var_list: list of variables to process
        :param skip: only process every skip-th frame
        """
        traceargs()

        # open the netCDF dataset
        d = nc4.Dataset(wrfout_path)

        # extract ESMF string times and identify timestamp of interest
        times = [''.join(x) for x in d.variables['Times'][:]]

        # build one KMZ per variable
        fixed_colorbars = {}
        for tndx, ts_esmf in enumerate(times[::skip]):
            print('Processing time %s ...' % ts_esmf)
            for var in var_list:
                try:
                    outpath_base = os.path.join(self.output_path, self.product_name + '-' + ts_esmf + '-' + var) 
                    if var in ['WINDVEC']:
                        kmz_path,raster_path,coords = self._vector2kmz(d, var, tndx, outpath_base, cleanup=False)
                        raster_name = osp.basename(raster_path)
                        kmz_name = osp.basename(kmz_path)
                        self._update_manifest(dom_id, ts_esmf, var, { 'raster' : raster_name, 'coords' : coords, 'kml' : kmz_name})
                    else:
                        kmz_path,raster_path,cb_path,coords = self._scalar2kmz(d, var, tndx, outpath_base, cleanup=False)
                        mf_upd = { 'raster' : osp.basename(raster_path), 'coords' : coords, 'kml' : osp.basename(kmz_path) }
                        if cb_path is not None:
                            # optimization for display when we know colorbar has fixed scale
                            # we memoize the colorbar computed at time 0 and use it for all frames
                            # although other colorbars are generated, they are deleted
                            scale = self.wisdom_update.get('scale', get_wisdom(var)['scale'])
                            if type(scale) == list and np.isfinite(scale[0]) and np.isfinite(scale[1]):
                                logging.info("Using fixed colorbar strategy for variable " + var)
                                if tndx == 0:
                                    fixed_colorbars[var] = osp.basename(cb_path)
                                else:
                                    os.remove(cb_path)
                                mf_upd['colorbar'] = fixed_colorbars[var]
                            else:
                                mf_upd['colorbar'] = osp.basename(cb_path)
                        self._update_manifest(dom_id, ts_esmf, var, mf_upd)
                except Exception as e:
                    logging.warning("Exception %s while postprocessing %s for time %s" % (e.message, var, ts_esmf))
                    logging.warning(traceback.print_exc())
Exemple #7
0
    def __init__(self, output_path, prod_name, tslist, num_doms):
        """
        Initialize timeseries with output parameters.

        :param output_path: path where timeseries files are stored
        :param prod_name: name of manifest json file and prefix of all output files
        :param tslist: dictionary with time series information
        :param num_doms: number of domains
        """
        logging.info("Timeseries: output_path=%s prod_name=%s" %
                     (output_path, prod_name))
        self.output_path = output_path
        self.product_name = prod_name
        self.stations = tslist['stations'].copy()
        self.variables = {
            var: get_wisdom(var).copy()
            for var in tslist['vars']
        }
        logging.info("Timeseries: stations=%s" %
                     [st['name'] for st in self.stations.values()])
        logging.info("Timeseries: variables=%s" % list(self.variables.keys()))
        self.num_doms = num_doms
        # initialize the CSV files for each station
        self.initialize_stations()
Exemple #8
0
    def process_file(self, wrfout_path, var_list, skip=1):
        """
        Process an entire file, all timestamps and generate images for var_instr.keys().

        :param wrfout_path: the wrfout to process
        :param var_list: list of variables to process
        :param skip: only process every skip-th frame
        """
        traceargs()

        # open the netCDF dataset
        d = nc4.Dataset(wrfout_path)

        # extract ESMF string times and identify timestamp of interest
        times = [''.join(x) for x in d.variables['Times'][:]]

        # build one KMZ per variable
        fixed_colorbars = {}
        for tndx, ts_esmf in enumerate(times[::skip]):
            print('Processing time %s ...' % ts_esmf)
            for var in var_list:
                try:
                    outpath_base = os.path.join(
                        self.output_path,
                        self.product_name + '-' + ts_esmf + '-' + var)
                    if is_windvec(var):
                        kmz_path, raster_path, coords = self._vector2kmz(
                            d, var, tndx, ts_esmf, outpath_base, cleanup=False)
                        raster_name = osp.basename(raster_path)
                        kmz_name = osp.basename(kmz_path)
                        self._update_manifest(
                            dom_id, ts_esmf, var, {
                                'raster': raster_name,
                                'coords': coords,
                                'kml': kmz_name
                            })
                    else:
                        kmz_path, raster_path, cb_path, coords = self._scalar2kmz(
                            d, var, tndx, ts_esmf, outpath_base, cleanup=False)
                        mf_upd = {
                            'raster': osp.basename(raster_path),
                            'coords': coords,
                            'kml': osp.basename(kmz_path)
                        }
                        if cb_path is not None:
                            # optimization for display when we know colorbar has fixed scale
                            # we memoize the colorbar computed at time 0 and use it for all frames
                            # although other colorbars are generated, they are deleted
                            scale = self.wisdom_update.get(
                                'scale',
                                get_wisdom(var)['scale'])
                            if type(scale) == list and np.isfinite(
                                    scale[0]) and np.isfinite(scale[1]):
                                logging.info(
                                    "Using fixed colorbar strategy for variable "
                                    + var)
                                if tndx == 0:
                                    fixed_colorbars[var] = osp.basename(
                                        cb_path)
                                else:
                                    os.remove(cb_path)
                                mf_upd['colorbar'] = fixed_colorbars[var]
                            else:
                                mf_upd['colorbar'] = osp.basename(cb_path)
                        self._update_manifest(dom_id, ts_esmf, var, mf_upd)
                except Exception as e:
                    logging.warning(
                        "Exception %s while postprocessing %s for time %s" %
                        (e.message, var, ts_esmf))
                    logging.warning(traceback.print_exc())
Exemple #9
0
    def _scalar2raster(self, d, var, tndx):
        """
        Convert a single variable into a raster and colorbar.

        :param d: the netcdf file
        :param var: the variable name
        :param tndx: the time index
        :return: two StringIO objects, first is the PNG raster, second is PNG colorbar
        """
        # gather wisdom about the variable
        wisdom = get_wisdom(var).copy()
        wisdom.update(self.wisdom_update.get(var, {}))
        native_unit = wisdom['native_unit']
        cmap_name = wisdom['colormap']
        cmap = mpl.cm.get_cmap(cmap_name)

        # extract variable
        fa = wisdom['retrieve_as'](
            d,
            tndx)  # this calls a lambda defined to read the required 2d field
        lat, lon = wisdom['grid'](d)

        if lat.shape != fa.shape:
            raise PostprocError(
                "Variable %s size does not correspond to grid size." % var)

        # check for 'transparent' color value and mask
        if 'transparent_values' in wisdom:
            rng = wisdom['transparent_values']
            logging.info(
                '_scalar2raster: variable %s min %s max %s masking transparent from %s to %s'
                % (var, np.nanmin(fa), np.nanmax(fa), rng[0], rng[1]))
            fa = np.ma.masked_array(fa,
                                    np.logical_and(fa >= rng[0], fa <= rng[1]))
        else:
            fa = np.ma.masked_array(fa)

        # create the raster & get coordinate bounds

        # look at mins and maxes
        # fa_min,fa_max = np.nanmin(fa),np.nanmax(fa)
        # look at mins and maxes, transparent don't count
        fa_min, fa_max = np.nanmin(fa), np.nanmax(fa)

        # determine if we will use the range in the variable or a fixed range
        scale = wisdom['scale']
        if scale != 'original':
            m = fa.mask.copy(
            )  # save mask, resetting values below will destroy the mask
            fa_min, fa_max = scale[0], scale[1]
            fa[fa < fa_min] = fa_min
            fa[fa > fa_max] = fa_max
            fa.mask = m  # restore the mask

        # only create the colorbar if requested
        cb_png_data = None
        if wisdom['colorbar'] is not None:
            cb_unit = wisdom['colorbar']
            cbu_min, cbu_max = convert_value(native_unit, cb_unit,
                                             fa_min), convert_value(
                                                 native_unit, cb_unit, fa_max)
            #  colorbar + add it to the KMZ as a screen overlay
            legend = wisdom['name'] + ' ' + cb_unit
            logging.info(
                '_scalar2raster: variable %s colorbar from %s to %s %s' %
                (var, cbu_min, cbu_max, legend))
            cb_png_data = make_colorbar([cbu_min, cbu_max], 'vertical', 2,
                                        cmap, legend)

        # replace masked values by nans just in case
        fa.data[fa.mask] = np.nan
        fa.fill_value = np.nan

        logging.info(
            '_scalar2raster: variable %s elements %s count %s not masked %s min %s max %s'
            % (var, fa.size, fa.count(), np.count_nonzero(fa.mask == False),
               np.nanmin(fa), np.nanmax(fa)))

        raster_png_data, corner_coords = basemap_raster_mercator(
            lon, lat, fa, fa_min, fa_max, cmap)

        return raster_png_data, corner_coords, cb_png_data