Exemplo n.º 1
0
def test_texture_all_excluded():
    radar = _make_constant_refl_radar()
    gatefilter = GateFilter(radar)
    gatefilter.exclude_all()
    texture._compute_texture(radar, 'reflectivity', gatefilter=gatefilter)
    refl_texture = radar.fields['reflectivity_texture']['data']

    assert np.all(refl_texture.mask)

    return
Exemplo n.º 2
0
def test_kdp_maesaka_all_excluded(first_guess=0.01, maxiter=100):
    radar = _make_linear_psidp_radar()
    gatefilter = GateFilter(radar)
    gatefilter.exclude_all()
    kdp_dict, phidpf_dict, phidpr_dict = kdp_proc.kdp_maesaka(
        radar, gatefilter=gatefilter, first_guess=first_guess, maxiter=maxiter,
        check_outliers=False)

    assert np.allclose(kdp_dict['data'][0], 0.0, atol=first_guess)

    return
Exemplo n.º 3
0
def process_radar(radar, domain, weight, outdir, gatefilter=None, debug=False,
                  verbose=False):
    """
    """

    if verbose:
        print('Processing file: {}'.format(os.path.basename(filename)))

    # Read radar data
    radar = read(filename, exclude_fields=EXCLUDE_FIELDS)

    # Create gatefilter from significant detection
    gf = GateFilter(radar)
    gf.exclude_below(SD_FIELD, 1, op='or', inclusive=False)

    if debug:
        print('Number of sweeps: {}'.format(radar.nsweeps))

    # Grid radar data
    grid = grid_radar(
        radar, domain, weight=weight, fields=FIELDS, gatefilter=gf, toa=TOA,
        max_range=MAX_RANGE, gqi_field=None, legacy=True, debug=debug,
        verbose=verbose)

    # Add new metadata
    _add_metadata(grid, filename)

    # ARM file name protocols
    date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S')
    fname = 'nexradwsr88d{}{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp)

    # Write MMCG NetCDF file
    grid_io.write_grid(
        os.path.join(outdir, fname), grid, format=FORMAT,
        write_proj_coord_sys=False, proj_coord_sys=None,
        arm_time_variables=True, write_point_x_y_z=False,
        write_point_lon_lat_alt=False)

    return
Exemplo n.º 4
0
def map_gates_to_grid(
        radars, grid_shape, grid_limits, grid_origin=None,
        grid_origin_alt=None, grid_projection=None,
        fields=None, gatefilters=False, map_roi=True,
        weighting_function='Barnes', toa=17000.0, roi_func='dist_beam',
        constant_roi=None, z_factor=0.05, xy_factor=0.02, min_radius=500.0,
        h_factor=1.0, nb=1.5, bsp=1.0, analysis_time=None, **kwargs):
    """
    Map gates from one or more radars to a Cartesian grid.

    Generate a Cartesian grid of points for the requested fields from the
    collected points from one or more radars. For each radar gate that is not
    filtered a radius of influence is calculated. The weighted field values
    for that gate are added to all grid points within that radius. This
    routine scaled linearly with the number of radar gates and the effective
    grid size.

    Parameters not defined below are identical to those in
    :py:func:`map_to_grid`.

    Parameters
    ----------
    roi_func : str or RoIFunction
        Radius of influence function. A function which takes an
        z, y, x grid location, in meters, and returns a radius (in meters)
        within which all collected points will be included in the weighting
        for that grid points. Examples can be found in the
        Typically following strings can use to specify a built in
        radius of influence function:

            * constant: constant radius of influence.
            * dist: radius grows with the distance from each radar.
            * dist_beam: radius grows with the distance from each radar
              and parameter are based of virtual beam sizes.

        A custom RoIFunction can be defined using the RoIFunction class
        and defining a get_roi method which returns the radius. For efficient
        mapping this class should be implemented in Cython.

    Returns
    -------
    grids : dict
        Dictionary of mapped fields. The keys of the dictionary are given by
        parameter fields. Each elements is a `grid_size` float64 array
        containing the interpolated grid for that field.

    """
    # make a tuple if passed a radar object as the first argument
    if isinstance(radars, Radar):
        radars = (radars, )

    skip_transform = False
    if len(radars) == 1 and grid_origin_alt is None and grid_origin is None:
        skip_transform = True

    if grid_origin_alt is None:
        try:
            grid_origin_alt = float(radars[0].altitude['data'])
        except TypeError:
            grid_origin_alt = np.mean(radars[0].altitude['data'])

    gatefilters = _parse_gatefilters(gatefilters, radars)
    cy_weighting_function = _determine_cy_weighting_func(weighting_function)
    projparams = _find_projparams(grid_origin, radars, grid_projection)
    fields = _determine_fields(fields, radars)
    grid_starts, grid_steps = _find_grid_params(grid_shape, grid_limits)
    offsets = _find_offsets(radars, projparams, grid_origin_alt)
    roi_func = _parse_roi_func(roi_func, constant_roi, z_factor, xy_factor,
                               min_radius, h_factor, nb, bsp, offsets)

    # prepare grid storage arrays
    nfields = len(fields)
    grid_sum = np.zeros(grid_shape + (nfields, ), dtype=np.float32)
    grid_wsum = np.zeros(grid_shape + (nfields, ), dtype=np.float32)
    gatemapper = GateToGridMapper(
        grid_shape, grid_starts, grid_steps, grid_sum, grid_wsum)

    # project gates from each radar onto the grid
    for radar, gatefilter in zip(radars, gatefilters):

        # Copy the field data and masks.
        # TODO method that does not copy field data into new array
        shape = (radar.nrays, radar.ngates, nfields)
        field_data = np.empty(shape, dtype='float32')
        field_mask = np.empty(shape, dtype='uint8')
        for i, field in enumerate(fields):
            fdata = radar.fields[field]['data']
            field_data[:, :, i] = np.ma.getdata(fdata)
            field_mask[:, :, i] = np.ma.getmaskarray(fdata)

        # find excluded gates from the gatefilter
        if gatefilter is False:
            gatefilter = GateFilter(radar)  # include all gates
        elif gatefilter is None:
            gatefilter = moment_based_gate_filter(radar, **kwargs)
        excluded_gates = gatefilter.gate_excluded.astype('uint8')

        # calculate gate locations relative to the grid origin
        if skip_transform:
            # single radar, grid centered at radar location
            gate_x = radar.gate_x['data']
            gate_y = radar.gate_y['data']
        else:
            gate_x, gate_y = geographic_to_cartesian(
                radar.gate_longitude['data'], radar.gate_latitude['data'],
                projparams)
        gate_z = radar.gate_altitude['data'] - grid_origin_alt

        # Range and offset params (OpenMosiac addition)
        gate_range = radar.range['data']
        if analysis_time is None:
            gate_timedelta = np.zeros(radar.nrays)
        else:
            gate_timedelta = np.array([
                (t - analysis_time.replace(tzinfo=None)).total_seconds()
                for t in num2date(radar.time['data'], radar.time['units'])
            ])

        # map the gates onto the grid (modified with OpenMosaic addition)
        gatemapper.map_gates_to_grid(
            radar.ngates, radar.nrays, gate_z.astype('float32'),
            gate_y.astype('float32'), gate_x.astype('float32'),
            gate_range.astype('float32'), gate_timedelta.astype('float32'),
            field_data, field_mask, excluded_gates,
            toa, roi_func, cy_weighting_function)

    # create and return the grid dictionary
    mweight = np.ma.masked_equal(grid_wsum, 0)
    msum = np.ma.masked_array(grid_sum, mweight.mask)
    grids = dict(
        [(f, msum[..., i] / mweight[..., i]) for i, f in enumerate(fields)])
    if map_roi:
        roi_array = np.empty(grid_shape, dtype=np.float32)
        gatemapper.find_roi_for_grid(roi_array, roi_func)
        grids['ROI'] = roi_array
    return grids
Exemplo n.º 5
0
    def map_gates_to_grid(self,
                          fields,
                          analysis_time=None,
                          weighting_function='GridRad',
                          gatefilters=False,
                          map_roi=False,
                          roi_func='dist_beam',
                          constant_roi=None,
                          z_factor=0.05,
                          xy_factor=0.02,
                          min_radius=500.0,
                          h_factor=1.0,
                          nb=1.5,
                          bsp=1.0,
                          filter_kwargs=None):
        """Run the actual regridding using Py-ART style routine.

        TODO: document
        """

        self.analysis_time = analysis_time

        filter_kwargs = {} if filter_kwargs is None else filter_kwargs
        gatefilters = _parse_gatefilters(
            gatefilters, [radar['radar'] for radar in self.radars])
        cy_weighting_function = _determine_cy_weighting_func(
            weighting_function)
        fields = _determine_fields(fields,
                                   [radar['radar'] for radar in self.radars])
        offsets = [(radar['radar'].altitude['data'].item(), radar['y_radar'],
                    radar['x_radar']) for radar in self.radars]
        roi_func_args = (roi_func, constant_roi, z_factor, xy_factor,
                         min_radius, h_factor, nb, bsp, offsets)

        nfields = len(fields)

        subgrid_args = []
        for radar, gatefilter in zip(self.radars, gatefilters):
            subgrid_shape = (self.grid_params['nz'],
                             radar['yi_max'] - radar['yi_min'],
                             radar['xi_max'] - radar['xi_min'], nfields)
            subgrid_starts = (self.grid_params['z_min'],
                              self.grid_params['y_min'] +
                              self.grid_params['dy'] * radar['yi_min'],
                              self.grid_params['x_min'] +
                              self.grid_params['dx'] * radar['xi_min'])
            subgrid_steps = (
                self.grid_params['dz'],
                self.grid_params['dy'],
                self.grid_params['dx'],
            )

            # Copy field data and masks
            field_shape = (radar['radar'].nrays, radar['radar'].ngates,
                           nfields)
            field_data = np.empty(field_shape, dtype='float32')
            field_mask = np.empty(field_shape, dtype='uint8')
            for i, field in enumerate(fields):
                fdata = radar['radar'].fields[field]['data']
                field_data[:, :, i] = np.ma.getdata(fdata)
                field_mask[:, :, i] = np.ma.getmaskarray(fdata)

            # Find excluded gates from gatefilter
            if gatefilter is False:
                gatefilter = GateFilter(radar['radar'])
            elif gatefilter is None:
                gatefilter = moment_based_gate_filter(radar['radar'],
                                                      **filter_kwargs)
            excluded_gates = gatefilter.gate_excluded.astype('uint8')

            # Range and offsets
            gate_range = radar['radar'].range['data'].astype('float32')
            if analysis_time is None:
                gate_timedelta = np.zeros(radar['radar'].nrays,
                                          dtype='float32')
            else:
                gate_timedelta = np.array(
                    [(t - analysis_time.replace(tzinfo=None)).total_seconds()
                     for t in num2date(radar['radar'].time['data'],
                                       radar['radar'].time['units'])],
                    dtype='float32')

            # Add args to list to be applied to map_gates_to_subgrid
            subgrid_args.append(
                (subgrid_shape, subgrid_starts, subgrid_steps, field_shape,
                 field_data, field_mask, excluded_gates, radar['gate_dest_z'],
                 radar['gate_dest_y'], radar['gate_dest_x'], gate_range,
                 gate_timedelta, self.grid_params['z_max'], roi_func_args,
                 cy_weighting_function))

        # Run the computation
        if self.pool is None:
            subgrids = []
            for args in subgrid_args:
                subgrids.append(map_gates_to_subgrid(*args))
        else:
            subgrids = list(
                self.pool.starmap(map_gates_to_subgrid, subgrid_args))

        # Sum the subgrids
        grid_shape = (self.grid_params['nz'], self.grid_params['ny'],
                      self.grid_params['nx'], nfields)
        grid_sum = np.zeros(grid_shape, dtype='float32')
        grid_wsum = np.zeros(grid_shape, dtype='float32')
        for subgrid, radar in zip(subgrids, self.radars):
            grid_sum[:, radar['yi_min']:radar['yi_max'],
                     radar['xi_min']:radar['xi_max'], :] += subgrid['sum']
            grid_wsum[:, radar['yi_min']:radar['yi_max'],
                      radar['xi_min']:radar['xi_max'], :] += subgrid['wsum']

        # Apply the weighting and masking
        mweight = np.ma.masked_equal(grid_wsum, 0)
        msum = np.ma.masked_array(grid_sum, mweight.mask)
        self.grids = dict(
            (f, msum[..., i] / mweight[..., i]) for i, f in enumerate(fields))

        # NOTE: Py-ART's map_roi option is ignored due to difference in gatemapper construction
        return self.grids
Exemplo n.º 6
0
    def process_nexrad_file_to_subgrids_and_weights(
            self,
            nexrad_file_path,
            radar_id,
            fields,
            sweep_interval,
            analysis_time=None,
            weighting_function='GridRad',
            gatefilter=False,
            map_roi=False,
            roi_func='dist_beam',
            constant_roi=None,
            z_factor=0.05,
            xy_factor=0.02,
            min_radius=500.0,
            h_factor=1.0,
            nb=1.5,
            bsp=1.0,
            filter_kwargs=None,
            cache_dir=None):
        """Do basically everything for a single radar file.

        Read the file, filter for sweeps, calcuate coordinates, and map to subgrid.
        
        PARAMETERS TODO.
        """
        ##############################
        # Read and filter radar file #
        ##############################
        if nexrad_file_path is None:
            return None
        try:
            radar = pyart.io.read_nexrad_archive(nexrad_file_path,
                                                 delay_field_loading=True)
        except:
            warnings.warn(f"Cannot read file {nexrad_file_path}")
            return None

        sweep_time_offsets = [
            np.median(radar.time['data'][s:e])
            for s, e in radar.iter_start_end()
        ]
        try:
            sweep_times = num2date(sweep_time_offsets, radar.time['units'])
        except:
            warnings.warn(
                f"Cannot read valid sweep times from file {nexrad_file_path}")
            return None
        valid_sweep_ids = [
            i for i, t in enumerate(sweep_times)
            if (analysis_time - sweep_interval <= t <= analysis_time +
                sweep_interval)
        ]
        if valid_sweep_ids:
            try:
                radar = radar.extract_sweeps(valid_sweep_ids)
            except:
                log.info(
                    f"File {nexrad_file_path} encountered a sweep extraction error"
                )
                return None
        else:
            log.info(
                f"File {nexrad_file_path} does not contain valid sweep times")
            del radar
            return None

        #############################
        # Compute Radar Coordinates #
        #############################
        crs_kwargs = {
            'proj': 'aeqd',
            'lat_0': radar.latitude['data'].item(),
            'lon_0': radar.longitude['data'].item()
        }
        gate_dest_x, gate_dest_y = radar_coords_to_grid_coords(
            radar.gate_x['data'],
            radar.gate_y['data'],
            site_id=radar_id,
            radar_crs_kwargs=crs_kwargs,
            target_crs_cf_attrs=self.cf_attrs,
            wait_for_cache=False,  # TODO validate assumption
            cache_dir=self.cache_dir)
        gate_dest_z = radar.gate_altitude['data']

        ##################
        # Map to Subgrid #
        ##################

        # Arguments
        filter_kwargs = {} if filter_kwargs is None else filter_kwargs
        cy_weighting_function = _determine_cy_weighting_func(
            weighting_function)
        offsets = [(radar.altitude['data'].item(),
                    self.subgrid_params[radar_id]['y_radar'],
                    self.subgrid_params[radar_id]['x_radar'])]
        roi_func_args = (roi_func, constant_roi, z_factor, xy_factor,
                         min_radius, h_factor, nb, bsp, offsets)
        nfields = len(fields)

        # Subgrid setup
        subgrid_shape = (self.grid_params['nz'],
                         self.subgrid_params[radar_id]['ny'],
                         self.subgrid_params[radar_id]['nx'], nfields)
        subgrid_starts = (self.grid_params['z_min'],
                          self.subgrid_params[radar_id]['y_min'],
                          self.subgrid_params[radar_id]['x_min'])
        subgrid_steps = (
            self.grid_params['dz'],
            self.grid_params['dy'],
            self.grid_params['dx'],
        )

        # Copy field data and masks
        field_shape = (radar.nrays, radar.ngates, nfields)
        field_data = np.empty(field_shape, dtype='float32')
        field_mask = np.empty(field_shape, dtype='uint8')
        for i, field in enumerate(fields):
            fdata = radar.fields[field]['data']
            field_data[:, :, i] = np.ma.getdata(fdata)
            field_mask[:, :, i] = np.ma.getmaskarray(fdata)

        # Find excluded gates from gatefilter
        if gatefilter is False:
            gatefilter = GateFilter(radar)
        elif gatefilter is None:
            gatefilter = moment_based_gate_filter(radar, **filter_kwargs)
        excluded_gates = gatefilter.gate_excluded.astype('uint8')

        # Range and offsets
        gate_range = radar.range['data'].astype('float32')
        if analysis_time is None:
            gate_timedelta = np.zeros(radar.nrays, dtype='float32')
        else:
            gate_timedelta = np.array(
                [(t - analysis_time.replace(tzinfo=None)).total_seconds()
                 for t in num2date(radar.time['data'], radar.time['units'])],
                dtype='float32')

        # Apply low-level map_gates_to_subgrid
        subgrid = map_gates_to_subgrid(
            subgrid_shape, subgrid_starts, subgrid_steps, field_shape,
            field_data, field_mask, excluded_gates, gate_dest_z, gate_dest_y,
            gate_dest_x, gate_range, gate_timedelta, self.grid_params['z_max'],
            roi_func_args, cy_weighting_function)

        # Inject subgrid info and return
        return {
            **subgrid,
            **self.subgrid_params[radar_id], 'field_metadata': {
                field: {
                    k: v
                    for k, v in radar.fields[field].items()
                    if k in ['units', 'standard_name', 'long_name']
                }
                for field in fields
            }
        }