def test_texture_all_excluded():
radar = _make_constant_refl_radar()
gatefilter = GateFilter(radar)
gatefilter.exclude_all()
texture._compute_texture(radar, 'reflectivity', gatefilter=gatefilter)
refl_texture = radar.fields['reflectivity_texture']['data']
assert np.all(refl_texture.mask)
return
def test_kdp_maesaka_all_excluded(first_guess=0.01, maxiter=100):
radar = _make_linear_psidp_radar()
gatefilter = GateFilter(radar)
gatefilter.exclude_all()
kdp_dict, phidpf_dict, phidpr_dict = kdp_proc.kdp_maesaka(
radar, gatefilter=gatefilter, first_guess=first_guess, maxiter=maxiter,
check_outliers=False)
assert np.allclose(kdp_dict['data'][0], 0.0, atol=first_guess)
return
def process_radar(radar, domain, weight, outdir, gatefilter=None, debug=False,
verbose=False):
"""
"""
if verbose:
print('Processing file: {}'.format(os.path.basename(filename)))
# Read radar data
radar = read(filename, exclude_fields=EXCLUDE_FIELDS)
# Create gatefilter from significant detection
gf = GateFilter(radar)
gf.exclude_below(SD_FIELD, 1, op='or', inclusive=False)
if debug:
print('Number of sweeps: {}'.format(radar.nsweeps))
# Grid radar data
grid = grid_radar(
radar, domain, weight=weight, fields=FIELDS, gatefilter=gf, toa=TOA,
max_range=MAX_RANGE, gqi_field=None, legacy=True, debug=debug,
verbose=verbose)
# Add new metadata
_add_metadata(grid, filename)
# ARM file name protocols
date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S')
fname = 'nexradwsr88d{}{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp)
# Write MMCG NetCDF file
grid_io.write_grid(
os.path.join(outdir, fname), grid, format=FORMAT,
write_proj_coord_sys=False, proj_coord_sys=None,
arm_time_variables=True, write_point_x_y_z=False,
write_point_lon_lat_alt=False)
return
def map_gates_to_grid(
radars, grid_shape, grid_limits, grid_origin=None,
grid_origin_alt=None, grid_projection=None,
fields=None, gatefilters=False, map_roi=True,
weighting_function='Barnes', toa=17000.0, roi_func='dist_beam',
constant_roi=None, z_factor=0.05, xy_factor=0.02, min_radius=500.0,
h_factor=1.0, nb=1.5, bsp=1.0, analysis_time=None, **kwargs):
"""
Map gates from one or more radars to a Cartesian grid.
Generate a Cartesian grid of points for the requested fields from the
collected points from one or more radars. For each radar gate that is not
filtered a radius of influence is calculated. The weighted field values
for that gate are added to all grid points within that radius. This
routine scaled linearly with the number of radar gates and the effective
grid size.
Parameters not defined below are identical to those in
:py:func:`map_to_grid`.
Parameters
----------
roi_func : str or RoIFunction
Radius of influence function. A function which takes an
z, y, x grid location, in meters, and returns a radius (in meters)
within which all collected points will be included in the weighting
for that grid points. Examples can be found in the
Typically following strings can use to specify a built in
radius of influence function:
* constant: constant radius of influence.
* dist: radius grows with the distance from each radar.
* dist_beam: radius grows with the distance from each radar
and parameter are based of virtual beam sizes.
A custom RoIFunction can be defined using the RoIFunction class
and defining a get_roi method which returns the radius. For efficient
mapping this class should be implemented in Cython.
Returns
-------
grids : dict
Dictionary of mapped fields. The keys of the dictionary are given by
parameter fields. Each elements is a `grid_size` float64 array
containing the interpolated grid for that field.
"""
# make a tuple if passed a radar object as the first argument
if isinstance(radars, Radar):
radars = (radars, )
skip_transform = False
if len(radars) == 1 and grid_origin_alt is None and grid_origin is None:
skip_transform = True
if grid_origin_alt is None:
try:
grid_origin_alt = float(radars[0].altitude['data'])
except TypeError:
grid_origin_alt = np.mean(radars[0].altitude['data'])
gatefilters = _parse_gatefilters(gatefilters, radars)
cy_weighting_function = _determine_cy_weighting_func(weighting_function)
projparams = _find_projparams(grid_origin, radars, grid_projection)
fields = _determine_fields(fields, radars)
grid_starts, grid_steps = _find_grid_params(grid_shape, grid_limits)
offsets = _find_offsets(radars, projparams, grid_origin_alt)
roi_func = _parse_roi_func(roi_func, constant_roi, z_factor, xy_factor,
min_radius, h_factor, nb, bsp, offsets)
# prepare grid storage arrays
nfields = len(fields)
grid_sum = np.zeros(grid_shape + (nfields, ), dtype=np.float32)
grid_wsum = np.zeros(grid_shape + (nfields, ), dtype=np.float32)
gatemapper = GateToGridMapper(
grid_shape, grid_starts, grid_steps, grid_sum, grid_wsum)
# project gates from each radar onto the grid
for radar, gatefilter in zip(radars, gatefilters):
# Copy the field data and masks.
# TODO method that does not copy field data into new array
shape = (radar.nrays, radar.ngates, nfields)
field_data = np.empty(shape, dtype='float32')
field_mask = np.empty(shape, dtype='uint8')
for i, field in enumerate(fields):
fdata = radar.fields[field]['data']
field_data[:, :, i] = np.ma.getdata(fdata)
field_mask[:, :, i] = np.ma.getmaskarray(fdata)
# find excluded gates from the gatefilter
if gatefilter is False:
gatefilter = GateFilter(radar) # include all gates
elif gatefilter is None:
gatefilter = moment_based_gate_filter(radar, **kwargs)
excluded_gates = gatefilter.gate_excluded.astype('uint8')
# calculate gate locations relative to the grid origin
if skip_transform:
# single radar, grid centered at radar location
gate_x = radar.gate_x['data']
gate_y = radar.gate_y['data']
else:
gate_x, gate_y = geographic_to_cartesian(
radar.gate_longitude['data'], radar.gate_latitude['data'],
projparams)
gate_z = radar.gate_altitude['data'] - grid_origin_alt
# Range and offset params (OpenMosiac addition)
gate_range = radar.range['data']
if analysis_time is None:
gate_timedelta = np.zeros(radar.nrays)
else:
gate_timedelta = np.array([
(t - analysis_time.replace(tzinfo=None)).total_seconds()
for t in num2date(radar.time['data'], radar.time['units'])
])
# map the gates onto the grid (modified with OpenMosaic addition)
gatemapper.map_gates_to_grid(
radar.ngates, radar.nrays, gate_z.astype('float32'),
gate_y.astype('float32'), gate_x.astype('float32'),
gate_range.astype('float32'), gate_timedelta.astype('float32'),
field_data, field_mask, excluded_gates,
toa, roi_func, cy_weighting_function)
# create and return the grid dictionary
mweight = np.ma.masked_equal(grid_wsum, 0)
msum = np.ma.masked_array(grid_sum, mweight.mask)
grids = dict(
[(f, msum[..., i] / mweight[..., i]) for i, f in enumerate(fields)])
if map_roi:
roi_array = np.empty(grid_shape, dtype=np.float32)
gatemapper.find_roi_for_grid(roi_array, roi_func)
grids['ROI'] = roi_array
return grids
def map_gates_to_grid(self,
fields,
analysis_time=None,
weighting_function='GridRad',
gatefilters=False,
map_roi=False,
roi_func='dist_beam',
constant_roi=None,
z_factor=0.05,
xy_factor=0.02,
min_radius=500.0,
h_factor=1.0,
nb=1.5,
bsp=1.0,
filter_kwargs=None):
"""Run the actual regridding using Py-ART style routine.
TODO: document
"""
self.analysis_time = analysis_time
filter_kwargs = {} if filter_kwargs is None else filter_kwargs
gatefilters = _parse_gatefilters(
gatefilters, [radar['radar'] for radar in self.radars])
cy_weighting_function = _determine_cy_weighting_func(
weighting_function)
fields = _determine_fields(fields,
[radar['radar'] for radar in self.radars])
offsets = [(radar['radar'].altitude['data'].item(), radar['y_radar'],
radar['x_radar']) for radar in self.radars]
roi_func_args = (roi_func, constant_roi, z_factor, xy_factor,
min_radius, h_factor, nb, bsp, offsets)
nfields = len(fields)
subgrid_args = []
for radar, gatefilter in zip(self.radars, gatefilters):
subgrid_shape = (self.grid_params['nz'],
radar['yi_max'] - radar['yi_min'],
radar['xi_max'] - radar['xi_min'], nfields)
subgrid_starts = (self.grid_params['z_min'],
self.grid_params['y_min'] +
self.grid_params['dy'] * radar['yi_min'],
self.grid_params['x_min'] +
self.grid_params['dx'] * radar['xi_min'])
subgrid_steps = (
self.grid_params['dz'],
self.grid_params['dy'],
self.grid_params['dx'],
)
# Copy field data and masks
field_shape = (radar['radar'].nrays, radar['radar'].ngates,
nfields)
field_data = np.empty(field_shape, dtype='float32')
field_mask = np.empty(field_shape, dtype='uint8')
for i, field in enumerate(fields):
fdata = radar['radar'].fields[field]['data']
field_data[:, :, i] = np.ma.getdata(fdata)
field_mask[:, :, i] = np.ma.getmaskarray(fdata)
# Find excluded gates from gatefilter
if gatefilter is False:
gatefilter = GateFilter(radar['radar'])
elif gatefilter is None:
gatefilter = moment_based_gate_filter(radar['radar'],
**filter_kwargs)
excluded_gates = gatefilter.gate_excluded.astype('uint8')
# Range and offsets
gate_range = radar['radar'].range['data'].astype('float32')
if analysis_time is None:
gate_timedelta = np.zeros(radar['radar'].nrays,
dtype='float32')
else:
gate_timedelta = np.array(
[(t - analysis_time.replace(tzinfo=None)).total_seconds()
for t in num2date(radar['radar'].time['data'],
radar['radar'].time['units'])],
dtype='float32')
# Add args to list to be applied to map_gates_to_subgrid
subgrid_args.append(
(subgrid_shape, subgrid_starts, subgrid_steps, field_shape,
field_data, field_mask, excluded_gates, radar['gate_dest_z'],
radar['gate_dest_y'], radar['gate_dest_x'], gate_range,
gate_timedelta, self.grid_params['z_max'], roi_func_args,
cy_weighting_function))
# Run the computation
if self.pool is None:
subgrids = []
for args in subgrid_args:
subgrids.append(map_gates_to_subgrid(*args))
else:
subgrids = list(
self.pool.starmap(map_gates_to_subgrid, subgrid_args))
# Sum the subgrids
grid_shape = (self.grid_params['nz'], self.grid_params['ny'],
self.grid_params['nx'], nfields)
grid_sum = np.zeros(grid_shape, dtype='float32')
grid_wsum = np.zeros(grid_shape, dtype='float32')
for subgrid, radar in zip(subgrids, self.radars):
grid_sum[:, radar['yi_min']:radar['yi_max'],
radar['xi_min']:radar['xi_max'], :] += subgrid['sum']
grid_wsum[:, radar['yi_min']:radar['yi_max'],
radar['xi_min']:radar['xi_max'], :] += subgrid['wsum']
# Apply the weighting and masking
mweight = np.ma.masked_equal(grid_wsum, 0)
msum = np.ma.masked_array(grid_sum, mweight.mask)
self.grids = dict(
(f, msum[..., i] / mweight[..., i]) for i, f in enumerate(fields))
# NOTE: Py-ART's map_roi option is ignored due to difference in gatemapper construction
return self.grids
def process_nexrad_file_to_subgrids_and_weights(
self,
nexrad_file_path,
radar_id,
fields,
sweep_interval,
analysis_time=None,
weighting_function='GridRad',
gatefilter=False,
map_roi=False,
roi_func='dist_beam',
constant_roi=None,
z_factor=0.05,
xy_factor=0.02,
min_radius=500.0,
h_factor=1.0,
nb=1.5,
bsp=1.0,
filter_kwargs=None,
cache_dir=None):
"""Do basically everything for a single radar file.
Read the file, filter for sweeps, calcuate coordinates, and map to subgrid.
PARAMETERS TODO.
"""
##############################
# Read and filter radar file #
##############################
if nexrad_file_path is None:
return None
try:
radar = pyart.io.read_nexrad_archive(nexrad_file_path,
delay_field_loading=True)
except:
warnings.warn(f"Cannot read file {nexrad_file_path}")
return None
sweep_time_offsets = [
np.median(radar.time['data'][s:e])
for s, e in radar.iter_start_end()
]
try:
sweep_times = num2date(sweep_time_offsets, radar.time['units'])
except:
warnings.warn(
f"Cannot read valid sweep times from file {nexrad_file_path}")
return None
valid_sweep_ids = [
i for i, t in enumerate(sweep_times)
if (analysis_time - sweep_interval <= t <= analysis_time +
sweep_interval)
]
if valid_sweep_ids:
try:
radar = radar.extract_sweeps(valid_sweep_ids)
except:
log.info(
f"File {nexrad_file_path} encountered a sweep extraction error"
)
return None
else:
log.info(
f"File {nexrad_file_path} does not contain valid sweep times")
del radar
return None
#############################
# Compute Radar Coordinates #
#############################
crs_kwargs = {
'proj': 'aeqd',
'lat_0': radar.latitude['data'].item(),
'lon_0': radar.longitude['data'].item()
}
gate_dest_x, gate_dest_y = radar_coords_to_grid_coords(
radar.gate_x['data'],
radar.gate_y['data'],
site_id=radar_id,
radar_crs_kwargs=crs_kwargs,
target_crs_cf_attrs=self.cf_attrs,
wait_for_cache=False, # TODO validate assumption
cache_dir=self.cache_dir)
gate_dest_z = radar.gate_altitude['data']
##################
# Map to Subgrid #
##################
# Arguments
filter_kwargs = {} if filter_kwargs is None else filter_kwargs
cy_weighting_function = _determine_cy_weighting_func(
weighting_function)
offsets = [(radar.altitude['data'].item(),
self.subgrid_params[radar_id]['y_radar'],
self.subgrid_params[radar_id]['x_radar'])]
roi_func_args = (roi_func, constant_roi, z_factor, xy_factor,
min_radius, h_factor, nb, bsp, offsets)
nfields = len(fields)
# Subgrid setup
subgrid_shape = (self.grid_params['nz'],
self.subgrid_params[radar_id]['ny'],
self.subgrid_params[radar_id]['nx'], nfields)
subgrid_starts = (self.grid_params['z_min'],
self.subgrid_params[radar_id]['y_min'],
self.subgrid_params[radar_id]['x_min'])
subgrid_steps = (
self.grid_params['dz'],
self.grid_params['dy'],
self.grid_params['dx'],
)
# Copy field data and masks
field_shape = (radar.nrays, radar.ngates, nfields)
field_data = np.empty(field_shape, dtype='float32')
field_mask = np.empty(field_shape, dtype='uint8')
for i, field in enumerate(fields):
fdata = radar.fields[field]['data']
field_data[:, :, i] = np.ma.getdata(fdata)
field_mask[:, :, i] = np.ma.getmaskarray(fdata)
# Find excluded gates from gatefilter
if gatefilter is False:
gatefilter = GateFilter(radar)
elif gatefilter is None:
gatefilter = moment_based_gate_filter(radar, **filter_kwargs)
excluded_gates = gatefilter.gate_excluded.astype('uint8')
# Range and offsets
gate_range = radar.range['data'].astype('float32')
if analysis_time is None:
gate_timedelta = np.zeros(radar.nrays, dtype='float32')
else:
gate_timedelta = np.array(
[(t - analysis_time.replace(tzinfo=None)).total_seconds()
for t in num2date(radar.time['data'], radar.time['units'])],
dtype='float32')
# Apply low-level map_gates_to_subgrid
subgrid = map_gates_to_subgrid(
subgrid_shape, subgrid_starts, subgrid_steps, field_shape,
field_data, field_mask, excluded_gates, gate_dest_z, gate_dest_y,
gate_dest_x, gate_range, gate_timedelta, self.grid_params['z_max'],
roi_func_args, cy_weighting_function)
# Inject subgrid info and return
return {
**subgrid,
**self.subgrid_params[radar_id], 'field_metadata': {
field: {
k: v
for k, v in radar.fields[field].items()
if k in ['units', 'standard_name', 'long_name']
}
for field in fields
}
}