def read_disdro(fname):
"""
Reads scattering parameters computed from disdrometer data contained in a
text file
Parameters
----------
fname : str
path of time series file
Returns
-------
date, preciptype, variable, scattering temperature: tuple
The read values
"""
try:
var = re.search('GHz_(.{,7})_el', fname).group(1)
except AttributeError:
# AAA, ZZZ not found in the original string
var = '' # apply your error handling
try:
with open(fname, 'r', newline='', encoding='utf-8',
errors='ignore') as csvfile:
# first count the lines
reader = csv.DictReader(
(row for row in csvfile if not row.startswith('#')),
delimiter=',')
nrows = sum(1 for row in reader)
variable = np.ma.empty(nrows, dtype='float32')
scatt_temp = np.ma.empty(nrows, dtype='float32')
# now read the data
csvfile.seek(0)
reader = csv.DictReader(
(row for row in csvfile if not row.startswith('#')),
delimiter=',')
i = 0
date = list()
preciptype = list()
for row in reader:
date.append(
datetime.datetime.strptime(row['date'],
'%Y-%m-%d %H:%M:%S'))
preciptype.append(row['Precip Code'])
variable[i] = float(row[var])
scatt_temp[i] = float(row['Scattering Temp [deg C]'])
i += 1
variable = np.ma.masked_values(variable, get_fillvalue())
np.ma.set_fill_value(variable, get_fillvalue())
csvfile.close()
return (date, preciptype, variable, scatt_temp)
except EnvironmentError as ee:
warn(str(ee))
warn('Unable to read file ' + fname)
return (None, None, None, None)
def add_textures(radar,
fields=None,
gatefilter=None,
texture_window=(3, 3),
texture_sample=5,
min_ncp=None,
min_sweep=None,
max_sweep=None,
min_range=None,
max_range=None,
rays_wrap_around=False,
fill_value=None,
ncp_field=None,
debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# Parse fields to compute textures
# If no fields are specified then the texture field of all available radar
# fields are computed
if fields is None:
fields = radar.fields.keys()
# Parse texture window parameters
ray_window, gate_window = texture_window
if debug:
print 'Number of rays in window: {}'.format(ray_window)
print 'Number of gates in window: {}'.format(gate_window)
for field in fields:
if verbose:
print 'Computing texture field: {}'.format(field)
_compute_field(radar,
field,
gatefilter=gatefilter,
ray_window=ray_window,
gate_window=gate_window,
min_sample=texture_sample,
min_ncp=min_ncp,
min_sweep=min_sweep,
max_sweep=max_sweep,
min_range=min_range,
max_range=max_range,
rays_wrap_around=rays_wrap_around,
fill_value=fill_value,
text_field=None,
ncp_field=ncp_field)
return
def populate_field(data, inds, shape, field, weights=None, mask=None,
fill_value=None):
"""
Create mapped radar field data dictionary.
Parameters
----------
data : ndarray
Input radar data.
inds : ndarray
Indices corresponding to the k-nearest neighbours.
shape : list-like
Shape of analysis grid.
field : str
Field name.
weights : ndarray, optional
Distance-dependent weights applied to k-nearest neighbours. Use default
None for nearest neighbor scheme. Must have same shape as inds.
mask : ndarray, optional
Masking will be applied where mask is True. Must have same shape as
flattened grid.
fill_value : float, optional
Value indicating missing or bad data in input data. If None, default
value in configuration file is used.
Returns
-------
field_dict : dict
Field dictionary containing data and metadata.
"""
if fill_value is None:
fill_value = get_fillvalue()
if weights is None:
fq = data[inds]
else:
fq = np.ma.average(data[inds], weights=weights, axis=1)
fq = np.ma.masked_where(mask, fq, copy=False)
fq.set_fill_value(fill_value)
# Populate field dictionary
field_dict = get_metadata(field)
field_dict['data'] = fq.reshape(shape).astype(np.float32)
if np.ma.is_masked(fq):
field_dict['_FillValue'] = fq.fill_value
return field_dict
def read_sband_archive(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
delay_field_loading=False,
station=None,
scans=None,
linear_interp=True,
**kwargs):
"""
Read a S band Archive file.
Parameters
----------
filename : str
Filename of S band Archive file.
field_names : dict, optional
Dictionary mapping S band moments to radar field names. If a
data type found in the file does not appear in this dictionary or has
a value of None it will not be placed in the radar.fields dictionary.
A value of None, the default, will use the mapping defined in the
metadata configuration file.
additional_metadata : dict of dicts, optional
Dictionary of dictionaries to retrieve metadata from during this read.
This metadata is not used during any successive file reads unless
explicitly included. A value of None, the default, will not
introduct any addition metadata and the file specific or default
metadata as specified by the metadata configuration file will be used.
file_field_names : bool, optional
True to use the S band field names for the field names. If this
case the field_names parameter is ignored. The field dictionary will
likely only have a 'data' key, unless the fields are defined in
`additional_metadata`.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters.
delay_field_loading : bool, optional
True to delay loading of field data from the file until the 'data'
key in a particular field dictionary is accessed. In this case
the field attribute of the returned Radar object will contain
LazyLoadDict objects not dict objects.
station : tuple or None, optional
Three float tuple, include latitude, longitude and height of S band radar,
first latitude, second longitude, third height (units: metre)
scans : list or None, optional
Read only specified scans from the file. None (the default) will read
all scans.
linear_interp : bool, optional
True (the default) to perform linear interpolation between valid pairs
of gates in low resolution rays in files mixed resolution rays.
False will perform a nearest neighbor interpolation. This parameter is
not used if the resolution of all rays in the file or requested sweeps
is constant.
Returns
-------
radar : Radar
Radar object containing all moments and sweeps/cuts in the volume.
Gates not collected are masked in the field data.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('nexrad_archive', field_names,
additional_metadata, file_field_names,
exclude_fields)
# open the file and retrieve scan information
nfile = SbandRadarFile(prepare_for_read(filename))
scan_info = nfile.scan_info(scans)
# time
time = filemetadata('time')
time_start, _time = nfile.get_times(scans)
time['data'] = _time
time['units'] = make_time_unit_str(time_start)
# range
_range = filemetadata('range')
first_gate, gate_spacing, last_gate = _find_range_params(
scan_info, filemetadata)
_range['data'] = np.arange(first_gate, last_gate, gate_spacing, 'float32')
_range['meters_to_center_of_first_gate'] = float(first_gate)
_range['meters_between_gates'] = float(gate_spacing)
# metadata
metadata = filemetadata('metadata')
metadata['original_container'] = 'S band'
vcp_pattern = nfile.get_vcp_pattern()
if vcp_pattern is not None:
metadata['vcp_pattern'] = vcp_pattern
# scan_type
scan_type = 'ppi'
# latitude, longitude, altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
if station is None:
lat, lon, alt = 0, 0, 0
else:
lat, lon, alt = station
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([alt], dtype='float64')
# sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index
# sweep_end_ray_index
sweep_number = filemetadata('sweep_number')
sweep_mode = filemetadata('sweep_mode')
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
if scans is None:
nsweeps = int(nfile.nscans)
else:
nsweeps = len(scans)
sweep_number['data'] = np.arange(nsweeps, dtype='int32')
sweep_mode['data'] = np.array(nsweeps * ['azimuth_surveillance'],
dtype='S')
rays_per_scan = [s['nrays'] for s in scan_info]
sweep_end_ray_index['data'] = np.cumsum(rays_per_scan, dtype='int32') - 1
rays_per_scan.insert(0, 0)
sweep_start_ray_index['data'] = np.cumsum(rays_per_scan[:-1],
dtype='int32')
# azimuth, elevation, fixed_angle
azimuth = filemetadata('azimuth')
elevation = filemetadata('elevation')
fixed_angle = filemetadata('fixed_angle')
azimuth['data'] = nfile.get_azimuth_angles(scans)
elevation['data'] = nfile.get_elevation_angles(scans).astype('float32')
fixed_angle['data'] = nfile.get_target_angles(scans)
# fields
max_ngates = len(_range['data'])
available_moments = set([m for scan in scan_info for m in scan['moments']])
interpolate = _find_scans_to_interp(scan_info, first_gate, gate_spacing,
filemetadata)
fields = {}
for moment in available_moments:
field_name = filemetadata.get_field_name(moment)
if field_name is None:
continue
dic = filemetadata(field_name)
dic['_FillValue'] = get_fillvalue()
if delay_field_loading and moment not in interpolate:
dic = LazyLoadDict(dic)
data_call = _NEXRADLevel2StagedField(nfile, moment, max_ngates,
scans)
dic.set_lazy('data', data_call)
else:
mdata = nfile.get_data(moment, max_ngates, scans=scans)
if moment in interpolate:
interp_scans = interpolate[moment]
warnings.warn(
"Gate spacing is not constant, interpolating data in " +
"scans %s for moment %s." % (interp_scans, moment),
UserWarning)
for scan in interp_scans:
idx = scan_info[scan]['moments'].index(moment)
moment_ngates = scan_info[scan]['ngates'][idx]
start = sweep_start_ray_index['data'][scan]
end = sweep_end_ray_index['data'][scan]
_interpolate_scan(mdata, start, end, moment_ngates,
linear_interp)
dic['data'] = mdata
fields[field_name] = dic
# instrument_parameters
nyquist_velocity = filemetadata('nyquist_velocity')
unambiguous_range = filemetadata('unambiguous_range')
nyquist_velocity['data'] = nfile.get_nyquist_vel(scans).astype('float32')
unambiguous_range['data'] = (
nfile.get_unambigous_range(scans).astype('float32'))
instrument_parameters = {
'unambiguous_range': unambiguous_range,
'nyquist_velocity': nyquist_velocity,
}
nfile.close()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters)
def add_textures(grid,
fields=None,
window=(3, 3),
min_sample=5,
fill_value=None,
debug=False,
verbose=False):
"""
Add texture fields to grid fields dictionary.
Parameters
----------
grid : Grid
Py-ART Grid containing specified fields.
fields : str or list or tuple, optional
Grid field(s) to compute texture field(s). If None, texture fields
for all available grid fields will be computed and added.
window : list or tuple, optional
The 2-D (x, y) texture window used to compute texture fields.
min_sample : int, optional
Minimum sample size within texture window required to define a valid
texture. Note that a minimum of 2 grid points are required to compute
the texture field.
fill_value : float, optional
The value indicating missing or bad data in the grid field data. If
None, the default value in the Py-ART configuration file is used.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print progress or identification information, False to
suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse the fields to compute textures from
if fields is None:
fields = grid.fields.keys()
elif isinstance(fields, str):
fields = [fields]
else:
fields = [field for field in fields if field in grid.fields]
# Parse texture window parameters
x_window, y_window = window
for field in fields:
if verbose:
print 'Computing texture field: {}'.format(field)
_add_texture(grid,
field,
x_window=x_window,
y_window=y_window,
min_sample=min_sample,
fill_value=fill_value,
text_field=None,
debug=debug,
verbose=verbose)
return
def add_textures(
radar, fields=None, gatefilter=None, window=(3, 3), min_sample=5,
min_sweep=None, max_sweep=None, min_range=None, max_range=None,
rays_wrap_around=False, fill_value=None, debug=False, verbose=False):
"""
Add texture fields to radar.
Parameters
----------
radar : Radar
Py-ART Radar containing specified field.
fields : str or list or tuple, optional
Radar field(s) to compute texture field(s). If None, texture fields
for all available radar fields will be computed and added.
gatefilter : GateFilter, optional
Py-ART GateFilter specifying radar gates which should be included when
computing the texture field.
window : list or tuple, optional
The 2-D (ray, gate) texture window used to compute texture fields.
min_sample : int, optional
Minimum sample size within texture window required to define a valid
texture. Note that a minimum of 2 radar gates are required to compute
the texture field.
min_sweep : int, optional
Minimum sweep number to compute texture field.
max_sweep : int, optional
Maximum sweep number to compute texture field.
min_range : float, optional
Minimum range in meters from radar to compute texture field.
max_range : float, optional
Maximum range in meters from radar to compute texture field.
fill_value : float, optional
Value indicating missing or bad data in radar field data. If None,
default value in Py-ART configuration file is used.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print progress and identification information, False to
suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse fields to compute textures
# If no fields are specified then the texture field of all available radar
# fields are computed
if fields is None:
fields = radar.fields.keys()
if isinstance(fields, str):
fields = [fields]
# Parse texture window parameters
ray_window, gate_window = window
if verbose:
print 'Number of rays in window: {}'.format(ray_window)
print 'Number of gates in window: {}'.format(gate_window)
for field in fields:
if verbose:
print 'Computing texture field: {}'.format(field)
_add_texture(
radar, field, gatefilter=gatefilter, ray_window=ray_window,
gate_window=gate_window, min_sample=min_sample,
min_sweep=min_sweep, max_sweep=max_sweep, min_range=min_range,
max_range=max_range, rays_wrap_around=rays_wrap_around,
fill_value=fill_value, debug=debug, verbose=verbose)
return
def interpolate_missing(radar,
fields=None,
interp_window=(3, 3),
interp_sample=8,
kind='mean',
rays_wrap_around=False,
fill_value=None,
debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names to interpolate
if fields is None:
fields = radar.fields.keys()
# Parse interpolation parameters
ray_window, gate_window = interp_window
# Loop over all fields and interpolate missing gates
for field in fields:
if verbose:
print 'Filling missing gates: {}'.format(field)
# Parse radar data and its original data type
data = radar.fields[field]['data']
dtype_orig = data.dtype
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size before fill: {}'.format(N)
# Parse sweep parameters
# Offset index arrays in order to be compatible with Fortran and avoid
# a segmentation fault
sweep_start = radar.sweep_start_ray_index['data'] + 1
sweep_end = radar.sweep_end_ray_index['data'] + 1
# Call Fortran routine
if kind.upper() == 'MEAN':
sweeps.mean_fill(data,
sweep_start,
sweep_end,
ray_window=ray_window,
gate_window=gate_window,
min_sample=interp_sample,
rays_wrap=rays_wrap_around,
fill_value=fill_value)
else:
raise ValueError('Unsupported interpolation method')
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size after fill: {}'.format(N)
# Mask invalid data
data = np.ma.masked_equal(data, fill_value, copy=False)
data.set_fill_value(fill_value)
# Add interpolated data to radar object
radar.fields[field]['data'] = data.astype(dtype_orig)
return
def velocity_coherency(
radar, gatefilter=None, text_bins=40, text_limits=(0, 20),
nyquist_vel=None, texture_window=(3, 3), texture_sample=5,
max_texture=None, min_sweep=None, rays_wrap_around=False,
remove_small_features=False, size_bins=100, size_limits=(0, 400),
fill_value=None, vdop_field=None, text_field=None, coherent_field=None,
debug=False, verbose=False):
"""
Parameters
----------
radar : Radar
Py-ART Radar containing Doppler velocity data.
gatefilter : GateFilter, optional
Py-ART GateFilter instance. If None, all radar gates will initially be
assumed valid.
Returns
-------
gf : GateFilter
Py-ART GateFilter.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if vdop_field is None:
vdop_field = get_field_name('velocity')
if text_field is None:
text_field = '{}_texture'.format(vdop_field)
if coherent_field is None:
coherent_field = '{}_coherency_mask'.format(vdop_field)
if verbose:
print 'Computing Doppler velocity coherency mask'
# Parse Nyquist velocity
if nyquist_vel is None:
nyquist = radar.get_nyquist_vel(0, check_uniform=True)
if debug:
print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist)
# Compute Doppler velocity texture field
ray_window, gate_window = texture_window
textures._compute_field(
radar, vdop_field, ray_window=ray_window, gate_window=gate_window,
min_sample=texture_sample, min_sqi=None, min_sweep=min_sweep,
max_sweep=None, fill_value=fill_value, sqi_field=None,
text_field=text_field)
# Automatically determine the maximum Doppler velocity texture value which
# brackets the coherent part of the Doppler velocity texture distribution
if max_texture is None:
# Bin and count Doppler velocity texture field
counts, bin_edges = np.histogram(
radar.fields[text_field]['data'].compressed(), bins=text_bins,
range=text_limits, normed=False, weights=None, density=False)
# Compute bin centers and bin width
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {:.3f} m/s'.format(bin_width)
# Determine the location of extrema in the Doppler velocity texture
# distribution
kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0]
kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0]
if debug:
print 'Minima located at: {} m/s'.format(bin_centers[kmin])
print 'Maxima located at: {} m/s'.format(bin_centers[kmax])
# Compute the theoretical noise peak location from Guassian noise
# statistics
noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0)
if debug:
print 'Theoretical noise peak: {:.3f} m/s'.format(
noise_peak_theory)
# Find the closest Doppler velocity texture distribution peak to the
# computed theoretical location. Here we assume that the Doppler
# velocity texture distribution has at least one primary mode which
# corresponds to the incoherent (noisy) part of the Doppler velocity
# texture distribution. Depending on the radar volume and the bin width
# used to define the distribution, the distribution may be bimodal,
# with the additional peak corresponding to the coherent part of the
# Doppler velocity texture distribution
idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin()
noise_peak = bin_centers[kmax][idx]
if debug:
print 'Computed noise peak: {:.3f} m/s'.format(noise_peak)
# Determine primary and secondary peak locations for debugging
# purposes
if kmax.size > 1:
counts_max = np.sort(counts[kmax], kind='mergesort')[::-1]
prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()]
sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()]
if debug:
print 'Primary peak: {:.3f} m/s'.format(prm_peak)
print 'Secondary peak: {:.3f} m/s'.format(sec_peak)
# Determine the left edge of the noise mode
# Where this mode becomes a minimum defines the separation between
# coherent and incoherent Doppler velocity texture values
# TODO: if the chosen bin width is very small than multiple extrema can
# exist such that the first minimum to the left of the noise peak is
# not the appropriate minimum and the left edge detection breaks down
is_left_side = bin_centers[kmin] < noise_peak
max_texture = bin_centers[kmin][is_left_side].min() + bin_width / 2.0
if debug:
_range = [0.0, round(max_texture, 3)]
print 'Doppler velocity coherent mode: {} m/s'.format(_range)
# Create the Doppler velocity texture coherency mask
is_coherent = np.logical_and(
radar.fields[text_field]['data'] >= 0.0,
radar.fields[text_field]['data'] <= max_texture)
is_coherent = np.ma.filled(is_coherent, False)
# Conditional sampling checks
for sweep, sweep_slice in enumerate(radar.iter_slice()):
if sweep < min_sweep:
is_coherent[sweep_slice] = False
coherent_dict = {
'data': is_coherent.astype(np.int8),
'long_name': 'Doppler velocity coherency mask',
'standard_name': coherent_field,
'valid_min': 0,
'valid_max': 1,
'_FillValue': None,
'units': 'unitless',
'comment': '0 = incoherent velocity, 1 = coherent velocity',
}
radar.add_field(coherent_field, coherent_dict, replace_existing=True)
# Remove insignificant features from Doppler velocity coherency mask
if remove_small_features:
basic_fixes._binary_significant_features(
radar, coherent_field, size_bins=size_bins,
size_limits=size_limits, structure=structure, debug=debug,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(coherent_field, 1, op='and')
return gatefilter
def _spectrum_width_coherency(radar,
gatefilter=None,
num_bins=10,
limits=None,
texture_window=(3, 3),
texture_sample=5,
min_sigma=None,
max_sigma=None,
rays_wrap_around=False,
remove_salt=False,
salt_window=(5, 5),
salt_sample=10,
fill_value=None,
width_field=None,
width_text_field=None,
cohere_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if width_field is None:
width_field = get_field_name('spectrum_width')
if width_text_field is None:
width_text_field = '{}_texture'.format(width_field)
if cohere_field is None:
cohere_field = '{}_coherency_mask'.format(width_field)
# Compute spectrum width texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(radar,
width_field,
ray_window=ray_window,
gate_window=gate_window,
min_sample=texture_sample,
min_ncp=None,
min_sweep=None,
max_sweep=None,
fill_value=fill_value,
ncp_field=None)
# Automatically bracket noise distribution
if min_sigma is None and max_sigma is None:
# Compute spectrum width texture frequency counts
# Normalize frequency counts and compute bin centers and bin width
width_sigma = radar.fields[width_text_field]['data']
hist, edges = np.histogram(width_sigma.compressed(),
bins=num_bins,
range=limits,
normed=False,
weights=None,
density=False)
hist = hist.astype(np.float64) / hist.max()
width = np.diff(edges).mean()
half_width = width / 2.0
bins = edges[:-1] + half_width
if verbose:
print 'Bin width = %.2f m/s' % width
# Determine distribution extrema locations
k_min = argrelextrema(hist, np.less, axis=0, order=1, mode='clip')[0]
k_max = argrelextrema(hist, np.greater, axis=0, order=1,
mode='clip')[0]
if verbose:
print 'Minima located at %s m/s' % bins[k_min]
print 'Maxima located at %s m/s' % bins[k_max]
# Potentially a clear air volume
if k_min.size <= 1 or k_max.size <= 1:
# Bracket noise distribution
# Add (left side) or subtract (right side) the half bin width to
# account for the bin width
max_sigma = bins.max() + half_width
# Account for the no coherent signal case
if k_min.size == 0:
min_sigma = bins.min() - half_width
else:
min_sigma = bins[k_min][0] + half_width
if verbose:
print 'Computed min_sigma = %.2f m/s' % min_sigma
print 'Computed max_sigma = %.2f m/s' % max_sigma
print 'Radar volume is likely a clear air volume'
# Typical volume containing sufficient scatterers (e.g., hydrometeors,
# insects, etc.)
else:
# Compute primary and secondary peak locations
hist_max = np.sort(hist[k_max], kind='mergesort')[::-1]
prm_peak = bins[np.abs(hist - hist_max[0]).argmin()]
sec_peak = bins[np.abs(hist - hist_max[1]).argmin()]
if verbose:
print 'Primary peak located at %.2f m/s' % prm_peak
print 'Secondary peak located at %.2f m/s' % sec_peak
# If the primary (secondary) peak velocity texture is greater than
# the secondary (primary) peak velocity texture, than the primary
# (secondary) peak defines the noise distribution
noise_peak = np.max([prm_peak, sec_peak])
if verbose:
print 'Noise peak located at %.2f m/s' % noise_peak
# Determine left/right sides of noise distribution
left_side = bins[k_min] < noise_peak
right_side = bins[k_min] > noise_peak
# Bracket noise distribution
# Add (left side) or subtract (right side) the half bin width to
# account for the bin width
min_sigma = bins[k_min][left_side].max() + half_width
max_sigma = bins.max() + half_width
if verbose:
print 'Computed min_sigma = %.2f m/s' % min_sigma
print 'Computed max_sigma = %.2f m/s' % max_sigma
# Create the spectrum width texture coherency mask
mask = np.logical_or(radar.fields[width_text_field]['data'] <= min_sigma,
radar.fields[width_text_field]['data'] >= max_sigma)
mask = np.ma.filled(mask, False)
mask_dict = {
'data':
mask.astype(np.int8),
'long_name':
'Spectrum width coherency mask',
'standard_name':
cohere_field,
'valid_min':
0,
'valid_max':
1,
'_FillValue':
None,
'units':
'unitless',
'comment': ('0 = incoherent spectrum width, '
'1 = coherent spectrum width'),
}
radar.add_field(cohere_field, mask_dict, replace_existing=True)
# Remove salt and pepper noise from mask
if remove_salt:
basic_fixes.remove_salt(radar,
fields=[cohere_field],
salt_window=salt_window,
salt_sample=salt_sample,
rays_wrap_around=rays_wrap_around,
fill_value=0,
mask_data=False,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(cohere_field, 1, op='and')
return gatefilter
def read_lightning_all(fname,
labels=[
'hydro [-]', 'KDPc [deg/Km]', 'dBZc [dBZ]',
'RhoHVc [-]', 'TEMP [deg C]', 'ZDRc [dB]'
]):
"""
Reads a file containing lightning data and co-located polarimetric data.
fields:
flashnr
time data
Time within flash (in seconds)
Latitude (decimal degrees)
Longitude (decimal degrees)
Altitude (m MSL)
Power (dBm)
Polarimetric values at flash position
Parameters
----------
fname : str
path of time series file
labels : list of str
The polarimetric variables labels
Returns
-------
flashnr, time_data, time_in_flash, lat, lon, alt, dBm,
pol_vals_dict : tupple
A tupple containing the read values. None otherwise
"""
try:
with open(fname, 'r', newline='') as csvfile:
# first count the lines
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
nrows = sum(1 for row in reader)
flashnr = np.ma.empty(nrows, dtype=int)
time_data = np.ma.empty(nrows, dtype=datetime.datetime)
time_in_flash = np.ma.empty(nrows, dtype=float)
lat = np.ma.empty(nrows, dtype=float)
lon = np.ma.empty(nrows, dtype=float)
alt = np.ma.empty(nrows, dtype=float)
dBm = np.ma.empty(nrows, dtype=float)
pol_vals_dict = dict()
for label in labels:
pol_vals_dict.update({label: np.ma.empty(nrows, dtype=float)})
# now read the data
csvfile.seek(0)
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
for i, row in enumerate(reader):
flashnr[i] = int(row['flashnr'])
time_data[i] = datetime.datetime.strptime(
row['time_data'], '%Y-%m-%d %H:%M:%S.%f')
time_in_flash[i] = float(row['time_in_flash'])
lat[i] = float(row['lat'])
lon[i] = float(row['lon'])
alt[i] = float(row['alt'])
dBm[i] = float(row['dBm'])
for label in labels:
pol_vals_dict[label][i] = float(row[label])
csvfile.close()
for label in labels:
pol_vals_dict[label] = np.ma.masked_values(
pol_vals_dict[label], get_fillvalue())
return flashnr, time_data, time_in_flash, lat, lon, alt, dBm, pol_vals_dict
except EnvironmentError as ee:
warn(str(ee))
warn('Unable to read file ' + fname)
return None, None, None, None, None, None, None, None
def histograms_from_radar(
radar, hist_dict, gatefilter=None, min_ncp=None, min_sweep=None,
max_sweep=None, min_range=None, max_range=None, fill_value=None,
ncp_field=None, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# TODO: check input histogram dictionary for proper keys
# Loop over all fields and compute histogram counts
for field in hist_dict:
# Parse radar fields
data = radar.fields[field]['data']
# Mask sweeps outside specified range
if min_sweep is not None:
i = radar.sweep_start_ray_index['data'][min_sweep]
data[:i+1,:] = np.ma.masked
if max_sweep is not None:
i = radar.sweep_end_ray_index['data'][max_sweep]
data[i+1:,:] = np.ma.masked
# Mask radar range gates outside specified range
if min_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - min_range).argmin()
data[:,:i+1] = np.ma.masked
if max_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - max_range).argmin()
data[:,i+1:] = np.ma.masked
# Mask incoherent echoes
if min_ncp is not None:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Mask excluded gates
if gatefilter is not None:
data = np.ma.masked_where(
gatefilter.gate_excluded, data)
# Parse histogram parameters
bins = hist_dict[field]['number of bins']
limits = hist_dict[field]['limits']
# Bin data and compute frequencies
counts, bin_edges = np.histogram(
data.compressed(), bins=bins, range=limits, normed=False,
weights=None, density=False)
hist_dict[field]['histogram counts'] += counts
# Parse bin edges and bin centers
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
hist_dict[field]['bin edges'] = bin_edges
hist_dict[field]['bin centers'] = bin_centers
return
def read_trt_cell_lightning(fname):
"""
Reads the lightning data of a TRT cell. The file has the following
fields:
traj_ID
yyyymmddHHMM
lon
lat
area
RANKr
nflashes
flash_dens
Parameters
----------
fname : str
path of the TRT data file
Returns
-------
A tupple containing the read values. None otherwise
"""
try:
with open(fname, 'r', newline='') as csvfile:
# first count the lines
reader = csv.DictReader(
(row for row in csvfile if not row.startswith('#')),
delimiter=',')
nrows = sum(1 for row in reader)
if nrows == 0:
warn('No data in file ' + fname)
return None, None, None, None, None, None, None, None
traj_ID = np.empty(nrows, dtype=int)
time_cell = np.empty(nrows, dtype=datetime.datetime)
lon_cell = np.empty(nrows, dtype=float)
lat_cell = np.empty(nrows, dtype=float)
area_cell = np.empty(nrows, dtype=float)
rank_cell = np.empty(nrows, dtype=float)
nflashes_cell = np.ma.empty(nrows, dtype=float)
flash_dens_cell = np.ma.empty(nrows, dtype=float)
# now read the data
csvfile.seek(0)
reader = csv.DictReader(
(row for row in csvfile if not row.startswith('#')),
delimiter=',')
for i, row in enumerate(reader):
traj_ID[i] = int(row['traj_ID'])
time_cell[i] = datetime.datetime.strptime(
row['yyyymmddHHMM'], '%Y%m%d%H%M')
lon_cell[i] = float(row['lon'])
lat_cell[i] = float(row['lat'])
area_cell[i] = float(row['area'])
rank_cell[i] = float(row['RANKr'])
nflashes_cell[i] = float(row['nflashes'])
flash_dens_cell[i] = float(row['flash_dens'])
csvfile.close()
nflashes_cell = np.ma.masked_values(nflashes_cell, get_fillvalue())
flash_dens_cell = np.ma.masked_values(flash_dens_cell,
get_fillvalue())
return (traj_ID, time_cell, lon_cell, lat_cell, area_cell,
rank_cell, nflashes_cell, flash_dens_cell)
except EnvironmentError as ee:
warn(str(ee))
warn('Unable to read file ' + fname)
return None, None, None, None, None, None, None, None
def histogram_from_json(
filename, field, inpdir=None, bins=10, limits=None, min_ncp=0.5,
vcp_sweeps=None, vcp_rays=None, min_sweep=None, max_sweep=None,
exclude_fields=None, fill_value=None, ncp_field=None, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# Parse files from JSON file
with open(filename, 'r') as fid:
files = json.load(fid)
# Append input directory if given
if inpdir is not None:
files = [os.path.join(inpdir, f) for f in files]
if verbose:
print 'Total number of radar files to process = %i' % len(files)
# Loop over all files
histogram = np.zeros(bins, dtype=np.float64)
for f in files:
# Read radar data
radar = read(f, exclude_fields=exclude_fields)
# Check radar VCP
if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps:
continue
if vcp_rays is not None and radar.nrays != vcp_rays:
continue
if verbose:
print 'Processing file %s' % os.path.basename(f)
# Parse radar fields
data = radar.fields[field]['data']
# Mask sweeps outside specified range
if min_sweep is not None:
i = radar.sweep_start_ray_index['data'][min_sweep]
data[:i+1,:] = np.ma.masked
if max_sweep is not None:
i = radar.sweep_end_ray_index['data'][max_sweep]
data[i+1:,:] = np.ma.masked
# Mask incoherent echoes
if min_ncp is not None:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Bin data and compute frequencies
hist, bin_edges = np.histogram(
data.compressed(), bins=bins, range=limits, normed=False,
weights=None, density=False)
histogram += hist
# Compute bin centers
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
# Compute normalized histogram and probability density
histogram_norm = histogram / histogram.max()
pdf = histogram_norm / np.sum(histogram_norm * np.diff(bin_edges))
return {
'field': field,
'histogram counts': histogram,
'normalized histogram': histogram_norm,
'probability density': pdf,
'number of bins': bins,
'limits': limits,
'bin edges': bin_edges,
'bin centers': bin_centers,
'radar files': [os.path.basename(f) for f in files],
'min sweep': min_sweep,
'max sweep': max_sweep,
'min normalized coherent power': min_ncp,
'sweeps in VCP': vcp_sweeps,
'rays in VCP': vcp_rays,
}
def c98dfile_archive(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
cutnum=None,
delay_field_loading=False,
**kwargs):
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('c98d_archive', field_names,
additional_metadata, file_field_names,
exclude_fields)
nfile = C98DRadFile(prepare_for_read(filename))
# scan_info = nfile.scan_info
if cutnum is None:
cutnum = nfile.scans
else:
cutnum = list(cutnum)
# time
time = filemetadata('time')
_time = nfile.radial_info['seconds']
time['data'] = _time
time['units'] = nfile.get_volume_start_time
# range
_range = filemetadata('range')
_range['data'] = nfile.get_range('dBZ', cutnum)
# _range['meters_to_center_of_first_gate'] = float(first_gate)
# _range['meters_between_gates'] = float(gate_spacing)
# fields
fields = {}
field_names = nfile.get_moment_type
for field_name in field_names:
dic = filemetadata(field_name)
for i, cn in enumerate(cutnum):
dic['_FillValue'] = get_fillvalue()
if i == 0:
dic['data'] = nfile.get_data(field_name, cn)
else:
fndata = nfile.get_data(field_name, cn)
if fndata.shape[1] != dic['data'].shape[1]:
if dic['data'].shape[1] - fndata.shape[1] > 0:
apps = np.ones(
(fndata.shape[0],
dic['data'].shape[1] - fndata.shape[1])) * np.nan
else:
raise ValueError('something wrong!')
fndata = np.c_[fndata, apps]
dic['data'] = np.append(dic['data'], fndata, axis=0)
fields.update({field_name: dic})
# scan_type
scan_type = nfile.scan_type
# latitude, longitude, altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
lat, lon, height = nfile.get_location
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([height], dtype='float64')
# sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index
# sweep_end_ray_index
sweep_number = filemetadata('sweep_number')
sweep_mode = filemetadata('sweep_mode')
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_number['data'] = np.arange(nfile.cutnum, dtype='int32')
sweep_mode['data'] = np.array(nfile.cutnum * ['azimuth_surveillance'],
dtype='S')
sweep_end_ray_index['data'] = np.array(nfile.cut_end)
sweep_start_ray_index['data'] = np.array(nfile.cut_start)
# azimuth, elevation, fixed_angle
azimuth = filemetadata('azimuth')
elevation = filemetadata('elevation')
fixed_angle = filemetadata('fixed_angle')
azimuth['data'] = np.array(nfile.get_azimuth)
elevation['data'] = np.array(nfile.get_elevation)
fixed_angle['data'] = nfile.get_target_angles
# instrument_parameters
nyquist_velocity = filemetadata('nyquist_velocity')
unambiguous_range = filemetadata('unambiguous_range')
nyquist_velocity['data'] = None
unambiguous_range['data'] = None
instrument_parameters = {
'unambiguous_range': unambiguous_range,
'nyquist_velocity': nyquist_velocity
}
return Radar(time,
_range,
fields,
filemetadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=None)
def read_sun_retrieval(fname):
"""
Reads sun retrieval data contained in a csv file
Parameters
----------
fname : str
path of time series file
Returns
-------
first_hit_time, last_hit_time, nhits_h, el_width_h, az_width_h, el_bias_h,
az_bias_h, dBm_sun_est, std_dBm_sun_est, sf_h,
nhits_v, el_width_v, az_width_v, el_bias_v, az_bias_v, dBmv_sun_est,
std_dBmv_sun_est, sf_v,
nhits_zdr, zdr_sun_est, std_zdr_sun_est,
sf_ref, ref_time : tupple
Each parameter is an array containing a time series of information on
a variable
"""
try:
with open(fname, 'r', newline='') as csvfile:
# first count the lines
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
nrows = sum(1 for row in reader)
nhits_h = np.empty(nrows, dtype=int)
el_width_h = np.ma.empty(nrows, dtype=float)
az_width_h = np.ma.empty(nrows, dtype=float)
el_bias_h = np.ma.empty(nrows, dtype=float)
az_bias_h = np.ma.empty(nrows, dtype=float)
dBm_sun_est = np.ma.empty(nrows, dtype=float)
std_dBm_sun_est = np.ma.empty(nrows, dtype=float)
sf_h = np.ma.empty(nrows, dtype=float)
nhits_v = np.empty(nrows, dtype=int)
el_width_v = np.ma.empty(nrows, dtype=float)
az_width_v = np.ma.empty(nrows, dtype=float)
el_bias_v = np.ma.empty(nrows, dtype=float)
az_bias_v = np.ma.empty(nrows, dtype=float)
dBmv_sun_est = np.ma.empty(nrows, dtype=float)
std_dBmv_sun_est = np.ma.empty(nrows, dtype=float)
sf_v = np.ma.empty(nrows, dtype=float)
nhits_zdr = np.empty(nrows, dtype=int)
zdr_sun_est = np.ma.empty(nrows, dtype=float)
std_zdr_sun_est = np.ma.empty(nrows, dtype=float)
sf_ref = np.ma.empty(nrows, dtype=float)
# now read the data
csvfile.seek(0)
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
first_hit_time = list()
last_hit_time = list()
ref_time = list()
for i, row in enumerate(reader):
first_hit_time.append(
datetime.datetime.strptime(row['first_hit_time'],
'%Y%m%d%H%M%S'))
last_hit_time.append(
datetime.datetime.strptime(row['last_hit_time'],
'%Y%m%d%H%M%S'))
nhits_h[i] = int(row['nhits_h'])
el_width_h[i] = float(row['el_width_h'])
az_width_h[i] = float(row['az_width_h'])
el_bias_h[i] = float(row['el_bias_h'])
az_bias_h[i] = float(row['az_bias_h'])
dBm_sun_est[i] = float(row['dBm_sun_est'])
std_dBm_sun_est[i] = float(row['std(dBm_sun_est)'])
sf_h[i] = float(row['sf_h'])
nhits_v[i] = int(row['nhits_v'])
el_width_v[i] = float(row['el_width_v'])
az_width_v[i] = float(row['az_width_v'])
el_bias_v[i] = float(row['el_bias_v'])
az_bias_v[i] = float(row['az_bias_v'])
dBmv_sun_est[i] = float(row['dBmv_sun_est'])
std_dBmv_sun_est[i] = float(row['std(dBmv_sun_est)'])
sf_v[i] = float(row['sf_v'])
nhits_zdr[i] = int(row['nhits_zdr'])
zdr_sun_est[i] = float(row['ZDR_sun_est'])
std_zdr_sun_est[i] = float(row['std(ZDR_sun_est)'])
sf_ref[i] = float(row['sf_ref'])
if row['ref_time'] == 'None':
ref_time.append(None)
else:
ref_time.append(
datetime.datetime.strptime(row['ref_time'],
'%Y%m%d%H%M%S'))
el_width_h = np.ma.masked_values(el_width_h, get_fillvalue())
az_width_h = np.ma.masked_values(az_width_h, get_fillvalue())
el_bias_h = np.ma.masked_values(el_bias_h, get_fillvalue())
az_bias_h = np.ma.masked_values(az_bias_h, get_fillvalue())
dBm_sun_est = np.ma.masked_values(dBm_sun_est, get_fillvalue())
std_dBm_sun_est = np.ma.masked_values(std_dBm_sun_est,
get_fillvalue())
sf_h = np.ma.masked_values(sf_h, get_fillvalue())
el_width_v = np.ma.masked_values(el_width_v, get_fillvalue())
az_width_v = np.ma.masked_values(az_width_v, get_fillvalue())
el_bias_v = np.ma.masked_values(el_bias_v, get_fillvalue())
az_bias_v = np.ma.masked_values(az_bias_v, get_fillvalue())
dBmv_sun_est = np.ma.masked_values(dBmv_sun_est, get_fillvalue())
std_dBmv_sun_est = np.ma.masked_values(std_dBmv_sun_est,
get_fillvalue())
sf_v = np.ma.masked_values(sf_v, get_fillvalue())
zdr_sun_est = np.ma.masked_values(zdr_sun_est, get_fillvalue())
std_zdr_sun_est = np.ma.masked_values(std_zdr_sun_est,
get_fillvalue())
sf_ref = np.ma.masked_values(sf_ref, get_fillvalue())
csvfile.close()
return (first_hit_time, last_hit_time, nhits_h, el_width_h,
az_width_h, el_bias_h, az_bias_h, dBm_sun_est,
std_dBm_sun_est, sf_h, nhits_v, el_width_v, az_width_v,
el_bias_v, az_bias_v, dBmv_sun_est, std_dBmv_sun_est, sf_v,
nhits_zdr, zdr_sun_est, std_zdr_sun_est, sf_ref, ref_time)
except EnvironmentError as ee:
warn(str(ee))
warn('Unable to read file ' + fname)
return (None, None, None, None, None, None, None, None, None, None,
None, None, None, None, None, None, None, None, None, None,
None, None, None)
def read_sun_hits(fname):
"""
Reads sun hits data contained in a csv file
Parameters
----------
fname : str
path of time series file
Returns
-------
date, ray, nrng, rad_el, rad_az, sun_el, sun_az, ph, ph_std, nph, nvalh,
pv, pv_std, npv, nvalv, zdr, zdr_std, nzdr, nvalzdr : tupple
Each parameter is an array containing a time series of information on
a variable
"""
try:
with open(fname, 'r', newline='') as csvfile:
# first count the lines
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
nrows = sum(1 for row in reader)
ray = np.empty(nrows, dtype=int)
nrng = np.empty(nrows, dtype=int)
rad_el = np.empty(nrows, dtype=float)
rad_az = np.empty(nrows, dtype=float)
sun_el = np.empty(nrows, dtype=float)
sun_az = np.empty(nrows, dtype=float)
ph = np.ma.empty(nrows, dtype=float)
ph_std = np.ma.empty(nrows, dtype=float)
nph = np.empty(nrows, dtype=int)
nvalh = np.empty(nrows, dtype=int)
pv = np.ma.empty(nrows, dtype=float)
pv_std = np.ma.empty(nrows, dtype=float)
npv = np.empty(nrows, dtype=int)
nvalv = np.empty(nrows, dtype=int)
zdr = np.ma.empty(nrows, dtype=float)
zdr_std = np.ma.empty(nrows, dtype=float)
nzdr = np.empty(nrows, dtype=int)
nvalzdr = np.empty(nrows, dtype=int)
# now read the data
csvfile.seek(0)
reader = csv.DictReader(row for row in csvfile
if not row.startswith('#'))
date = list()
for i, row in enumerate(reader):
date.append(
datetime.datetime.strptime(row['time'],
'%Y-%m-%d %H:%M:%S.%f'))
ray[i] = int(row['ray'])
nrng[i] = int(row['NPrng'])
rad_el[i] = float(row['rad_el'])
rad_az[i] = float(row['rad_az'])
sun_el[i] = float(row['sun_el'])
sun_az[i] = float(row['sun_az'])
ph[i] = float(row['dBm_sun_hit'])
ph_std[i] = float(row['std(dBm_sun_hit)'])
nph[i] = int(row['NPh'])
nvalh[i] = int(row['NPhval'])
pv[i] = float(row['dBmv_sun_hit'])
pv_std[i] = float(row['std(dBmv_sun_hit)'])
npv[i] = int(row['NPv'])
nvalv[i] = int(row['NPvval'])
zdr[i] = float(row['ZDR_sun_hit'])
zdr_std[i] = float(row['std(ZDR_sun_hit)'])
nzdr[i] = int(row['NPzdr'])
nvalzdr[i] = int(row['NPzdrval'])
ph = np.ma.masked_values(ph, get_fillvalue())
ph_std = np.ma.masked_values(ph_std, get_fillvalue())
pv = np.ma.masked_values(pv, get_fillvalue())
pv_std = np.ma.masked_values(pv_std, get_fillvalue())
zdr = np.ma.masked_values(zdr, get_fillvalue())
zdr_std = np.ma.masked_values(zdr_std, get_fillvalue())
csvfile.close()
return (date, ray, nrng, rad_el, rad_az, sun_el, sun_az, ph,
ph_std, nph, nvalh, pv, pv_std, npv, nvalv, zdr, zdr_std,
nzdr, nvalzdr)
except EnvironmentError as ee:
warn(str(ee))
warn('Unable to read file ' + fname)
return (None, None, None, None, None, None, None, None, None, None,
None, None, None, None, None, None, None, None, None)
def grid_radar(radar, domain, weight=None, fields=None, gatefilter=None,
toa=17000.0, max_range=None, legacy=False, fill_value=None,
dist_field=None, weight_field=None, time_field=None,
gqi_field=None, range_field=None, azimuth_field=None,
elevation_field=None, debug=False, verbose=False):
"""
Map volumetric radar data to a rectilinear grid. This routine uses a k-d
tree space-partitioning data structure for the efficient searching of the
k-nearest neighbours.
Parameters
----------
radar : pyart.core.Radar
Radar containing the fields to be mapped.
domain : Domain
Grid domain.
weight : Weight, optional
Weight defining the radar data objective analysis parameters and
available kd-tree information. If None, a one-pass isotropic
distance-dependent Barnes weight with a constant smoothing parameter
is used.
fields : sequence of str, optional
Radar fields to be mapped. If None, all available radar fields are
mapped.
gatefilter : pyart.filters.GateFilter, optional
GateFilter used to determine the grid quality index. If None, no grid
quality index field is returned.
Optional parameters
-------------------
toa : float, optional
Top of the atmosphere in meters. Radar gates above this altitude are
ignored. Lower heights will increase processing time but may also
produce poor results if the height is similar to the top level of the
grid.
max_range : float, optional
Grid points further than `max_range` from radar are excluded from
mapping. If None, the maximum range of the radar is used.
legacy : bool, optional
True to return a legacy Py-ART Grid. Note that the legacy Grid is
planned for removal altogether in future Py-ART releases.
proc : int, optional
Number of processes to use when querying the k-d tree.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print relevant information, False to suppress.
Return
------
grid : pyart.core.Grid
Grid containing the mapped volumetric radar data.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if dist_field is None:
dist_field = get_field_name('nearest_neighbor_distance')
if weight_field is None:
weight_field = get_field_name('nearest_neighbor_weight')
if time_field is None:
time_field = get_field_name('nearest_neighbor_time')
if gqi_field is None:
gqi_field = get_field_name('grid_quality_index')
if range_field is None:
range_field = get_field_name('range')
if azimuth_field is None:
azimuth_field = get_field_name('azimuth')
if elevation_field is None:
elevation_field = get_field_name('elevation')
# Parse fields to map
if fields is None:
fields = radar.fields.keys()
elif isinstance(fields, str):
fields = [fields]
fields = [field for field in fields if field in radar.fields]
# Parse radar data objective analysis weight
if weight is None:
weight = Weight(radar)
# Parse maximum range
if max_range is None:
max_range = radar.range['data'].max()
# Calculate radar offset relative to the analysis grid origin
domain.compute_radar_offset_from_origin(radar, debug=debug)
# Compute Cartesian coordinates of radar gates relative to specified origin
# Add reference gate locations and current gate locations to weight object
# which will help determine if the kd-tree needs to be requeried or not
z_g, y_g, x_g = transform.equivalent_earth_model(
radar, offset=domain.radar_offset, debug=debug, verbose=verbose)
weight._add_gate_reference([z_g, y_g, x_g], replace_existing=False)
weight._add_gate_coordinates([z_g, y_g, x_g])
if debug:
print 'Number of radar gates before pruning: {}'.format(z_g.size)
# Do not consider radar gates that are above the "top of the atmosphere"
is_below_toa = z_g <= toa
if debug:
N = is_below_toa.sum()
print 'Number of radar gates below TOA: {}'.format(N)
# Slice radar coordinates below the TOA
z_g = z_g[is_below_toa]
y_g = y_g[is_below_toa]
x_g = x_g[is_below_toa]
# Slice radar data fields below the TOA but preserve original radar data
radar_data = {}
for field in fields:
data = radar.fields[field]['data'].copy().flatten()
radar_data[field] = data[is_below_toa]
# Parse coordinates of analysis grid
z_a, y_a, x_a = domain.z, domain.y, domain.x
nz, ny, nx = domain.nz, domain.ny, domain.nx
if debug:
print 'Number of x grid points: {}'.format(nx)
print 'Number of y grid points: {}'.format(ny)
print 'Number of z grid points: {}'.format(nz)
# Create analysis domain coordinates mesh
z_a, y_a, x_a = np.meshgrid(z_a, y_a, x_a, indexing='ij')
z_a, y_a, x_a = z_a.flatten(), y_a.flatten(), x_a.flatten()
if debug:
print 'Grid 1-D array shape: {}'.format(z_a.shape)
# Query the radar gate k-d tree for the k-nearest analysis grid points.
# Also compute the distance-dependent weights
# This is the step that consumes the most processing time, but it can be
# skipped if results from a similar radar volume have already computed and
# stored in the weight object
if weight.requery(verbose=verbose):
# Create k-d tree object from radar gate locations
# Depending on the number of radar gates this can be resource intensive
# but nonetheless should take on the order of 1 second to create
weight.create_radar_tree(
zip(z_g, y_g, x_g), replace_existing=True, debug=debug,
verbose=verbose)
_, _ = weight.query_tree(
zip(z_a, y_a, x_a), store=True, debug=debug, verbose=verbose)
# Compute distance-dependent weights
_ = weight.compute_weights(weight.dists, store=True, verbose=verbose)
# Reset reference radar gate coordinates
weight._reset_gate_reference()
# Missing neighbors are indicated with an index set to tree.n
# This condition will not be met for the nearest neighbor scheme, but
# it can be met for the Cressman and Barnes schemes if the cutoff radius
# is not large enough
is_bad_index = weight.inds == weight.radar_tree.n
if debug:
N = is_bad_index.sum()
print 'Number of invalid indices: {}'.format(N)
# Grid points which are further than the specified maximum range away from
# the radar should not contribute
z_r, y_r, x_r = domain.radar_offset
_range = np.sqrt((z_a - z_r)**2 + (y_a - y_r)**2 + (x_a - x_r)**2)
is_far = _range > max_range
if debug:
N = is_far.sum()
print('Number of analysis points too far from radar: {}'.format(N))
# Populate grid fields
map_fields = {}
for field in fields:
if verbose:
print('Mapping radar field: {}'.format(field))
map_fields[field] = common.populate_field(
radar_data[field], weight.inds, (nz, ny, nx), field,
weights=weight.wq, mask=is_far, fill_value=None)
# Add grid quality index field
if gatefilter is not None:
# Compute distance-dependent weighted average of k-nearest neighbors
# for included gates
sqi = gatefilter.gate_included.flatten()[is_below_toa]
gqi = np.average(sqi[weight.inds], weights=weight.wq, axis=1)
gqi[is_far] = 0.0
map_fields[gqi_field] = get_metadata(gqi_field)
map_fields[gqi_field]['data'] = gqi.reshape(
nz, ny, nx).astype(np.float32)
# Add nearest neighbor distance field
map_fields[dist_field] = get_metadata(dist_field)
map_fields[dist_field]['data'] = weight.dists[:,0].reshape(
nz, ny, nx).astype(np.float32)
# Add nearest neighbor weight field
map_fields[weight_field] = get_metadata(weight_field)
map_fields[weight_field]['data'] = weight.wq[:,0].reshape(
nz, ny, nx).astype(np.float32)
# Add nearest neighbor time field
time = radar.time['data'][:,np.newaxis].repeat(
radar.ngates, axis=1).flatten()[is_below_toa][weight.inds]
map_fields[time_field] = get_metadata(time_field)
map_fields[time_field]['data'] = time[:,0].reshape(
nz, ny, nx).astype(np.float32)
map_fields[time_field]['units'] = radar.time['units']
# Populate grid metadata
metadata = common._populate_metadata(radar, weight=weight)
if legacy:
axes = common._populate_legacy_axes(radar, domain)
grid = Grid.from_legacy_parameters(map_fields, axes, metadata)
else:
grid = None
return grid
def height_histogram_from_radar(radar,
hist_dict,
gatefilter=None,
min_ncp=None,
min_sweep=None,
max_sweep=None,
min_range=None,
max_range=None,
fill_value=None,
ncp_field=None,
verbose=False,
debug=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# TODO: check input histogram dictionary for proper keys
# Compute locations of radar gates and convert to kilometers
x, y, heights = common.standard_refraction(radar, use_km=True)
if debug:
print '(Min, Max) radar gate height = ({:.3f}, {:.3f}) km'.format(
heights.min(), heights.max())
# Mask sweeps outside specified range
if min_sweep is not None:
i = radar.sweep_start_ray_index['data'][min_sweep]
heights[:i + 1, :] = np.ma.masked
if max_sweep is not None:
i = radar.sweep_end_ray_index['data'][max_sweep]
heights[i + 1:, :] = np.ma.masked
# Mask radar range gates outside specified range
if min_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - min_range).argmin()
heights[:, :i + 1] = np.ma.masked
if max_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - max_range).argmin()
heights[:, i + 1:] = np.ma.masked
# Mask incoherent echoes
if min_ncp is not None:
ncp = radar.fields[ncp_field]['data']
heights = np.ma.masked_where(ncp < min_ncp, heights)
# Mask excluded gates
if gatefilter is not None:
heights = np.ma.masked_where(gatefilter.gate_excluded, heights)
# Parse histogram parameters
bins = hist_dict['number of bins']
limits = hist_dict['limits']
# Compute histogram counts
counts, bin_edges = np.histogram(heights.compressed(),
bins=bins,
range=limits,
normed=False,
weights=None,
density=False)
hist_dict['histogram counts'] += counts
# Parse bin edges and bin centers
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
hist_dict['bin edges'] = bin_edges
hist_dict['bin centers'] = bin_centers
return
def remove_salt(radar, fields=None, salt_window=(3, 3), salt_sample=5,
rays_wrap_around=False, mask_data=True, fill_value=None,
debug=False, verbose=False):
"""
Remove basic salt and pepper noise from radar fields. Noise removal is done
in-place for each radar field, i.e., no new field is created, rather
the original field is changed.
Parameters
----------
radar : Radar
Radar object containing the specified fields for noise removal.
fields : str, list or tuple, optional
The field(s) which will have basic salt and pepper noised removed.
salt_window : tuple or list, optional
The 2-D (ray, gate) window filter used to determine whether a radar
gate is isolated (noise) or part of a larger feature.
salt_sample : int, optional
The minimum sample size within 'salt_window' for a radar gate to be
considered part of a larger feature. If the sample size within
salt_window is below this value, then the radar gate is considered to
be isolated and therefore salt and pepper noise.
rays_wrap_around : bool, optional
mask_data : bool, optional
Whether the radar field(s) should be masked after salt and pepper noise
is removed. This should be set to False when the field in question is
a binary (mask) field, e.g., radar significant detection mask.
fill_value : float, optional
The fill value for radar fields. If not specified, the default fill
value from the Py-ART configuration is used.
debug, verbose : bool, optional
True to print debugging and progress information, respectively, False
to suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if fields is None:
fields = radar.fields.keys()
# Check if input fields is single field
if isinstance(fields, str):
fields = [fields]
# Parse sweep start/end indices
# Offset indices in order to be compatible with Fortran and avoid a
# segmentation fault
sweep_start = radar.sweep_start_ray_index['data'] + 1
sweep_end = radar.sweep_end_ray_index['data'] + 1
# Parse window size
ray_window, gate_window = salt_window
# Remove salt and pepper noise for each field
for field in fields:
if debug:
print 'Removing salt and pepper noise: {}'.format(field)
# Parse radar data and its original data type
data = radar.fields[field]['data']
dtype_orig = data.dtype
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size before salt removal: {}'.format(N)
# Fortran wrapper
sweeps.remove_salt(
data, sweep_start, sweep_end, ray_window=ray_window,
gate_window=gate_window, min_sample=salt_sample,
rays_wrap=rays_wrap_around, fill_value=fill_value)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size after salt removal: {}'.format(N)
# Mask invalid data
if mask_data:
data = np.ma.masked_equal(data, fill_value, copy=False)
data.set_fill_value(fill_value)
radar.fields[field]['data'] = data.astype(dtype_orig)
return
def echo_boundaries(radar,
gatefilter=None,
texture_window=(3, 3),
texture_sample=5,
min_texture=None,
bounds_percentile=95.0,
remove_small_features=False,
size_bins=75,
size_limits=(0, 300),
rays_wrap_around=False,
fill_value=None,
sqi_field=None,
text_field=None,
bounds_field=None,
debug=False,
verbose=False):
"""
Objectively determine the location of significant echo boundaries through
analysis of signal quality index (SQI) texture. The a priori assumption is
that at echo boundaries (e.g., cloud boundaries), the SQI field decreases
substantially and therefore the SQI texture field is large near echo
boundaries.
Parameters
----------
radar : Radar
Radar object containing the SQI field used to derive significant echo
boundaries.
Returns
-------
gatefilter : GateFilter
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if sqi_field is None:
sqi_field = get_field_name('normalized_coherent_power')
if text_field is None:
text_field = '{}_texture'.format(sqi_field)
if bounds_field is None:
bounds_field = 'echo_boundaries_mask'
if verbose:
print 'Computing significant echo boundaries mask'
# Compute signal quality index texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(radar,
sqi_field,
ray_window=ray_window,
gate_window=gate_window,
min_sample=texture_sample,
min_ncp=None,
min_sweep=None,
max_sweep=None,
rays_wrap_around=rays_wrap_around,
fill_value=fill_value,
text_field=text_field,
ncp_field=None)
if min_texture is None:
# The specified boundary percentile defines the minimum SQI texture
# value for significant echo boundaries
min_texture = np.percentile(
radar.fields[text_field]['data'].compressed(),
bounds_percentile,
overwrite_input=False,
interpolation='linear')
if debug:
max_texture = radar.fields[text_field]['data'].max()
_range = [round(min_texture, 3), round(max_texture, 3)]
print 'Echo boundary SQI texture range: {}'.format(_range)
# Compute percentiles for debugging purposes
percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 100]
textures = np.percentile(radar.fields[text_field]['data'].compressed(),
percentiles,
overwrite_input=False,
interpolation='linear')
if debug:
for p, texture in zip(percentiles, textures):
print '{}% SQI texture = {:.5f}'.format(p, texture)
# Determine radar gates which meet minimum normalized coherent power
# texture
is_boundary = radar.fields[text_field]['data'] >= min_texture
is_boundary = np.ma.filled(is_boundary, False)
# Create significant echo boundaries field dictionary
bounds_dict = {
'data': is_boundary.astype(np.int8),
'standard_name': bounds_field,
'long_name': 'Significant echo boundaries mask',
'_FillValue': None,
'units': 'unitless',
'comment': '0 = not an echo boundary, 1 = echo boundary',
}
radar.add_field(bounds_field, bounds_dict, replace_existing=True)
# Remove insignificant features from significant echo boundaries mask
if remove_small_features:
basic_fixes._binary_significant_features(radar,
bounds_field,
size_bins=size_bins,
size_limits=size_limits,
structure=structure,
debug=debug,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(bounds_field, 1, op='and')
return gatefilter
def histograms_from_radar(radar,
hist_dict,
gatefilter=None,
texture_window=(3, 3),
texture_sample=5,
min_ncp=None,
min_sweep=None,
max_sweep=None,
min_range=None,
max_range=None,
rays_wrap_around=False,
fill_value=None,
ncp_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# TODO: check input histogram dictionary for proper keys
# Parse texture window parameters
ray_window, gate_window = texture_window
# Loop over all fields and compute histogram counts
for field in hist_dict:
# Compute texture fields
_compute_field(radar,
field,
gatefilter=gatefilter,
ray_window=ray_window,
gate_window=gate_window,
min_sample=texture_sample,
min_ncp=min_ncp,
min_sweep=min_sweep,
max_sweep=max_sweep,
min_range=min_range,
max_range=max_range,
rays_wrap_around=rays_wrap_around,
fill_value=fill_value,
ncp_field=ncp_field)
# Parse histogram parameters
bins = hist_dict[field]['number of bins']
limits = hist_dict[field]['limits']
# Parse data and compute histogram
data = radar.fields['{}_texture'.format(field)]['data']
counts, bin_edges = np.histogram(data.compressed(),
bins=bins,
range=limits,
normed=False,
weights=None,
density=False)
hist_dict[field]['histogram counts'] += counts
# Compute bin centers and add to dictionary
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
hist_dict[field]['bin edges'] = bin_edges
hist_dict[field]['bin centers'] = bin_centers
return
def significant_features(
radar, fields, gatefilter=None, size_bins=100, size_limits=(0, 400),
structure=None, save_size_field=False, fill_value=None, debug=False,
verbose=False):
"""
Determine significant radar echo features on a sweep by sweep basis by
computing the size each echo feature. Here an echo feature is defined as
multiple connected radar gates with valid data, where the connection
structure is defined by the user.
Parameters
----------
radar : Radar
Py-ART Radar containing
fields : str or list or tuple
Radar fields to be used to identify significant echo featues.
gatefilter : GateFilter
Py-ART GateFilter instance.
size_bins : int, optional
Number of size bins used to bin feature size distribution.
size_limits : list or tuple, optional
Lower and upper limits of the feature size distribution. This together
with size_bins defines the bin width of the feature size distribution.
structure : array_like, optional
Binary structuring element used to define connected radar gates. The
default defines a structuring element in which diagonal radar gates are
not considered connected.
save_size_field : bool, optional
True to save size fields in the radar object, False to discard.
debug : bool, optional
True to print debugging information, False to suppress.
Returns
-------
gf : GateFilter
Py-ART GateFilter.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse radar fields
if isinstance(fields, str):
fields = [fields]
# Parse gate filter
if gatefilter is None:
gf = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
for field in fields:
if verbose:
print 'Processing echo features: {}'.format(field)
# Initialize echo feature size array
size_data = np.zeros_like(
radar.fields[field]['data'], subok=False, dtype=np.int32)
feature_sizes = []
for sweep, _slice in enumerate(radar.iter_slice()):
# Parse radar sweep data and define only valid gates
data = radar.get_field(sweep, field, copy=False)
is_valid_gate = ~np.ma.getmaskarray(data)
# Label the connected features in the radar sweep data and create
# index array which defines each unique label (feature)
labels, nlabels = ndimage.label(
is_valid_gate, structure=structure, output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Unique features in sweep {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero,
np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[_slice][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Bin and compute feature size occurrences
counts, bin_edges = np.histogram(
feature_sizes, bins=size_bins, range=size_limits, normed=False,
weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution. We expect the
# peak of the echo feature size distribution to be close to 1 radar
# gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak. This will
# define the minimum echo feature size
is_zero_size = np.logical_and(
bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-1))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Parse echo feature size field name
size_field = '{}_feature_size'.format(field)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'long_name': 'Echo feature size in number of radar gates',
'_FillValue': size_data.fill_value,
'units': 'unitless',
'comment': None,
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gf.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field if specified
if not save_size_field:
radar.fields.pop(size_field, None)
return gf
def hildebrand_noise(radar,
gatefilter=None,
scale=1.0,
remove_small_features=False,
size_bins=75,
size_limits=(0, 300),
rays_wrap_around=False,
fill_value=None,
power_field=None,
noise_field=None,
mask_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if power_field is None:
power_field = get_field_name('signal_to_noise_ratio')
if noise_field is None:
noise_field = 'radar_noise_floor'
if mask_field is None:
mask_field = 'radar_noise_floor_mask'
# Parse radar power data
power = radar.fields[power_field]['data']
# Prepare data for ingest into Fortran wrapper
power = np.ma.filled(power, fill_value)
power = np.asfortranarray(power, dtype=np.float64)
# Convert power units to linear and sort in descending order
# TODO: check if units are already linear
power = np.where(power != fill_value, 10.0**(power / 10.0), power)
power = np.sort(power, axis=0, kind='mergesort')[::-1]
# Fortran wrapper to Hildebrand and Sekhon (1974) algorithm
P, Q, R2, N = sweeps.hildebrand(power, fill_value=fill_value)
# Mask invalid data
P = np.ma.masked_equal(P, fill_value)
Q = np.ma.masked_equal(Q, fill_value)
R2 = np.ma.masked_equal(R2, fill_value)
N = np.ma.masked_equal(N, fill_value)
# Estimate noise floor in decibels and tile to proper dimensions
noise = 10.0 * np.ma.log10(P + scale * np.ma.sqrt(Q))
noise = np.tile(noise, (radar.nrays, 1))
noise.set_fill_value(fill_value)
# Add Hildebrand noise floor field to radar
noise_dict = {
'data':
noise.astype(np.float32),
'long_name':
'Radar noise floor estimate',
'standard_name':
noise_field,
'units':
'dB',
'_FillValue':
noise.fill_value,
'comment': ('Noise floor is estimated using Hildebrand and '
'Sekhon (1974) algorithm'),
}
radar.add_field(noise_field, noise_dict, replace_existing=True)
# Compute noise floor mask
power = radar.fields[power_field]['data']
noise = radar.fields[noise_field]['data']
is_noise = np.ma.filled(power >= noise, False)
# Create radar noise floor mask dictionary
mask_dict = {
'data': is_noise.astype(np.int8),
'long_name': 'Noise floor mask',
'standard_name': mask_field,
'valid_min': 0,
'valid_max': 1,
'units': 'unitless',
'_FillValue': None,
'comment': '0 = below noise floor, 1 = at or above noise floor',
}
radar.add_field(mask_field, mask_dict, replace_existing=True)
# Remove insignificant features from noise floor mask
if remove_small_features:
basic_fixes._binary_significant_features(radar,
mask_field,
size_bins=size_bins,
size_limits=size_limits,
structure=structure,
debug=debug,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(mask_field, 1, op='and')
return gatefilter
def _compute_field(radar,
field,
gatefilter=None,
ray_window=3,
gate_window=3,
min_sample=5,
min_ncp=None,
min_sweep=None,
max_sweep=None,
min_range=None,
max_range=None,
rays_wrap_around=False,
fill_value=None,
text_field=None,
ncp_field=None):
"""
Compute the texture (standard deviation) within the 2-D window for the
specified field.
Parameters
----------
Optional Parameters
----------------
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
if text_field is None:
text_field = '{}_texture'.format(field)
# Parse radar data
data = radar.fields[field]['data']
# Mask sweeps outside of specified range
if min_sweep is not None:
i = radar.sweep_start_ray_index['data'][min_sweep]
data[:i + 1, :] = np.ma.masked
if max_sweep is not None:
i = radar.sweep_end_ray_index['data'][max_sweep]
data[i + 1:, :] = np.ma.masked
# Mask radar range gates outside specified range
if min_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - min_range).argmin()
data[:, :i + 1] = np.ma.masked
if max_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - max_range).argmin()
data[:, i + 1:] = np.ma.masked
# Mask incoherent echoes
if min_ncp is not None and ncp_field in radar.fields:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Mask excluded gates
if gatefilter is not None:
data = np.ma.masked_where(gatefilter.gate_excluded, data)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Parse sweep start/end indices
# Offset indices in order to be compatible with Fortran and avoid a
# segmentation fault
sweep_start = radar.sweep_start_ray_index['data'] + 1
sweep_end = radar.sweep_end_ray_index['data'] + 1
# Compute texture field
sample_size, texture = compute_texture.compute(data,
sweep_start,
sweep_end,
ray_window=ray_window,
gate_window=gate_window,
fill_value=fill_value)
# Mask pixels (gates) where the sample size used to compute the texture
# field was too small
if min_sample is not None:
texture = np.ma.masked_where(sample_size < min_sample,
texture,
copy=False)
# Mask invalid values
texture = np.ma.masked_equal(texture, fill_value, copy=False)
texture = np.ma.masked_invalid(texture, copy=False)
texture.set_fill_value(fill_value)
# Create texture field dictionary and add it to the radar object
texture = {
'data':
texture.astype(np.float32),
'long_name':
'{} texture'.format(radar.fields[field]['long_name']),
'standard_name':
text_field,
'valid_min':
0.0,
'_FillValue':
texture.fill_value,
'units':
radar.fields[field]['units'],
'comment_1': ('Texture field is defined as the standard deviation '
'within a prescribed 2D window'),
'comment_2':
'{} x {} window'.format(gate_window, ray_window),
}
radar.add_field(text_field, texture, replace_existing=True)
return
def velocity_phasor_coherency(radar,
gatefilter=None,
text_bins=40,
text_limits=(0, 20),
nyquist=None,
texture_window=(3, 3),
texture_sample=5,
max_texture=None,
rays_wrap_around=False,
remove_small_features=False,
size_bins=75,
size_limits=(0, 300),
fill_value=None,
vdop_field=None,
phasor_field=None,
text_field=None,
coherent_field=None,
debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if vdop_field is None:
vdop_field = get_field_name('velocity')
if phasor_field is None:
phasor_field = '{}_phasor_real'.format(vdop_field)
if text_field is None:
text_field = '{}_texture'.format(phasor_field)
if coherent_field is None:
coherent_field = '{}_coherency_mask'.format(phasor_field)
if verbose:
print 'Computing Doppler velocity phasor coherency mask'
# Parse Nyquist velocity
if nyquist is None:
nyquist = radar.get_nyquist_vel(0, check_uniform=True)
if debug:
print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist)
# Compute the real part of Doppler velocity phasor
# Normalize real part of phasor to the Nyquist interval
vdop = radar.fields[vdop_field]['data']
phasor_real = nyquist * np.cos(np.radians(360.0 * vdop / nyquist))
# Mask invalid values
phasor_real = np.ma.masked_invalid(phasor_real)
phasor_real.set_fill_value(fill_value)
# Create Doppler velocity phasor field dictionary
phasor_dict = {
'data':
phasor_real.astype(np.float32),
'long_name':
'Real part of Doppler velocity phasor',
'standard_name':
phasor_field,
'valid_min':
-nyquist,
'valid_max':
nyquist,
'_FillValue':
phasor_real.fill_value,
'units':
'meters_per_second',
'comment': ('Real part of Doppler velocity phasor normalized to the '
'Nyquist interval'),
}
radar.add_field(phasor_field, phasor_dict, replace_existing=True)
# Compute Doppler velocity phasor texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(radar,
phasor_field,
ray_window=ray_window,
gate_window=gate_window,
min_sample=texture_sample,
min_ncp=None,
min_sweep=None,
max_sweep=None,
fill_value=fill_value,
ncp_field=None)
# Automatically bracket coherent part of Doppler velocity phasor texture
# distribution
if max_texture is None:
# Bin Doppler velocity phasor texture data and count occurrences
# Compute bin centers and bin width
counts, bin_edges = np.histogram(
radar.fields[text_field]['data'].compressed(),
bins=text_bins,
range=text_limits,
normed=False,
weights=None,
density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {:.3f} m/s'.format(bin_width)
# Determine positions of the extrema in the Doppler velocity phasor
# texture distribution
kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0]
kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0]
if debug:
print 'Minima located at: {} m/s'.format(bin_centers[kmin])
print 'Maxima located at: {} m/s'.format(bin_centers[kmax])
# Compute the theoretical noise peak location from Guassian noise
# statistics
noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0)
if debug:
print 'Theoretical noise peak: {:.3f} m/s'.format(
noise_peak_theory)
# Find the closest Doppler velocity phasor texture distribution peak to
# the computed theoretical location
# Here we assume that the Doppler velocity phasor texture distribution
# has at least one primary mode which correspondes to the incoherent
# (noisy) part of the Doppler velocity phasor texture distribution
# Depending on the radar volume and the bin width used to define the
# distribution, the distribution may be bimodal, with the new peak
# corresponding to the coherent part of the Doppler velocity phasor
# texture distribution
idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin()
noise_peak = bin_centers[kmax][idx]
if debug:
print 'Computed noise peak: {:.3f} m/s'.format(noise_peak)
# Determine primary and secondary peak locations for debugging
# purposes
if kmax.size > 1:
counts_max = np.sort(counts[kmax], kind='mergesort')[::-1]
prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()]
sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()]
if debug:
print 'Primary peak: {:.3f} m/s'.format(prm_peak)
print 'Secondary peak: {:.3f} m/s'.format(sec_peak)
# Determine the left edge of the noise distribution
# Where this distribution becomes a minimum will define the separation
# between coherent and incoherent Doppler velocity phasor values
is_left_side = bin_centers[kmin] < noise_peak
max_texture = bin_centers[kmin][is_left_side].max() + bin_width / 2.0
if debug:
_range = [0.0, round(max_texture, 3)]
print 'Doppler velocity phasor coherency mode: {} m/s'.format(
_range)
# Create Doppler velocity phasor coherency mask
is_coherent = np.logical_and(
radar.fields[text_field]['data'] >= 0.0,
radar.fields[text_field]['data'] <= max_texture)
is_coherent = np.ma.filled(is_coherent, False)
# Create Doppler velocity phasor coherency mask dictionary
coherent_dict = {
'data': is_coherent.astype(np.int8),
'long_name': 'Doppler velocity phasor coherency',
'standard_name': coherent_field,
'valid_min': 0,
'valid_max': 1,
'_FillValue': None,
'units': 'unitless',
'comment': '0 = incoherent value, 1 = coherent value',
}
radar.add_field(coherent_field, coherent_dict, replace_existing=True)
# Remove insignificant features from Doppler velocity phasor coherency mask
if remove_small_features:
basic_fixes._binary_significant_features(radar,
coherent_field,
size_bins=size_bins,
size_limits=size_limits,
structure=structure,
debug=debug,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(coherent_field, 1, op='and')
return gatefilter
def histogram_from_json(filename,
field,
inpdir=None,
texture_window=(3, 3),
min_sample=5,
num_bins=10,
limits=None,
min_ncp=0.5,
vcp_sweeps=None,
vcp_rays=None,
min_sweep=None,
max_sweep=None,
min_range=None,
max_range=None,
rays_wrap_around=False,
exclude_fields=None,
fill_value=None,
ncp_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# Parse files from JSON file
with open(filename, 'r') as fid:
files = json.load(fid)
# Append input directory if given
if inpdir is not None:
files = [os.path.join(inpdir, f) for f in files]
if verbose:
print 'Total number of radar files to process = %i' % len(files)
# Parse texture window parameters
ray_window, gate_window = texture_window
# Loop over all files
counts = np.zeros(num_bins, dtype=np.float64)
for f in files:
# Read radar data
radar = read(f, exclude_fields=exclude_fields)
# Check radar VCP
if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps:
continue
if vcp_rays is not None and radar.nrays != vcp_rays:
continue
if verbose:
print 'Processing file %s' % os.path.basename(f)
# Compute texture fields
_compute_field(radar,
field,
ray_window=ray_window,
gate_window=gate_window,
min_sample=min_sample,
min_ncp=min_ncp,
min_sweep=min_sweep,
max_sweep=max_sweep,
min_range=min_range,
max_range=max_range,
rays_wrap_around=rays_wrap_around,
fill_value=fill_value,
ncp_field=ncp_field)
# Parse data and compute histogram
data = radar.fields['{}_texture'.format(field)]['data']
hist, bin_edges = np.histogram(data.compressed(),
bins=num_bins,
range=limits,
normed=False,
weights=None,
density=False)
counts += hist
# Compute bin centers
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
# Compute normalized histogram and probability density
counts_norm = counts / counts.max()
pdf = counts_norm / np.sum(counts_norm * np.diff(bin_edges))
return {
'field': '{}_texture'.format(field),
'histogram counts': counts,
'normalized histogram': counts_norm,
'probability density': pdf,
'number of bins': num_bins,
'limits': limits,
'bin edges': bin_edges,
'bin centers': bin_centers,
'radar files': [os.path.basename(f) for f in files],
'min sweep': min_sweep,
'max sweep': max_sweep,
'min range': min_range,
'max range': max_range,
'min normalized coherent power': min_ncp,
'sweeps in VCP': vcp_sweeps,
'rays in VCP': vcp_rays,
'ray window size': ray_window,
'gate window size': gate_window,
}
def _significant_features(radar,
field,
gatefilter=None,
min_size=None,
size_bins=75,
size_limits=(0, 300),
structure=None,
remove_size_field=True,
fill_value=None,
size_field=None,
debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if size_field is None:
size_field = '{}_feature_size'.format(field)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Initialize echo feature size array
size_data = np.zeros_like(radar.fields[field]['data'],
subok=False,
dtype=np.int32)
# Loop over all sweeps
feature_sizes = []
for sweep in radar.iter_slice():
# Parse radar sweep data and define only valid gates
is_valid_gate = ~radar.fields[field]['data'][sweep].mask
# Label the connected features in radar sweep data and create index
# array which defines each unique label (feature)
labels, nlabels = ndimage.label(is_valid_gate,
structure=structure,
output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Number of unique features for {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero, np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Compute histogram of echo feature sizes, bin centers and bin
# width
counts, bin_edges = np.histogram(feature_sizes,
bins=size_bins,
range=size_limits,
normed=False,
weights=None,
density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution
# We expect the peak of the echo feature size distribution to be close to 1
# radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak
# This will define the minimum echo feature size
is_zero_size = np.logical_and(bin_centers > peak_size,
np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'standard_name': size_field,
'long_name': '',
'_FillValue': size_data.fill_value,
'units': 'unitless',
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gatefilter.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field
if remove_size_field:
radar.fields.pop(size_field, None)
return gatefilter
def hildebrand_noise(
radar, gatefilter=None, scale=1.0, remove_small_features=False,
size_bins=75, size_limits=(0, 300), rays_wrap_around=False,
fill_value=None, power_field=None, noise_field=None, mask_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if power_field is None:
power_field = get_field_name('signal_to_noise_ratio')
if noise_field is None:
noise_field = 'radar_noise_floor'
if mask_field is None:
mask_field = 'radar_noise_floor_mask'
# Parse radar power data
power = radar.fields[power_field]['data']
# Prepare data for ingest into Fortran wrapper
power = np.ma.filled(power, fill_value)
power = np.asfortranarray(power, dtype=np.float64)
# Convert power units to linear and sort in descending order
# TODO: check if units are already linear
power = np.where(power != fill_value, 10.0**(power / 10.0), power)
power = np.sort(power, axis=0, kind='mergesort')[::-1]
# Fortran wrapper to Hildebrand and Sekhon (1974) algorithm
P, Q, R2, N = sweeps.hildebrand(power, fill_value=fill_value)
# Mask invalid data
P = np.ma.masked_equal(P, fill_value)
Q = np.ma.masked_equal(Q, fill_value)
R2 = np.ma.masked_equal(R2, fill_value)
N = np.ma.masked_equal(N, fill_value)
# Estimate noise floor in decibels and tile to proper dimensions
noise = 10.0 * np.ma.log10(P + scale * np.ma.sqrt(Q))
noise = np.tile(noise, (radar.nrays, 1))
noise.set_fill_value(fill_value)
# Add Hildebrand noise floor field to radar
noise_dict = {
'data': noise.astype(np.float32),
'long_name': 'Radar noise floor estimate',
'standard_name': noise_field,
'units': 'dB',
'_FillValue': noise.fill_value,
'comment': ('Noise floor is estimated using Hildebrand and '
'Sekhon (1974) algorithm'),
}
radar.add_field(noise_field, noise_dict, replace_existing=True)
# Compute noise floor mask
power = radar.fields[power_field]['data']
noise = radar.fields[noise_field]['data']
is_noise = np.ma.filled(power >= noise, False)
# Create radar noise floor mask dictionary
mask_dict = {
'data': is_noise.astype(np.int8),
'long_name': 'Noise floor mask',
'standard_name': mask_field,
'valid_min': 0,
'valid_max': 1,
'units': 'unitless',
'_FillValue': None,
'comment': '0 = below noise floor, 1 = at or above noise floor',
}
radar.add_field(mask_field, mask_dict, replace_existing=True)
# Remove insignificant features from noise floor mask
if remove_small_features:
basic_fixes._binary_significant_features(
radar, mask_field, size_bins=size_bins, size_limits=size_limits,
structure=structure, debug=debug, verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(mask_field, 1, op='and')
return gatefilter
def _add_texture(
radar, field, gatefilter=None, ray_window=3, gate_window=3,
min_sample=5, min_sweep=None, max_sweep=None, min_range=None,
max_range=None, rays_wrap_around=False, fill_value=None,
text_field=None, debug=False, verbose=False):
"""
Compute the texture field (standard deviation) of the input radar field
within a 1-D or 2-D window.
Parameters
----------
radar : Radar
Py-ART Radar containing specified field.
field : str
Radar field to compute texture field.
gatefilter : GateFilter, optional
Py-ART GateFilter specifying radar gates which should be included when
computing the texture field.
ray_window : int, optional
Number of rays in texture window.
gate_window : int, optional
Number of range gates in texture window.
min_sample : int, optional
Minimum sample size within texture window required to define a valid
texture. Note that a minimum of 2 radar gates are required to compute
the texture field.
min_sweep : int, optional
Minimum sweep number to compute texture field.
max_sweep : int, optional
Maximum sweep number to compute texture field.
min_range : float, optional
Minimum range in meters from radar to compute texture field.
max_range : float, optional
Maximum range in meters from radar to compute texture field.
fill_value : float, optional
Value indicating missing or bad data in radar field data. If None,
default value in Py-ART configuration file is used.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print relevant information, False to suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if text_field is None:
text_field = '{}_texture'.format(field)
# Parse radar data
data = radar.fields[field]['data'].copy()
# Mask sweeps outside of specified sweep range
for sweep, slc in enumerate(radar.iter_slice()):
if min_sweep is not None and sweep < min_sweep:
data[slc] = np.ma.masked
if max_sweep is not None and sweep > max_sweep:
data[slc] = np.ma.masked
# Mask radar range gates outside specified gate range
if min_range is not None:
idx = np.abs(radar.range['data'] - min_range).argmin()
data[:,:idx+1] = np.ma.masked
if max_range is not None:
idx = np.abs(radar.range['data'] - max_range).argmin()
data[:,idx+1:] = np.ma.masked
# Parse gate filter information
if gatefilter is not None:
data = np.ma.masked_where(gatefilter.gate_excluded, data)
if debug:
N = np.ma.count(data)
print 'Sample size of data field: {}'.format(N)
# Parse sweep start and end indices
sweep_start = radar.sweep_start_ray_index['data']
sweep_end = radar.sweep_end_ray_index['data']
# Record starting time
start = time.time()
# Compute texture field
sigma, sample_size = compute_texture(
np.ma.filled(data, fill_value), sweep_start, sweep_end,
ray_window=ray_window, gate_window=gate_window,
rays_wrap_around=rays_wrap_around, fill_value=fill_value,
debug=debug, verbose=verbose)
# Record elapsed time to compute texture
elapsed = time.time() - start
if debug:
print('Elapsed time to compute texture: {:.2f} sec'.format(elapsed))
if min_sample is not None:
sigma = np.ma.masked_where(sample_size < min_sample, sigma)
sigma = np.ma.masked_invalid(sigma)
sigma = np.ma.masked_values(sigma, fill_value, atol=1.0e-5)
if debug:
N = np.ma.count(sigma)
print 'Sample size of texture field: {}'.format(N)
sigma_dict = {
'data': sigma,
'units': '',
'valid_min': 0.0,
'number_of_rays': ray_window,
'number_of_gates': gate_window,
}
radar.add_field(text_field, sigma_dict, replace_existing=True)
return
def echo_boundaries(
radar, gatefilter=None, texture_window=(3, 3), texture_sample=5,
min_texture=None, bounds_percentile=95.0, remove_small_features=False,
size_bins=75, size_limits=(0, 300), rays_wrap_around=False,
fill_value=None, sqi_field=None, text_field=None, bounds_field=None,
debug=False, verbose=False):
"""
Objectively determine the location of significant echo boundaries through
analysis of signal quality index (SQI) texture. The a priori assumption is
that at echo boundaries (e.g., cloud boundaries), the SQI field decreases
substantially and therefore the SQI texture field is large near echo
boundaries.
Parameters
----------
radar : Radar
Radar object containing the SQI field used to derive significant echo
boundaries.
Returns
-------
gatefilter : GateFilter
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if sqi_field is None:
sqi_field = get_field_name('normalized_coherent_power')
if text_field is None:
text_field = '{}_texture'.format(sqi_field)
if bounds_field is None:
bounds_field = 'echo_boundaries_mask'
if verbose:
print 'Computing significant echo boundaries mask'
# Compute signal quality index texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(
radar, sqi_field, ray_window=ray_window, gate_window=gate_window,
min_sample=texture_sample, min_ncp=None, min_sweep=None,
max_sweep=None, rays_wrap_around=rays_wrap_around,
fill_value=fill_value, text_field=text_field, ncp_field=None)
if min_texture is None:
# The specified boundary percentile defines the minimum SQI texture
# value for significant echo boundaries
min_texture = np.percentile(
radar.fields[text_field]['data'].compressed(), bounds_percentile,
overwrite_input=False, interpolation='linear')
if debug:
max_texture = radar.fields[text_field]['data'].max()
_range = [round(min_texture, 3), round(max_texture, 3)]
print 'Echo boundary SQI texture range: {}'.format(_range)
# Compute percentiles for debugging purposes
percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 100]
textures = np.percentile(
radar.fields[text_field]['data'].compressed(), percentiles,
overwrite_input=False, interpolation='linear')
if debug:
for p, texture in zip(percentiles, textures):
print '{}% SQI texture = {:.5f}'.format(p, texture)
# Determine radar gates which meet minimum normalized coherent power
# texture
is_boundary = radar.fields[text_field]['data'] >= min_texture
is_boundary = np.ma.filled(is_boundary, False)
# Create significant echo boundaries field dictionary
bounds_dict = {
'data': is_boundary.astype(np.int8),
'standard_name': bounds_field,
'long_name': 'Significant echo boundaries mask',
'_FillValue': None,
'units': 'unitless',
'comment': '0 = not an echo boundary, 1 = echo boundary',
}
radar.add_field(bounds_field, bounds_dict, replace_existing=True)
# Remove insignificant features from significant echo boundaries mask
if remove_small_features:
basic_fixes._binary_significant_features(
radar, bounds_field, size_bins=size_bins, size_limits=size_limits,
structure=structure, debug=debug, verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(bounds_field, 1, op='and')
return gatefilter
def remove_salt(radar,
fields=None,
salt_window=(3, 3),
salt_sample=5,
rays_wrap_around=False,
mask_data=True,
fill_value=None,
debug=False,
verbose=False):
"""
Remove basic salt and pepper noise from radar fields. Noise removal is done
in-place for each radar field, i.e., no new field is created, rather
the original field is changed.
Parameters
----------
radar : Radar
Radar object containing the specified fields for noise removal.
fields : str, list or tuple, optional
The field(s) which will have basic salt and pepper noised removed.
salt_window : tuple or list, optional
The 2-D (ray, gate) window filter used to determine whether a radar
gate is isolated (noise) or part of a larger feature.
salt_sample : int, optional
The minimum sample size within 'salt_window' for a radar gate to be
considered part of a larger feature. If the sample size within
salt_window is below this value, then the radar gate is considered to
be isolated and therefore salt and pepper noise.
rays_wrap_around : bool, optional
mask_data : bool, optional
Whether the radar field(s) should be masked after salt and pepper noise
is removed. This should be set to False when the field in question is
a binary (mask) field, e.g., radar significant detection mask.
fill_value : float, optional
The fill value for radar fields. If not specified, the default fill
value from the Py-ART configuration is used.
debug, verbose : bool, optional
True to print debugging and progress information, respectively, False
to suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if fields is None:
fields = radar.fields.keys()
# Check if input fields is single field
if isinstance(fields, str):
fields = [fields]
# Parse sweep start/end indices
# Offset indices in order to be compatible with Fortran and avoid a
# segmentation fault
sweep_start = radar.sweep_start_ray_index['data'] + 1
sweep_end = radar.sweep_end_ray_index['data'] + 1
# Parse window size
ray_window, gate_window = salt_window
# Remove salt and pepper noise for each field
for field in fields:
if debug:
print 'Removing salt and pepper noise: {}'.format(field)
# Parse radar data and its original data type
data = radar.fields[field]['data']
dtype_orig = data.dtype
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size before salt removal: {}'.format(N)
# Fortran wrapper
sweeps.remove_salt(data,
sweep_start,
sweep_end,
ray_window=ray_window,
gate_window=gate_window,
min_sample=salt_sample,
rays_wrap=rays_wrap_around,
fill_value=fill_value)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size after salt removal: {}'.format(N)
# Mask invalid data
if mask_data:
data = np.ma.masked_equal(data, fill_value, copy=False)
data.set_fill_value(fill_value)
radar.fields[field]['data'] = data.astype(dtype_orig)
return
def velocity_phasor_coherency(
radar, gatefilter=None, text_bins=40, text_limits=(0, 20),
nyquist=None, texture_window=(3, 3), texture_sample=5,
max_texture=None, rays_wrap_around=False, remove_small_features=False,
size_bins=75, size_limits=(0, 300), fill_value=None, vdop_field=None,
phasor_field=None, text_field=None, coherent_field=None, debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if vdop_field is None:
vdop_field = get_field_name('velocity')
if phasor_field is None:
phasor_field = '{}_phasor_real'.format(vdop_field)
if text_field is None:
text_field = '{}_texture'.format(phasor_field)
if coherent_field is None:
coherent_field = '{}_coherency_mask'.format(phasor_field)
if verbose:
print 'Computing Doppler velocity phasor coherency mask'
# Parse Nyquist velocity
if nyquist is None:
nyquist = radar.get_nyquist_vel(0, check_uniform=True)
if debug:
print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist)
# Compute the real part of Doppler velocity phasor
# Normalize real part of phasor to the Nyquist interval
vdop = radar.fields[vdop_field]['data']
phasor_real = nyquist * np.cos(np.radians(360.0 * vdop / nyquist))
# Mask invalid values
phasor_real = np.ma.masked_invalid(phasor_real)
phasor_real.set_fill_value(fill_value)
# Create Doppler velocity phasor field dictionary
phasor_dict = {
'data': phasor_real.astype(np.float32),
'long_name': 'Real part of Doppler velocity phasor',
'standard_name': phasor_field,
'valid_min': -nyquist,
'valid_max': nyquist,
'_FillValue': phasor_real.fill_value,
'units': 'meters_per_second',
'comment': ('Real part of Doppler velocity phasor normalized to the '
'Nyquist interval'),
}
radar.add_field(phasor_field, phasor_dict, replace_existing=True)
# Compute Doppler velocity phasor texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(
radar, phasor_field, ray_window=ray_window, gate_window=gate_window,
min_sample=texture_sample, min_ncp=None, min_sweep=None,
max_sweep=None, fill_value=fill_value, ncp_field=None)
# Automatically bracket coherent part of Doppler velocity phasor texture
# distribution
if max_texture is None:
# Bin Doppler velocity phasor texture data and count occurrences
# Compute bin centers and bin width
counts, bin_edges = np.histogram(
radar.fields[text_field]['data'].compressed(), bins=text_bins,
range=text_limits, normed=False, weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {:.3f} m/s'.format(bin_width)
# Determine positions of the extrema in the Doppler velocity phasor
# texture distribution
kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0]
kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0]
if debug:
print 'Minima located at: {} m/s'.format(bin_centers[kmin])
print 'Maxima located at: {} m/s'.format(bin_centers[kmax])
# Compute the theoretical noise peak location from Guassian noise
# statistics
noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0)
if debug:
print 'Theoretical noise peak: {:.3f} m/s'.format(
noise_peak_theory)
# Find the closest Doppler velocity phasor texture distribution peak to
# the computed theoretical location
# Here we assume that the Doppler velocity phasor texture distribution
# has at least one primary mode which correspondes to the incoherent
# (noisy) part of the Doppler velocity phasor texture distribution
# Depending on the radar volume and the bin width used to define the
# distribution, the distribution may be bimodal, with the new peak
# corresponding to the coherent part of the Doppler velocity phasor
# texture distribution
idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin()
noise_peak = bin_centers[kmax][idx]
if debug:
print 'Computed noise peak: {:.3f} m/s'.format(noise_peak)
# Determine primary and secondary peak locations for debugging
# purposes
if kmax.size > 1:
counts_max = np.sort(counts[kmax], kind='mergesort')[::-1]
prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()]
sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()]
if debug:
print 'Primary peak: {:.3f} m/s'.format(prm_peak)
print 'Secondary peak: {:.3f} m/s'.format(sec_peak)
# Determine the left edge of the noise distribution
# Where this distribution becomes a minimum will define the separation
# between coherent and incoherent Doppler velocity phasor values
is_left_side = bin_centers[kmin] < noise_peak
max_texture = bin_centers[kmin][is_left_side].max() + bin_width / 2.0
if debug:
_range = [0.0, round(max_texture, 3)]
print 'Doppler velocity phasor coherency mode: {} m/s'.format(
_range)
# Create Doppler velocity phasor coherency mask
is_coherent = np.logical_and(
radar.fields[text_field]['data'] >= 0.0,
radar.fields[text_field]['data'] <= max_texture)
is_coherent = np.ma.filled(is_coherent, False)
# Create Doppler velocity phasor coherency mask dictionary
coherent_dict = {
'data': is_coherent.astype(np.int8),
'long_name': 'Doppler velocity phasor coherency',
'standard_name': coherent_field,
'valid_min': 0,
'valid_max': 1,
'_FillValue': None,
'units': 'unitless',
'comment': '0 = incoherent value, 1 = coherent value',
}
radar.add_field(coherent_field, coherent_dict, replace_existing=True)
# Remove insignificant features from Doppler velocity phasor coherency mask
if remove_small_features:
basic_fixes._binary_significant_features(
radar, coherent_field, size_bins=size_bins,
size_limits=size_limits, structure=structure, debug=debug,
verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(coherent_field, 1, op='and')
return gatefilter
def read_uf(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
delay_field_loading=False,
**kwargs):
"""
Read a UF File.
Parameters
----------
filename : str or file-like
Name of Universal format file to read data from.
field_names : dict, optional
Dictionary mapping UF data type names to radar field names. If a
data type found in the file does not appear in this dictionary or has
a value of None it will not be placed in the radar.fields dictionary.
A value of None, the default, will use the mapping defined in the
Py-ART configuration file.
additional_metadata : dict of dicts, optional
Dictionary of dictionaries to retrieve metadata from during this read.
This metadata is not used during any successive file reads unless
explicitly included. A value of None, the default, will not
introduct any addition metadata and the file specific or default
metadata as specified by the Py-ART configuration file will be used.
file_field_names : bool, optional
True to force the use of the field names from the file in which
case the `field_names` parameter is ignored. False will use to
`field_names` parameter to rename fields.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters.
delay_field_loading : bool
This option is not implemented in the function but included for
compatability.
Returns
-------
radar : Radar
Radar object.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('uf', field_names, additional_metadata,
file_field_names, exclude_fields)
# Open UF file and get handle
ufile = UFFile(filename)
first_ray = ufile.rays[0]
# time
dts = ufile.get_datetimes()
units = make_time_unit_str(min(dts))
time = filemetadata('time')
time['units'] = units
time['data'] = date2num(dts, units).astype('float32')
# range
_range = filemetadata('range')
# assume that the number of gates and spacing from the first ray is
# representative of the entire volume
field_header = first_ray.field_headers[0]
ngates = field_header['nbins']
start = field_header['range_start_m']
step = field_header['range_spacing_m']
# this gives distances to the start of each gate, add step/2 for center
_range['data'] = np.arange(ngates, dtype='float32') * step + start
_range['meters_to_center_of_first_gate'] = start
_range['meters_between_gates'] = step
# latitude, longitude and altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
lat, lon, height = first_ray.get_location()
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([height], dtype='float64')
# metadata
metadata = filemetadata('metadata')
metadata['original_container'] = 'UF'
metadata['site_name'] = first_ray.mandatory_header['site_name']
metadata['radar_name'] = first_ray.mandatory_header['radar_name']
# sweep_start_ray_index, sweep_end_ray_index
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_start_ray_index['data'] = ufile.first_ray_in_sweep
sweep_end_ray_index['data'] = ufile.last_ray_in_sweep
# sweep number
sweep_number = filemetadata('sweep_number')
sweep_number['data'] = np.arange(ufile.nsweeps, dtype='int32')
# sweep_type
scan_type = _UF_SWEEP_MODES[first_ray.mandatory_header['sweep_mode']]
# sweep_mode
sweep_mode = filemetadata('sweep_mode')
sweep_mode['data'] = np.array(ufile.nsweeps * [_SWEEP_MODE_STR[scan_type]])
# elevation
elevation = filemetadata('elevation')
elevation['data'] = ufile.get_elevations()
# azimuth
azimuth = filemetadata('azimuth')
azimuth['data'] = ufile.get_azimuths()
# fixed_angle
fixed_angle = filemetadata('fixed_angle')
fixed_angle['data'] = ufile.get_sweep_fixed_angles()
# fields
fields = {}
for uf_field_number, uf_field_dic in enumerate(first_ray.field_positions):
uf_field_name = uf_field_dic['data_type']
field_name = filemetadata.get_field_name(uf_field_name)
if field_name is None:
continue
field_dic = filemetadata(field_name)
field_dic['data'] = ufile.get_field_data(uf_field_number)
field_dic['_FillValue'] = get_fillvalue()
fields[field_name] = field_dic
# instrument_parameters
instrument_parameters = _get_instrument_parameters(ufile, filemetadata)
# scan rate
scan_rate = filemetadata('scan_rate')
scan_rate['data'] = ufile.get_sweep_rates()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
scan_rate=scan_rate,
instrument_parameters=instrument_parameters)
def _spectrum_width_coherency(
radar, gatefilter=None, num_bins=10, limits=None,
texture_window=(3, 3), texture_sample=5, min_sigma=None,
max_sigma=None, rays_wrap_around=False, remove_salt=False,
salt_window=(5, 5), salt_sample=10, fill_value=None, width_field=None,
width_text_field=None, cohere_field=None, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if width_field is None:
width_field = get_field_name('spectrum_width')
if width_text_field is None:
width_text_field = '{}_texture'.format(width_field)
if cohere_field is None:
cohere_field = '{}_coherency_mask'.format(width_field)
# Compute spectrum width texture field
ray_window, gate_window = texture_window
texture_fields._compute_field(
radar, width_field, ray_window=ray_window, gate_window=gate_window,
min_sample=texture_sample, min_ncp=None, min_sweep=None,
max_sweep=None, fill_value=fill_value, ncp_field=None)
# Automatically bracket noise distribution
if min_sigma is None and max_sigma is None:
# Compute spectrum width texture frequency counts
# Normalize frequency counts and compute bin centers and bin width
width_sigma = radar.fields[width_text_field]['data']
hist, edges = np.histogram(
width_sigma.compressed(), bins=num_bins, range=limits,
normed=False, weights=None, density=False)
hist = hist.astype(np.float64) / hist.max()
width = np.diff(edges).mean()
half_width = width / 2.0
bins = edges[:-1] + half_width
if verbose:
print 'Bin width = %.2f m/s' % width
# Determine distribution extrema locations
k_min = argrelextrema(
hist, np.less, axis=0, order=1, mode='clip')[0]
k_max = argrelextrema(
hist, np.greater, axis=0, order=1, mode='clip')[0]
if verbose:
print 'Minima located at %s m/s' % bins[k_min]
print 'Maxima located at %s m/s' % bins[k_max]
# Potentially a clear air volume
if k_min.size <= 1 or k_max.size <= 1:
# Bracket noise distribution
# Add (left side) or subtract (right side) the half bin width to
# account for the bin width
max_sigma = bins.max() + half_width
# Account for the no coherent signal case
if k_min.size == 0:
min_sigma = bins.min() - half_width
else:
min_sigma = bins[k_min][0] + half_width
if verbose:
print 'Computed min_sigma = %.2f m/s' % min_sigma
print 'Computed max_sigma = %.2f m/s' % max_sigma
print 'Radar volume is likely a clear air volume'
# Typical volume containing sufficient scatterers (e.g., hydrometeors,
# insects, etc.)
else:
# Compute primary and secondary peak locations
hist_max = np.sort(hist[k_max], kind='mergesort')[::-1]
prm_peak = bins[np.abs(hist - hist_max[0]).argmin()]
sec_peak = bins[np.abs(hist - hist_max[1]).argmin()]
if verbose:
print 'Primary peak located at %.2f m/s' % prm_peak
print 'Secondary peak located at %.2f m/s' % sec_peak
# If the primary (secondary) peak velocity texture is greater than
# the secondary (primary) peak velocity texture, than the primary
# (secondary) peak defines the noise distribution
noise_peak = np.max([prm_peak, sec_peak])
if verbose:
print 'Noise peak located at %.2f m/s' % noise_peak
# Determine left/right sides of noise distribution
left_side = bins[k_min] < noise_peak
right_side = bins[k_min] > noise_peak
# Bracket noise distribution
# Add (left side) or subtract (right side) the half bin width to
# account for the bin width
min_sigma = bins[k_min][left_side].max() + half_width
max_sigma = bins.max() + half_width
if verbose:
print 'Computed min_sigma = %.2f m/s' % min_sigma
print 'Computed max_sigma = %.2f m/s' % max_sigma
# Create the spectrum width texture coherency mask
mask = np.logical_or(
radar.fields[width_text_field]['data'] <= min_sigma,
radar.fields[width_text_field]['data'] >= max_sigma)
mask = np.ma.filled(mask, False)
mask_dict = {
'data': mask.astype(np.int8),
'long_name': 'Spectrum width coherency mask',
'standard_name': cohere_field,
'valid_min': 0,
'valid_max': 1,
'_FillValue': None,
'units': 'unitless',
'comment': ('0 = incoherent spectrum width, '
'1 = coherent spectrum width'),
}
radar.add_field(cohere_field, mask_dict, replace_existing=True)
# Remove salt and pepper noise from mask
if remove_salt:
basic_fixes.remove_salt(
radar, fields=[cohere_field], salt_window=salt_window,
salt_sample=salt_sample, rays_wrap_around=rays_wrap_around,
fill_value=0, mask_data=False, verbose=verbose)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Update gate filter
gatefilter.include_equal(cohere_field, 1, op='and')
return gatefilter
def _add_texture(grid,
field,
x_window=3,
y_window=3,
min_sample=5,
fill_value=None,
text_field=None,
debug=False,
verbose=False):
"""
Compute the texture field (standard deviation) of the input grid field
within a 1-D or 2-D window.
Parameters
----------
grid : Grid
Py-ART Grid containing input field.
field : str
Input radar field used to compute the texture field.
x_window : int, optional
Number of x grid points in texture window.
y_window : int, optional
Number of y grid points in texture window.
min_sample : int, optional
Minimum sample size within texture window required to define a valid
texture. Note that a minimum of 2 grid points are required to compute
the texture field.
fill_value : float, optional
The value indicating missing or bad data in the grid field data. If
None, the default value in the Py-ART configuration file is used.
text_field : str, optional
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print progress or identification information, False to
suppress.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if text_field is None:
text_field = '{}_texture'.format(field)
# Parse grid data
data = grid.fields[field]['data'].copy()
if debug:
N = np.ma.count(data)
print 'Sample size of data field: {}'.format(N)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
sigma, sample_size = _texture.compute_texture(data,
x_window=x_window,
y_window=y_window,
fill_value=fill_value)
# Mask grid points where sample size is insufficient
if min_sample is not None:
np.ma.masked_where(sample_size < min_sample, sigma, copy=False)
np.ma.masked_values(sigma, fill_value, atol=1.0e-5, copy=False)
np.ma.masked_invalid(sigma, copy=False)
sigma.set_fill_value(fill_value)
if debug:
N = np.ma.count(sigma)
print 'Sample size of texture field: {}'.format(N)
# Create texture field dictionary
sigma_dict = {
'data': sigma,
'units': '',
'_FillValue': sigma.fill_value,
}
grid.add_field(text_field, sigma_dict, replace_existing=True)
return
def _significant_features(
radar, field, gatefilter=None, min_size=None, size_bins=75,
size_limits=(0, 300), structure=None, remove_size_field=True,
fill_value=None, size_field=None, debug=False, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if size_field is None:
size_field = '{}_feature_size'.format(field)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Initialize echo feature size array
size_data = np.zeros_like(
radar.fields[field]['data'], subok=False, dtype=np.int32)
# Loop over all sweeps
feature_sizes = []
for sweep in radar.iter_slice():
# Parse radar sweep data and define only valid gates
is_valid_gate = ~radar.fields[field]['data'][sweep].mask
# Label the connected features in radar sweep data and create index
# array which defines each unique label (feature)
labels, nlabels = ndimage.label(
is_valid_gate, structure=structure, output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Number of unique features for {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero, np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Compute histogram of echo feature sizes, bin centers and bin
# width
counts, bin_edges = np.histogram(
feature_sizes, bins=size_bins, range=size_limits, normed=False,
weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution
# We expect the peak of the echo feature size distribution to be close to 1
# radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak
# This will define the minimum echo feature size
is_zero_size = np.logical_and(
bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'standard_name': size_field,
'long_name': '',
'_FillValue': size_data.fill_value,
'units': 'unitless',
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gatefilter.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field
if remove_size_field:
radar.fields.pop(size_field, None)
return gatefilter
def interpolate_missing(
radar, fields=None, interp_window=(3, 3), interp_sample=8,
kind='mean', rays_wrap_around=False, fill_value=None, debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names to interpolate
if fields is None:
fields = radar.fields.keys()
# Parse interpolation parameters
ray_window, gate_window = interp_window
# Loop over all fields and interpolate missing gates
for field in fields:
if verbose:
print 'Filling missing gates: {}'.format(field)
# Parse radar data and its original data type
data = radar.fields[field]['data']
dtype_orig = data.dtype
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size before fill: {}'.format(N)
# Parse sweep parameters
# Offset index arrays in order to be compatible with Fortran and avoid
# a segmentation fault
sweep_start = radar.sweep_start_ray_index['data'] + 1
sweep_end = radar.sweep_end_ray_index['data'] + 1
# Call Fortran routine
if kind.upper() == 'MEAN':
sweeps.mean_fill(
data, sweep_start, sweep_end, ray_window=ray_window,
gate_window=gate_window, min_sample=interp_sample,
rays_wrap=rays_wrap_around, fill_value=fill_value)
else:
raise ValueError('Unsupported interpolation method')
if debug:
N = np.count_nonzero(~np.isclose(data, fill_value, atol=1.0e-5))
print 'Sample size after fill: {}'.format(N)
# Mask invalid data
data = np.ma.masked_equal(data, fill_value, copy=False)
data.set_fill_value(fill_value)
# Add interpolated data to radar object
radar.fields[field]['data'] = data.astype(dtype_orig)
return
def classify(radar,
textures=None,
moments=None,
heights=None,
nonprecip_map=None,
gatefilter=None,
weights=1.0,
class_prob='equal',
min_inputs=1,
min_ncp=None,
zero=1.0e-10,
ignore_inputs=None,
use_insects=True,
fill_value=None,
ncp_field=None,
cloud_field=None,
ground_field=None,
insect_field=None,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
if cloud_field is None:
cloud_field = 'cloud'
if ground_field is None:
ground_field = 'ground'
if insect_field is None:
insect_field = 'insect'
# Parse ignore fields
if ignore_inputs is None:
ignore_inputs = []
# Check if at least one input is available
if textures is None and moments is None:
raise ValueError('No inputs specified')
# Parse classification labels
if use_insects:
labels = [cloud_field, ground_field, insect_field]
else:
labels = [cloud_field, ground_field]
if textures is not None:
textures.pop(insect_field, None)
if moments is not None:
moments.pop(insect_field, None)
if heights is not None:
heights.pop(insect_field, None)
# Determine total number of inputs available for each class
inputs = {label: 0 for label in labels}
if textures is not None:
for label, texture in textures.iteritems():
fields = [field for field in texture if field not in ignore_inputs]
inputs[label] += len(fields)
if moments is not None:
for label, moment in moments.iteritems():
fields = [field for field in moment if field not in ignore_inputs]
inputs[label] += len(fields)
if heights is not None:
for label in labels:
inputs[label] += 1
if ground_map is not None:
inputs[ground_field] += 1
if verbose:
for label in labels:
print 'Total number of inputs for {} = {}'.format(
label, inputs[label])
# Parse class probability P(c)
if class_prob.upper() == 'EQUAL':
P_c = 0.5
# Parse input probabilities P(x1, x2, ... , xn)
if isinstance(weights, float):
P_xi = weights
# Initialize total probability and number of inputs arrays
P_tot = {
label: np.ones(radar.fields[ncp_field]['data'].shape, dtype=np.float64)
for label in labels
}
num_inputs = {
label: np.zeros(radar.fields[ncp_field]['data'].shape, dtype=np.int32)
for label in labels
}
# Process radar texture fields
if textures is not None:
for label, texture in textures.iteritems():
for field, histogram in texture.iteritems():
if field in ignore_inputs:
continue
# Parse radar texture data and its distribution
data = radar.fields[field]['data']
pdf = histogram['probability density']
bins = histogram['bin centers']
# Mask incoherent gates
if min_ncp is not None and ncp_field in radar.fields:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(data,
pdf,
bins,
zero=zero,
fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process radar moments
if moments is not None:
for label, moment in moments.iteritems():
for field, histogram in moment.iteritems():
if field in ignore_inputs:
continue
# Parse radar moment data and its distribution
data = radar.fields[field]['data']
pdf = histogram['probability density']
bins = histogram['bin centers']
# Mask incoherent gates
if min_ncp is not None and ncp_field in radar.fields:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(data,
pdf,
bins,
zero=zero,
fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process radar gate heights
if heights is not None:
for label in labels:
# Parse height distribution data
pdf = heights[label]['probability density']
bins = heights[label]['bin centers']
# Compute radar gate heights in kilometers
x, y, data = common.standard_refraction(radar, use_km=True)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(data,
pdf,
bins,
zero=zero,
fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process ground frequency map
if ground_map is not None:
pdf = ground_map['ground frequency map']
# Mask gates where not enough inputs were available to properly classify
for label, sample_size in num_inputs.iteritems():
P_tot[label] = np.ma.masked_where(sample_size < min_inputs,
P_tot[label])
# Mask excluded gates from gate filter
if gatefilter is not None:
for label in P_tot:
P_tot[label] = np.ma.masked_where(gatefilter.gate_excluded,
P_tot[label])
# Determine where each class is most probable
echo = np.zeros(P_tot[cloud_field].shape, dtype=np.int8)
if use_insects:
is_ground = np.logical_and(P_tot[ground_field] > P_tot[cloud_field],
P_tot[ground_field] > P_tot[insect_field])
is_insect = np.logical_and(P_tot[insect_field] > P_tot[ground_field],
P_tot[insect_field] > P_tot[cloud_field])
is_missing = P_tot[ground_field].mask
echo[is_ground] = 1
echo[is_insect] = 2
echo[is_missing] = -1
else:
is_ground = P_tot[ground_field] > P_tot[cloud_field]
is_missing = P_tot[ground_field].mask
echo[is_ground] = 1
echo[is_missing] = -1
# Create echo classification dictionary and add it to the radar object
echo = {
'data':
echo.astype(np.int8),
'long_name':
'Radar echo classification',
'standard_name':
'radar_echo_classification',
'_FillValue':
None,
'units':
'unitless',
'comment': ('-1 = Missing gate, 0 = Cloud or precipitation, '
'1 = Ground clutter, 2 = Insects')
}
radar.add_field('radar_echo_classification', echo, replace_existing=True)
return
def classify(radar, textures=None, moments=None, heights=None,
nonprecip_map=None, gatefilter=None, weights=1.0,
class_prob='equal', min_inputs=1, min_ncp=None, zero=1.0e-10,
ignore_inputs=None, use_insects=True, fill_value=None,
ncp_field=None, cloud_field=None, ground_field=None,
insect_field=None, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
if cloud_field is None:
cloud_field = 'cloud'
if ground_field is None:
ground_field = 'ground'
if insect_field is None:
insect_field = 'insect'
# Parse ignore fields
if ignore_inputs is None:
ignore_inputs = []
# Check if at least one input is available
if textures is None and moments is None:
raise ValueError('No inputs specified')
# Parse classification labels
if use_insects:
labels = [cloud_field, ground_field, insect_field]
else:
labels = [cloud_field, ground_field]
if textures is not None:
textures.pop(insect_field, None)
if moments is not None:
moments.pop(insect_field, None)
if heights is not None:
heights.pop(insect_field, None)
# Determine total number of inputs available for each class
inputs = {label: 0 for label in labels}
if textures is not None:
for label, texture in textures.iteritems():
fields = [field for field in texture if field not in ignore_inputs]
inputs[label] += len(fields)
if moments is not None:
for label, moment in moments.iteritems():
fields = [field for field in moment if field not in ignore_inputs]
inputs[label] += len(fields)
if heights is not None:
for label in labels:
inputs[label] += 1
if ground_map is not None:
inputs[ground_field] += 1
if verbose:
for label in labels:
print 'Total number of inputs for {} = {}'.format(
label, inputs[label])
# Parse class probability P(c)
if class_prob.upper() == 'EQUAL':
P_c = 0.5
# Parse input probabilities P(x1, x2, ... , xn)
if isinstance(weights, float):
P_xi = weights
# Initialize total probability and number of inputs arrays
P_tot = {
label: np.ones(radar.fields[ncp_field]['data'].shape, dtype=np.float64)
for label in labels
}
num_inputs = {
label: np.zeros(radar.fields[ncp_field]['data'].shape, dtype=np.int32)
for label in labels
}
# Process radar texture fields
if textures is not None:
for label, texture in textures.iteritems():
for field, histogram in texture.iteritems():
if field in ignore_inputs:
continue
# Parse radar texture data and its distribution
data = radar.fields[field]['data']
pdf = histogram['probability density']
bins = histogram['bin centers']
# Mask incoherent gates
if min_ncp is not None and ncp_field in radar.fields:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(
data, pdf, bins, zero=zero, fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process radar moments
if moments is not None:
for label, moment in moments.iteritems():
for field, histogram in moment.iteritems():
if field in ignore_inputs:
continue
# Parse radar moment data and its distribution
data = radar.fields[field]['data']
pdf = histogram['probability density']
bins = histogram['bin centers']
# Mask incoherent gates
if min_ncp is not None and ncp_field in radar.fields:
ncp = radar.fields[ncp_field]['data']
data = np.ma.masked_where(ncp < min_ncp, data)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(
data, pdf, bins, zero=zero, fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process radar gate heights
if heights is not None:
for label in labels:
# Parse height distribution data
pdf = heights[label]['probability density']
bins = heights[label]['bin centers']
# Compute radar gate heights in kilometers
x, y, data = common.standard_refraction(radar, use_km=True)
# Prepare data for ingest into Fortran wrapper
data = np.ma.filled(data, fill_value)
data = np.asfortranarray(data, dtype=np.float64)
# Compute conditional probability for each radar gate
P_cond = member.conditional_all(
data, pdf, bins, zero=zero, fill_value=fill_value)
# Determine where conditional probability is valid
valid_prob = P_cond != fill_value
num_inputs[label] += valid_prob
# Bayes classifier
P_tot[label][valid_prob] *= P_c * P_cond[valid_prob] / P_xi
# Process ground frequency map
if ground_map is not None:
pdf = ground_map['ground frequency map']
# Mask gates where not enough inputs were available to properly classify
for label, sample_size in num_inputs.iteritems():
P_tot[label] = np.ma.masked_where(
sample_size < min_inputs, P_tot[label])
# Mask excluded gates from gate filter
if gatefilter is not None:
for label in P_tot:
P_tot[label] = np.ma.masked_where(
gatefilter.gate_excluded, P_tot[label])
# Determine where each class is most probable
echo = np.zeros(P_tot[cloud_field].shape, dtype=np.int8)
if use_insects:
is_ground = np.logical_and(
P_tot[ground_field] > P_tot[cloud_field],
P_tot[ground_field] > P_tot[insect_field])
is_insect = np.logical_and(
P_tot[insect_field] > P_tot[ground_field],
P_tot[insect_field] > P_tot[cloud_field])
is_missing = P_tot[ground_field].mask
echo[is_ground] = 1
echo[is_insect] = 2
echo[is_missing] = -1
else:
is_ground = P_tot[ground_field] > P_tot[cloud_field]
is_missing = P_tot[ground_field].mask
echo[is_ground] = 1
echo[is_missing] = -1
# Create echo classification dictionary and add it to the radar object
echo = {
'data': echo.astype(np.int8),
'long_name': 'Radar echo classification',
'standard_name': 'radar_echo_classification',
'_FillValue': None,
'units': 'unitless',
'comment': ('-1 = Missing gate, 0 = Cloud or precipitation, '
'1 = Ground clutter, 2 = Insects')
}
radar.add_field('radar_echo_classification', echo, replace_existing=True)
return
def height_histogram_from_radar(
radar, hist_dict, gatefilter=None, min_ncp=None, min_sweep=None,
max_sweep=None, min_range=None, max_range=None, fill_value=None,
ncp_field=None, verbose=False, debug=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
# TODO: check input histogram dictionary for proper keys
# Compute locations of radar gates and convert to kilometers
x, y, heights = common.standard_refraction(radar, use_km=True)
if debug:
print '(Min, Max) radar gate height = ({:.3f}, {:.3f}) km'.format(
heights.min(), heights.max())
# Mask sweeps outside specified range
if min_sweep is not None:
i = radar.sweep_start_ray_index['data'][min_sweep]
heights[:i+1,:] = np.ma.masked
if max_sweep is not None:
i = radar.sweep_end_ray_index['data'][max_sweep]
heights[i+1:,:] = np.ma.masked
# Mask radar range gates outside specified range
if min_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - min_range).argmin()
heights[:,:i+1] = np.ma.masked
if max_range is not None:
i = np.abs(radar.range['data'] / 1000.0 - max_range).argmin()
heights[:,i+1:] = np.ma.masked
# Mask incoherent echoes
if min_ncp is not None:
ncp = radar.fields[ncp_field]['data']
heights = np.ma.masked_where(ncp < min_ncp, heights)
# Mask excluded gates
if gatefilter is not None:
heights = np.ma.masked_where(gatefilter.gate_excluded, heights)
# Parse histogram parameters
bins = hist_dict['number of bins']
limits = hist_dict['limits']
# Compute histogram counts
counts, bin_edges = np.histogram(
heights.compressed(), bins=bins, range=limits, normed=False,
weights=None, density=False)
hist_dict['histogram counts'] += counts
# Parse bin edges and bin centers
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
hist_dict['bin edges'] = bin_edges
hist_dict['bin centers'] = bin_centers
return
