def velocity_phasor_coherency(radar, gatefilter=None, text_bins=40, text_limits=(0, 20), nyquist=None, texture_window=(3, 3), texture_sample=5, max_texture=None, rays_wrap_around=False, remove_small_features=False, size_bins=75, size_limits=(0, 300), fill_value=None, vdop_field=None, phasor_field=None, text_field=None, coherent_field=None, debug=False, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if vdop_field is None: vdop_field = get_field_name('velocity') if phasor_field is None: phasor_field = '{}_phasor_real'.format(vdop_field) if text_field is None: text_field = '{}_texture'.format(phasor_field) if coherent_field is None: coherent_field = '{}_coherency_mask'.format(phasor_field) if verbose: print 'Computing Doppler velocity phasor coherency mask' # Parse Nyquist velocity if nyquist is None: nyquist = radar.get_nyquist_vel(0, check_uniform=True) if debug: print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist) # Compute the real part of Doppler velocity phasor # Normalize real part of phasor to the Nyquist interval vdop = radar.fields[vdop_field]['data'] phasor_real = nyquist * np.cos(np.radians(360.0 * vdop / nyquist)) # Mask invalid values phasor_real = np.ma.masked_invalid(phasor_real) phasor_real.set_fill_value(fill_value) # Create Doppler velocity phasor field dictionary phasor_dict = { 'data': phasor_real.astype(np.float32), 'long_name': 'Real part of Doppler velocity phasor', 'standard_name': phasor_field, 'valid_min': -nyquist, 'valid_max': nyquist, '_FillValue': phasor_real.fill_value, 'units': 'meters_per_second', 'comment': ('Real part of Doppler velocity phasor normalized to the ' 'Nyquist interval'), } radar.add_field(phasor_field, phasor_dict, replace_existing=True) # Compute Doppler velocity phasor texture field ray_window, gate_window = texture_window texture_fields._compute_field(radar, phasor_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, fill_value=fill_value, ncp_field=None) # Automatically bracket coherent part of Doppler velocity phasor texture # distribution if max_texture is None: # Bin Doppler velocity phasor texture data and count occurrences # Compute bin centers and bin width counts, bin_edges = np.histogram( radar.fields[text_field]['data'].compressed(), bins=text_bins, range=text_limits, normed=False, weights=None, density=False) bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 bin_width = np.diff(bin_edges).mean() if debug: print 'Bin width: {:.3f} m/s'.format(bin_width) # Determine positions of the extrema in the Doppler velocity phasor # texture distribution kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0] kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0] if debug: print 'Minima located at: {} m/s'.format(bin_centers[kmin]) print 'Maxima located at: {} m/s'.format(bin_centers[kmax]) # Compute the theoretical noise peak location from Guassian noise # statistics noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0) if debug: print 'Theoretical noise peak: {:.3f} m/s'.format( noise_peak_theory) # Find the closest Doppler velocity phasor texture distribution peak to # the computed theoretical location # Here we assume that the Doppler velocity phasor texture distribution # has at least one primary mode which correspondes to the incoherent # (noisy) part of the Doppler velocity phasor texture distribution # Depending on the radar volume and the bin width used to define the # distribution, the distribution may be bimodal, with the new peak # corresponding to the coherent part of the Doppler velocity phasor # texture distribution idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin() noise_peak = bin_centers[kmax][idx] if debug: print 'Computed noise peak: {:.3f} m/s'.format(noise_peak) # Determine primary and secondary peak locations for debugging # purposes if kmax.size > 1: counts_max = np.sort(counts[kmax], kind='mergesort')[::-1] prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()] sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()] if debug: print 'Primary peak: {:.3f} m/s'.format(prm_peak) print 'Secondary peak: {:.3f} m/s'.format(sec_peak) # Determine the left edge of the noise distribution # Where this distribution becomes a minimum will define the separation # between coherent and incoherent Doppler velocity phasor values is_left_side = bin_centers[kmin] < noise_peak max_texture = bin_centers[kmin][is_left_side].max() + bin_width / 2.0 if debug: _range = [0.0, round(max_texture, 3)] print 'Doppler velocity phasor coherency mode: {} m/s'.format( _range) # Create Doppler velocity phasor coherency mask is_coherent = np.logical_and( radar.fields[text_field]['data'] >= 0.0, radar.fields[text_field]['data'] <= max_texture) is_coherent = np.ma.filled(is_coherent, False) # Create Doppler velocity phasor coherency mask dictionary coherent_dict = { 'data': is_coherent.astype(np.int8), 'long_name': 'Doppler velocity phasor coherency', 'standard_name': coherent_field, 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': '0 = incoherent value, 1 = coherent value', } radar.add_field(coherent_field, coherent_dict, replace_existing=True) # Remove insignificant features from Doppler velocity phasor coherency mask if remove_small_features: basic_fixes._binary_significant_features(radar, coherent_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(coherent_field, 1, op='and') return gatefilter
def _spectrum_width_coherency(radar, gatefilter=None, num_bins=10, limits=None, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=False, salt_window=(5, 5), salt_sample=10, fill_value=None, width_field=None, width_text_field=None, cohere_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if width_field is None: width_field = get_field_name('spectrum_width') if width_text_field is None: width_text_field = '{}_texture'.format(width_field) if cohere_field is None: cohere_field = '{}_coherency_mask'.format(width_field) # Compute spectrum width texture field ray_window, gate_window = texture_window texture_fields._compute_field(radar, width_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, fill_value=fill_value, ncp_field=None) # Automatically bracket noise distribution if min_sigma is None and max_sigma is None: # Compute spectrum width texture frequency counts # Normalize frequency counts and compute bin centers and bin width width_sigma = radar.fields[width_text_field]['data'] hist, edges = np.histogram(width_sigma.compressed(), bins=num_bins, range=limits, normed=False, weights=None, density=False) hist = hist.astype(np.float64) / hist.max() width = np.diff(edges).mean() half_width = width / 2.0 bins = edges[:-1] + half_width if verbose: print 'Bin width = %.2f m/s' % width # Determine distribution extrema locations k_min = argrelextrema(hist, np.less, axis=0, order=1, mode='clip')[0] k_max = argrelextrema(hist, np.greater, axis=0, order=1, mode='clip')[0] if verbose: print 'Minima located at %s m/s' % bins[k_min] print 'Maxima located at %s m/s' % bins[k_max] # Potentially a clear air volume if k_min.size <= 1 or k_max.size <= 1: # Bracket noise distribution # Add (left side) or subtract (right side) the half bin width to # account for the bin width max_sigma = bins.max() + half_width # Account for the no coherent signal case if k_min.size == 0: min_sigma = bins.min() - half_width else: min_sigma = bins[k_min][0] + half_width if verbose: print 'Computed min_sigma = %.2f m/s' % min_sigma print 'Computed max_sigma = %.2f m/s' % max_sigma print 'Radar volume is likely a clear air volume' # Typical volume containing sufficient scatterers (e.g., hydrometeors, # insects, etc.) else: # Compute primary and secondary peak locations hist_max = np.sort(hist[k_max], kind='mergesort')[::-1] prm_peak = bins[np.abs(hist - hist_max[0]).argmin()] sec_peak = bins[np.abs(hist - hist_max[1]).argmin()] if verbose: print 'Primary peak located at %.2f m/s' % prm_peak print 'Secondary peak located at %.2f m/s' % sec_peak # If the primary (secondary) peak velocity texture is greater than # the secondary (primary) peak velocity texture, than the primary # (secondary) peak defines the noise distribution noise_peak = np.max([prm_peak, sec_peak]) if verbose: print 'Noise peak located at %.2f m/s' % noise_peak # Determine left/right sides of noise distribution left_side = bins[k_min] < noise_peak right_side = bins[k_min] > noise_peak # Bracket noise distribution # Add (left side) or subtract (right side) the half bin width to # account for the bin width min_sigma = bins[k_min][left_side].max() + half_width max_sigma = bins.max() + half_width if verbose: print 'Computed min_sigma = %.2f m/s' % min_sigma print 'Computed max_sigma = %.2f m/s' % max_sigma # Create the spectrum width texture coherency mask mask = np.logical_or(radar.fields[width_text_field]['data'] <= min_sigma, radar.fields[width_text_field]['data'] >= max_sigma) mask = np.ma.filled(mask, False) mask_dict = { 'data': mask.astype(np.int8), 'long_name': 'Spectrum width coherency mask', 'standard_name': cohere_field, 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': ('0 = incoherent spectrum width, ' '1 = coherent spectrum width'), } radar.add_field(cohere_field, mask_dict, replace_existing=True) # Remove salt and pepper noise from mask if remove_salt: basic_fixes.remove_salt(radar, fields=[cohere_field], salt_window=salt_window, salt_sample=salt_sample, rays_wrap_around=rays_wrap_around, fill_value=0, mask_data=False, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(cohere_field, 1, op='and') return gatefilter
def significant_detection(radar, gatefilter=None, remove_small_features=True, size_bins=75, size_limits=(0, 300), fill_holes=False, dilate=False, structure=None, iterations=1, rays_wrap_around=False, min_ncp=None, ncp_field=None, detect_field=None, debug=False, verbose=False): """ Determine the significant detection of a radar. Note that significant detection can still include other non-meteorological echoes that the user may still have to remove further down the processing chain. Parameters ---------- radar : Radar Radar object used to determine the appropriate GateFilter. gatefilter : GateFilter, optional If None, all radar gates will initially be assumed valid. remove_small_features : bool, optional True to remove insignificant echo features (e.g., salt and pepper noise) from significant detection mask. size_bins : int, optional Number of bins used to bin echo feature sizes and thus define its distribution. size_limits : list or tuple, optional Limits of the echo feature size distribution. The upper limit needs to be large enough to include the minimum feature size. fill_holes : bool, optional Fill any holes in the significant detection mask. For most radar volumes this should not be used since the default structuring element will automatically fill any sized hole. dilate : bool, optional Use binary dilation to fill in edges of the significant detection mask. structure : array_like, optional The binary structuring element used for all morphology routines. See SciPy's ndimage documentation for more information. iterations : int, optional The number of iterations to repeat binary dilation. If iterations is less than 1, binary dilation is repeated until the result does not change anymore. rays_wrap_around : bool, optional Whether the rays at the beginning and end of a sweep are connected (e.g., PPI VCP). min_ncp : float, optional Minimum normalized coherent power (signal quality) value used to indicate a significant echo. ncp_field : str, optional Minimum normalized coherent power (signal quality) field name. The default uses the Py-ART configuation file. detect_field : str, optional Radar significant detection mask field name. debug : bool, optional True to print debugging information, False to suppress. verbose : bool, optional True to print progress information, False to suppress. Returns ------- gatefilter : GateFilter Py-ART GateFilter object indicating which radar gates are valid and invalid. """ # Parse field names if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') if detect_field is None: detect_field = 'significant_detection_mask' # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Exclude gates with poor signal quality if min_ncp is not None and ncp_field in radar.fields: gatefilter.include_above(ncp_field, min_ncp, op='and', inclusive=True) detect_dict = { 'data': gatefilter.gate_included.astype(np.int8), 'long_name': 'Radar significant detection mask', 'standard_name': 'significant_detection_mask', 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': '0 = no significant detection, 1 = significant detection', } radar.add_field(detect_field, detect_dict, replace_existing=True) # Remove insignificant features from significant detection mask if remove_small_features: basic_fixes._binary_significant_features(radar, detect_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Fill holes in significant detection mask if fill_holes: basic_fixes._binary_fill(radar, detect_field, structure=structure) # Dilate significant detection mask if dilate: basic_fixes._binary_dilation(radar, detect_field, structure=structure, iterations=iterations, debug=debug, verbose=verbose) # Update gate filter gatefilter.include_equal(detect_field, 1, op='new') return gatefilter
def echo_boundaries(radar, gatefilter=None, texture_window=(3, 3), texture_sample=5, min_texture=None, bounds_percentile=95.0, remove_small_features=False, size_bins=75, size_limits=(0, 300), rays_wrap_around=False, fill_value=None, sqi_field=None, text_field=None, bounds_field=None, debug=False, verbose=False): """ Objectively determine the location of significant echo boundaries through analysis of signal quality index (SQI) texture. The a priori assumption is that at echo boundaries (e.g., cloud boundaries), the SQI field decreases substantially and therefore the SQI texture field is large near echo boundaries. Parameters ---------- radar : Radar Radar object containing the SQI field used to derive significant echo boundaries. Returns ------- gatefilter : GateFilter """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if sqi_field is None: sqi_field = get_field_name('normalized_coherent_power') if text_field is None: text_field = '{}_texture'.format(sqi_field) if bounds_field is None: bounds_field = 'echo_boundaries_mask' if verbose: print 'Computing significant echo boundaries mask' # Compute signal quality index texture field ray_window, gate_window = texture_window texture_fields._compute_field(radar, sqi_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, rays_wrap_around=rays_wrap_around, fill_value=fill_value, text_field=text_field, ncp_field=None) if min_texture is None: # The specified boundary percentile defines the minimum SQI texture # value for significant echo boundaries min_texture = np.percentile( radar.fields[text_field]['data'].compressed(), bounds_percentile, overwrite_input=False, interpolation='linear') if debug: max_texture = radar.fields[text_field]['data'].max() _range = [round(min_texture, 3), round(max_texture, 3)] print 'Echo boundary SQI texture range: {}'.format(_range) # Compute percentiles for debugging purposes percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 100] textures = np.percentile(radar.fields[text_field]['data'].compressed(), percentiles, overwrite_input=False, interpolation='linear') if debug: for p, texture in zip(percentiles, textures): print '{}% SQI texture = {:.5f}'.format(p, texture) # Determine radar gates which meet minimum normalized coherent power # texture is_boundary = radar.fields[text_field]['data'] >= min_texture is_boundary = np.ma.filled(is_boundary, False) # Create significant echo boundaries field dictionary bounds_dict = { 'data': is_boundary.astype(np.int8), 'standard_name': bounds_field, 'long_name': 'Significant echo boundaries mask', '_FillValue': None, 'units': 'unitless', 'comment': '0 = not an echo boundary, 1 = echo boundary', } radar.add_field(bounds_field, bounds_dict, replace_existing=True) # Remove insignificant features from significant echo boundaries mask if remove_small_features: basic_fixes._binary_significant_features(radar, bounds_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(bounds_field, 1, op='and') return gatefilter
def _spectrum_width_coherency( radar, gatefilter=None, num_bins=10, limits=None, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=False, salt_window=(5, 5), salt_sample=10, fill_value=None, width_field=None, width_text_field=None, cohere_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if width_field is None: width_field = get_field_name('spectrum_width') if width_text_field is None: width_text_field = '{}_texture'.format(width_field) if cohere_field is None: cohere_field = '{}_coherency_mask'.format(width_field) # Compute spectrum width texture field ray_window, gate_window = texture_window texture_fields._compute_field( radar, width_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, fill_value=fill_value, ncp_field=None) # Automatically bracket noise distribution if min_sigma is None and max_sigma is None: # Compute spectrum width texture frequency counts # Normalize frequency counts and compute bin centers and bin width width_sigma = radar.fields[width_text_field]['data'] hist, edges = np.histogram( width_sigma.compressed(), bins=num_bins, range=limits, normed=False, weights=None, density=False) hist = hist.astype(np.float64) / hist.max() width = np.diff(edges).mean() half_width = width / 2.0 bins = edges[:-1] + half_width if verbose: print 'Bin width = %.2f m/s' % width # Determine distribution extrema locations k_min = argrelextrema( hist, np.less, axis=0, order=1, mode='clip')[0] k_max = argrelextrema( hist, np.greater, axis=0, order=1, mode='clip')[0] if verbose: print 'Minima located at %s m/s' % bins[k_min] print 'Maxima located at %s m/s' % bins[k_max] # Potentially a clear air volume if k_min.size <= 1 or k_max.size <= 1: # Bracket noise distribution # Add (left side) or subtract (right side) the half bin width to # account for the bin width max_sigma = bins.max() + half_width # Account for the no coherent signal case if k_min.size == 0: min_sigma = bins.min() - half_width else: min_sigma = bins[k_min][0] + half_width if verbose: print 'Computed min_sigma = %.2f m/s' % min_sigma print 'Computed max_sigma = %.2f m/s' % max_sigma print 'Radar volume is likely a clear air volume' # Typical volume containing sufficient scatterers (e.g., hydrometeors, # insects, etc.) else: # Compute primary and secondary peak locations hist_max = np.sort(hist[k_max], kind='mergesort')[::-1] prm_peak = bins[np.abs(hist - hist_max[0]).argmin()] sec_peak = bins[np.abs(hist - hist_max[1]).argmin()] if verbose: print 'Primary peak located at %.2f m/s' % prm_peak print 'Secondary peak located at %.2f m/s' % sec_peak # If the primary (secondary) peak velocity texture is greater than # the secondary (primary) peak velocity texture, than the primary # (secondary) peak defines the noise distribution noise_peak = np.max([prm_peak, sec_peak]) if verbose: print 'Noise peak located at %.2f m/s' % noise_peak # Determine left/right sides of noise distribution left_side = bins[k_min] < noise_peak right_side = bins[k_min] > noise_peak # Bracket noise distribution # Add (left side) or subtract (right side) the half bin width to # account for the bin width min_sigma = bins[k_min][left_side].max() + half_width max_sigma = bins.max() + half_width if verbose: print 'Computed min_sigma = %.2f m/s' % min_sigma print 'Computed max_sigma = %.2f m/s' % max_sigma # Create the spectrum width texture coherency mask mask = np.logical_or( radar.fields[width_text_field]['data'] <= min_sigma, radar.fields[width_text_field]['data'] >= max_sigma) mask = np.ma.filled(mask, False) mask_dict = { 'data': mask.astype(np.int8), 'long_name': 'Spectrum width coherency mask', 'standard_name': cohere_field, 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': ('0 = incoherent spectrum width, ' '1 = coherent spectrum width'), } radar.add_field(cohere_field, mask_dict, replace_existing=True) # Remove salt and pepper noise from mask if remove_salt: basic_fixes.remove_salt( radar, fields=[cohere_field], salt_window=salt_window, salt_sample=salt_sample, rays_wrap_around=rays_wrap_around, fill_value=0, mask_data=False, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(cohere_field, 1, op='and') return gatefilter
def hildebrand_noise(radar, gatefilter=None, scale=1.0, remove_small_features=False, size_bins=75, size_limits=(0, 300), rays_wrap_around=False, fill_value=None, power_field=None, noise_field=None, mask_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if power_field is None: power_field = get_field_name('signal_to_noise_ratio') if noise_field is None: noise_field = 'radar_noise_floor' if mask_field is None: mask_field = 'radar_noise_floor_mask' # Parse radar power data power = radar.fields[power_field]['data'] # Prepare data for ingest into Fortran wrapper power = np.ma.filled(power, fill_value) power = np.asfortranarray(power, dtype=np.float64) # Convert power units to linear and sort in descending order # TODO: check if units are already linear power = np.where(power != fill_value, 10.0**(power / 10.0), power) power = np.sort(power, axis=0, kind='mergesort')[::-1] # Fortran wrapper to Hildebrand and Sekhon (1974) algorithm P, Q, R2, N = sweeps.hildebrand(power, fill_value=fill_value) # Mask invalid data P = np.ma.masked_equal(P, fill_value) Q = np.ma.masked_equal(Q, fill_value) R2 = np.ma.masked_equal(R2, fill_value) N = np.ma.masked_equal(N, fill_value) # Estimate noise floor in decibels and tile to proper dimensions noise = 10.0 * np.ma.log10(P + scale * np.ma.sqrt(Q)) noise = np.tile(noise, (radar.nrays, 1)) noise.set_fill_value(fill_value) # Add Hildebrand noise floor field to radar noise_dict = { 'data': noise.astype(np.float32), 'long_name': 'Radar noise floor estimate', 'standard_name': noise_field, 'units': 'dB', '_FillValue': noise.fill_value, 'comment': ('Noise floor is estimated using Hildebrand and ' 'Sekhon (1974) algorithm'), } radar.add_field(noise_field, noise_dict, replace_existing=True) # Compute noise floor mask power = radar.fields[power_field]['data'] noise = radar.fields[noise_field]['data'] is_noise = np.ma.filled(power >= noise, False) # Create radar noise floor mask dictionary mask_dict = { 'data': is_noise.astype(np.int8), 'long_name': 'Noise floor mask', 'standard_name': mask_field, 'valid_min': 0, 'valid_max': 1, 'units': 'unitless', '_FillValue': None, 'comment': '0 = below noise floor, 1 = at or above noise floor', } radar.add_field(mask_field, mask_dict, replace_existing=True) # Remove insignificant features from noise floor mask if remove_small_features: basic_fixes._binary_significant_features(radar, mask_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(mask_field, 1, op='and') return gatefilter
def velocity_phasor_coherency( radar, gatefilter=None, text_bins=40, text_limits=(0, 20), nyquist=None, texture_window=(3, 3), texture_sample=5, max_texture=None, rays_wrap_around=False, remove_small_features=False, size_bins=75, size_limits=(0, 300), fill_value=None, vdop_field=None, phasor_field=None, text_field=None, coherent_field=None, debug=False, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if vdop_field is None: vdop_field = get_field_name('velocity') if phasor_field is None: phasor_field = '{}_phasor_real'.format(vdop_field) if text_field is None: text_field = '{}_texture'.format(phasor_field) if coherent_field is None: coherent_field = '{}_coherency_mask'.format(phasor_field) if verbose: print 'Computing Doppler velocity phasor coherency mask' # Parse Nyquist velocity if nyquist is None: nyquist = radar.get_nyquist_vel(0, check_uniform=True) if debug: print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist) # Compute the real part of Doppler velocity phasor # Normalize real part of phasor to the Nyquist interval vdop = radar.fields[vdop_field]['data'] phasor_real = nyquist * np.cos(np.radians(360.0 * vdop / nyquist)) # Mask invalid values phasor_real = np.ma.masked_invalid(phasor_real) phasor_real.set_fill_value(fill_value) # Create Doppler velocity phasor field dictionary phasor_dict = { 'data': phasor_real.astype(np.float32), 'long_name': 'Real part of Doppler velocity phasor', 'standard_name': phasor_field, 'valid_min': -nyquist, 'valid_max': nyquist, '_FillValue': phasor_real.fill_value, 'units': 'meters_per_second', 'comment': ('Real part of Doppler velocity phasor normalized to the ' 'Nyquist interval'), } radar.add_field(phasor_field, phasor_dict, replace_existing=True) # Compute Doppler velocity phasor texture field ray_window, gate_window = texture_window texture_fields._compute_field( radar, phasor_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, fill_value=fill_value, ncp_field=None) # Automatically bracket coherent part of Doppler velocity phasor texture # distribution if max_texture is None: # Bin Doppler velocity phasor texture data and count occurrences # Compute bin centers and bin width counts, bin_edges = np.histogram( radar.fields[text_field]['data'].compressed(), bins=text_bins, range=text_limits, normed=False, weights=None, density=False) bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 bin_width = np.diff(bin_edges).mean() if debug: print 'Bin width: {:.3f} m/s'.format(bin_width) # Determine positions of the extrema in the Doppler velocity phasor # texture distribution kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0] kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0] if debug: print 'Minima located at: {} m/s'.format(bin_centers[kmin]) print 'Maxima located at: {} m/s'.format(bin_centers[kmax]) # Compute the theoretical noise peak location from Guassian noise # statistics noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0) if debug: print 'Theoretical noise peak: {:.3f} m/s'.format( noise_peak_theory) # Find the closest Doppler velocity phasor texture distribution peak to # the computed theoretical location # Here we assume that the Doppler velocity phasor texture distribution # has at least one primary mode which correspondes to the incoherent # (noisy) part of the Doppler velocity phasor texture distribution # Depending on the radar volume and the bin width used to define the # distribution, the distribution may be bimodal, with the new peak # corresponding to the coherent part of the Doppler velocity phasor # texture distribution idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin() noise_peak = bin_centers[kmax][idx] if debug: print 'Computed noise peak: {:.3f} m/s'.format(noise_peak) # Determine primary and secondary peak locations for debugging # purposes if kmax.size > 1: counts_max = np.sort(counts[kmax], kind='mergesort')[::-1] prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()] sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()] if debug: print 'Primary peak: {:.3f} m/s'.format(prm_peak) print 'Secondary peak: {:.3f} m/s'.format(sec_peak) # Determine the left edge of the noise distribution # Where this distribution becomes a minimum will define the separation # between coherent and incoherent Doppler velocity phasor values is_left_side = bin_centers[kmin] < noise_peak max_texture = bin_centers[kmin][is_left_side].max() + bin_width / 2.0 if debug: _range = [0.0, round(max_texture, 3)] print 'Doppler velocity phasor coherency mode: {} m/s'.format( _range) # Create Doppler velocity phasor coherency mask is_coherent = np.logical_and( radar.fields[text_field]['data'] >= 0.0, radar.fields[text_field]['data'] <= max_texture) is_coherent = np.ma.filled(is_coherent, False) # Create Doppler velocity phasor coherency mask dictionary coherent_dict = { 'data': is_coherent.astype(np.int8), 'long_name': 'Doppler velocity phasor coherency', 'standard_name': coherent_field, 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': '0 = incoherent value, 1 = coherent value', } radar.add_field(coherent_field, coherent_dict, replace_existing=True) # Remove insignificant features from Doppler velocity phasor coherency mask if remove_small_features: basic_fixes._binary_significant_features( radar, coherent_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(coherent_field, 1, op='and') return gatefilter
def echo_boundaries( radar, gatefilter=None, texture_window=(3, 3), texture_sample=5, min_texture=None, bounds_percentile=95.0, remove_small_features=False, size_bins=75, size_limits=(0, 300), rays_wrap_around=False, fill_value=None, sqi_field=None, text_field=None, bounds_field=None, debug=False, verbose=False): """ Objectively determine the location of significant echo boundaries through analysis of signal quality index (SQI) texture. The a priori assumption is that at echo boundaries (e.g., cloud boundaries), the SQI field decreases substantially and therefore the SQI texture field is large near echo boundaries. Parameters ---------- radar : Radar Radar object containing the SQI field used to derive significant echo boundaries. Returns ------- gatefilter : GateFilter """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if sqi_field is None: sqi_field = get_field_name('normalized_coherent_power') if text_field is None: text_field = '{}_texture'.format(sqi_field) if bounds_field is None: bounds_field = 'echo_boundaries_mask' if verbose: print 'Computing significant echo boundaries mask' # Compute signal quality index texture field ray_window, gate_window = texture_window texture_fields._compute_field( radar, sqi_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_ncp=None, min_sweep=None, max_sweep=None, rays_wrap_around=rays_wrap_around, fill_value=fill_value, text_field=text_field, ncp_field=None) if min_texture is None: # The specified boundary percentile defines the minimum SQI texture # value for significant echo boundaries min_texture = np.percentile( radar.fields[text_field]['data'].compressed(), bounds_percentile, overwrite_input=False, interpolation='linear') if debug: max_texture = radar.fields[text_field]['data'].max() _range = [round(min_texture, 3), round(max_texture, 3)] print 'Echo boundary SQI texture range: {}'.format(_range) # Compute percentiles for debugging purposes percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 100] textures = np.percentile( radar.fields[text_field]['data'].compressed(), percentiles, overwrite_input=False, interpolation='linear') if debug: for p, texture in zip(percentiles, textures): print '{}% SQI texture = {:.5f}'.format(p, texture) # Determine radar gates which meet minimum normalized coherent power # texture is_boundary = radar.fields[text_field]['data'] >= min_texture is_boundary = np.ma.filled(is_boundary, False) # Create significant echo boundaries field dictionary bounds_dict = { 'data': is_boundary.astype(np.int8), 'standard_name': bounds_field, 'long_name': 'Significant echo boundaries mask', '_FillValue': None, 'units': 'unitless', 'comment': '0 = not an echo boundary, 1 = echo boundary', } radar.add_field(bounds_field, bounds_dict, replace_existing=True) # Remove insignificant features from significant echo boundaries mask if remove_small_features: basic_fixes._binary_significant_features( radar, bounds_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(bounds_field, 1, op='and') return gatefilter
def significant_detection( radar, gatefilter=None, remove_small_features=True, size_bins=75, size_limits=(0, 300), fill_holes=False, dilate=False, structure=None, iterations=1, rays_wrap_around=False, min_ncp=None, ncp_field=None, detect_field=None, debug=False, verbose=False): """ Determine the significant detection of a radar. Note that significant detection can still include other non-meteorological echoes that the user may still have to remove further down the processing chain. Parameters ---------- radar : Radar Radar object used to determine the appropriate GateFilter. gatefilter : GateFilter, optional If None, all radar gates will initially be assumed valid. remove_small_features : bool, optional True to remove insignificant echo features (e.g., salt and pepper noise) from significant detection mask. size_bins : int, optional Number of bins used to bin echo feature sizes and thus define its distribution. size_limits : list or tuple, optional Limits of the echo feature size distribution. The upper limit needs to be large enough to include the minimum feature size. fill_holes : bool, optional Fill any holes in the significant detection mask. For most radar volumes this should not be used since the default structuring element will automatically fill any sized hole. dilate : bool, optional Use binary dilation to fill in edges of the significant detection mask. structure : array_like, optional The binary structuring element used for all morphology routines. See SciPy's ndimage documentation for more information. iterations : int, optional The number of iterations to repeat binary dilation. If iterations is less than 1, binary dilation is repeated until the result does not change anymore. rays_wrap_around : bool, optional Whether the rays at the beginning and end of a sweep are connected (e.g., PPI VCP). min_ncp : float, optional Minimum normalized coherent power (signal quality) value used to indicate a significant echo. ncp_field : str, optional Minimum normalized coherent power (signal quality) field name. The default uses the Py-ART configuation file. detect_field : str, optional Radar significant detection mask field name. debug : bool, optional True to print debugging information, False to suppress. verbose : bool, optional True to print progress information, False to suppress. Returns ------- gatefilter : GateFilter Py-ART GateFilter object indicating which radar gates are valid and invalid. """ # Parse field names if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') if detect_field is None: detect_field = 'significant_detection_mask' # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Exclude gates with poor signal quality if min_ncp is not None and ncp_field in radar.fields: gatefilter.include_above(ncp_field, min_ncp, op='and', inclusive=True) detect_dict = { 'data': gatefilter.gate_included.astype(np.int8), 'long_name': 'Radar significant detection mask', 'standard_name': 'significant_detection_mask', 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': '0 = no significant detection, 1 = significant detection', } radar.add_field(detect_field, detect_dict, replace_existing=True) # Remove insignificant features from significant detection mask if remove_small_features: basic_fixes._binary_significant_features( radar, detect_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Fill holes in significant detection mask if fill_holes: basic_fixes._binary_fill(radar, detect_field, structure=structure) # Dilate significant detection mask if dilate: basic_fixes._binary_dilation( radar, detect_field, structure=structure, iterations=iterations, debug=debug, verbose=verbose) # Update gate filter gatefilter.include_equal(detect_field, 1, op='new') return gatefilter
def hildebrand_noise( radar, gatefilter=None, scale=1.0, remove_small_features=False, size_bins=75, size_limits=(0, 300), rays_wrap_around=False, fill_value=None, power_field=None, noise_field=None, mask_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if power_field is None: power_field = get_field_name('signal_to_noise_ratio') if noise_field is None: noise_field = 'radar_noise_floor' if mask_field is None: mask_field = 'radar_noise_floor_mask' # Parse radar power data power = radar.fields[power_field]['data'] # Prepare data for ingest into Fortran wrapper power = np.ma.filled(power, fill_value) power = np.asfortranarray(power, dtype=np.float64) # Convert power units to linear and sort in descending order # TODO: check if units are already linear power = np.where(power != fill_value, 10.0**(power / 10.0), power) power = np.sort(power, axis=0, kind='mergesort')[::-1] # Fortran wrapper to Hildebrand and Sekhon (1974) algorithm P, Q, R2, N = sweeps.hildebrand(power, fill_value=fill_value) # Mask invalid data P = np.ma.masked_equal(P, fill_value) Q = np.ma.masked_equal(Q, fill_value) R2 = np.ma.masked_equal(R2, fill_value) N = np.ma.masked_equal(N, fill_value) # Estimate noise floor in decibels and tile to proper dimensions noise = 10.0 * np.ma.log10(P + scale * np.ma.sqrt(Q)) noise = np.tile(noise, (radar.nrays, 1)) noise.set_fill_value(fill_value) # Add Hildebrand noise floor field to radar noise_dict = { 'data': noise.astype(np.float32), 'long_name': 'Radar noise floor estimate', 'standard_name': noise_field, 'units': 'dB', '_FillValue': noise.fill_value, 'comment': ('Noise floor is estimated using Hildebrand and ' 'Sekhon (1974) algorithm'), } radar.add_field(noise_field, noise_dict, replace_existing=True) # Compute noise floor mask power = radar.fields[power_field]['data'] noise = radar.fields[noise_field]['data'] is_noise = np.ma.filled(power >= noise, False) # Create radar noise floor mask dictionary mask_dict = { 'data': is_noise.astype(np.int8), 'long_name': 'Noise floor mask', 'standard_name': mask_field, 'valid_min': 0, 'valid_max': 1, 'units': 'unitless', '_FillValue': None, 'comment': '0 = below noise floor, 1 = at or above noise floor', } radar.add_field(mask_field, mask_dict, replace_existing=True) # Remove insignificant features from noise floor mask if remove_small_features: basic_fixes._binary_significant_features( radar, mask_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(mask_field, 1, op='and') return gatefilter
def velocity_coherency( radar, gatefilter=None, text_bins=40, text_limits=(0, 20), nyquist_vel=None, texture_window=(3, 3), texture_sample=5, max_texture=None, min_sweep=None, rays_wrap_around=False, remove_small_features=False, size_bins=100, size_limits=(0, 400), fill_value=None, vdop_field=None, text_field=None, coherent_field=None, debug=False, verbose=False): """ Parameters ---------- radar : Radar Py-ART Radar containing Doppler velocity data. gatefilter : GateFilter, optional Py-ART GateFilter instance. If None, all radar gates will initially be assumed valid. Returns ------- gf : GateFilter Py-ART GateFilter. """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if vdop_field is None: vdop_field = get_field_name('velocity') if text_field is None: text_field = '{}_texture'.format(vdop_field) if coherent_field is None: coherent_field = '{}_coherency_mask'.format(vdop_field) if verbose: print 'Computing Doppler velocity coherency mask' # Parse Nyquist velocity if nyquist_vel is None: nyquist = radar.get_nyquist_vel(0, check_uniform=True) if debug: print 'Radar Nyquist velocity: {:.3f} m/s'.format(nyquist) # Compute Doppler velocity texture field ray_window, gate_window = texture_window textures._compute_field( radar, vdop_field, ray_window=ray_window, gate_window=gate_window, min_sample=texture_sample, min_sqi=None, min_sweep=min_sweep, max_sweep=None, fill_value=fill_value, sqi_field=None, text_field=text_field) # Automatically determine the maximum Doppler velocity texture value which # brackets the coherent part of the Doppler velocity texture distribution if max_texture is None: # Bin and count Doppler velocity texture field counts, bin_edges = np.histogram( radar.fields[text_field]['data'].compressed(), bins=text_bins, range=text_limits, normed=False, weights=None, density=False) # Compute bin centers and bin width bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 bin_width = np.diff(bin_edges).mean() if debug: print 'Bin width: {:.3f} m/s'.format(bin_width) # Determine the location of extrema in the Doppler velocity texture # distribution kmin = argrelextrema(counts, np.less, order=1, mode='clip')[0] kmax = argrelextrema(counts, np.greater, order=1, mode='clip')[0] if debug: print 'Minima located at: {} m/s'.format(bin_centers[kmin]) print 'Maxima located at: {} m/s'.format(bin_centers[kmax]) # Compute the theoretical noise peak location from Guassian noise # statistics noise_peak_theory = 2.0 * nyquist / np.sqrt(12.0) if debug: print 'Theoretical noise peak: {:.3f} m/s'.format( noise_peak_theory) # Find the closest Doppler velocity texture distribution peak to the # computed theoretical location. Here we assume that the Doppler # velocity texture distribution has at least one primary mode which # corresponds to the incoherent (noisy) part of the Doppler velocity # texture distribution. Depending on the radar volume and the bin width # used to define the distribution, the distribution may be bimodal, # with the additional peak corresponding to the coherent part of the # Doppler velocity texture distribution idx = np.abs(bin_centers[kmax] - noise_peak_theory).argmin() noise_peak = bin_centers[kmax][idx] if debug: print 'Computed noise peak: {:.3f} m/s'.format(noise_peak) # Determine primary and secondary peak locations for debugging # purposes if kmax.size > 1: counts_max = np.sort(counts[kmax], kind='mergesort')[::-1] prm_peak = bin_centers[np.abs(counts - counts_max[0]).argmin()] sec_peak = bin_centers[np.abs(counts - counts_max[1]).argmin()] if debug: print 'Primary peak: {:.3f} m/s'.format(prm_peak) print 'Secondary peak: {:.3f} m/s'.format(sec_peak) # Determine the left edge of the noise mode # Where this mode becomes a minimum defines the separation between # coherent and incoherent Doppler velocity texture values # TODO: if the chosen bin width is very small than multiple extrema can # exist such that the first minimum to the left of the noise peak is # not the appropriate minimum and the left edge detection breaks down is_left_side = bin_centers[kmin] < noise_peak max_texture = bin_centers[kmin][is_left_side].min() + bin_width / 2.0 if debug: _range = [0.0, round(max_texture, 3)] print 'Doppler velocity coherent mode: {} m/s'.format(_range) # Create the Doppler velocity texture coherency mask is_coherent = np.logical_and( radar.fields[text_field]['data'] >= 0.0, radar.fields[text_field]['data'] <= max_texture) is_coherent = np.ma.filled(is_coherent, False) # Conditional sampling checks for sweep, sweep_slice in enumerate(radar.iter_slice()): if sweep < min_sweep: is_coherent[sweep_slice] = False coherent_dict = { 'data': is_coherent.astype(np.int8), 'long_name': 'Doppler velocity coherency mask', 'standard_name': coherent_field, 'valid_min': 0, 'valid_max': 1, '_FillValue': None, 'units': 'unitless', 'comment': '0 = incoherent velocity, 1 = coherent velocity', } radar.add_field(coherent_field, coherent_dict, replace_existing=True) # Remove insignificant features from Doppler velocity coherency mask if remove_small_features: basic_fixes._binary_significant_features( radar, coherent_field, size_bins=size_bins, size_limits=size_limits, structure=structure, debug=debug, verbose=verbose) # Parse gate filter if gatefilter is None: gatefilter = GateFilter(radar, exclude_based=False) # Update gate filter gatefilter.include_equal(coherent_field, 1, op='and') return gatefilter