def process_radar(radar, domain, weight, outdir, gatefilter=None, debug=False, verbose=False): """ """ if verbose: print('Processing file: {}'.format(os.path.basename(filename))) # Read radar data radar = read(filename, exclude_fields=EXCLUDE_FIELDS) # Create gatefilter from significant detection gf = GateFilter(radar) gf.exclude_below(SD_FIELD, 1, op='or', inclusive=False) if debug: print('Number of sweeps: {}'.format(radar.nsweeps)) # Grid radar data grid = grid_radar( radar, domain, weight=weight, fields=FIELDS, gatefilter=gf, toa=TOA, max_range=MAX_RANGE, gqi_field=None, legacy=True, debug=debug, verbose=verbose) # Add new metadata _add_metadata(grid, filename) # ARM file name protocols date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S') fname = 'nexradwsr88d{}{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp) # Write MMCG NetCDF file grid_io.write_grid( os.path.join(outdir, fname), grid, format=FORMAT, write_proj_coord_sys=False, proj_coord_sys=None, arm_time_variables=True, write_point_x_y_z=False, write_point_lon_lat_alt=False) return
def _loop_over_dict(json_file, pickle_file, inpdir=None, outdir=None, verbose=False, debug=False): """ """ # Parse files from JSON with open(json_file, "r") as fid: files = json.load(fid) if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] # Loop over all files for f in files: # Read radar data radar = read(f, exclude_fields=EXCLUDE_FIELDS) if VCP_SWEEPS is not None and radar.nsweeps != VCP_SWEEPS: continue if VCP_RAYS is not None and radar.nrays != VCP_RAYS: continue if verbose: print "Processing file %s" % os.path.basename(f) # Determine significant detection of the radar gatefilter = noise.velocity_coherency( radar, gatefilter=None, num_bins=BINS_VDOP_COHER, limits=LIMITS_VDOP_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, nyquist=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=verbose, ) gatefilter = noise.spectrum_width_coherency( radar, gatefilter=gatefilter, num_bins=BINS_SW_COHER, limits=LIMITS_SW_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=True, ) gatefilter = noise.significant_detection( radar, gatefilter=gatefilter, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, min_ncp=MIN_NCP, detect_field=None, verbose=verbose, ) # Compute histogram counts for each field geo.height_histogram_from_radar( radar, HIST_DICT, gatefilter=gatefilter, min_ncp=MIN_NCP, min_sweep=MIN_SWEEP, max_sweep=MAX_SWEEP, min_range=MIN_RANGE, max_range=MAX_RANGE, fill_value=None, ncp_field=NCP_FIELD, verbose=verbose, debug=debug, ) # Parse bin edges and histogram counts bin_edges = HIST_DICT["bin edges"] counts = HIST_DICT["histogram counts"] # Compute normalized histogram and probability density # Add these to the histogram dictionary counts_norm = counts.astype(np.float64) / counts.max() pdf = counts_norm / np.sum(counts_norm * np.diff(bin_edges)) HIST_DICT["normalized histogram"] = counts_norm HIST_DICT["probability density"] = pdf # Include other parameters in the histogram dictionary HIST_DICT["radar files"] = files HIST_DICT["min sweep"] = MIN_SWEEP HIST_DICT["max sweep"] = MAX_SWEEP HIST_DICT["min range"] = MIN_RANGE HIST_DICT["max range"] = MAX_RANGE HIST_DICT["sweeps in VCP"] = VCP_SWEEPS HIST_DICT["rays in VCP"] = VCP_RAYS HIST_DICT["min NCP"] = MIN_NCP # Pickle histogram data geo._pickle_histograms(HIST_DICT, pickle_file, outdir=outdir) return
if args.verbose: print 'MAX_RANGE -> {} km'.format(MAX_RANGE) # Parse files to plot files = [os.path.join(args.inpdir, f) for f in sorted(os.listdir(args.inpdir)) if args.stamp in f] if args.verbose: print 'Number of files to plot: {}'.format(len(files)) # Loop over all files for filename in files: if args.verbose: print 'Plotting file: {}'.format(os.path.basename(filename)) # Read radar data radar = read(filename, exclude_fields=EXCLUDE_FIELDS) start = time.time() # Call desired plotting function multipanel(radar, args.outdir, dpi=args.dpi, debug=args.debug, verbose=args.verbose) # Record elapsed time if args.verbose: elapsed = time.time() - start print('Elapsed time to save plot: {:.0f} sec'.format(elapsed))
def _loop_over_dict( json_file, pickle_file, inpdir=None, outdir=None, verbose=False, debug=False): """ """ # Parse files from JSON with open(json_file, 'r') as fid: files = json.load(fid) if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] # Loop over all files for f in files: # Read radar data radar = read(f, exclude_fields=EXCLUDE_FIELDS) if VCP_SWEEPS is not None and radar.nsweeps != VCP_SWEEPS: continue if VCP_RAYS is not None and radar.nrays != VCP_RAYS: continue if verbose: print 'Processing file %s' % os.path.basename(f) # Determine significant detection of the radar gatefilter = noise.velocity_coherency( radar, gatefilter=None, num_bins=BINS_VDOP_COHER, limits=LIMITS_VDOP_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, nyquist=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=verbose) gatefilter = noise.spectrum_width_coherency( radar, gatefilter=gatefilter, num_bins=BINS_SW_COHER, limits=LIMITS_SW_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=True) gatefilter = noise.significant_detection( radar, gatefilter=gatefilter, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, min_ncp=MIN_NCP, detect_field=None, verbose=verbose) # Compute histogram counts for each field geo.height_histogram_from_radar( radar, HIST_DICT, gatefilter=gatefilter, min_ncp=MIN_NCP, min_sweep=MIN_SWEEP, max_sweep=MAX_SWEEP, min_range=MIN_RANGE, max_range=MAX_RANGE, fill_value=None, ncp_field=NCP_FIELD, verbose=verbose, debug=debug) # Parse bin edges and histogram counts bin_edges = HIST_DICT['bin edges'] counts = HIST_DICT['histogram counts'] # Compute normalized histogram and probability density # Add these to the histogram dictionary counts_norm = counts.astype(np.float64) / counts.max() pdf = counts_norm / np.sum(counts_norm * np.diff(bin_edges)) HIST_DICT['normalized histogram'] = counts_norm HIST_DICT['probability density'] = pdf # Include other parameters in the histogram dictionary HIST_DICT['radar files'] = files HIST_DICT['min sweep'] = MIN_SWEEP HIST_DICT['max sweep'] = MAX_SWEEP HIST_DICT['min range'] = MIN_RANGE HIST_DICT['max range'] = MAX_RANGE HIST_DICT['sweeps in VCP'] = VCP_SWEEPS HIST_DICT['rays in VCP'] = VCP_RAYS HIST_DICT['min NCP'] = MIN_NCP # Pickle histogram data geo._pickle_histograms( HIST_DICT, pickle_file, outdir=outdir) return
if args.verbose: print('MAX_RANGE -> {}'.format(MAX_RANGE)) # Parse radar files to plot files = [os.path.join(args.inpdir, f) for f in sorted(os.listdir(args.inpdir)) if args.stamp in f] if args.verbose: print('Number of files to plot: {}'.format(len(files))) # Loop over all files for filename in files: if args.verbose: print('Plotting file: {}'.format(os.path.basename(filename))) # Read radar data radar = read(filename, exclude_fields=None) if args.verbose: start = time.time() # Call desired plotting function multipanel(radar, args.outdir, dpi=args.dpi, debug=args.debug, verbose=args.verbose) if args.verbose: elapsed = time.time() - start print('Elapsed time to save plot: {:.0f} sec'.format(elapsed))
def map_from_json( filename, inpdir=None, vcp_sweeps=None, vcp_rays=None, vcp_gates=None, min_ncp=None, use_filter=True, texture_window=(3, 3), texture_sample=5, vdop_bins=100, vdop_limits=(0, 20), sw_bins=50, sw_limits=(0, 5), remove_salt=True, salt_window=(5, 5), salt_sample=10, exclude_fields=None, ncp_field=None, debug=False, verbose=False): """ Compute the non-precipitating frequency (probability) map from the files listed in a JSON file. The listed files should define a non-precipitating time period where (most) echoes present must be, by definition, not precipitation or cloud. Parameters ---------- Optional Parameters ------------------- Returns ------- """ if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') # Parse files from JSON file with open(filename, 'r') as fid: files = json.load(fid) # Append input directory if provided if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] if verbose: print 'Total number of radar files to process = %i' % len(files) # Parse non-precipitating frequency map if vcp_rays is not None and vcp_gates is not None: nonprecip = np.zeros((vcp_rays, vcp_gates), dtype=np.float64) else: nonprecip = None # Loop over all files sample_size = 0 for i, f in enumerate(files): # Read radar data radar = read(f, exclude_fields=exclude_fields) # Check radar VCP parameters if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps: continue if vcp_rays is not None and radar.nrays != vcp_rays: continue if vcp_gates is not None and radar.ngates != vcp_gates: continue if verbose: print 'Processing file %s' % os.path.basename(f) # Initialize the non-precipitation frequency map if it does not exist if i == 0 and nonprecip is None: vcp_sweeps = radar.nsweeps vcp_rays = radar.nrays vcp_gates = radar.ngates nonprecip = np.zeros((vcp_rays, vcp_gates), dtype=np.float64) if verbose: print 'VCP sweeps = {}'.format(vcp_sweeps) print 'VCP rays = {}'.format(vcp_rays) print 'VCP gates = {}'.format(vcp_gates) # Increase sample size sample_size += 1 # Determine significant detection if use_filter: # Doppler velocity coherency gatefilter = noise.velocity_coherency( radar, gatefilter=None, num_bins=vdop_bins, limits=vdop_limits, texture_window=texture_window, texture_sample=texture_sample, min_sigma=None, max_sigma=None, nyquist=None, rays_wrap_around=False, remove_salt=remove_salt, salt_window=salt_window, salt_sample=salt_sample, fill_value=None, verbose=verbose) # Spectrum width coherency gatefilter = noise.spectrum_width_coherency( radar, gatefilter=gatefilter, num_bins=sw_bins, limits=sw_limits, texture_window=texture_window, texture_sample=texture_sample, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=remove_salt, salt_window=salt_window, salt_sample=salt_sample, fill_value=None, verbose=verbose) # Significant detection gatefilter = noise.significant_detection( radar, gatefilter=gatefilter, remove_salt=remove_salt, salt_window=salt_window, salt_sample=salt_sample, min_ncp=min_ncp, detect_field=None, verbose=verbose) # Parse gate filter is_coherent = gatefilter.gate_included.astype(np.float64) elif min_ncp is not None: is_coherent = radar.fields[ncp_field]['data'] >= min_ncp is_coherent = np.ma.filled(is_coherent, False).astype(np.float64) else: raise ValueError('No way to determine significant detection') # Increase the non-precipitation map for all coherent gates (pixels) nonprecip += is_coherent # Compute the probability a gate (pixel) has a valid echo during # non-precipitating events nonprecip_map = nonprecip / sample_size # Add clutter frequency map to (last) radar object nonprecip = { 'data': nonprecip_map, 'long_name': 'Non-precipitating (clutter) frequency map', 'standard_name': 'clutter_map', 'valid_min': 0.0, 'valid_max': 1.0, '_FillValue': None, 'units': None, } radar.add_field('clutter_map', nonprecip, replace_existing=False) return { 'non-precipitating map': nonprecip_map, 'last radar': radar, 'sample size': sample_size, 'radar files': [os.path.basename(f) for f in files], 'vcp_sweeps': vcp_sweeps, 'vcp_rays': vcp_rays, 'vcp_gates': vcp_gates, 'min_ncp': min_ncp, 'use_filter': use_filter, 'texture_window': texture_window, 'texture_sample': texture_sample, 'remove_salt': remove_salt, 'salt_window': salt_window, 'salt_sample': salt_sample, }
def map_date_range(start, stop, stamp, inpdir, date_str='[0-9]{12}', date_fmt='%y%m%d%H%M%S', min_ncp=None, vcp_sweeps=None, vcp_rays=None, exclude_fields=None, ncp_field=None, debug=False, verbose=False): """ Compute the clutter frequency (probability) map within the specified date range. The start and stop times should define a non-precipitating time period where (most) echoes present must be, by definition, clutter. Parameters ---------- Optional Parameters ------------------- Returns ------- """ # Parse field names if refl_field is None: refl_field = get_field_name('reflectivity') if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') # Get all files with stamp in directory files = [os.path.join(inpdir, f) for f in sorted(os.listdir(inpdir)) if stamp in f] if verbose: print 'Total number of radar files found = %i' % len(files) # Remove files outside date range time_str = [re.search(date_str, f).group() for f in files] times = [datetime.strptime(string, date_fmt) for string in time_str] files = [ f for f, time in zip(files, times) if time >= start and time <= stop] if verbose: print 'Number of radar files after within date range = %i' % len(files) if vcp_sweeps is not None and vcp_rays is not None: nonprecip = np.zeros((vcp_sweeps, vcp_rays), dtype=np.float64) else: nonprecip = None # Loop over all files sample_size = 0 for i, f in enumerate(files): if verbose: print 'Processing file %s' % os.path.basename(f) # Read radar data radar = read(f, exclude_fields=exclude_fields) # Check radar VCP if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps: continue if vcp_rays is not None and radar.nrays != vcp_rays: continue # Initialize the non-precipitation map if not already done so if i == 0 and nonprecip is None: nonprecip = np.zeros((radar.nrays, radar.ngates), dtype=np.float64) # Increase sample size sample_size += 1 # Find coherent pixels if min_ncp is not None: is_coherent = radar.fields[ncp_field]['data'] >= min_ncp is_coherent = np.ma.filled(is_coherent, Fales).astype(np.float64) else: is_coherent = np.zeros(nonprecip.shape, dtype=np.float64) # Find pixels that have a coherent signal nonprecip += is_coherent # Compute the probability a pixel (gate) has a valid echo during # non-precipitating events nonprecip_map = nonprecip / sample_size # Add clutter frequency map to radar object nonprecip = { 'data': nonprecip_map, 'long_name': 'Non-precipitating frequency map', 'standard_name': 'nonprecip_map', 'valid_min': 0.0, 'valid_max': 1.0, '_FillValue': None, 'units': None, } radar.add_field('nonprecip_map', nonprecip, replace_existing=False) return { 'non-precipitating map': nonprecip_map, 'last radar': radar, 'sample size': sample_size, 'radar files': [os.path.basename(f) for f in files], 'sweeps in VCP': vcp_sweeps, 'rays in VCP': vcp_rays, 'min NCP': min_ncp, }
def histogram_from_json(filename, field, inpdir=None, texture_window=(3, 3), min_sample=5, num_bins=10, limits=None, min_ncp=0.5, vcp_sweeps=None, vcp_rays=None, min_sweep=None, max_sweep=None, min_range=None, max_range=None, rays_wrap_around=False, exclude_fields=None, fill_value=None, ncp_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') # Parse files from JSON file with open(filename, 'r') as fid: files = json.load(fid) # Append input directory if given if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] if verbose: print 'Total number of radar files to process = %i' % len(files) # Parse texture window parameters ray_window, gate_window = texture_window # Loop over all files counts = np.zeros(num_bins, dtype=np.float64) for f in files: # Read radar data radar = read(f, exclude_fields=exclude_fields) # Check radar VCP if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps: continue if vcp_rays is not None and radar.nrays != vcp_rays: continue if verbose: print 'Processing file %s' % os.path.basename(f) # Compute texture fields _compute_field(radar, field, ray_window=ray_window, gate_window=gate_window, min_sample=min_sample, min_ncp=min_ncp, min_sweep=min_sweep, max_sweep=max_sweep, min_range=min_range, max_range=max_range, rays_wrap_around=rays_wrap_around, fill_value=fill_value, ncp_field=ncp_field) # Parse data and compute histogram data = radar.fields['{}_texture'.format(field)]['data'] hist, bin_edges = np.histogram(data.compressed(), bins=num_bins, range=limits, normed=False, weights=None, density=False) counts += hist # Compute bin centers bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 # Compute normalized histogram and probability density counts_norm = counts / counts.max() pdf = counts_norm / np.sum(counts_norm * np.diff(bin_edges)) return { 'field': '{}_texture'.format(field), 'histogram counts': counts, 'normalized histogram': counts_norm, 'probability density': pdf, 'number of bins': num_bins, 'limits': limits, 'bin edges': bin_edges, 'bin centers': bin_centers, 'radar files': [os.path.basename(f) for f in files], 'min sweep': min_sweep, 'max sweep': max_sweep, 'min range': min_range, 'max range': max_range, 'min normalized coherent power': min_ncp, 'sweeps in VCP': vcp_sweeps, 'rays in VCP': vcp_rays, 'ray window size': ray_window, 'gate window size': gate_window, }
def _loop_over_dict( json_file, pickle_file, inpdir=None, outdir=None, verbose=False): """ """ # Parse files from JSON file with open(json_file, 'r') as fid: files = json.load(fid) if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] # Loop over all files for f in files: # Read radar data radar = read(f, exclude_fields=EXCLUDE_FIELDS) if VCP_SWEEPS is not None and radar.nsweeps != VCP_SWEEPS: continue if VCP_RAYS is not None and radar.nrays != VCP_RAYS: continue if verbose: print 'Processing file %s' % os.path.basename(f) # Determine significant detection of the radar gatefilter = noise.velocity_coherency( radar, gatefilter=None, num_bins=BINS_VDOP_COHER, limits=LIMITS_VDOP_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, nyquist=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=verbose) gatefilter = noise.spectrum_width_coherency( radar, gatefilter=gatefilter, num_bins=BINS_SW_COHER, limits=LIMITS_SW_COHER, texture_window=(3, 3), texture_sample=5, min_sigma=None, max_sigma=None, rays_wrap_around=False, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_value=None, verbose=verbose) gatefilter = noise.significant_detection( radar, gatefilter=gatefilter, remove_salt=REMOVE_SALT, salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, min_ncp=MIN_NCP, detect_field=None, verbose=verbose) # Compute histogram counts for each texture field texture_fields.histograms_from_radar( radar, HIST_DICT, gatefilter=gatefilter, texture_window=TEXTURE_WINDOW, texture_sample=TEXTURE_SAMPLE, min_ncp=MIN_NCP, min_sweep=MIN_SWEEP, max_sweep=MAX_SWEEP, min_range=MIN_RANGE, max_range=MAX_RANGE, rays_wrap_around=False, fill_value=None, ncp_field=NCP_FIELD, verbose=verbose) # Normalize histograms for each field and compute probability densities for field in HIST_DICT: # Parse bin edges and histogram counts bin_edges = HIST_DICT[field]['bin edges'] counts = HIST_DICT[field]['histogram counts'] # Compute normalized histogram and probability density # Add these to the histogram dictionary counts_norm = counts.astype(np.float64) / counts.max() pdf = counts_norm / np.sum(counts_norm * np.diff(bin_edges)) HIST_DICT[field]['normalized histogram'] = counts_norm HIST_DICT[field]['probability density'] = pdf # Include other parameters in the histogram dictionary HIST_DICT[field]['radar files'] = files HIST_DICT[field]['min sweep'] = MIN_SWEEP HIST_DICT[field]['max sweep'] = MAX_SWEEP HIST_DICT[field]['min range'] = MIN_RANGE HIST_DICT[field]['max range'] = MAX_RANGE HIST_DICT[field]['sweeps in VCP'] = VCP_SWEEPS HIST_DICT[field]['rays in VCP'] = VCP_RAYS HIST_DICT[field]['minimum normalized coherent power'] = MIN_NCP # Change dictionary field names to include texture for field in HIST_DICT.keys(): HIST_DICT['{}_texture'.format(field)] = HIST_DICT.pop(field) # Pickle histogram data texture_fields._pickle_histograms( HIST_DICT, pickle_file, outdir=outdir) return
if args.verbose: print("Number of files to process: {}".format(len(files))) for filename in files: if args.verbose: print("Processing file: {}".format(os.path.basename(filename))) # Create tar file instance tf = tarfile.TarFile(name=filename, mode="r", format=None) for member in tf.getmembers(): if args.verbose: print("Plotting file: {}".format(member.name)) # Read radar data from file object radar = read(tf.extractfile(member), exclude_fields=EXCLUDE_FIELDS) # Record start time start = time.time() # Call desired plotting function multipanel(radar, args.outdir, dpi=args.dpi, debug=args.debug, verbose=args.verbose) # Record elapsed time elapsed = time.time() - start if args.verbose: print("Elapsed time to save plot: {:.0f} sec".format(elapsed))
def process_file(filename, outdir, debug=False, verbose=False): """ """ # Read radar data if USE_RADX: radar = read_radx(filename) else: radar = read(filename, exclude_fields=None) # Radar VCP check if CHECK_VCP: if NSWEEPS is not None and radar.nsweeps != NSWEEPS: return if verbose: print 'Processing file: {}'.format(os.path.basename(filename)) if debug: print 'Number of sweeps: {}'.format(radar.nsweeps) if USE_RADX: # Create file metadata object meta = FileMetadata( 'nexrad_archive', field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None) # Remove unnecessary fields for field in REMOVE_FIELDS: radar.fields.pop(field, None) # Rename fields to default Py-ART names for field in radar.fields.keys(): default_field = meta.get_field_name(field) radar.fields[default_field] = radar.fields.pop(field, None) # Step 1: Determine radar significant detection # Since NEXRAD WSR-88D Level II data is already processed to some degree, # this amounts to essentially removing salt and pepper noise gf = noise._significant_features( radar, REFL_FIELD, gatefilter=None, size_bins=SIZE_BINS, size_limits=SIZE_LIMITS, structure=STRUCTURE, remove_size_field=False, fill_value=None, size_field=None, debug=debug) gf = noise.significant_detection( radar, gatefilter=gf, remove_small_features=False, size_bins=SIZE_BINS, size_limits=SIZE_LIMITS, fill_holes=FILL_HOLES, dilate=DILATE, structure=STRUCTURE, iterations=1, rays_wrap_around=False, min_ncp=None, detect_field=None, debug=debug, verbose=verbose) # Step 2: Doppler velocity correction if DEALIAS == 'phase': vdop_corr = dealias_unwrap_phase( radar, gatefilter=gf, unwrap_unit='sweep', nyquist_vel=None, rays_wrap_around=True, keep_original=False, vel_field=None, corr_vel_field=VDOP_CORR_FIELD) elif DEALIAS == 'region': vdop_corr = dealias_region_based( radar, gatefilter=gf, interval_splits=INTERVAL_SPLITS, interval_limits=None, skip_between_rays=2, skip_along_ray=2, centered=True, nyquist_vel=None, rays_wrap_around=True, keep_original=False, vel_field=None, corr_vel_field=VDOP_CORR_FIELD) else: raise ValueError('Unsupported velocity correction routine') radar.add_field(VDOP_CORR_FIELD, vdop_corr, replace_existing=True) # Step 3: Reflectivity correction # Currently no correction procedures are applied to the reflectivity field # due to minimal attenuation at S-band refl_corr = radar.fields[REFL_FIELD].copy() radar.add_field(REFL_CORR_FIELD, refl_corr, replace_existing=True) # Step 4: Interpolate missing gates basic_fixes.interpolate_missing( radar, fields=FILL_FIELDS, interp_window=FILL_WINDOW, interp_sample=FILL_SAMPLE, kind='mean', rays_wrap_around=False, fill_value=None, debug=debug, verbose=verbose) # Add metadata _add_metadata(radar, filename) # ARM file name protocols date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S') fname = 'nexradwsr88d{}cmac{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp) # Write CMAC NetCDF file write_cfradial(os.path.join(outdir, fname), radar, format=FORMAT, arm_time_variables=True) return
def histogram_from_json( filename, field, inpdir=None, bins=10, limits=None, min_ncp=0.5, vcp_sweeps=None, vcp_rays=None, min_sweep=None, max_sweep=None, exclude_fields=None, fill_value=None, ncp_field=None, verbose=False): """ """ # Parse fill value if fill_value is None: fill_value = get_fillvalue() # Parse field names if ncp_field is None: ncp_field = get_field_name('normalized_coherent_power') # Parse files from JSON file with open(filename, 'r') as fid: files = json.load(fid) # Append input directory if given if inpdir is not None: files = [os.path.join(inpdir, f) for f in files] if verbose: print 'Total number of radar files to process = %i' % len(files) # Loop over all files histogram = np.zeros(bins, dtype=np.float64) for f in files: # Read radar data radar = read(f, exclude_fields=exclude_fields) # Check radar VCP if vcp_sweeps is not None and radar.nsweeps != vcp_sweeps: continue if vcp_rays is not None and radar.nrays != vcp_rays: continue if verbose: print 'Processing file %s' % os.path.basename(f) # Parse radar fields data = radar.fields[field]['data'] # Mask sweeps outside specified range if min_sweep is not None: i = radar.sweep_start_ray_index['data'][min_sweep] data[:i+1,:] = np.ma.masked if max_sweep is not None: i = radar.sweep_end_ray_index['data'][max_sweep] data[i+1:,:] = np.ma.masked # Mask incoherent echoes if min_ncp is not None: ncp = radar.fields[ncp_field]['data'] data = np.ma.masked_where(ncp < min_ncp, data) # Bin data and compute frequencies hist, bin_edges = np.histogram( data.compressed(), bins=bins, range=limits, normed=False, weights=None, density=False) histogram += hist # Compute bin centers bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0 # Compute normalized histogram and probability density histogram_norm = histogram / histogram.max() pdf = histogram_norm / np.sum(histogram_norm * np.diff(bin_edges)) return { 'field': field, 'histogram counts': histogram, 'normalized histogram': histogram_norm, 'probability density': pdf, 'number of bins': bins, 'limits': limits, 'bin edges': bin_edges, 'bin centers': bin_centers, 'radar files': [os.path.basename(f) for f in files], 'min sweep': min_sweep, 'max sweep': max_sweep, 'min normalized coherent power': min_ncp, 'sweeps in VCP': vcp_sweeps, 'rays in VCP': vcp_rays, }