def vad_to_csv(fname, site, radar, u, v, rms, hgts, elevs):
"""
Export VAD data to csv format
Parameters
----------
fname - Filename to write csv to
site - Radar site
radar - PyArt radar object
u - Numpy array of U
v - Numpy array of V
rms - Numpy array of RMS
hgts - Numpy array of RMS
elevs - List of elevation angles used
Returns
-------
"""
with open(fname, 'wb') as csv_file:
headers = ['site', 'time', 'lat', 'lon', 'elev', 'u', 'v', 'rms', 'hgt', 'radar_hgt']
writer = DictWriter(csv_file, fieldnames=headers)
writer.writeheader()
lat = radar.latitude['data'][0]
lon = radar.longitude['data'][0]
time = datetime_from_radar(radar)
time = time.isoformat()
radar_elev = radar.altitude['data'][0]
u = u.filled(FILL_VALUE)
v = v.filled(FILL_VALUE)
rms = rms.filled(FILL_VALUE)
for i, elev in enumerate(elevs):
for j, hgt in enumerate(hgts):
if u[j][i] != FILL_VALUE:
# print headers
dic = {'site': site,
'time': time,
'lat': lat,
'lon': lon,
'elev': elev,
'u': u[j][i],
'v': v[j][i],
'rms': rms[j][i],
'hgt': hgt[i],
'radar_hgt': radar_elev}
# print dic
writer.writerow(dic)
def process_NEXRAD(start_hour, end_hour, csv=False):
# start_hour = datetime(2015, 06, 2, 12, 0, 0)
# end_hour = datetime(2015, 06, 2, 13, 0, 0)
td = timedelta(hours=1)
while start_hour < end_hour:
for i, site in enumerate(radars):
# Setup output directories
image_dir = "%s/images/%s/%s" % (os.getcwd(), site, start_hour.strftime("%Y%m%d"))
csv_dir = "%s/csv_vad/%s/%s" % (os.getcwd(), site, start_hour.strftime("%Y%m%d"))
if not os.path.isdir(image_dir):
os.makedirs(image_dir)
if not os.path.isdir(csv_dir):
os.makedirs(csv_dir)
d = 'D:\\TMBell\\projects\\VADAnalysis\\data\\%s\\raw\\%s' % (site, start_hour.strftime("%Y%m%d"))
file_glob = "%s%s" % (site, start_hour.strftime("%Y%m%d_%H*"))
in_files = glob(os.path.join(d, file_glob))
for in_file in in_files:
logging.info(in_file)
try:
# Read in the file and dealias
radar = io.nexrad_archive.read_nexrad_archive(in_file)
logging.info("Dealiasing %s using PyART package..." % in_file)
VEL2 = correct.dealias_unwrap_phase(radar)
radar.add_field('VEL2', VEL2)
# Get some parameters
time = datetime_from_radar(radar)
time_str = "%s_%s" % (site, time.strftime("%Y%m%d_%H%M%S"))
elevs = utils.get_elevs(radar)
radar_elev = radar.altitude['data'][0]
# print "Trying 'our' method"
plt.figure(1, figsize=(15, 7))
our_u, our_v, hgt, gates = vad.get_uv_vs_hgt(radar, 'VEL2')
our_RMSE = utils.calc_RMSE(radar, 'VEL2', our_u, our_v)
display.u_v_rmse_plot(our_u, our_v, our_RMSE, hgt, title=time_str, elevs=elevs, radar_elev=radar_elev)
img_name = "%s_%s" % (time_str, 'fig1')
plt.savefig(os.path.join(image_dir, img_name))
plt.clf()
if csv:
csv_name = time_str + ".csv"
csv_name = os.path.join(csv_dir, csv_name)
logging.info("Exporting VAD as CSV: " + csv_name)
vad_csv.vad_to_csv(csv_name, site, radar, our_u, our_v, our_RMSE, hgt, elevs)
except IOError:
logging.warning("Error Reading NEXRAD lvlII file: " + str(in_file))
except KeyError, e:
logging.warning("Key not found " + str(e))
except Exception, e:
logging.warning(e)
def process_file(filename, outdir, dl='b1', verbose=False):
"""
"""
if verbose:
print 'Processing file: {}'.format(os.path.basename(filename))
# Read radar data
radar = read_kazr(filename, exclude_fields=None)
# Step 1: Radar significant detection
# Includes Hildebrand noise floor estimate and Doppler velocity coherency
gf = noise.velocity_coherency(
radar, gatefilter=None, num_bins=VDOP_COHER_BINS,
limits=VDOP_COHER_LIMITS, texture_window=TEXTURE_WINDOW,
texture_sample=TEXTURE_SAMPLE, min_sigma=None, max_sigma=None,
nyquist=None, rays_wrap_around=None, remove_salt=False,
fill_value=None, vdop_field=VDOP_FIELD, vdop_text_field=None,
cohere_field=None, verbose=verbose)
gf = noise.hildebrand_noise(
radar, gatefilter=gf, scale=1.0, remove_salt=False,
rays_wrap_around=False, fill_value=None, power_field=POWER_FIELD,
noise_field=None, verbose=verbose)
gf = noise.significant_detection(
radar, gatefilter=gf, min_ncp=None, remove_salt=True,
salt_window=SALT_WINDOW, salt_sample=SALT_SAMPLE, fill_holes=False,
dilate=False, structure=None, rays_wrap_around=False, ncp_field=None,
detect_field=None, verbose=verbose)
# Step 2: Doppler velocity correction
if DEALIAS == 'phase':
vdop_corr = dealias_unwrap_phase(
radar, gatefilter=gf, unwrap_unit='sweep', nyquist_vel=None,
rays_wrap_around=False, keep_original=False, skip_checks=True,
vel_field=VDOP_FIELD, corr_vel_field=None)
elif DEALIAS == 'region':
vdop_corr = dealias_region_based(
radar, gatefilter=gf, interval_splits=INTERVAL_SPLITS,
interval_limits=None, skip_between_rays=2, skip_along_ray=2,
centered=True, nyquist_vel=None, rays_wrap_around=False,
keep_original=False, vel_field=VDOP_FIELD, corr_vel_field=None)
else:
raise ValueError('Unsupported velocity correction routine')
radar.add_field(CORR_VDOP_FIELD, vdop_corr, replace_existing=True)
# TODO
# Step 3: Reflectivity correction
# Parse metadata
radar.metadata = _create_metadata(radar, filename)
# ARM file name protocols
date = datetime_from_radar(radar).strftime('%Y%m%d.%H%M%S')
filename = 'sgpkazrgecmacC1.{}.{}.cdf'.format(dl, date)
# Write CMAC NetCDF file
write_cfradial(os.path.join(outdir, filename), radar, format=FORMAT,
arm_time_variables=True)
return
def process_file(filename, outdir, verbose=False):
"""
"""
if verbose:
print 'Processing file: {}'.format(os.path.basename(filename))
# Read radar data
radar = read_sigmet(filename, exclude_fields=EXLUDE_FIELDS)
# Radar significant detection
# Includes Doppler velocity coherency, spectrum width coherency, and
# minimum normalized coherent power
gf = noise.velocity_coherency(
radar, gatefilter=None, num_bins=VDOP_COHER_BINS,
limits=VDOP_COHER_LIMITS, texture_window=TEXTURE_WINDOW,
texture_sample=TEXTURE_SAMPLE, min_sigma=None, max_sigma=None,
rays_wrap_around=False, remove_salt=False, fill_value=None,
vdop_field=VDOP_FIELD, vdop_text_field=None, cohere_field=None,
verbose=verbose)
gf = noise.velocity_phasor_coherency(
radar, gatefilter=gf, num_bins=PHASE_COHER_BINS,
limits=PHASE_COHER_LIMITS, texture_window=TEXTURE_WINDOW,
texture_sample=TEXTURE_SAMPLE, min_sigma=None, max_sigma=None,
rays_wrap_around=False, remove_salt=False, fill_value=None,
vdop_field=VDOP_FIELD, vdop_phase_field=None, phase_text_field=None,
cohere_field=None, verbose=verbose)
gf = noise.spectrum_width_coherency(
radar, gatefilter=gf, num_bins=SW_COHER_BINS,
limits=SW_COHER_LIMITS, texture_window=TEXTURE_WINDOW,
texture_sample=TEXTURE_SAMPLE, min_sigma=None, max_sigma=None,
rays_wrap_around=False, remove_salt=False, fill_value=None,
width_field=SW_FIELD, width_text_field=None, cohere_field=None,
verbose=verbose)
gf = noise.significant_detection(
radar, gatefilter=gf, remove_salt=True, salt_window=SALT_WINDOW,
salt_sample=SALT_SAMPLE, fill_holes=False, dilate=DILATE,
structure=None, min_ncp=MIN_NCP, ncp_field=NCP_FIELD,
detect_field=None, verbose=verbose)
# Compute radar texture fields
texture_fields.add_textures(
radar, fields=TEXTURE_FIELDS, gatefilter=None,
texture_window=TEXTURE_WINDOW, texture_sample=TEXTURE_SAMPLE,
min_sweep=None, max_sweep=None, min_range=None, max_range=None,
min_ncp=None, rays_wrap_around=False, fill_value=None,
ncp_field=NCP_FIELD)
# Echo classification
bayes.classify(
radar, textures=TEXTURES, moments=MOMENTS, heights=HEIGHTS,
nonprecip_map=None, gatefilter=gf, weights=1.0, class_prob='equal',
min_inputs=3, zero=ZERO, ignore_inputs=IGNORE_INPUTS, use_insects=True,
fill_value=None, cloud_field=CLOUD_FIELD, ground_field=GROUND_FIELD,
insect_field=INSECT_FIELD, ncp_field=NCP_FIELD, verbose=verbose)
# Filter ground clutter gates
gf.exclude_equal(
'radar_echo_classification', 1, exclude_masked=True, op='or')
# Doppler velocity correction
vdop_corr = dealias_region_based(
radar, gatefilter=gf, interval_splits=3, interval_limits=None,
skip_between_rays=2, skip_along_ray=2, centered=True, nyquist_vel=None,
rays_wrap_around=True, keep_original=False, vel_field=VDOP_FIELD,
corr_vel_field=CORR_VDOP_FIELD)
radar.add_field(CORR_VDOP_FIELD, vdop_corr, replace_existing=False)
# TODO: reflectivity correction
# Parse metadata
radar.metadata = _create_metadata(radar, filename)
# ARM file name protocols
date = datetime_from_radar(radar).strftime('%Y%m%d.%H%M%S')
filename = 'sgpxsaprppicmac{}.{}.{}.cdf'.format(FN, DL, date)
# Write CMAC NetCDF file
write_cfradial(os.path.join(outdir, filename), radar, format=FORMAT,
arm_time_variables=True)
return
def process_SR(in_dir, image_dir, csv=None):
# image_dir = "/Users/tbupper90/Desktop/NEXRAD_BLISS/VADAnalysis/images/SR1_test/"
if not os.path.isdir(image_dir):
os.makedirs(image_dir)
"""
"""
# d = '/Users/tbupper90/Desktop/NEXRAD_BLISS/data/SR1/pecan/SR1/20150619/corrected/'
# d = '/Users/tbupper90/Desktop/NEXRAD_BLISS/data/SR1/pecan/SR1/20150619/corr2/'
in_files = glob(os.path.join(in_dir, "*"))
for in_file in in_files:
print in_file
try:
radar = io.read_cfradial(in_file)
# print "Dealiasing Radar using PyART package..."
# VEL2 = correct.dealias_unwrap_phase(radar)
# radar.add_field('VEL2', VEL2)
# Get some parameters
time = datetime_from_radar(radar)
time_str = "SR2_%s" % (time.strftime("%Y%m%d_%H%M%S"))
elevs = utils.get_elevs(radar)
radar_elev = radar.altitude['data'][0]
# Mask out the sector where the cab blocks the beam
radar = utils.mask_sector(radar, 'VEL2', 250, 290)
# plt.figure(111)
# display.plot_velocity(radar, 'VEL2', sweep_number=3)
# img_name = "%s_%s" % (time_str, 'fig2')
# plt.savefig(os.path.join(image_dir, img_name))
# plt.clf()
print "Trying 'our' method"
plt.figure(1, figsize=(15, 7))
our_u, our_v, hgt, gates = vad.get_uv_vs_hgt(radar, 'VEL2')
our_RMSE = utils.calc_RMSE(radar, 'VEL2', our_u, our_v)
display.u_v_rmse_plot(our_u, our_v, our_RMSE, hgt, title="'Our' Method -- " + time_str, elevs=elevs,
radar_elev=radar_elev)
img_name = "%s_%s" % (time_str, 'fig1')
plt.savefig(os.path.join(image_dir, img_name))
plt.clf()
if csv is True:
csv_dir = 'D:\\TMBell\\projects\\VADAnalysis\\csv_vad2\\SR2\\{0}'
csv_dir = csv_dir.format(time.strftime('%Y%m%d'))
csv_name = time_str + ".csv"
csv_name = os.path.join(csv_dir, csv_name)
if not os.path.isdir(csv_dir): os.makedirs(csv_dir)
vad_csv.vad_to_csv(csv_name, 'SR2', radar, our_u, our_v, our_RMSE, hgt, elevs)
except IOError:
print "Error Reading file"
except KeyError, e:
print "Key not found " + str(e)
