def get_month(radar, year, month, odir):
templocation = tempfile.mkdtemp()
conn = nexradaws.NexradAwsInterface()
days = conn.get_avail_days(year, month)
fls = []
for day in days:
print('doing ', day)
# stage to temp dir
scans = conn.get_avail_scans(year, month, day, radar)
localfiles = conn.download(scans, templocation)
# loop thought files and determine destination
try:
for downloaded_file in localfiles.success:
infile = downloaded_file.filepath
# os.path.join(downloaded_file.filepath,
# downloaded_file.filename)
print(downloaded_file.filepath)
outpath = os.path.join(
odir, radar,
downloaded_file.scan_time.strftime('%Y/%m/%d/'))
try:
os.makedirs(outpath)
except FileExistsError:
pass # directory exists
# move to destination
print(os.path.join(outpath, downloaded_file.filename))
shutil.move(infile,
os.path.join(outpath, downloaded_file.filename))
fls.append(outpath)
except TypeError: #catching the case when no files that day
fls.append("NO data on " + day)
return fls
def callback(ch, method, properties, body):
print("Recieved with routing key:", method.routing_key, "Data recieved:",
body)
# Input recieved as a message
body = json.loads(body)
year = body['year']
day = body['day']
month = body['month']
station = body['radar']
key = body['key']
print("Inputs in DI:", year, day, month, station, key)
conn = nexradaws.NexradAwsInterface()
availscans = conn.get_avail_scans(year, month, day, station)
serialized_obj = pickle.dumps({'key': key, 'message': availscans[0]})
connection_send = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel_send = connection_send.channel()
channel_send.exchange_declare(exchange='Broker', exchange_type='direct')
channel_send.basic_publish(exchange='Broker',
routing_key='get_objects',
body=serialized_obj)
print("Sent nexrad object from Data ingestor:", serialized_obj)
connection_send.close()
def pull_data(self):
aws_interface = nexradaws.NexradAwsInterface()
try:
self.local_data = aws_interface.download(
[self.scan], LOC_FOLS['nexrad'])._successfiles[0]
except:
self.local_data = None
del aws_interface
def get_results():
data_handling = DataHandling(time_zone=pytz.timezone('US/Central'))
data_handling.get_inputs()
conn = nexradaws.NexradAwsInterface()
scans = conn.get_avail_scans_in_range(data_handling.start_time,
data_handling.end_time,
data_handling.site)
print("There are {} scans available between {} and {}\n".format(
len(scans), data_handling.start_time, data_handling.end_time))
temp_location = tempfile.mkdtemp()
results = conn.download(scans[0:], temp_location)
print(results)
def callback(ch, method, properties, body):
import pickle
body = pickle.loads(body)
key = body['key']
print("got key:", key)
radar_object = body['message']
conn = nexradaws.NexradAwsInterface()
results = conn.download(radar_object, os.getcwd())
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(16, 12))
for i, scan in enumerate(results.iter_success(), start=1):
radar = scan.open_pyart()
display = pyart.graph.RadarDisplay(radar)
display.plot('reflectivity', 0, title="{} {}".format(scan.radar_id, scan.scan_time))
display.set_limits((-150, 150), (-150, 150))
plt.savefig(key + '.png')
# for item in os.getcwd():
# if item.endswith(".png"):
# os.remove(os.path.join(os.getcwd(), item))
print("Recieved with routing key:", method.routing_key, "Latest Data:", radar_object)
# img = Image.open(key + '.png')
# output_plot = io.BytesIO()
# img.save(output_plot, format=img.format)
# output_plot.close()
# output_plot = base64.b64encode(output_plot)
with open(key + ".png", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
dir_name = os.getcwd()
dir = os.listdir(dir_name)
for item in dir:
if item.endswith(".png"):
os.remove(os.path.join(dir_name, item))
if item.endswith(".gz"):
os.remove(os.path.join(dir_name, item))
connection_send = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel_send = connection_send.channel()
channel_send.exchange_declare(exchange='Broker', exchange_type='direct')
channel_send.basic_publish(exchange='Broker', routing_key='API_send', properties=pika.BasicProperties(
headers={'key': key} # Add a key/value header
), body=encoded_string)
print("Sent from Model Analysis:")
connection_send.close()
def get_WSR_from_AWS(config, day, start, end, radar_id):
''' Retrieve the NEXRAD files that fall within a timerange for a specified radar site from the AWS server
----------
INPUTS radar_id : string, four letter radar designation
start & end: datetime, start & end of the desired timerange
-------
RETURN radar_list : Py-ART Radar Objects
'''
# Create this at the point of use # Otherwise it saves everything and eventually crashes
conn = nexradaws.NexradAwsInterface()
#Determine the radar scans that fall within the time range for a given radar site
scans = conn.get_avail_scans_in_range(start, end, radar_id)
print("There are {} scans available between {} and {}\n".format(len(scans), start, end))
# Don't download files that you already have...
path = config.g_download_directory+ day +'/radar/Nexrad/Nexrad_files/'
# If you dont have the path already make it and download the files
if not os.path.exists(path): Path(path).mkdir(parents=True, exist_ok=True)
# Remove all files ending in _MDM
scans = list(filter(lambda x: not fnmatch.fnmatch(x.create_filepath(path, False)[-1], '*_MDM') , scans))
# missing_scans is a list of scans we don't have and need to download
# create_filepath returns tuple of (directory, directory+filename)
# [-1] returns the directory+filename
missing_scans = list(filter(lambda x: not Path(x.create_filepath(path,False)[-1]).exists(), scans))
# missing files is the list of filenames of files we need to download
missing_files = list(map(lambda x: x.create_filepath(path,False)[-1], missing_scans))
print("missing ", len(missing_files), "of ", len(scans), " files")
print(missing_files)
# Download the files
results = conn.download(missing_scans, path, keep_aws_folders=False)
print(results.success)
print("{} downloads failed: {}\n".format(results.failed_count,results.failed))
#print("Results.iter_success : {}\n".format(results.iter_success()))
# missing_scans_after is a list of scans we don't have (download failed)
# create_filepath returns tuple of (directory, directory+filename)
# [-1] returns the directory+filename
missing_files_after = list(filter(lambda x: not Path(x.create_filepath(path,False)[-1]).exists(), scans))
if len(missing_files_after) > 0:
print("ERROR: Some Radar Scans Missing \n ", missing_files_after)
exit()
# Return list of files
radar_files = list(map(lambda x: x.create_filepath(path,False)[-1], scans))
return radar_files
def getDataAWS(radar,sYear,sDay,sMonth,sStartH,sStartM,sEndH,sEndM):
conn = nexradaws.NexradAwsInterface()
downloadDirectory = os.getcwd()
central_timezone = pytz.timezone('US/Central')
start = central_timezone.localize(datetime(sYear,sMonth,sDay,sStartH,sStartM))
end = central_timezone.localize (datetime(sYear,sMonth,sDay,sEndH,sEndM))
scans = conn.get_avail_scans_in_range(start, end, radar)
print('There are {} scans available between {} and {}\n'.format(len(scans), start, end))
results = conn.download(scans, downloadDirectory)
print(results.success)
print(results.failed)
return results.success
def download_radar_data(download_folder, unzipped_data_folder, scan):
# Download a file
conn = nexradaws.NexradAwsInterface()
localfiles = conn.download(scan, download_folder)
radar_filename = sorted(os.listdir(download_folder))[0]
radar_filepath = download_folder + radar_filename
# Remove MDM files as we do not need them
if '_MDM' in radar_filename:
os.remove(radar_filepath)
else:
# Unzip downloaded data if it is zipped
if (radar_filename[-3:] == '.gz'):
with gzip.open(radar_filepath, 'rb') as f_in:
with open(unzipped_data_folder + radar_filename[:-3],
'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
radar_filename = radar_filename[:-3]
else:
shutil.move(radar_filepath, unzipped_data_folder + radar_filename)
return radar_filename
def pull_new_data(self):
year = self.__sim_time.year
month = self.__sim_time.month
day = self.__sim_time.day
self.cl_wd()
new_relevant_stations = []
logging.info(TAG + 'starting a pull of new data')
aws_interface = nexradaws.NexradAwsInterface()
radar_list = aws_interface.get_avail_radars(year, month, day)
# Updates the scans for each station to ones closest to the target time
for station in self.__relevant_stations:
name = station.icao
if name in radar_list:
scans = aws_interface.get_avail_scans(s_ext(str(year), 4),
s_ext(str(month), 2),
s_ext(str(day), 2), name)
closest_time_delta = abs(
(self.__sim_time - scans[0].scan_time).total_seconds())
closest_scan = scans[0]
for index in range(1, len(scans) - 1):
delta = abs((self.__sim_time -
scans[index].scan_time).total_seconds())
if delta > closest_time_delta:
break
else:
closest_time_delta = delta
closest_scan = scans[index]
logging.info(TAG + name)
logging.info(TAG + 'target time: ' + str(self.__sim_time))
logging.info(TAG + 'scan time: ' + str(closest_scan.scan_time))
station.scan = closest_scan
new_relevant_stations.append(station)
# Have each RadarStation update itself, downloading its respective scan
self.__relevant_stations = new_relevant_stations
for station in self.__relevant_stations:
station.pull_data()
logging.info(TAG +
'pulled data for all the stations in the station manager')
del aws_interface
def get_month(year, month, odir):
c = cdsapi.Client()
conn = nexradaws.NexradAwsInterface()
days = conn.get_avail_days(year, month)
fls = []
for day in days:
print('doing ', day)
# stage to temp dir
fname = 'era5_seusa' + year + month + day + '.nc'
outpath = os.path.join(odir, 'era5', year, month)
try:
os.makedirs(outpath)
except FileExistsError:
pass # directory exists
myreq = make_a_request(year, month, day)
c.retrieve("reanalysis-era5-pressure-levels", myreq,
os.path.join(outpath, fname))
fls.append(fname)
return fls
def callback(ch, method, properties, body):
import pickle
body = pickle.loads(body)
key = body['key']
print("got key:", key)
radar_object = body['message']
print("Recieved with routing key:", method.routing_key, "Latest Data:",
radar_object)
conn = nexradaws.NexradAwsInterface()
results = conn.download(radar_object, os.getcwd())
for i, scan in enumerate(results.iter_success(), start=1):
radar = scan.open_pyart()
max_spectrum_width = radar.fields['spectrum_width']['valid_max']
print("max:", max_spectrum_width)
dir_name = os.getcwd()
dir = os.listdir(dir_name)
for item in dir:
if item.endswith(".gz"):
os.remove(os.path.join(dir_name, item))
connection_send = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel_send = connection_send.channel()
channel_send.exchange_declare(exchange='Broker', exchange_type='direct')
channel_send.basic_publish(
exchange='Broker',
routing_key='API_send_pp',
properties=pika.BasicProperties(
headers={'key': key} # Add a key/value header
),
body=str(max_spectrum_width))
print("Sent from Model Analysis:")
connection_send.close()
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 6 16:03:52 2019
@author: lmtomkin
"""
import nexradaws
import os
date = '20200218'
radar = 'KBUF'
conn = nexradaws.NexradAwsInterface()
availscans = conn.get_avail_scans(date[0:4], date[4:6], date[6:8], radar)
if not os.path.exists('G:\\My Drive\\phd\\plotly\\data\\NEXRAD\\'+radar+'\\'+date):
os.makedirs('G:\\My Drive\\phd\\plotly\\data\\NEXRAD\\'+radar+'\\'+date)
results = conn.download(availscans, 'G:\\My Drive\\phd\\plotly\\data\\NEXRAD\\'+radar+'\\'+date)
def multi_case_algorithm_ML1_dev(storm_relative_dir, zdrlev, kdplev, REFlev, REFlev1, big_storm, zero_z_trigger, storm_to_track, year, month, day, hour, start_min, duration, calibration, station, h_Z0C, track_dis=10):
#Set vector perpendicular to FFD Z gradient
storm_relative_dir = storm_relative_dir
#Set ZDR Threshold for outlining arcs
zdrlev = [zdrlev]
#Set KDP Threshold for finding KDP feet
kdplev = [kdplev]
#Set reflectivity thresholds for storm tracking algorithm
REFlev = [REFlev]
REFlev1 = [REFlev1]
#Set storm size threshold that triggers subdivision of big storms
big_storm = big_storm #km^2
Z0C = h_Z0C
Outer_r = 30 #km
Inner_r = 6 #km
#Set trigger to ignore strangely-formatted files right before 00Z
#Pre-SAILS #: 17
#SAILS #: 25
zero_z_trigger = zero_z_trigger
storm_to_track = storm_to_track
zdr_outlines = []
#Here, set the initial time of the archived radar loop you want.
#Our specified time
dt = datetime(year,month, day, hour, start_min)
station = station
end_dt = dt + timedelta(hours=duration)
#Set up nexrad interface
conn = nexradaws.NexradAwsInterface()
scans = conn.get_avail_scans_in_range(dt,end_dt,station)
results = conn.download(scans, 'RadarFolder')
#Setting counters for figures and Pandas indices
f = 27
n = 1
storm_index = 0
scan_index = 0
tracking_index = 0
#Create geod object for later distance and area calculations
g = Geod(ellps='sphere')
#Open the placefile
f = open("NEWSPORK"+station+str(dt.year)+str(dt.month)+str(dt.day)+str(dt.hour)+str(dt.minute)+"_Placefile.txt", "w+")
f.write("Title: SPORK Placefile \n")
f.write("Refresh: 8 \n \n")
#Load ML algorithm
forest_loaded = pickle.load(open('BestRandomForest.pkl', 'rb'))
forest_loaded_col = pickle.load(open('BestRandomForestColumnsLEN200.pkl', 'rb'))
#Actual algorithm code starts here
#Create a list for the lists of arc outlines
zdr_out_list = []
tracks_dataframe = []
for i,scan in enumerate(results.iter_success(),start=1):
#Local file option:
#Loop over all files in the dataset and pull out each 0.5 degree tilt for analysis
try:
radar1 = scan.open_pyart()
except:
print('bad radar file')
continue
#Local file option
print('File Reading')
#Make sure the file isn't a strange format
if radar1.nsweeps > zero_z_trigger:
continue
#Calling quality_control from ungridded_section.py; See separate function for break-down
[radar,n,range_2d,last_height,rlons_h,rlats_h,ungrid_lons,ungrid_lats] = quality_control(radar1,n,calibration)
time_start = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
object_number=0.0
month = time_start.month
if month < 10:
month = '0'+str(month)
hour = time_start.hour
if hour < 10:
hour = '0'+str(hour)
minute = time_start.minute
if minute < 10:
minute = '0'+str(minute)
day = time_start.day
if day < 10:
day = '0'+str(day)
time_beg = time_start - timedelta(minutes=0.5)
time_end = time_start + timedelta(minutes=5.0)
sec_beg = time_beg.second
sec_end = time_end.second
min_beg = time_beg.minute
min_end = time_end.minute
h_beg = time_beg.hour
h_end = time_end.hour
d_beg = time_beg.day
d_end = time_end.day
if sec_beg < 10:
sec_beg = '0'+str(sec_beg)
if sec_end < 10:
sec_end = '0'+str(sec_end)
if min_beg < 10:
min_beg = '0'+str(min_beg)
if min_end < 10:
min_end = '0'+str(min_end)
if h_beg < 10:
h_beg = '0'+str(h_beg)
if h_end < 10:
h_end = '0'+str(h_end)
if d_beg < 10:
d_beg = '0'+str(d_beg)
if d_end < 10:
d_end = '0'+str(d_end)
#Calling kdp_section; See separate function for break-down
kdp_nwsdict = kdp_genesis(radar)
#Add field to radar
radar.add_field('KDP', kdp_nwsdict)
kdp_ungridded_nws = radar.fields['KDP']['data']
#Calling gridding from grid_section.py; See separate function for break-down
[Zint,REF,KDP,CC,CC_c,CCall,ZDRmasked1,ZDRrmasked1,REFmasked,REFrmasked,KDPmasked,KDPrmasked,rlons,rlats,rlons_2d,rlats_2d,cenlat,cenlon] = gridding(radar,Z0C)
#Calling grad_mask from gradient_section.py; See separate function for break-down
[grad_mag,grad_ffd,ZDRmasked,ZDRallmasked,ZDRrmasked] = grad_mask(Zint,REFmasked,REF,storm_relative_dir,ZDRmasked1,ZDRrmasked1,CC,CCall)
#Let's create the ZDR column depth field as in Snyder et al. (2015)
ZDR_count = np.copy(ZDRallmasked)
ZDR_count[ZDR_count > 1.0] = 1
ZDR_count[ZDR_count < 1.0] = 0
ZDR_sum_stuff = np.zeros((ZDR_count.shape[1], ZDR_count.shape[2]))
ZDR_top = np.copy(ZDR_count[(Zint-4):,:,:])
for i in range(ZDR_top.shape[0]):
ZDR_new_sum = ZDR_sum_stuff + ZDR_top[i,:,:]
ZDR_same = np.where(ZDR_new_sum-ZDR_sum_stuff==0)
ZDR_top[i:,ZDR_same[0],ZDR_same[1]] = 0
ZDR_sum_stuff = ZDR_new_sum
#Let's create a field for inferred hail
REF_Hail = np.copy(REFmasked)
REF_Hail1 = ma.masked_where(ZDRmasked1 > 1.0, REF_Hail)
REF_Hail2 = ma.masked_where(CC > 1.0, REF_Hail1)
REF_Hail2 = ma.filled(REF_Hail2, fill_value = 1)
#Let's set up the map projection!
crs = ccrs.LambertConformal(central_longitude=-100.0, central_latitude=45.0)
#Set up our array of latitude and longitude values and transform our data to the desired projection.
tlatlons = crs.transform_points(ccrs.LambertConformal(central_longitude=265, central_latitude=25, standard_parallels=(25.,25.)),rlons[0,:,:],rlats[0,:,:])
tlons = tlatlons[:,:,0]
tlats = tlatlons[:,:,1]
#Limit the extent of the map area, must convert to proper coords.
LL = (cenlon-1.0,cenlat-1.0,ccrs.PlateCarree())
UR = (cenlon+1.0,cenlat+1.0,ccrs.PlateCarree())
print(LL)
#Get data to plot state and province boundaries
states_provinces = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_1_states_provinces_lakes',
scale='50m',
facecolor='none')
#Make sure these shapefiles are in the same directory as the script
#fname = 'cb_2016_us_county_20m/cb_2016_us_county_20m.shp'
#fname2 = 'cb_2016_us_state_20m/cb_2016_us_state_20m.shp'
#counties = ShapelyFeature(Reader(fname).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')
#states = ShapelyFeature(Reader(fname2).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')
#Create a figure and plot up the initial data and contours for the algorithm
fig=plt.figure(n,figsize=(30.,25.))
ax = plt.subplot(111,projection=ccrs.PlateCarree())
ax.coastlines('50m',edgecolor='black',linewidth=0.75)
#ax.add_feature(counties, edgecolor = 'black', linewidth = 0.5)
#ax.add_feature(states, edgecolor = 'black', linewidth = 1.5)
ax.set_extent([LL[0],UR[0],LL[1],UR[1]])
REFlevels = np.arange(20,73,2)
depth_levels= np.arange(0.01,23,1)
#Options for Z backgrounds/contours
#refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_c, cmap=plt.cm.gist_ncar, vmin = 10, vmax = 73)
#refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_ungridded_base, cmap='HomeyerRainbow', vmin = 10, vmax = 73)
#refp = ax.pcolormesh(rlons_2d, rlats_2d, REFrmasked, cmap=pyart.graph.cm_colorblind.HomeyerRainbow, vmin = 10, vmax = 73)
refp2 = ax.contour(rlons_2d, rlats_2d, REFmasked, [40], colors='grey', linewidths=5, zorder=1)
#refp3 = ax.contour(rlons_2d, rlats_2d, REFmasked, [45], color='r')
#plt.contourf(rlons_2d, rlats_2d, ZDR_sum_stuff, depth_levels, cmap=plt.cm.viridis)
#Option to have a ZDR background instead of Z:
#zdrp = ax.pcolormesh(ungrid_lons, ungrid_lats, zdr_c, cmap=plt.cm.nipy_spectral, vmin = -2, vmax = 6)
#Storm tracking algorithm starts here
#Reflectivity smoothed for storm tracker
smoothed_ref = ndi.gaussian_filter(REFmasked, sigma = 3, order = 0)
#1st Z contour plotted
refc = ax.contour(rlons[0,:,:],rlats[0,:,:],smoothed_ref,REFlev, alpha=.01)
#Set up projection for area calculations
proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
pyproj.Proj(init='epsg:3857'))
#Main part of storm tracking algorithm starts by looping through all contours looking for Z centroids
#This method for breaking contours into polygons based on this stack overflow tutorial:
#https://gis.stackexchange.com/questions/99917/converting-matplotlib-contour-objects-to-shapely-objects
#Calling storm_objects from stormid_section.py
[storm_ids,max_lons_c,max_lats_c,ref_areas,storm_index] = storm_objects(refc,proj,REFlev,REFlev1,big_storm,smoothed_ref,ax,rlons,rlats,storm_index,tracking_index,scan_index,tracks_dataframe,track_dis)
#Setup tracking index for storm of interest
tracking_ind=np.where(np.asarray(storm_ids)==storm_to_track)[0]
max_lons_c = np.asarray(max_lons_c)
max_lats_c = np.asarray(max_lats_c)
ref_areas = np.asarray(ref_areas)
#Create the ZDR and KDP contours which will later be broken into polygons
if np.max(ZDRmasked) > zdrlev:
zdrc = ax.contour(rlons[0,:,:],rlats[0,:,:],ZDRmasked,zdrlev,linewidths = 2, colors='purple', alpha = .5)
else:
zdrc=[]
if np.max(ZDRrmasked) > 1.0:
zdrrc = ax.contour(rlons[0,:,:],rlats[0,:,:],ZDRrmasked,[1.0],linewidths = 4, colors='cyan', alpha = 0.4)
else:
zdrrc=[]
if np.max(KDPmasked) > kdplev:
kdpc = ax.contour(rlons[0,:,:],rlats[0,:,:],KDPmasked,kdplev,linewidths = 2, colors='green', alpha = 0.01)
else:
kdpc=[]
if np.max(REF_Hail2) > 50.0:
hailc = ax.contour(rlons[0,:,:],rlats[0,:,:],REF_Hail2,[50],linewidths = 4, colors='pink', alpha = 0.01)
else:
hailc=[]
if np.max(REFmasked) > 35.0:
zhhc = ax.contour(rlons[0,:,:],rlats[0,:,:],REFmasked,[35.0],linewidths = 3,colors='orange', alpha = 0.8)
else:
zhhc=[]
plt.contour(ungrid_lons, ungrid_lats, range_2d, [73000], linewidths=7, colors='r')
plt.contour(rlons_h, rlats_h, last_height, [Z0C], linewidths=7, colors='g')
plt.savefig('testfig.png')
print('Testfig Saved')
if len(max_lons_c) > 0:
#Calling zdr_arc_section; Create ZDR arc objects using a similar method as employed in making the storm objects
[zdr_storm_lon,zdr_storm_lat,zdr_dist,zdr_forw,zdr_back,zdr_areas,zdr_centroid_lon,zdr_centroid_lat,zdr_mean,zdr_cc_mean,zdr_max,zdr_masks,zdr_outlines,ax,f] = zdrarc(zdrc,ZDRmasked,CC,REF,grad_ffd,grad_mag,KDP,forest_loaded,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,zdrlev,proj,storm_relative_dir,Outer_r,Inner_r,tracking_ind)
#Calling hail_section; Identify Hail core objects in a similar way to the ZDR arc objects
[hail_areas,hail_centroid_lon,hail_centroid_lat,hail_storm_lon,hail_storm_lat,ax,f] = hail_objects(hailc,REF_Hail2,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,proj)
#Calling zhh_section; Identify 35dBz storm area in a similar way to the ZDR arc objects
[zhh_areas,zhh_centroid_lon,zhh_centroid_lat,zhh_storm_lon,zhh_storm_lat,zhh_max,zhh_core_avg] = zhh_objects(zhhc,REFmasked,rlons,rlats,max_lons_c,max_lats_c,proj)
#Calling kdpfoot_section; Identify KDP foot objects in a similar way to the ZDR arc objects
[kdp_areas,kdp_centroid_lon,kdp_centroid_lat,kdp_storm_lon,kdp_storm_lat,kdp_max,ax,f] = kdp_objects(kdpc,KDPmasked,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,kdplev,proj)
#Calling zdr_col_section; Identify ZDR columns in a similar way to the ZDR arc objects
[col_areas,col_maxdepths,col_depths,col_centroid_lon,col_centroid_lat,col_storm_lon,col_storm_lat,ax,col_masks,f] = zdrcol(zdrrc,ZDRrmasked,CC_c,REFrmasked,grad_ffd,grad_mag,KDP,ZDR_sum_stuff,KDPrmasked,depth_levels,forest_loaded_col,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,ref_areas,proj,storm_relative_dir,tracking_ind,object_number)
#Consolidating the arc objects associated with each storm:
zdr_areas_arr = np.zeros((len(zdr_areas)))
zdr_max_arr = np.zeros((len(zdr_max)))
zdr_mean_arr = np.zeros((len(zdr_mean)))
for i in range(len(zdr_areas)):
zdr_areas_arr[i] = zdr_areas[i].magnitude
zdr_max_arr[i] = zdr_max[i]
zdr_mean_arr[i] = zdr_mean[i]
zdr_centroid_lons = np.asarray(zdr_centroid_lon)
zdr_centroid_lats = np.asarray(zdr_centroid_lat)
zdr_con_areas = []
zdr_con_maxes = []
zdr_con_means = []
zdr_con_centroid_lon = []
zdr_con_centroid_lat = []
zdr_con_max_lon = []
zdr_con_max_lat = []
zdr_con_storm_lon = []
zdr_con_storm_lat = []
zdr_con_masks = []
zdr_con_dev = []
zdr_con_10max = []
zdr_con_mode = []
zdr_con_median = []
zdr_masks = np.asarray(zdr_masks)
#Consolidate KDP objects as well
kdp_areas_arr = np.zeros((len(kdp_areas)))
kdp_max_arr = np.zeros((len(kdp_max)))
for i in range(len(kdp_areas)):
kdp_areas_arr[i] = kdp_areas[i].magnitude
kdp_max_arr[i] = kdp_max[i]
kdp_centroid_lons = np.asarray(kdp_centroid_lon)
kdp_centroid_lats = np.asarray(kdp_centroid_lat)
kdp_con_areas = []
kdp_con_maxes = []
kdp_con_centroid_lon = []
kdp_con_centroid_lat = []
kdp_con_max_lon = []
kdp_con_max_lat = []
kdp_con_storm_lon = []
kdp_con_storm_lat = []
#Consolidate Hail objects as well
hail_areas_arr = np.zeros((len(hail_areas)))
for i in range(len(hail_areas)):
hail_areas_arr[i] = hail_areas[i].magnitude
hail_centroid_lons = np.asarray(hail_centroid_lon)
hail_centroid_lats = np.asarray(hail_centroid_lat)
hail_con_areas = []
hail_con_centroid_lon = []
hail_con_centroid_lat = []
hail_con_storm_lon = []
hail_con_storm_lat = []
#Consolidate Zhh objects as well
zhh_areas_arr = np.zeros((len(zhh_areas)))
zhh_max_arr = np.zeros((len(zhh_max)))
zhh_core_avg_arr = np.zeros((len(zhh_core_avg)))
for i in range(len(zhh_areas)):
zhh_areas_arr[i] = zhh_areas[i].magnitude
zhh_max_arr[i] = zhh_max[i]
zhh_core_avg_arr[i] = zhh_core_avg[i]
zhh_centroid_lons = np.asarray(zhh_centroid_lon)
zhh_centroid_lats = np.asarray(zhh_centroid_lat)
zhh_con_areas = []
zhh_con_maxes = []
zhh_con_core_avg = []
zhh_con_centroid_lon = []
zhh_con_centroid_lat = []
zhh_con_max_lon = []
zhh_con_max_lat = []
zhh_con_storm_lon = []
zhh_con_storm_lat = []
#Consolidate ZDR Column objects as well
col_areas_arr = np.zeros((len(col_areas)))
col_peaks_arr = np.zeros((len(col_areas)))
col_depths_arr = np.zeros((len(col_areas)))
for i in range(len(col_areas)):
col_areas_arr[i] = col_areas[i].magnitude
col_peaks_arr[i] = col_maxdepths[i]
col_depths_arr[i] = col_depths[i]
col_centroid_lons = np.asarray(col_centroid_lon)
col_centroid_lats = np.asarray(col_centroid_lat)
col_con_areas = []
col_con_peaks = []
col_con_depths = []
col_con_masks = []
col_con_centroid_lon = []
col_con_centroid_lat = []
col_con_storm_lon = []
col_con_storm_lat = []
col_masks = np.asarray(col_masks)
for i in enumerate(max_lons_c):
try:
#Find the arc objects associated with this storm:
zdr_objects_lons = zdr_centroid_lons[np.where(zdr_storm_lon == max_lons_c[i[0]])]
zdr_objects_lats = zdr_centroid_lats[np.where(zdr_storm_lon == max_lons_c[i[0]])]
#Get the sum of their areas
zdr_con_areas.append(np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#print("consolidated area", np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
zdr_con_maxes.append(np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#print("consolidated max", np.max(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
zdr_con_means.append(np.mean(zdr_mean_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#print("consolidated mean", np.mean(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
zdr_con_max_lon.append(rlons_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])
zdr_con_max_lat.append(rlats_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])
#Find the actual centroids
weighted_lons = zdr_objects_lons * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
zdr_con_centroid_lon.append(np.sum(weighted_lons) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
weighted_lats = zdr_objects_lats * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
zdr_con_centroid_lat.append(np.sum(weighted_lats) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
zdr_con_storm_lon.append(max_lons_c[i[0]])
zdr_con_storm_lat.append(max_lats_c[i[0]])
zdr_con_masks.append(np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])],axis=0, dtype=bool))
mask_con = np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])], axis=0, dtype=bool)
zdr_con_dev.append(np.std(ZDRmasked[mask_con]))
ZDRsorted = np.sort(ZDRmasked[mask_con])[::-1]
zdr_con_10max.append(np.mean(ZDRsorted[0:10]))
zdr_con_mode.append(stats.mode(ZDRmasked[mask_con]))
zdr_con_median.append(np.median(ZDRmasked[mask_con]))
except:
zdr_con_maxes.append(0)
zdr_con_means.append(0)
zdr_con_centroid_lon.append(0)
zdr_con_centroid_lat.append(0)
zdr_con_max_lon.append(0)
zdr_con_max_lat.append(0)
zdr_con_storm_lon.append(max_lons_c[i[0]])
zdr_con_storm_lat.append(max_lats_c[i[0]])
zdr_con_masks.append(0)
zdr_con_dev.append(0)
zdr_con_10max.append(0)
zdr_con_mode.append(0)
zdr_con_median.append(0)
try:
#Find the kdp objects associated with this storm:
kdp_objects_lons = kdp_centroid_lons[np.where(kdp_storm_lon == max_lons_c[i[0]])]
kdp_objects_lats = kdp_centroid_lats[np.where(kdp_storm_lon == max_lons_c[i[0]])]
#Get the sum of their areas
kdp_con_areas.append(np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
kdp_con_maxes.append(np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
kdp_con_max_lon.append(rlons_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
kdp_con_max_lat.append(rlats_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
#Find the actual centroids
weighted_lons_kdp = kdp_objects_lons * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
kdp_con_centroid_lon.append(np.sum(weighted_lons_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
weighted_lats_kdp = kdp_objects_lats * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
kdp_con_centroid_lat.append(np.sum(weighted_lats_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
kdp_con_storm_lon.append(max_lons_c[i[0]])
kdp_con_storm_lat.append(max_lats_c[i[0]])
except:
kdp_con_maxes.append(0)
kdp_con_max_lon.append(0)
kdp_con_max_lat.append(0)
kdp_con_centroid_lon.append(0)
kdp_con_centroid_lat.append(0)
kdp_con_storm_lon.append(0)
kdp_con_storm_lat.append(0)
try:
#Find the hail core objects associated with this storm:
hail_objects_lons = hail_centroid_lons[np.where(hail_storm_lon == max_lons_c[i[0]])]
hail_objects_lats = hail_centroid_lats[np.where(hail_storm_lon == max_lons_c[i[0]])]
#Get the sum of their areas
hail_con_areas.append(np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
#Find the actual centroids
weighted_lons_hail = hail_objects_lons * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
hail_con_centroid_lon.append(np.sum(weighted_lons_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
weighted_lats_hail = hail_objects_lats * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
hail_con_centroid_lat.append(np.sum(weighted_lats_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
hail_con_storm_lon.append(max_lons_c[i[0]])
hail_con_storm_lat.append(max_lats_c[i[0]])
except:
hail_con_centroid_lon.append(0)
hail_con_centroid_lat.append(0)
hail_con_storm_lon.append(0)
hail_con_storm_lat.append(0)
try:
#Find the zhh objects associated with this storm:
zhh_objects_lons = zhh_centroid_lons[np.where(zhh_storm_lon == max_lons_c[i[0]])]
zhh_objects_lats = zhh_centroid_lats[np.where(zhh_storm_lon == max_lons_c[i[0]])]
#Get the sum of their areas
zhh_con_areas.append(np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
zhh_con_maxes.append(np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
zhh_con_core_avg.append(np.max(zhh_core_avg_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
zhh_con_max_lon.append(rlons_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
zhh_con_max_lat.append(rlats_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
#Find the actual centroids
weighted_lons_zhh = zhh_objects_lons * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
zhh_con_centroid_lon.append(np.sum(weighted_lons_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
weighted_lats_zhh = zhh_objects_lats * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
zhh_con_centroid_lat.append(np.sum(weighted_lats_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
zhh_con_storm_lon.append(max_lons_c[i[0]])
zhh_con_storm_lat.append(max_lats_c[i[0]])
except:
zhh_con_maxes.append(0)
zhh_con_core_avg.append(0)
zhh_con_max_lon.append(0)
zhh_con_max_lat.append(0)
zhh_con_centroid_lon.append(0)
zhh_con_centroid_lat.append(0)
zhh_con_storm_lon.append(0)
zhh_con_storm_lat.append(0)
try:
#Find the kdp objects associated with this storm:
col_objects_lons = col_centroid_lons[np.where(col_storm_lon == max_lons_c[i[0]])]
col_objects_lats = col_centroid_lats[np.where(col_storm_lon == max_lons_c[i[0]])]
#Get the sum of their areas
col_con_storm_lon.append(max_lons_c[i[0]])
col_con_storm_lat.append(max_lats_c[i[0]])
col_con_areas.append(np.sum(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
weighted_lons_col = col_objects_lons * col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]
col_con_centroid_lon.append(np.sum(weighted_lons_col) / np.sum(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
weighted_lats_col = col_objects_lats * col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]
col_con_centroid_lat.append(np.sum(weighted_lats_col) / np.sum(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
col_con_peaks.append(np.max(col_peaks_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
mask_con_col = np.sum(col_masks[np.where(col_storm_lon == max_lons_c[i[0]])], axis=0, dtype=bool)
col_con_depths.append(np.mean(ZDR_sum_stuff[mask_con_col]))
#if len(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])])==0:
# col_con_areas.append(0)
#elif col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])].shape[0] == 1:
#col_con_areas.append(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])])
#col_con_maxes.append(np.max(col_max_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
#col_con_max_lon.append(rlons_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
#col_con_max_lat.append(rlats_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
#Find the actual centroids
#col_ind = np.where(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])] == np.max(col_areas_arr[np.where(col_storm_lon == max_lons_c[i[0]])]))
#col_con_centroid_lon.append(col_objects_lons[col_ind][0])
#col_con_centroid_lat.append(col_objects_lats[col_ind][0])
#Find the actual centroids
except:
#col_con_areas.append(0)
#kdp_con_maxes.append(0)
#kdp_con_max_lon.append(0)
#kdp_con_max_lat.append(0)
#uncomment
#col_con_centroid_lon.append(0)
#col_con_centroid_lat.append(0)
#col_con_storm_lon.append(0)
#col_con_storm_lat.append(0)
col_con_peaks.append(0)
col_con_depths.append(0)
if len(col_con_areas) < len(col_con_centroid_lon):
col_con_areas.append(0)
#Calculate KDP-ZDR separation
# kdp_con_centroid_lons1 = np.asarray(kdp_con_centroid_lon)
# kdp_con_centroid_lats1 = np.asarray(kdp_con_centroid_lat)
# zdr_con_centroid_lons1 = np.asarray(zdr_con_centroid_lon)
# zdr_con_centroid_lats1 = np.asarray(zdr_con_centroid_lat)
# #Eliminate consolidated arcs smaller than a specified area
# area = 2 #km*2
# zdr_con_areas_arr = np.asarray(zdr_con_areas)
# zdr_con_centroid_lats = zdr_con_centroid_lats1[zdr_con_areas_arr > area]
# zdr_con_centroid_lons = zdr_con_centroid_lons1[zdr_con_areas_arr > area]
# kdp_con_centroid_lats = kdp_con_centroid_lats1[zdr_con_areas_arr > area]
# kdp_con_centroid_lons = kdp_con_centroid_lons1[zdr_con_areas_arr > area]
# zdr_con_max_lons1 = np.asarray(zdr_con_max_lon)[zdr_con_areas_arr > area]
# zdr_con_max_lats1 = np.asarray(zdr_con_max_lat)[zdr_con_areas_arr > area]
# kdp_con_max_lons1 = np.asarray(kdp_con_max_lon)[zdr_con_areas_arr > area]
# kdp_con_max_lats1 = np.asarray(kdp_con_max_lat)[zdr_con_areas_arr > area]
# zdr_con_max1 = np.asarray(zdr_con_maxes)[zdr_con_areas_arr > area]
# zdr_con_areas1 = zdr_con_areas_arr[zdr_con_areas_arr > area]
kdp_con_centroid_lat = np.asarray(kdp_con_centroid_lat)
kdp_con_centroid_lon = np.asarray(kdp_con_centroid_lon)
zdr_con_centroid_lat = np.asarray(zdr_con_centroid_lat)
zdr_con_centroid_lon = np.asarray(zdr_con_centroid_lon)
kdp_inds = np.where(kdp_con_centroid_lat*zdr_con_centroid_lat > 0)
distance_kdp_zdr = g.inv(kdp_con_centroid_lon[kdp_inds], kdp_con_centroid_lat[kdp_inds], zdr_con_centroid_lon[kdp_inds], zdr_con_centroid_lat[kdp_inds])
dist_kdp_zdr = distance_kdp_zdr[2] / 1000.
#Now make an array for the distances which will have the same shape as the lats to prevent errors
shaped_dist = np.zeros((np.shape(zdr_con_areas)))
shaped_dist[kdp_inds] = dist_kdp_zdr
#Get separation angle for KDP-ZDR centroids
back_k = distance_kdp_zdr[1]
for i in range(back_k.shape[0]):
if distance_kdp_zdr[1][i] < 0:
back_k[i] = distance_kdp_zdr[1][i] + 360
forw_k = np.abs(back_k - storm_relative_dir)
rawangle_k = back_k - storm_relative_dir
#Account for weird angles
for i in range(back_k.shape[0]):
if forw_k[i] > 180:
forw_k[i] = 360 - forw_k[i]
rawangle_k[i] = (360-forw_k[i])*(-1)
rawangle_k = rawangle_k*(-1)
#Now make an array for the distances which will have the same shape as the lats to prevent errors
shaped_ang = np.zeros((np.shape(zdr_con_areas)))
shaped_ang[kdp_inds] = rawangle_k
shaped_ang = (180-np.abs(shaped_ang))*(shaped_ang/np.abs(shaped_ang))
###Now let's consolidate everything to fit the Pandas dataframe!
p_zdr_areas = []
p_zdr_maxes = []
p_zdr_means = []
p_zdr_devs = []
p_zdr_10max = []
p_zdr_mode = []
p_zdr_median = []
p_hail_areas = []
p_zhh_areas = []
p_zhh_maxes = []
p_zhh_core_avgs = []
p_separations = []
p_sp_angle = []
p_col_areas = []
p_col_max_depths = []
p_col_depths = []
for storm in enumerate(max_lons_c):
matching_ind = np.flatnonzero(np.isclose(max_lons_c[storm[0]], zdr_con_storm_lon, rtol=1e-05))
if matching_ind.shape[0] > 0:
p_zdr_areas.append((zdr_con_areas[matching_ind[0]]))
p_zdr_maxes.append((zdr_con_maxes[matching_ind[0]]))
p_zdr_means.append((zdr_con_means[matching_ind[0]]))
p_zdr_devs.append((zdr_con_dev[matching_ind[0]]))
p_zdr_10max.append((zdr_con_10max[matching_ind[0]]))
p_zdr_mode.append((zdr_con_mode[matching_ind[0]]))
p_zdr_median.append((zdr_con_median[matching_ind[0]]))
p_separations.append((shaped_dist[matching_ind[0]]))
p_sp_angle.append((shaped_ang[matching_ind[0]]))
else:
p_zdr_areas.append((0))
p_zdr_maxes.append((0))
p_zdr_means.append((0))
p_zdr_devs.append((0))
p_zdr_10max.append((0))
p_zdr_mode.append((0))
p_zdr_median.append((0))
p_separations.append((0))
p_sp_angle.append((0))
matching_ind_hail = np.flatnonzero(np.isclose(max_lons_c[storm[0]], hail_con_storm_lon, rtol=1e-05))
if matching_ind_hail.shape[0] > 0:
p_hail_areas.append((hail_con_areas[matching_ind_hail[0]]))
else:
p_hail_areas.append((0))
matching_ind_zhh = np.flatnonzero(np.isclose(max_lons_c[storm[0]],zhh_con_storm_lon, rtol=1e-05))
if matching_ind_zhh.shape[0] > 0:
p_zhh_maxes.append((zhh_con_maxes[matching_ind_zhh[0]]))
p_zhh_areas.append((zhh_con_areas[matching_ind_zhh[0]]))
p_zhh_core_avgs.append((zhh_con_core_avg[matching_ind_zhh[0]]))
else:
p_zhh_areas.append((0))
p_zhh_maxes.append((0))
p_zhh_core_avgs.append((0))
matching_ind_col = np.flatnonzero(np.isclose(max_lons_c[storm[0]], col_con_storm_lon, rtol=1e-05))
if matching_ind_col.shape[0] > 0:
p_col_areas.append((col_con_areas[matching_ind_col[0]]))
p_col_max_depths.append((col_con_peaks[matching_ind_col[0]]))
p_col_depths.append((col_con_depths[matching_ind_col[0]]))
else:
p_hail_areas.append((0))
p_col_max_depths.append((0))
p_col_depths.append((0))
#Now start plotting stuff!
if np.asarray(zdr_centroid_lon).shape[0] > 0:
ax.scatter(zdr_centroid_lon, zdr_centroid_lat, marker = '*', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
if np.asarray(kdp_centroid_lon).shape[0] > 0:
ax.scatter(kdp_centroid_lon, kdp_centroid_lat, marker = '^', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
#Uncomment to print all object areas
#for i in enumerate(zdr_areas):
# plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2" %(zdr_areas[i[0]].magnitude), size = 23)
#plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2 / %.2f km / %.2f dB" %(zdr_areas[i[0]].magnitude, zdr_dist[i[0]], zdr_forw[i[0]]), size = 23)
#plt.annotate(zdr_areas[i[0]], (zdr_centroid_lon[i[0]],zdr_centroid_lat[i[0]]))
#ax.contourf(rlons[0,:,:],rlats[0,:,:],KDPmasked,KDPlevels1,linewide = .01, colors ='b', alpha = .5)
#plt.tight_layout()
#plt.savefig('ZDRarcannotated.png')
storm_times = []
for l in range(len(max_lons_c)):
storm_times.append((time_start))
tracking_index = tracking_index + 1
#If there are no storms, set everything to empty arrays!
else:
storm_ids = []
storm_ids = []
max_lons_c = []
max_lats_c = []
p_zdr_areas = []
p_zdr_maxes = []
p_zdr_means = []
p_zdr_devs = []
p_zdr_10max = []
p_zdr_mode = []
p_zdr_median = []
p_hail_areas = []
p_zhh_areas = []
p_zhh_maxes = []
p_zhh_core_avgs = []
p_separations = []
p_sp_angle = []
zdr_con_areas1 = []
p_col_areas = []
p_col_max_depths = []
p_col_depths = []
storm_times = time_start
#Now record all data in a Pandas dataframe.
new_cells = pd.DataFrame({
'scan': scan_index,
'storm_id' : storm_ids,
'storm_id1' : storm_ids,
'storm_lon' : max_lons_c,
'storm_lat' : max_lats_c,
'zdr_area' : p_zdr_areas,
'zdr_max' : p_zdr_maxes,
'zdr_mean' : p_zdr_means,
'zdr_std' : p_zdr_devs,
'zdr_10max' : p_zdr_10max,
'zdr_mode' : p_zdr_mode,
'zdr_median' : p_zdr_median,
'hail_area' : p_hail_areas,
'zhh_area' : p_zhh_areas,
'zhh_max' : p_zhh_maxes,
'zhh_core_avg' : p_zhh_core_avgs,
'kdp_zdr_sep' : p_separations,
'kdp_zdr_angle' : p_sp_angle,
'column_area' : p_col_areas,
'column_max_depth' : p_col_max_depths,
'column_mean_depth' : p_col_depths,
'times' : storm_times
})
new_cells.set_index(['scan', 'storm_id'], inplace=True)
if scan_index == 0:
tracks_dataframe = new_cells
else:
tracks_dataframe = tracks_dataframe.append(new_cells)
n = n+1
scan_index = scan_index + 1
#Plot the consolidated stuff!
#Write some text objects for the ZDR arc attributes to add to the placefile
f.write("Color: 139 000 000 \n")
f.write('Font: 1, 30, 1,"Arial" \n')
for y in range(len(p_zdr_areas)):
#f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Arc Area: '+str(p_zdr_areas[y])+'\\n Arc Mean: '+str(p_zdr_means[y])+'\\n KDP-ZDR Separation: '+str(p_separations[y])+'\\n Separation Angle: '+str(p_sp_angle[y])+'" \n')
f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Arc Area: %.2f km^2 \\n Arc Mean: %.2f dB \\n Arc 10 Max Mean: %.2f dB \\n KDP-ZDR Separation: %.2f km \\n Separation Angle: %.2f degrees \\n ZDR Column Area: %.2f km^2 \\n ZDR Column Depth: %.2f m \\n Hail Area: %.2f km^2" \n' %(p_zdr_areas[y], p_zdr_means[y], p_zdr_10max[y], p_separations[y], p_sp_angle[y], p_col_areas[y], p_col_max_depths[y]*250, p_hail_areas[y]))
title_plot = plt.title(station+' Radar Reflectivity, ZDR, and KDP '+str(time_start.year)+'-'+str(time_start.month)+'-'+str(time_start.day)+
' '+str(hour)+':'+str(minute)+' UTC', size = 25)
try:
plt.plot([zdr_con_centroid_lon[kdp_inds], kdp_con_centroid_lon[kdp_inds]], [zdr_con_centroid_lat[kdp_inds],kdp_con_centroid_lat[kdp_inds]], color = 'k', linewidth = 5, transform=ccrs.PlateCarree())
except:
print('Separation Angle Failure')
ref_centroid_lon = max_lons_c
ref_centroid_lat = max_lats_c
if len(max_lons_c) > 0:
ax.scatter(max_lons_c,max_lats_c, marker = "o", color = 'k', s = 500, alpha = .6)
for i in enumerate(ref_centroid_lon):
plt.text(ref_centroid_lon[i[0]]+.016, ref_centroid_lat[i[0]]+.016, "storm_id: %.1f" %(storm_ids[i[0]]), size = 25)
#Comment out this line if not plotting tornado tracks
#plt.plot([start_torlons, end_torlons], [start_torlats, end_torlats], color = 'purple', linewidth = 5, transform=ccrs.PlateCarree())
#Add legend stuff
zdr_outline = mlines.Line2D([], [], color='blue', linewidth = 5, linestyle = 'solid', label='ZDR Arc Outline(Area/Max)')
kdp_outline = mlines.Line2D([], [], color='green', linewidth = 5,linestyle = 'solid', label='"KDP Foot" Outline')
separation_vector = mlines.Line2D([], [], color='black', linewidth = 5,linestyle = 'solid', label='KDP/ZDR Centroid Separation Vector (Red Text=Distance)')
#tor_track = mlines.Line2D([], [], color='purple', linewidth = 5,linestyle = 'solid', label='Tornado Tracks')
elevation = mlines.Line2D([], [], color='grey', linewidth = 5,linestyle = 'solid', label='Height AGL (m)')
plt.legend(handles=[zdr_outline, kdp_outline, separation_vector, elevation], loc = 3, fontsize = 25)
alt_levs = [1000, 2000]
plt.savefig('Machine_Learning/SPORK_DEV'+station+str(time_start.year)+str(time_start.month)+str(day)+str(hour)+str(minute)+'.png')
print('Figure Saved')
plt.close()
zdr_out_list.append(zdr_outlines)
#except:
# traceback.print_exc()
# continue
f.close()
plt.show()
print('Fin')
#export_csv = tracks_dataframe.to_csv(r'C:\Users\Nick\Downloads\tracksdataframe.csv',index=None,header=True)
return tracks_dataframe, zdr_out_list, col_con_areas, col_con_centroid_lon, col_con_storm_lon
def setUp(self):
self.query = nexradaws.NexradAwsInterface()
self.templocation = tempfile.mkdtemp()
def setUp(self):
query = nexradaws.NexradAwsInterface()
self.test_scan = query.get_avail_scans('2013', '05', '31', 'KTLX')[0]
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
conn = nexradaws.NexradAwsInterface()
data = json.loads(body)
central_timezone = pytz.timezone('US/Central')
radar_id = data['radar_Id']
year = data['year']
month = data['month']
day = data['day']
#start = central_timezone.localize(datetime(2013,5,31,17,0))
#end = central_timezone.localize (datetime(2013,5,31,19,0))
# responseQ = pika.BlockingConnection(pika.ConnectionParameters(host='rabbitmq'))
# responseQ = responseQ.channel()
# responseQ.queue_declare(queue='status')
# try:
# scans = conn.get_avail_scans(year, month, day, radar_id)
# except:
# responseQ.basic_publish(exchange='',
# routing_key='status',
# body="Error: Please check the information entered")
# else:
# responseQ.basic_publish(exchange='',
# routing_key='status',
# body = "Downloading files")
# responseQ.close()
#print("There are {} scans available between {} and {}\n".format(len(scans), start, end))
scans = conn.get_avail_scans(year, month, day, radar_id)
print("scans: ", scans[0:int(data["n"])])
results = conn.download(scans[0:int(data["n"])], templocation)
for scan in results.iter_success():
print("{} volume scan time {}".format(scan.radar_id, scan.scan_time))
send_connection = pika.BlockingConnection(
pika.ConnectionParameters('rabbitmq-test'))
send_channel = send_connection.channel()
send_channel.queue_declare(queue='hello')
scansdata = [scan for scan in results.iter_success()]
scansdata = []
for scan in results.iter_success():
scansdata.append((scan.filepath, scan.filename))
jsonObj = {
"year": year,
"month": month,
"day": day,
"n": data["n"],
"date": year + "-" + month + "-" + day,
"radar": radar_id,
"username": data['username']
}
send_channel.basic_publish(exchange='',
routing_key='sendData',
body=json.dumps(jsonObj))
session_connection = pika.BlockingConnection(
pika.ConnectionParameters('rabbitmq-test'))
session_channel = session_connection.channel()
session_channel.queue_declare(queue='receiveData')
sessionObj = {
"username":
data['username'],
"status":
"retrieved",
"timestamp":
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
}
session_channel.basic_publish(exchange='',
routing_key='',
body=json.dumps(sessionObj))
print(" [x] Sent 'Hello World!'")
send_connection.close()
session_connection.close()
def setUp(self):
self.query = nexradaws.NexradAwsInterface()
self.templocation = tempfile.mkdtemp()
start = datetime(2013, 5, 20, 18, 45)
end = datetime(2013, 5, 20, 19, 00)
self.scans = self.query.get_avail_scans_in_range(start, end, 'KTLX')
def handle_delivery(channel, method, header, body):
"""Called when we receive a message from RabbitMQ"""
#print("Method: {}".format(method))
#print("Properties: {}".format(header))
#print(body)
print("message recieved")
print("message recieved")
try:
print("creating ecting to db")
##conn = psycopg2.connect("dbname='dataretrieval_db' user='******' host='localhost' password='******'")
conn = psycopg2.connect(
"dbname='postgres' user='******' host='postgres' password='******'"
)
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
# Obtain a DB Cursor
cursor = conn.cursor()
name_Database = "dataresult_db"
sqlCreateDatabase = "create database " + name_Database + ";"
cursor.execute(sqlCreateDatabase)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
if cursor is not None:
cursor.close()
try:
print("connecting to db")
conn = psycopg2.connect(
"dbname='dataresult_db' user='******' host='postgres' password='******'"
)
print("connected to db")
cur = conn.cursor()
command = create_tables()
print("executin command")
cur.execute(command)
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
if cur is not None:
cur.close()
try:
print("reading data")
data = json.loads(body)
userid = (data['userid'])
correlationid = (data['correlationid'])
year = (data['year'])
month = (data['month'])
day = (data['day'])
starthour = (data['starthour'])
startmin = (data['startmin'])
endhour = (data['endhour'])
endmin = (data['endmin'])
station = (data['station'])
central_timezone = pytz.timezone('US/Central')
radar_id = station
start = central_timezone.localize(
datetime(int(year), int(month), int(day), int(starthour),
int(startmin)))
end = central_timezone.localize(
datetime(int(year), int(month), int(day), int(endhour),
int(endmin)))
conn_nexrad = nexradaws.NexradAwsInterface()
scans = conn_nexrad.get_avail_scans_in_range(start, end, radar_id)
print("There are {} scans available between {} and {}\n".format(
len(scans), start, end))
print(scans[0:4])
templocation = tempfile.mkdtemp()
results = conn_nexrad.download(scans[0:1], templocation)
return_api(format(len(scans)), correlationid, userid)
conn = psycopg2.connect(
"dbname='dataresult_db' user='******' host='postgres' password='******'"
)
cur = conn.cursor()
userid = (data['userid'])
correlationid = (data['correlationid'])
status = "forwarded"
print(userid)
print(correlationid)
sql = "INSERT INTO analysis_status (userid,correlationid,request,status) VALUES(%s,%s,%s,%s);"
record_to_insert = (userid, correlationid, str(data), status)
cur.execute(sql, record_to_insert)
#get the generated id back
#id = cur.fetchone()[0]
conn.commit()
print("data entered")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
if cur is not None:
cur.close()
def ref_point(begin_time,
end_time,
outfile,
radar_id='KFTG',
glon=-105.242117,
glat=40.010494,
elv=0):
"""
Function name: ref_point.py
Purpose: To retireve nexrad data from the input radar and collect dual-pol
data from the gate of interest. The gate is automatically computed
based on the input gauge latitude and longitude.
Inputs:
begin_time: (String) The earliest time to search and retireve radar data in YYYYMMDDHH format
end_time: (String) The latest time to search and retireve radar data in YYYYMMDDHH format
outfile: (String) The path and filename to save the data to. Needs .csv extension
Optional Inputs:
radar_id: (String) The four letter code assigned by the NWS for the radar of interest
glon: (Float) The longitude of the gauge location
glat: (Float) The latitude of the gauge location
elv: (Int) The elevation scan INDEX. elv=0 is the lowest scan,
elv=1 is the second-lowest, etc.
Outputs:
A csv file with the date and time and dual-polarization data at the input
gauge location
Notes:
The deault inputs are for collecting data obtained by KFTG at the snow gauge
located at SkyWatch at SEEC (CU Boulder) in Boulder, CO.
New users must input the filepath to the radarlocations.csv file. Future
development of this routine will add an automatic search for the csv file.
"""
# The failepath to the csv that contains the radar locations.
# These data are provided in the github, but also can be obtained from
# http://apollo.lsc.vsc.edu/classes/remote/lecture_notes/radar/88d/88D_locations.html
# and need to be copied into a csv file.
radarlocations_filepath = 'path/to/radarlocations.csv'
# Set the beginning and end times for the period of analysis in YYYYMMDDHH format
radar_begin_time = datetime.datetime.strptime(begin_time, '%Y%m%d%H%M')
radar_end_time = datetime.datetime.strptime(end_time, '%Y%m%d%H%M')
if radar_begin_time > radar_end_time:
raise ValueError(
"Timespan for search invalid. End time needs to be after the start time"
)
# You can change the time zone if it's more conveient for you.
timezone = pytz.timezone('UTC')
# Set the begin and end times to a format the nexrad data call funtion can use
start = timezone.localize(radar_begin_time)
end = timezone.localize(radar_end_time)
templocation = tempfile.mkdtemp()
conn = nexradaws.NexradAwsInterface()
# This will scan the AWS servers for data between the specified time frame
scans = conn.get_avail_scans_in_range(start, end, radar_id)
if len(scans) == 0:
raise ValueError(
"There are {} scans available between: \n{} and {}\n### Try expanding your search timeframe ###"
.format(len(scans), start, end))
else:
print("There are {} scans available between {} and {}\n".format(
len(scans), start, end))
# This will download the data to a temporary folder. On Windows this is in:
# C:/Users/<user>/AppData/Local/Temp/<randomly generated temp folder name>
results = conn.download(scans, templocation)
print(' ')
# Initialize empty lists
ref_vector = ['Reflectivity (dBZ)']
ref_avg_vector = ['Average Reflectivity (dBZ)']
zdr_vector = ['Differential Reflectivity (dB)']
zdr_avg_vector = ['Average Differential Reflectivity (dB)']
rhv_vector = ['Cross Correlation Coefficient']
vel_vector = ['Velocity (m/s)']
yer_vec = ['Year']
mon_vec = ['Month']
day_vec = ['Day']
hor_vec = ['Hour']
min_vec = ['Minute']
sec_vec = ['Second']
# Get the radar latitude and longitude based on the input radar_id
rlon, rlat = radar_lonlat(radar_id, radarlocations_filepath)
print('Gauge Latitude: ' + str(round(glat, 2)) + ' / Gauge Longitude: ' +
str(round(glon, 2)))
# Get the distance and azimuth between the radar and the gauge location
point_dist, point_azim = calcdistbear(rlon, rlat, glon, glat)
print(' ')
for scan in results.iter_success():
# Pyart doesn't like the MDM files provided at the top of the hour, so we skip them
if 'MDM' in str(scan): continue
print('Reading in: ' + scan.filename)
vol = read_format_nexrad_lvl2(scan.filepath)
if vol is None:
print('Unable to decode current volume. Skipping volume')
continue
cur_time = datetime.datetime.strptime(
vol['year'] + vol['month'] + vol['day'] + vol['hour'] +
vol['minute'], '%Y%m%d%H%M')
# Ignore any data downloaded that isn't in the indicated timeframe. This
# is most useful if the code is written to include a manually downloaded
# large dataset.
if cur_time < radar_begin_time: continue
if cur_time > radar_end_time: break
# Find the nearest azimuth and gate index (since these need to be integers)
azim = find_nearest(vol['azim'], point_azim)
gate = find_nearest(vol['gate'], point_dist)
# If the user puts in an elevation scan that exceeds the number of
# elevations, sidestep the resultant index error by reducing the
# scan height index to the second lowest (so the average can take
# the gate above as well).
if elv >= len(vol['elv']):
print(
'Chosen Elevation Scan was too large. Selecting second lowest scan.'
)
print(' ')
elv = len(vol['elv']) - 2
# Fill in the exact reflectivity and differential reflectivity at the gate
# closest to the input location and at the lowest elevation scan
ref_vector.append(vol['ref'][azim, gate, elv])
zdr_vector.append(vol['zdr'][azim, gate, elv])
rhv_vector.append(vol['rhv'][azim, gate, elv])
vel_vector.append(vol['velocity'][azim, gate, elv])
# We can compute an average of reflectivity using the adjacent gates
# This may provide more realistic values for the precip that reaches
# the ground. Note: this does not account for temporal differences
# in the time radar data and the ground data are respectively observed.
ref_avg_vector.append(
log_avg([
vol['ref'][azim - 1, gate, elv],
vol['ref'][azim + 1, gate, elv], vol['ref'][azim, gate - 1,
elv],
vol['ref'][azim, gate + 1, elv], vol['ref'][azim, gate,
elv + 1]
]))
""" There is something funky about averaging the ZDR values. """
zdr_avg_vector.append(
log_avg([
vol['zdr'][azim - 1, gate, elv],
vol['zdr'][azim + 1, gate, elv], vol['zdr'][azim, gate - 1,
elv],
vol['zdr'][azim, gate + 1, elv], vol['zdr'][azim, gate,
elv + 1]
]))
yer_vec.append(vol['year'])
mon_vec.append(vol['month'])
day_vec.append(vol['day'])
hor_vec.append(vol['hour'])
min_vec.append(vol['minute'])
sec_vec.append(vol['second'][0:2])
# Write out the computed data to a csv file
with open(outfile, mode='w', newline='') as targ:
writer = csv.writer(targ)
writer.writerows([
yer_vec, mon_vec, day_vec, hor_vec, min_vec, sec_vec, ref_vector,
ref_avg_vector, zdr_vector, zdr_avg_vector, rhv_vector, vel_vector
])
