def ipw_diff_distribution_monthly():
x = DFWnet.CommonData()
# test may diff
x.date2doy(14, 5, 1)
start_day = x.doy
x.date2doy(14, 6, 1)
end_day = x.doy
ipw_2014 = x.IPWvals_2014
for i in range(44):
stn_diff = np.diff(ipw_2014[start_day:end_day, i,
1:].reshape(-1, ).astype('float'))
stn_diff[np.isnan(stn_diff)] = 0.
stn_diff = np.array([abs(num) for num in stn_diff])
bins_ = [10 * b for b in range(5)] # ,bins = bins_
counts, bins = np.histogram(stn_diff)
print ipw_2014[0, i, 0], counts, bins
stn_diff = stn_diff[stn_diff > 20.]
print stn_diff
def download_files(doy, net):
print doy
DFW = DFWnet.CommonData()
data = DFW.sites
ftp_noaa = ftplib.FTP('www.ngs.noaa.gov', 'anonymous', '*****@*****.**')
ftp_noaa.cwd('/cors/rinex/2016' + os.sep + doy)
sites = ftp_noaa.nlst()
initial = os.getcwd()
data = data[data[:, -1] == net, :]
for tx_site in data[:, 0]:
if tx_site in sites:
os.chdir('/home/aditya/UMASS/DFWnetwork' + os.sep +
data[data[:, 0] == tx_site, -1][0] + '/2016/rinex')
ftp_noaa.cwd('/cors/rinex/2016' + os.sep + doy + os.sep + tx_site)
dyna_file_name = tx_site + doy + '0' + '.' + '16o.gz'
print dyna_file_name
siteList = ftp_noaa.nlst()
print siteList
print dyna_file_name in siteList
if dyna_file_name in siteList:
gfile = open(dyna_file_name, 'wb')
ftp_noaa.retrbinary('RETR ' + dyna_file_name, gfile.write)
gfile.close()
print 'RINEX obs found site: ' + tx_site
os.chdir(initial)
ftp_noaa.close()
def NormalizeIPW_Normal(DAT,doy):
DFW = DFWnet.CommonData()
DFW.doytodate(14,doy)
SummaryStats = pd.read_csv('output/SitesSummary.csv',index_col=0)
IPWdataN = np.array(map(lambda x: (x[1:].astype('float') - SummaryStats[SummaryStats.site.values == x[0]][DFW.mon + '_Avg'].values)/SummaryStats[SummaryStats.site.values == x[0]][DFW.mon + '_Std'].values,DAT))
return IPWdataN.astype('float')
def reflectivity_monthly_distribution():
'''Plots the distribution for each month in the data set. Given an
archive of NEXRAD files we are going to seek to find the days where
rainfall was present to pupulate out data set. Try and compute the
monthly distribution of rainfall the prediction domain.'''
DFW = DFWnet.CommonData()
reflectivity = BuildDataSet.reflectivity_fields()
# Loop thru all the days in the data set 121 - 144
# Download each file keep the level 3 data throw the rest
# then check for each day whether there was a storm or not
# if there was a storm keep that folder
# else delete that folder and its contents
order_dict = {}
order_dict = {14: 'HAS010777764', 15: 'HAS010777767'}
initial = os.getcwd()
for yr in [14]:
for d in range(159, 244):
DFW.doytodate(int(yr), d)
# file_to_get = 'NWS_NEXRAD_NXL3_KFWS_20' +DFW.yr + DFW.mon + DFW.day + '000000_20' + DFW.yr + DFW.mon + DFW.day + '235959.tar.gz'
new_dir = '../data/RadarData/NEXRAD/20' + str(
yr) + os.sep + Months[int(DFW.mon) - 1] + DFW.day + os.sep
if not os.path.exists(new_dir):
os.mkdir(new_dir)
os.chdir(new_dir)
reflectivity.FTPNEXRADfile(DFW.mon, DFW.day, DFW.yr,
order_dict[yr])
os.chdir(initial)
reflectivity.keepLevel3files(new_dir)
reflectivity.ConvertToNETCDF(new_dir)
# Develop a logic here which takes files that have an average greater than
# 20 dBZ for its rainy days
file_list = os.listdir(new_dir)
# define an array the with size of the number of files in that day
# and a 100x100 grid to hold each time step worth of data
out_array = np.zeros((len(file_list), 100, 100))
time_array = []
for i, fl in enumerate(file_list):
rad = Dataset(new_dir + fl)
out_array[
i, ...] = reflectivity.reflectivity_polar_to_cartesian(rad)
time_array.append(rad.time_coverage_end.split('T')[1])
os.remove(new_dir + fl)
np.save(
new_dir + 'reflectivity_array_' + str(yr) + '_' + str(d) +
'.npy', out_array)
np.save(new_dir + 'time_array_' + str(yr) + '_' + str(d) + '.npy',
time_array)
def main():
DFW = DFWnet.CommonData()
print storm_dates
dates_list = storm_dates.split('\n')
num_dates = len(dates_list)
storm_dates_2014 = np.zeros((num_dates,4))
ctr=0
for d in dates_list:
k = re.findall('\d+',d)
print k
DFW.date2doy(14,int(k[1]),int(k[2]))
storm_dates_2014[ctr,:] = np.array((DFW.doy,14,int(k[1]),int(k[2])))
ctr+=1
print storm_dates_2014
np.save('../data/storm_dates_2014.npy',storm_dates_2014)
def call_metutil(net, doy, ofile):
x = DFWnet.CommonData()
base_path = '/home/aditya/UMASS/DFWnetwork/'
tempdir = base_path + net + '/2016/' + doy + os.sep
for s in x.sites[x.sites[:, -1] == net, 0]:
zfile = 'z' + s + '6.' + doy
if os.path.exists(tempdir + zfile):
os.chdir(tempdir)
subprocess.call(
['sh_metutil', '-f', ofile, '-z', zfile, '-i', '1800'])
else:
print 'Missing data: ' + doy + '/' + s
def ipw_diff_distribution():
x = DFWnet.CommonData()
ipw_2014 = x.IPWvals_2014
print ipw_2014.shape
plt.figure()
print ipw_2014[0, 35, 0]
plt.plot(ipw_2014[:, 35, 1:].reshape(365 * 48))
for i in range(44):
stn_diff = np.diff(ipw_2014[:, i,
1:].reshape(365 * 48).astype('float'))
stn_diff[np.isnan(stn_diff)] = 0.
stn_diff = [abs(num) for num in stn_diff]
counts, bins = np.histogram(stn_diff)
bins1 = bins[:-1] + np.diff(bins) / 2.
print bins1, counts / (counts.sum() * 1.)
plt.figure()
plt.bar(bins1, counts / (counts.sum() * 1.), width=np.diff(bins1)[0])
plt.title('IPW difference distribution for station: ' +
ipw_2014[0, i, 0])
plt.grid()
plt.savefig('../output/ipw_difference_distribution/Plot_' + str(i) +
'.png')
plt.show()
def main(yr,storm_dates,make_summary_stats = True,normalize = 'monthly'):
DFW_network = DFWnet.CommonData()
if yr == 14:
date_range = range(120,273) # DOY 122 - 244 -- 121,244
IPWvals = DFW_network.IPWvals_2014
elif yr == 15:
date_range = range(120,273) # DOY 122 - 244 -- 121,244
IPWvals = DFW_network.IPWvals_2015
IPWvals[IPWvals == '-9.9'] = str(np.nan)
elif yr == 16:
date_range = range(122,213) # DOY 122 - 244 -- index 122,244
IPWvals = np.load('../data/2016IPW_data.npy')
# Get stations with nan values in them. Resurnes the dictionary
# missing_value_dict = nan_stations(yr,IPWvals)
missing_value_dict = nan_stations(yr,IPWvals,date_range)
# set the missing values to nan for days with corresponding stations
# with corrupt values
set_ipw_nan(IPWvals,missing_value_dict)
if make_summary_stats:
# Define a data frame to hold monthly average and std
SummaryStats = make_summary_dataframe(yr,IPWvals,missing_value_dict)
SummaryStats.to_csv('../data/DFWsites_summary_20'+ str(yr) + '.csv')
SummaryStats = pd.read_csv('../data/DFWsites_summary_20'+ str(yr) + '.csv',index_col=0)
else:
# average out the saved summary stats
SummaryStats_2014 = pd.read_csv('../data/DFWsites_summary_20'+ str(14) + '.csv',index_col=0)
SummaryStats_2015 = pd.read_csv('../data/DFWsites_summary_20'+ str(15) + '.csv',index_col=0)
cols = ['site']
cols.extend(['Yearly_AVG','Yearly_STD'])
cols.extend(['{0}_Avg'.format(str(x).zfill(2)) for x in range(1,13)])
cols.extend(['{0}_Std'.format(str(x).zfill(2)) for x in range(1,13)])
SummaryStats = pd.DataFrame(columns = cols)
SummaryStats.ix[:,'site'] = SummaryStats_2014.site.values
# txsr and okar stations are f****d so impute one from an another
SummaryStats_2014.ix[34,:] = SummaryStats_2015.ix[34,:]
SummaryStats_2015.ix[0,['06_Avg','07_Avg','08_Avg']] = SummaryStats_2014.ix[0,['06_Avg','07_Avg','08_Avg']]
SummaryStats_2015.ix[0,['06_Std','07_Std','08_Std']] = SummaryStats_2014.ix[0,['06_Std','07_Std','08_Std']]
for key in ['{0}_Avg'.format(str(x).zfill(2)) for x in range(5,9)]:
SummaryStats.ix[:,key] = np.mean((SummaryStats_2015[key].values,SummaryStats_2014[key].values),axis = 0)
for key in ['{0}_Std'.format(str(x).zfill(2)) for x in range(5,9)]:
SummaryStats.ix[:,key] = np.sqrt(
np.mean((
SummaryStats_2014[key].apply(lambda x: x*x).values,
SummaryStats_2015[key].apply(lambda x: x*x).values),
axis = 0))
SummaryStats.to_csv('../data/DFWsites_summary.csv')
SummaryStats = pd.read_csv('../data/DFWsites_summary.csv',index_col=0)
for d in storm_dates[:1]:
# add to the above dictionary the dates with bad vals
# drop_stations(yr,d[0],IPWvals,missing_value_dict[d[0] - 1])
print d,missing_value_dict[d[0] - 1]
IPWvals_day = IPWvals[int(d[0]) - 1,:,1:].astype('float')
# imgs = []
# fig = plt.figure()
gpsX,gpsY,IPWsites,IPWvals_day = get_GPS_cartesian(missing_value_dict[int(d[0]) -1],IPWvals_day,DFW_network.sites)
if normalize == 'monthly':
IPWvals_day = NormalizeIPW_Normal(SummaryStats,IPWvals_day,int(d[0]),missing_value_dict,IPWsites,yr)
elif normalize == 'seasonal':
IPWvals_day = NormalizeIPW_Seasonal(SummaryStats,IPWvals_day,int(d[0]),missing_value_dict,IPWsites,yr)
if os.path.exists('../output/Weights/20' +str(yr) + '/Weights_' + str(int(d[0])) + '.npy'):
print 'Weights exist loading weights...'
W = np.load('../output/Weights/20' +str(yr) + '/Weights_' + str(int(d[0])) + '.npy')
else:
print 'Making Multiquadratic weights'
W = get_Mulit_Quadratic_Weights(gpsX,gpsY,IPWsites,int(d[0]),yr)
for t in range(48):
plot_radarIPW(t,d[0],IPWvals_day,W,missing_value_dict[int(d[0]) -1],yr,gpsX,gpsY,IPWsites)
import matplotlib
matplotlib.use('Agg')
#matplotlib.rcParams['animation.html'] = 'html5'
import DFWnet
import os
import numpy as np
import pandas as pd
from netCDF4 import Dataset
from matplotlib import pyplot as plt
import time
import subprocess
DFW = DFWnet.CommonData()
Months = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC']
def cart2pol(x,y):
r = np.sqrt(np.power(x,2) + np.power(y,2))
theta = np.degrees(np.arctan2(y,x))
return theta,r
#def get_IPW_vals(doy):
# global DFW
# print 'Obtain IPW values for DOY: ' + str(doy)
#
# IPWvals = DFW.IPWvals
#
# return IPWvals[doy -1,:,:]
Created on Tue Mar 1 17:00:20 2016
@author: adityanagarajan
This script converts the ipw and reflectivit fields from floating point to
8 bit int
"""
import numpy as np
import BuildDataSet
from matplotlib import pyplot as plt
import matplotlib.cm as cm
import DFWnet
file_path = 'data/TrainTest/'
data_builder = BuildDataSet.dataset(yr='2014')
network = DFWnet.CommonData()
map_ipw_array = np.linspace(-5, 5, 256)
map_refl_array = np.linspace(0, 90, 256)
#Trust me and uncomment this code for converting IPW to image
#for f in data_builder.IPWfiles:
# arr = np.load(file_path + f)
# new_array = np.zeros((100,100),dtype='uint8')
# for i in range(arr.shape[0]):
# for j in range(arr.shape[1]):
# new_array[i,j] = np.argmin(np.abs(arr[i,j] - map_ipw_array))
# np.save(file_path + f.split('.')[0] + 'img.npy',new_array)
#
#
#for r in data_builder.Radarfiles: