def call_nowcast_storm():
# Based of the optimal epoch measured against CSI
# 2CNN_0maxpool_2048neural_network_p20_special_0_40.pkl
# 2CNN_0maxpool_2048neural_network_p20_special_1_50.pkl
# ------------------------------------------------------#
# 2CNN Network list for IPW + refl
# network_list = ['2CNN_0maxpool_2048neural_network_p20_special_0_40.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_1_50.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_2_140.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_3_110.pkl']
# ------------------------------------------------------#
# 2CNN Network list for refl
network_list = [
'2CNN_0maxpool_2048neural_network_p20_refl_0_79.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_1_99.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_2_39.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_3_19.pkl'
]
# ------------------------------------------------------#
data_builder = BuildDataSet.dataset(num_points=500)
base_path = '../output/thesis_results/'
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
for network_file in network_list:
network_path = base_path + network_file
temp_file = network_path.split('/')[-1]
val_set = int(re.findall('\d+', temp_file)[-2])
print 'Running Validation Set %d' % val_set
prediction_list = nowcast_storm(data_builder,
validation_blocks[val_set],
network_path)
f1 = file('2CNN_prediction_file_refl_' + str(val_set) + '.pkl', 'wb')
pkl.dump(prediction_list, f1, protocol=pkl.HIGHEST_PROTOCOL)
f1.close()
def main(make_data_set=True,
prediction_lead='60',
feature_set='ipw_refl',
n_estimators=100,
file_name='RF_experiment_1.pkl'):
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
data_builder = BuildDataSet.dataset(num_points=500)
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
train, validation = arrange_training_validation(training_blocks,
validation_blocks)
if make_data_set:
if prediction_lead == '60':
print 'Building data set for 60 minute prediction'
make_dataset_RF(data_builder)
else:
print 'Building data set for 30 minute prediction'
make_dataset_RF_30minPrediction(data_builder)
performance = RF_classifier(train, validation, n_estimators, feature_set)
f1 = file('../output/' + str(n_estimators) + '_' + file_name, 'wb')
pkl.dump(performance, f1, protocol=pkl.HIGHEST_PROTOCOL)
f1.close()
def main(make_data_set = False, prediction_lead = '30',feature_set = 'ipw_refl',n_estimators = 300,file_name = 'RF_experiment_1.pkl',max_depth = 6,max_features = None):
print 'Saving file to ' + '../output/RF_experiments/' + str(n_estimators) + file_name + str(max_features) + '_max_depth' + str(max_depth) + '.pkl'
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
if max_features == 'None':
print 'Setting max_features to None!'
max_features = None
data_builder = BuildDataSet.dataset(num_points = 500)
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
if make_data_set:
if prediction_lead == '60':
print 'Building data set for 60 minute prediction'
make_dataset_RF(data_builder)
else:
print 'Building data set for 30 minute prediction'
make_dataset_RF_30minPrediction(data_builder)
train,validation = arrange_training_validation(training_blocks,validation_blocks)
performance = RF_classifier(train,validation,n_estimators,max_depth,feature_set,max_features)
# calculate the averages here before you pkl
f1 = file('../output/RF_experiments/' + str(n_estimators) + file_name + str(max_features) + '_max_depth' + str(max_depth) + '.pkl','wb')
pkl.dump(performance,f1,protocol = pkl.HIGHEST_PROTOCOL)
f1.close()
def call_nowcast_storm():
# Based of the optimal epoch measured against CSI
# 2CNN_0maxpool_2048neural_network_p20_special_0_40.pkl
# 2CNN_0maxpool_2048neural_network_p20_special_1_50.pkl
# ------------------------------------------------------#
# 2CNN Network list for IPW + refl
# network_list = ['2CNN_0maxpool_2048neural_network_p20_special_0_40.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_1_50.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_2_140.pkl',
# '2CNN_0maxpool_2048neural_network_p20_special_3_110.pkl']
# ------------------------------------------------------#
# 2CNN Network list for refl
network_list = ['2CNN_0maxpool_2048neural_network_p20_refl_0_79.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_1_99.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_2_39.pkl',
'2CNN_0maxpool_2048neural_network_p20_refl_3_19.pkl']
# ------------------------------------------------------#
data_builder = BuildDataSet.dataset(num_points = 500)
base_path = '../output/thesis_results/'
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
for network_file in network_list:
network_path = base_path + network_file
temp_file = network_path.split('/')[-1]
val_set = int(re.findall('\d+',temp_file)[-2])
print 'Running Validation Set %d'%val_set
prediction_list = nowcast_storm(data_builder,validation_blocks[val_set],network_path)
f1 = file('2CNN_prediction_file_refl_' + str(val_set) + '.pkl','wb')
pkl.dump(prediction_list,f1,protocol = pkl.HIGHEST_PROTOCOL)
f1.close()
def main():
print 'Plot reflectivity'
data_builder = BuildDataSet.dataset(num_points = 500)
yr = 14
storm_dates = data_builder.load_storm_days(yr)
for d in storm_dates[:1]:
for t in range(48):
plot_reflectivity(t,d[0],yr)
def main():
print 'Plot reflectivity'
data_builder = BuildDataSet.dataset(num_points=500)
yr = 14
storm_dates = data_builder.load_storm_days(yr)
for d in storm_dates[:1]:
for t in range(48):
plot_reflectivity(t, d[0], yr)
def main(make_data_set = False, prediction_lead = '60',feature_set = 'ipw_refl',n_estimators = 300,file_name = 'RF_experiment_1.pkl'):
print 'Saving file to ' + '../output/RF_experiments/' + str(n_estimators) + file_name
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
data_builder = BuildDataSet.dataset(num_points = 500)
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
cca_mdl = fit_CCA(training_blocks[0],data_builder)
transform_predict_RF(cca_mdl,training_blocks[0],validation_blocks[0],data_builder)
def main():
data_builder = BuildDataSet.dataset(num_points=500)
pixels = data_builder.sample_random_pixels()
base_path = "../data/RadarData/Decimated/"
file_list = os.listdir(base_path)
file_list = filter(lambda x: x[-4:] == ".npy", file_list)
full_data = np.zeros((500, len(file_list), 48))
pt_ctr = 0
for x_, y_ in pixels:
print x_, y_
for f_ctr, f in enumerate(file_list):
temp_array = np.load(base_path + f)
full_data[pt_ctr, f_ctr, :] = temp_array[:, y_, x_]
pt_ctr += 1
np.save(base_path + "full_data_decimated.npy", full_data)
def main():
data_builder = BuildDataSet.dataset(num_points=500)
pixels = data_builder.sample_random_pixels()
base_path = '../data/RadarData/Decimated/'
file_list = os.listdir(base_path)
file_list = filter(lambda x: x[-4:] == '.npy', file_list)
full_data = np.zeros((500, len(file_list), 48))
pt_ctr = 0
for x_, y_ in pixels:
print x_, y_
for f_ctr, f in enumerate(file_list):
temp_array = np.load(base_path + f)
full_data[pt_ctr, f_ctr, :] = temp_array[:, y_, x_]
pt_ctr += 1
np.save(base_path + 'full_data_decimated.npy', full_data)
def calc_metrics(network_path):
temp_file = network_path.split('/')[-1]
data_builder = BuildDataSet.dataset(num_points = 500)
val_set = int(re.findall('\d+',temp_file)[-2])
# network_params['num_conv_layers'] = num_conv_layers
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
performance = get_frames_points(training_blocks[val_set],validation_blocks[val_set],data_builder,network_path)
return (performance.POD,performance.FAR,performance.CSI,performance.average_precision)
def main(convert_to_images=False):
data_builder = BuildDataSet.dataset(num_points=500)
training_blocks, validation_blocks = build_training_validation_sets(data_builder)
# make_dataset_NN_2(data_builder)
if convert_to_images:
# convert_fields_to_images(data_builder)
make_dataset_NN(data_builder)
train, validation = arrange_training_validation(training_blocks, validation_blocks)
exp_no = 0
for tr, val in zip(train, validation):
print "Train Test Split %d " % (exp_no + 1)
neural_network_model_new(tr, val, exp_no + 1)
# convolution_neural_network_model(train[0],validation[0], exp_no + 1,100)
# empty_network(tr,val,exp_no+1,1)
exp_no += 1
def main(convert_to_images=False):
data_builder = BuildDataSet.dataset(num_points=500)
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
# make_dataset_NN_2(data_builder)
if convert_to_images:
# convert_fields_to_images(data_builder)
make_dataset_NN(data_builder)
train, validation = arrange_training_validation(training_blocks,
validation_blocks)
exp_no = 0
for tr, val in zip(train, validation):
print 'Train Test Split %d ' % (exp_no + 1)
neural_network_model_new(tr, val, exp_no + 1)
# convolution_neural_network_model(train[0],validation[0], exp_no + 1,100)
# empty_network(tr,val,exp_no+1,1)
exp_no += 1
def main(make_data_set=False,
prediction_lead='60',
feature_set='ipw_refl',
n_estimators=300,
file_name='RF_experiment_1.pkl'):
print 'Saving file to ' + '../output/RF_experiments/' + str(
n_estimators) + file_name
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
data_builder = BuildDataSet.dataset(num_points=500)
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
cca_mdl = fit_CCA(training_blocks[0], data_builder)
transform_predict_RF(cca_mdl, training_blocks[0], validation_blocks[0],
data_builder)
def main(make_data_set = True, prediction_lead = '60',feature_set = 'ipw_refl',n_estimators = 100,file_name = 'RF_experiment_1.pkl'):
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
data_builder = BuildDataSet.dataset(num_points = 500)
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
train,validation = arrange_training_validation(training_blocks,validation_blocks)
if make_data_set:
if prediction_lead == '60':
print 'Building data set for 60 minute prediction'
make_dataset_RF(data_builder)
else:
print 'Building data set for 30 minute prediction'
make_dataset_RF_30minPrediction(data_builder)
performance = RF_classifier(train,validation,n_estimators,feature_set)
f1 = file('../output/' + str(n_estimators)+ '_' + file_name,'wb')
pkl.dump(performance,f1,protocol = pkl.HIGHEST_PROTOCOL)
f1.close()
def calc_metrics(network_path):
temp_file = network_path.split('/')[-1]
data_builder = BuildDataSet.dataset(num_points=500)
val_set = int(re.findall('\d+', temp_file)[-2])
# network_params['num_conv_layers'] = num_conv_layers
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
performance = get_frames_points(training_blocks[val_set],
validation_blocks[val_set], data_builder,
network_path)
return (performance.POD, performance.FAR, performance.CSI,
performance.average_precision)
def main(make_data_set=False,
prediction_lead='30',
feature_set='ipw_refl',
n_estimators=300,
file_name='RF_experiment_1.pkl',
max_depth=6,
max_features=None):
print 'Saving file to ' + '../output/RF_experiments/' + str(
n_estimators) + file_name + str(max_features) + '_max_depth' + str(
max_depth) + '.pkl'
if make_data_set == 'False':
make_data_set = False
else:
make_data_set = True
if max_features == 'None':
print 'Setting max_features to None!'
max_features = None
data_builder = BuildDataSet.dataset(num_points=500)
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
if make_data_set:
if prediction_lead == '60':
print 'Building data set for 60 minute prediction'
make_dataset_RF(data_builder)
else:
print 'Building data set for 30 minute prediction'
make_dataset_RF_30minPrediction(data_builder)
train, validation = arrange_training_validation(training_blocks,
validation_blocks)
performance = RF_classifier(train, validation, n_estimators, max_depth,
feature_set, max_features)
# calculate the averages here before you pkl
f1 = file(
'../output/RF_experiments/' + str(n_estimators) + file_name +
str(max_features) + '_max_depth' + str(max_depth) + '.pkl', 'wb')
pkl.dump(performance, f1, protocol=pkl.HIGHEST_PROTOCOL)
f1.close()
def make_predictions(yr=14):
base_path = '../output/1_CNN_experiments/'
# /Users/adityanagarajan/projects/nowcaster/output/1_CNN_experiments
print os.path.exists(base_path + 'CPU_1_CNN_layer_max_pool_0_200.pkl')
params = np.load(base_path + 'CPU_1_CNN_layer_max_pool_0_200.pkl')
print len(params)
input_var = T.tensor4('inputs')
network, hidden_1 = build_DCNN_maxpool_softmax(input_var)
lasagne.layers.set_all_param_values(network, params)
prediction_ = lasagne.layers.get_output(network, deterministic=True)
predict_function = theano.function([input_var], prediction_)
data_builder = BuildDataSet.dataset(num_points=1000)
# This is the range of points in the central domain which can
# have a 33x33 around it.
num_points = 4356
domain_points = (range(17, 83), range(17, 83))
PixelPoints = [(x, y) for x in domain_points[0] for y in domain_points[1]]
PixelPoints = np.array(PixelPoints)
storm_dates_all = data_builder.load_storm_days(yr)
doy_strings = data_builder.club_days(storm_dates_all)
days_in_sorted = doy_strings.keys()
days_in_sorted.sort()
print doy_strings
ipw_files, refl_files = data_builder.sort_IPW_refl_files_imgs(yr)
temp_ipw_files = filter(
lambda x: re.findall('\d+', x)[1] in doy_strings['129'], ipw_files)
#
temp_refl_files = filter(
lambda x: re.findall('\d+', x)[1] in doy_strings['129'], refl_files)
temp_ipw_files = map(lambda x: '../data/dataset/20' + str(yr) + os.sep + x,
temp_ipw_files)
temp_refl_files = map(
lambda x: '../data/dataset/20' + str(yr) + os.sep + x, temp_refl_files)
# We define an array which contains the ground truth in column 1
# and the prediction probabilities in column 2 and 3
real_predictions = np.zeros((num_points, 91, 3))
pt_ctr = 0
for x_, y_ in zip(PixelPoints[:, 0], PixelPoints[:, 1]):
print 'Predicting for point: (%d,%d)' % (x_, y_)
temp_array = data_builder.build_features_and_truth_imgs(
temp_ipw_files, temp_refl_files, x_, y_)
ipw_refl_tensors = data_builder.arrange_frames_single(temp_array)
X_test = ipw_refl_tensors[1]
Y_test = ipw_refl_tensors[2]
Y_pred = predict_function(X_test)
real_predictions[pt_ctr, :, 0] = Y_test.reshape(91, )
# print Y_test.reshape(91,)
# print np.argmax(Y_pred,axis=1)
real_predictions[pt_ctr, :, 1:] = Y_pred
real_predictions.shape
pt_ctr += 1
return real_predictions
def make_predictions(yr = 14):
base_path = '../output/1_CNN_experiments/'
# /Users/adityanagarajan/projects/nowcaster/output/1_CNN_experiments
print os.path.exists(base_path + 'CPU_1_CNN_layer_max_pool_0_200.pkl')
params = np.load(base_path + 'CPU_1_CNN_layer_max_pool_0_200.pkl')
print len(params)
input_var = T.tensor4('inputs')
network,hidden_1 = build_DCNN_maxpool_softmax(input_var)
lasagne.layers.set_all_param_values(network, params)
prediction_ = lasagne.layers.get_output(network, deterministic=True)
predict_function = theano.function([input_var], prediction_)
data_builder = BuildDataSet.dataset(num_points = 1000)
# This is the range of points in the central domain which can
# have a 33x33 around it.
num_points = 4356
domain_points = (range(17,83),range(17,83))
PixelPoints = [(x,y) for x in domain_points[0]
for y in domain_points[1]]
PixelPoints = np.array(PixelPoints)
storm_dates_all = data_builder.load_storm_days(yr)
doy_strings = data_builder.club_days(storm_dates_all)
days_in_sorted = doy_strings.keys()
days_in_sorted.sort()
print doy_strings
ipw_files,refl_files = data_builder.sort_IPW_refl_files_imgs(yr)
temp_ipw_files = filter(lambda x: re.findall('\d+',x)[1] in doy_strings['129'],ipw_files)
#
temp_refl_files = filter(lambda x: re.findall('\d+',x)[1] in doy_strings['129'],refl_files)
temp_ipw_files = map(lambda x: '../data/dataset/20' + str(yr) + os.sep + x,temp_ipw_files)
temp_refl_files = map(lambda x: '../data/dataset/20' + str(yr) + os.sep + x,temp_refl_files)
# We define an array which contains the ground truth in column 1
# and the prediction probabilities in column 2 and 3
real_predictions = np.zeros((num_points,91,3))
pt_ctr = 0
for x_,y_ in zip(PixelPoints[:,0],PixelPoints[:,1]):
print 'Predicting for point: (%d,%d)'%(x_,y_)
temp_array = data_builder.build_features_and_truth_imgs(temp_ipw_files,temp_refl_files,x_,y_)
ipw_refl_tensors = data_builder.arrange_frames_single(temp_array)
X_test = ipw_refl_tensors[1]
Y_test = ipw_refl_tensors[2]
Y_pred = predict_function(X_test)
real_predictions[pt_ctr,:,0] = Y_test.reshape(91,)
# print Y_test.reshape(91,)
# print np.argmax(Y_pred,axis=1)
real_predictions[pt_ctr,:,1:] = Y_pred
real_predictions.shape
pt_ctr+=1
return real_predictions
"""
Created on Tue Mar 1 17:00:20 2016
@author: adityanagarajan
This script converts the ipw and reflectivit fields from floating point to
8 bit int
"""
import numpy as np
import BuildDataSet
from matplotlib import pyplot as plt
import matplotlib.cm as cm
import DFWnet
file_path = 'data/TrainTest/'
data_builder = BuildDataSet.dataset(yr = '2014')
network = DFWnet.CommonData()
map_ipw_array = np.linspace(-5,5,256)
map_refl_array = np.linspace(0,90,256)
#Trust me and uncomment this code for converting IPW to image
#for f in data_builder.IPWfiles:
# arr = np.load(file_path + f)
# new_array = np.zeros((100,100),dtype='uint8')
# for i in range(arr.shape[0]):
# for j in range(arr.shape[1]):
# new_array[i,j] = np.argmin(np.abs(arr[i,j] - map_ipw_array))
# np.save(file_path + f.split('.')[0] + 'img.npy',new_array)
#
network_file_name = '1CNN_1maxpool_neural_network_ipw_refl'
network_file = file(
'../output/' + network_file_name + '_' + str(exp_no) + '_' +
str(ep + 1) + '.pkl', 'wb')
pkl.dump(params, network_file, protocol=pkl.HIGHEST_PROTOCOL)
network_file.close()
f1 = file(
'../output/performance_metrics_' + network_file_name +
str(exp_no) + '.pkl', 'wb')
pkl.dump(performance_metrics, f1, protocol=pkl.HIGHEST_PROTOCOL)
f1.close()
def main(data_builder, make_data_set=False):
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
if make_data_set:
make_dataset_NN_2(data_builder)
for i in range(4):
print '-' * 50
print 'Validation year = %s Month = %s' % tuple(blocks[i].split('_'))
conv_net(training_blocks[i], validation_blocks[i], 200, i)
if __name__ == '__main__':
data_builder = BuildDataSet.dataset(num_points=500)
main(data_builder)
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 26 18:19:51 2016
@author: adityanagarajan
This file visually verifies the slices
"""
import BuildDataSet
import cPickle
import os
import sys
import numpy as np
data_builder = BuildDataSet.dataset(Threshold = 'binary')
PixelPoints = data_builder.sample_random_pixels()
#sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
def build_points_fields():
num_points = 80
prev_pt_tr = 0
for tr_pt in range(0,num_points,40):
print 'Building set %d '%tr_pt
train_set = data_builder.make_points_frames(PixelPoints[prev_pt_tr:tr_pt + 40])
print 'Done making training set'
X_Y_train = data_builder.arrange_frames(train_set)
temp_file = open('data/dataset2/points_' + str(prev_pt_tr) + '_'+ str(tr_pt + 40) + '.pkl','wb')
cPickle.dump((X_Y_train,PixelPoints[prev_pt_tr:tr_pt + 40]),temp_file,protocol=cPickle.HIGHEST_PROTOCOL)
temp_file.close()
# training_points.append(X_Y_train,PixelPoints[10:20])
input_var = T.tensor4('inputs')
# Load the deep neural network module
neural_nets = DCNN_network.DCNN_network()
# Build the deep neural net (see the package in includes/)
network = neural_nets.build_CNN_3_softmax(input_var)
lasagne.layers.set_all_param_values(network, params)
# Build theano function which inturn generates and compiles C code to run the network
prediction_ = lasagne.layers.get_output(network, deterministic=True)
predict_function = theano.function([input_var], prediction_)
# Load the data builder object
data_builder = BuildDataSet.dataset(Threshold=None)
# This is the range of points in the central domain which can have a 33x33 around it.
num_points = 4356
domain_points = (range(17, 83), range(17, 83))
# Arrange all pixel points to a list
PixelPoints = [(x, y) for x in domain_points[0] for y in domain_points[1]]
# Call the function to club consecutive days together
sorted_days = data_builder.club_days()
# Get the end of each consecutive string of days
days_in_sorted = sorted_days.keys()
# Sort the days
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 26 18:19:51 2016
@author: adityanagarajan
This file visually verifies the slices
"""
import BuildDataSet
import cPickle
import os
import sys
import numpy as np
data_builder = BuildDataSet.dataset(Threshold='binary')
PixelPoints = data_builder.sample_random_pixels()
#sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
def build_points_fields():
num_points = 80
prev_pt_tr = 0
for tr_pt in range(0, num_points, 40):
print 'Building set %d ' % tr_pt
train_set = data_builder.make_points_frames(
PixelPoints[prev_pt_tr:tr_pt + 40])
print 'Done making training set'
X_Y_train = data_builder.arrange_frames(train_set)
temp_file = open(
'data/dataset2/points_' + str(prev_pt_tr) + '_' + str(tr_pt + 40) +
"""
Created on Tue Mar 1 17:00:20 2016
@author: adityanagarajan
This script converts the ipw and reflectivit fields from floating point to
8 bit int
"""
import numpy as np
import BuildDataSet
from matplotlib import pyplot as plt
import matplotlib.cm as cm
import DFWnet
file_path = 'data/TrainTest/'
data_builder = BuildDataSet.dataset(yr='2014')
network = DFWnet.CommonData()
map_ipw_array = np.linspace(-5, 5, 256)
map_refl_array = np.linspace(0, 90, 256)
#Trust me and uncomment this code for converting IPW to image
#for f in data_builder.IPWfiles:
# arr = np.load(file_path + f)
# new_array = np.zeros((100,100),dtype='uint8')
# for i in range(arr.shape[0]):
# for j in range(arr.shape[1]):
# new_array[i,j] = np.argmin(np.abs(arr[i,j] - map_ipw_array))
# np.save(file_path + f.split('.')[0] + 'img.npy',new_array)
#
#
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import os
import cPickle
import BuildDataSet
import nowcast
average_fields = []
indices_i = [(x*1089,(x+1)*1089)for x in range(4) ]
indices_r = [(x*1089,(x+1)*1089)for x in range(4,8) ]
data_builder = BuildDataSet.dataset()
'''
>>> f = file('obj.save', 'rb')
>>> loaded_obj = cPickle.load(f)
>>> f.close()
'''
base_file = 'data/TrainTest/RandomPoints/'
file_list = os.listdir(base_file)
file_list = filter(lambda x: x[-4:] == '.pkl' and x[:3] == 'IPW',file_list)
print file_list
for pkl_file in file_list:
print pkl_file
elif yr == 16:
storm_dates = np.load('../data/storm_dates_2016.npy').astype('int')
for d in storm_dates:
print d
DFW.doytodate(int(yr),d[0])
new_dir = '../data/RadarData/NEXRAD/20' + str(yr) + os.sep + Months[int(DFW.mon) -1] + DFW.day
if not os.path.exists(new_dir):
os.mkdir(new_dir)
os.chdir(new_dir)
GetNEXRADfile(DFW.mon,DFW.day,DFW.yr,order_id = order_dict[yr])
os.chdir(initial)
nexrad_files = KeepRequiredFiles(d[0],yr)
# This commented code here delets the files which are inbetween
# 00 and 30.
Deletefiles(d[0],yr,nexrad_files)
## DFW.doytodate(int(yr),d[0])
## file_path = '../data/RadarData/NEXRAD/20' + str(yr) + os.sep + Months[int(DFW.mon) -1] + DFW.day + os.sep
## nexrad_files = os.listdir(file_path)
ConvertToNETCDF(d[0],yr,nexrad_files)
# check_refl_files(new_dir)
# decimated = det30minuteDecimated(d[0],yr)
# averages = get30minuteaverages(d[0],yr)
# np.save('../data/RadarData/Decimated/' +str(d[1]) + str(d[0]) + 'refl_decimated.npy',decimated)
# np.save('../data/RadarData/Averages/' + str(d[1]) + str(d[0]) + 'refl_averages.npy',averages)
if __name__ == '__main__':
yr = [16]
data_builder = BuildDataSet.dataset(num_points = 500)
pixels = data_builder.sample_random_pixels()
for y in yr:
main(y)
input_var = T.tensor4('inputs')
# Load the deep neural network module
neural_nets = DCNN_network.DCNN_network()
# Build the deep neural net (see the package in includes/)
network = neural_nets.build_CNN_3_softmax(input_var)
lasagne.layers.set_all_param_values(network, params)
# Build theano function which inturn generates and compiles C code to run the network
prediction_ = lasagne.layers.get_output(network, deterministic=True)
predict_function = theano.function([input_var], prediction_)
# Load the data builder object
data_builder = BuildDataSet.dataset(Threshold = None)
# This is the range of points in the central domain which can have a 33x33 around it.
num_points = 4356
domain_points = (range(17,83),range(17,83))
# Arrange all pixel points to a list
PixelPoints = [(x,y) for x in domain_points[0] for y in domain_points[1]]
# Call the function to club consecutive days together
sorted_days = data_builder.club_days()
# Get the end of each consecutive string of days
days_in_sorted = sorted_days.keys()
# Sort the days
network_file = file('../output/'+ network_file_name + '_' + str(exp_no) +'_' + str(ep + 1) + '.pkl','wb')
pkl.dump(params,network_file,protocol = pkl.HIGHEST_PROTOCOL)
network_file.close()
f1 = file('../output/performance_metrics_' + network_file_name + '_' + str(exp_no) + '.pkl','wb')
pkl.dump(performance_metrics,f1,protocol = pkl.HIGHEST_PROTOCOL)
f1.close()
def main(data_builder,make_data_set = False):
training_blocks,validation_blocks = build_training_validation_sets(data_builder)
if make_data_set:
make_dataset_NN_2(data_builder)
for i in range(4):
print '-'*50
print 'Validation year = %s Month = %s'%tuple(blocks[i].split('_'))
conv_net(training_blocks[i],validation_blocks[i],200,i)
if __name__ == '__main__':
data_builder = BuildDataSet.dataset(Threshold = 'real_valued',num_points = 500)
main(data_builder)
from matplotlib import pyplot as plt
import random
import cPickle
import nowcast
import BuildDataSet
region_dict = {}
region_dict['region1'] = (range(33, 50), range(50, 66), 'r*')
region_dict['region2'] = (range(50, 66), range(50, 66), 'g*')
region_dict['region3'] = (range(33, 50), range(33, 50), 'b*')
region_dict['region4'] = (range(50, 66), range(33, 50), 'y*')
now_caster = nowcast.BuildNowcaster()
data_builder = BuildDataSet.dataset()
#fill_domain = ([x for x in range(33,66)if x not in range(50,55)],[y for y in range(33,66) if y not in range(50,55)])
def plot_domains(PixelX, PixelY, marker):
central_chunk = (range(46, 54), range(46, 54))
#Pull out the noisy regions of the radar
central_chunk_points = [(x_, y_) for x_ in central_chunk[0]
for y_ in central_chunk[1]]
PixelPoints = [(x, y) for x in PixelX for y in PixelY]
PixelPoints = [
'../output/performance_metrics_' + network_file_name + '_' +
str(exp_no) + '.pkl', 'wb')
pkl.dump(performance_metrics, f1, protocol=pkl.HIGHEST_PROTOCOL)
f1.close()
def main(data_builder, make_data_set=False):
training_blocks, validation_blocks = build_training_validation_sets(
data_builder)
if make_data_set:
make_dataset_NN_2(data_builder)
for i in range(4):
print '-' * 50
print 'Validation year = %s Month = %s' % tuple(blocks[i].split('_'))
conv_net(training_blocks[i], validation_blocks[i], 200, i)
if __name__ == '__main__':
data_builder = BuildDataSet.dataset(Threshold='real_valued',
num_points=500)
main(data_builder)
