def train():
print sys.argv
n_args = len(sys.argv)
nb_epoch = 300
batch_size = 100
if n_args > 1:
nb_epoch = int(sys.argv[1])
if n_args > 2:
batch_size = int(sys.argv[2])
alldata_original = np.loadtxt("../data/muestras.csv",
delimiter=",",
skiprows=1)
scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9))
alldata = scaler_data.fit_transform(alldata_original)
locations = alldata[:, 0:3]
data_original = alldata[:, 3:4]
#scaler = sklearn.preprocessing.MinMaxScaler()
data = data_original #scaler.fit_transform(data_original)
if len(data.shape) < 2:
data = np.expand_dims(data, axis=1)
print locations.shape
print data.shape
n, m = data.shape
ret = utils.generate_kfold(range(n),
n_folds=5,
shuffle=True,
random_state=1634120)
train_index, test_index = ret[0]
print len(train_index), len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
model = make_model_locations(nh=1000)
X_training = np.empty((n_training, 3))
y_training = np.empty((n_training, m))
X_testing = np.empty((n_testing, 3))
y_testing = np.empty((n_testing, m))
X_training[:, :] = locations[train_index, :]
y_training[:, :] = data[train_index, :]
X_testing[:, :] = locations[test_index, :]
y_testing[:, :] = data[test_index, :]
history = model.fit(X_training,
y_training,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_testing, y_testing),
shuffle=True)
save_model(model, "muestras-model-locations")
score = model.predict(X_testing)
print np.mean(y_testing[:, 0])
print np.mean(score[:, 0])
print np.std(y_testing[:, 0])
print np.std(score[:, 0])
print "r2", sklearn.metrics.r2_score(score, y_testing)
quit()
print "Testing values"
for true_value, prediction in zip(y_testing[:, 0], score[:, 0]):
print true_value, ',', prediction
print "Training values"
score = model.predict(X_training)
for true_value, prediction in zip(y_training[:, 0], score[:, 0]):
print true_value, ',', prediction
def train():
print sys.argv
n_args = len(sys.argv)
nb_epoch = 300
batch_size = 100
if n_args > 1:
nb_epoch = int(sys.argv[1])
if n_args > 2:
batch_size = int(sys.argv[2])
alldata_original = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1)
scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9))
alldata = scaler_data.fit_transform(alldata_original)
locations = alldata[:,0:3]
data = alldata[:,3:4]
if len(data.shape) < 2:
data = np.expand_dims(data, axis=1)
print locations.shape
print data.shape
n,m = data.shape
ret = utils.generate_kfold(range(n),n_folds=5,shuffle=True,random_state=1634120)
train_index, test_index = ret[0]
print len(train_index),len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
kdtree = cKDTree(locations)
k = 50
model = make_model_locations(k*(3+m),nh=1000)
X_training= preprocess.get_neighbours(locations[train_index,:],locations,data,k,kdtree,distance=np.inf)
y_training = data[train_index,:]
X_testing = preprocess.get_neighbours(locations[test_index,:],locations,data,k,kdtree,distance=np.inf)
y_testing = data[test_index,:]
#reshape
X_training = X_training.reshape((n_training,k*(3+m)))
X_testing = X_testing.reshape((n_testing,k*(3+m)))
print X_training[:10,:], X_testing[:10,:]
#quit()
history = model.fit(X_training, y_training,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, validation_data=(X_testing, y_testing),shuffle=True)
save_model(model,"muestras-model-neighbour")
score = model.predict(X_testing)
print np.mean(y_testing[:,0])
print np.mean(score[:,0])
print np.std(y_testing[:,0])
print np.std(score[:,0])
print "r2", sklearn.metrics.r2_score(score, y_testing)
alldata = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1)
locations = alldata[:,0:3]
data = alldata[:,3:5]
scaler = sklearn.preprocessing.MinMaxScaler()
locations = scaler.fit_transform(locations)
print locations.shape
print data.shape
n,nin = locations.shape
n,nout = data.shape
ret = utils.generate_kfold(range(n),n_folds=10,shuffle=True,random_state=1634120)
train_index, test_index = ret[0]
print len(train_index),len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
model,encoder = make_model(nin,[100,100] ,nout)
X_training = locations[train_index,:]
y_training = data[train_index,:]
X_testing = locations[test_index,:]
y_testing = data[test_index,:]
if len(data.shape) < 2:
data = np.expand_dims(data, axis=1)
kdtree = cKDTree(locations)
nodes = (25, 25, 25)
sizes = (10.0, 10.0, 10.0)
print locations.shape
print data.shape
n, m = data.shape
ret = utils.generate_kfold(range(n),
n_folds=5,
shuffle=True,
random_state=1634120)
train_index, test_index = ret[0]
print len(train_index), len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
print "loading test data..."
X_testing = np.empty(
(n_testing, nodes[0] * 2 + 1, nodes[1] * 2 + 1, nodes[2] * 2 + 1, 1))
y_testing = np.empty((n_testing, 1))
for k in range(n_testing):
def train():
print sys.argv
n_args = len(sys.argv)
nb_epoch = 300
batch_size = 100
if n_args > 1:
nb_epoch = int(sys.argv[1])
if n_args > 2:
batch_size = int(sys.argv[2])
alldata_original = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1)
scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9))
alldata = scaler_data.fit_transform(alldata_original)
locations = alldata[:,0:3]
data_original = alldata[:,3:4]
#scaler = sklearn.preprocessing.MinMaxScaler()
data = data_original #scaler.fit_transform(data_original)
if len(data.shape) < 2:
data = np.expand_dims(data, axis=1)
print locations.shape
print data.shape
n,m = data.shape
ret = utils.generate_kfold(range(n),n_folds=5,shuffle=True,random_state=1634120)
train_index, test_index = ret[0]
print len(train_index),len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
model = make_model_locations(nh=1000)
X_training = np.empty((n_training,3))
y_training = np.empty((n_training,m))
X_testing = np.empty((n_testing,3))
y_testing = np.empty((n_testing,m))
X_training[:,:] = locations[train_index,:]
y_training[:,:] = data[train_index,:]
X_testing[:,:] = locations[test_index,:]
y_testing[:,:] = data[test_index,:]
history = model.fit(X_training, y_training,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, validation_data=(X_testing, y_testing),shuffle=True)
save_model(model,"muestras-model-locations")
score = model.predict(X_testing)
print np.mean(y_testing[:,0])
print np.mean(score[:,0])
print np.std(y_testing[:,0])
print np.std(score[:,0])
print "r2", sklearn.metrics.r2_score(score, y_testing)
quit()
print "Testing values"
for true_value, prediction in zip(y_testing[:,0],score[:,0]):
print true_value,',', prediction
print "Training values"
score = model.predict(X_training)
for true_value, prediction in zip(y_training[:,0],score[:,0]):
print true_value,',', prediction
def predict(locations,data):
print sys.argv
n_args = len(sys.argv)
nb_epoch = 1
batch_size = 50
neurons_full_layer = 100
if n_args > 1:
nb_epoch = int(sys.argv[1])
if n_args > 2:
batch_size = int(sys.argv[2])
if n_args > 3:
neurons_full_layer = int(sys.argv[3])
alldata = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1)
locations = alldata[:,0:3]
data_original = alldata[:,3:4]
#scaler = sklearn.preprocessing.MinMaxScaler()
data = data_original #scaler.fit_transform(data_original)
if len(data.shape) < 2:
data = np.expand_dims(data, axis=1)
print locations.shape
print data.shape
n,m = data.shape
ret = utils.generate_kfold(range(n),n_folds=10,shuffle=True,random_state=1634120)
train_index, test_index = ret[0]
print len(train_index),len(test_index)
n_training = len(train_index)
n_testing = len(test_index)
wsize = np.array((20,20,20),dtype=np.int32)
nwsize = wsize*2+1
print wsize
print nwsize
model = make_model_2(m, nwsize[0], nwsize[1],nwsize[2],neurons_full_layer)
X_training = None
y_training = None
X_testing = None
y_testing = None
if os.path.exists("X_training.npy"):
X_training = np.load("X_training.npy")
if os.path.exists("y_training.npy"):
y_training = np.load("y_training.npy")
if os.path.exists("X_testing.npy"):
X_testing = np.load("X_testing.npy")
if os.path.exists("y_testing.npy"):
y_testing = np.load("y_testing.npy")
if X_training is None:
X_training = np.empty((n_training,m,nwsize[0],nwsize[1],nwsize[2]))
y_training = np.empty((n_training,m))
X_testing = np.empty((n_testing,m,nwsize[0],nwsize[1],nwsize[2]))
y_testing = np.empty((n_testing,m))
kdtree = cKDTree(locations)
k = 1000
print "processing training data"
for i,index in enumerate(train_index):
location = locations[index]
image = preprocess.create_image_from_neighbours_3d(location,locations,data,k,kdtree,(wsize[0],wsize[1],wsize[2]),(10.0,10.0,10.0),distance=np.inf)
X_training[i,:,:,:,:] = image
y_training[i,:] = data[index,:]
print "processing testing data"
for i,index in enumerate(test_index):
location = locations[index]
image = preprocess.create_image_from_neighbours_3d(location,locations,data,k,kdtree,(wsize[0],wsize[1],wsize[2]),(10.0,10.0,10.0),distance=np.inf)
X_testing[i,:,:,:,:] = image
y_testing[i,:] = data[index,:]
np.save("X_training",X_training)
np.save("y_training",y_training)
np.save("X_testing",X_testing)
np.save("y_testing",y_testing)
else:
pass
history = model.fit(X_training, y_training,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, validation_data=(X_testing, y_testing))
score = model.predict(X_testing)
print np.mean(y_testing[:,0])
print np.mean(score[:,0])
print np.std(y_testing[:,0])
print np.std(score[:,0])
print "Testing values"
for true_value, prediction in zip(y_testing[:,0],score[:,0]):
print true_value,',', prediction
print "Training values"
score = model.predict(X_training)
for true_value, prediction in zip(y_training[:,0],score[:,0]):
print true_value,',', prediction