def train(): print sys.argv n_args = len(sys.argv) nb_epoch = 300 batch_size = 100 if n_args > 1: nb_epoch = int(sys.argv[1]) if n_args > 2: batch_size = int(sys.argv[2]) alldata_original = np.loadtxt("../data/muestras.csv", delimiter=",", skiprows=1) scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9)) alldata = scaler_data.fit_transform(alldata_original) locations = alldata[:, 0:3] data_original = alldata[:, 3:4] #scaler = sklearn.preprocessing.MinMaxScaler() data = data_original #scaler.fit_transform(data_original) if len(data.shape) < 2: data = np.expand_dims(data, axis=1) print locations.shape print data.shape n, m = data.shape ret = utils.generate_kfold(range(n), n_folds=5, shuffle=True, random_state=1634120) train_index, test_index = ret[0] print len(train_index), len(test_index) n_training = len(train_index) n_testing = len(test_index) model = make_model_locations(nh=1000) X_training = np.empty((n_training, 3)) y_training = np.empty((n_training, m)) X_testing = np.empty((n_testing, 3)) y_testing = np.empty((n_testing, m)) X_training[:, :] = locations[train_index, :] y_training[:, :] = data[train_index, :] X_testing[:, :] = locations[test_index, :] y_testing[:, :] = data[test_index, :] history = model.fit(X_training, y_training, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_testing, y_testing), shuffle=True) save_model(model, "muestras-model-locations") score = model.predict(X_testing) print np.mean(y_testing[:, 0]) print np.mean(score[:, 0]) print np.std(y_testing[:, 0]) print np.std(score[:, 0]) print "r2", sklearn.metrics.r2_score(score, y_testing) quit() print "Testing values" for true_value, prediction in zip(y_testing[:, 0], score[:, 0]): print true_value, ',', prediction print "Training values" score = model.predict(X_training) for true_value, prediction in zip(y_training[:, 0], score[:, 0]): print true_value, ',', prediction
def train(): print sys.argv n_args = len(sys.argv) nb_epoch = 300 batch_size = 100 if n_args > 1: nb_epoch = int(sys.argv[1]) if n_args > 2: batch_size = int(sys.argv[2]) alldata_original = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1) scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9)) alldata = scaler_data.fit_transform(alldata_original) locations = alldata[:,0:3] data = alldata[:,3:4] if len(data.shape) < 2: data = np.expand_dims(data, axis=1) print locations.shape print data.shape n,m = data.shape ret = utils.generate_kfold(range(n),n_folds=5,shuffle=True,random_state=1634120) train_index, test_index = ret[0] print len(train_index),len(test_index) n_training = len(train_index) n_testing = len(test_index) kdtree = cKDTree(locations) k = 50 model = make_model_locations(k*(3+m),nh=1000) X_training= preprocess.get_neighbours(locations[train_index,:],locations,data,k,kdtree,distance=np.inf) y_training = data[train_index,:] X_testing = preprocess.get_neighbours(locations[test_index,:],locations,data,k,kdtree,distance=np.inf) y_testing = data[test_index,:] #reshape X_training = X_training.reshape((n_training,k*(3+m))) X_testing = X_testing.reshape((n_testing,k*(3+m))) print X_training[:10,:], X_testing[:10,:] #quit() history = model.fit(X_training, y_training, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_testing, y_testing),shuffle=True) save_model(model,"muestras-model-neighbour") score = model.predict(X_testing) print np.mean(y_testing[:,0]) print np.mean(score[:,0]) print np.std(y_testing[:,0]) print np.std(score[:,0]) print "r2", sklearn.metrics.r2_score(score, y_testing)
alldata = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1) locations = alldata[:,0:3] data = alldata[:,3:5] scaler = sklearn.preprocessing.MinMaxScaler() locations = scaler.fit_transform(locations) print locations.shape print data.shape n,nin = locations.shape n,nout = data.shape ret = utils.generate_kfold(range(n),n_folds=10,shuffle=True,random_state=1634120) train_index, test_index = ret[0] print len(train_index),len(test_index) n_training = len(train_index) n_testing = len(test_index) model,encoder = make_model(nin,[100,100] ,nout) X_training = locations[train_index,:] y_training = data[train_index,:] X_testing = locations[test_index,:] y_testing = data[test_index,:]
if len(data.shape) < 2: data = np.expand_dims(data, axis=1) kdtree = cKDTree(locations) nodes = (25, 25, 25) sizes = (10.0, 10.0, 10.0) print locations.shape print data.shape n, m = data.shape ret = utils.generate_kfold(range(n), n_folds=5, shuffle=True, random_state=1634120) train_index, test_index = ret[0] print len(train_index), len(test_index) n_training = len(train_index) n_testing = len(test_index) print "loading test data..." X_testing = np.empty( (n_testing, nodes[0] * 2 + 1, nodes[1] * 2 + 1, nodes[2] * 2 + 1, 1)) y_testing = np.empty((n_testing, 1)) for k in range(n_testing):
def train(): print sys.argv n_args = len(sys.argv) nb_epoch = 300 batch_size = 100 if n_args > 1: nb_epoch = int(sys.argv[1]) if n_args > 2: batch_size = int(sys.argv[2]) alldata_original = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1) scaler_data = sklearn.preprocessing.MinMaxScaler(feature_range=(0.1, 0.9)) alldata = scaler_data.fit_transform(alldata_original) locations = alldata[:,0:3] data_original = alldata[:,3:4] #scaler = sklearn.preprocessing.MinMaxScaler() data = data_original #scaler.fit_transform(data_original) if len(data.shape) < 2: data = np.expand_dims(data, axis=1) print locations.shape print data.shape n,m = data.shape ret = utils.generate_kfold(range(n),n_folds=5,shuffle=True,random_state=1634120) train_index, test_index = ret[0] print len(train_index),len(test_index) n_training = len(train_index) n_testing = len(test_index) model = make_model_locations(nh=1000) X_training = np.empty((n_training,3)) y_training = np.empty((n_training,m)) X_testing = np.empty((n_testing,3)) y_testing = np.empty((n_testing,m)) X_training[:,:] = locations[train_index,:] y_training[:,:] = data[train_index,:] X_testing[:,:] = locations[test_index,:] y_testing[:,:] = data[test_index,:] history = model.fit(X_training, y_training, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_testing, y_testing),shuffle=True) save_model(model,"muestras-model-locations") score = model.predict(X_testing) print np.mean(y_testing[:,0]) print np.mean(score[:,0]) print np.std(y_testing[:,0]) print np.std(score[:,0]) print "r2", sklearn.metrics.r2_score(score, y_testing) quit() print "Testing values" for true_value, prediction in zip(y_testing[:,0],score[:,0]): print true_value,',', prediction print "Training values" score = model.predict(X_training) for true_value, prediction in zip(y_training[:,0],score[:,0]): print true_value,',', prediction
def predict(locations,data): print sys.argv n_args = len(sys.argv) nb_epoch = 1 batch_size = 50 neurons_full_layer = 100 if n_args > 1: nb_epoch = int(sys.argv[1]) if n_args > 2: batch_size = int(sys.argv[2]) if n_args > 3: neurons_full_layer = int(sys.argv[3]) alldata = np.loadtxt("../data/muestras.csv",delimiter=",",skiprows=1) locations = alldata[:,0:3] data_original = alldata[:,3:4] #scaler = sklearn.preprocessing.MinMaxScaler() data = data_original #scaler.fit_transform(data_original) if len(data.shape) < 2: data = np.expand_dims(data, axis=1) print locations.shape print data.shape n,m = data.shape ret = utils.generate_kfold(range(n),n_folds=10,shuffle=True,random_state=1634120) train_index, test_index = ret[0] print len(train_index),len(test_index) n_training = len(train_index) n_testing = len(test_index) wsize = np.array((20,20,20),dtype=np.int32) nwsize = wsize*2+1 print wsize print nwsize model = make_model_2(m, nwsize[0], nwsize[1],nwsize[2],neurons_full_layer) X_training = None y_training = None X_testing = None y_testing = None if os.path.exists("X_training.npy"): X_training = np.load("X_training.npy") if os.path.exists("y_training.npy"): y_training = np.load("y_training.npy") if os.path.exists("X_testing.npy"): X_testing = np.load("X_testing.npy") if os.path.exists("y_testing.npy"): y_testing = np.load("y_testing.npy") if X_training is None: X_training = np.empty((n_training,m,nwsize[0],nwsize[1],nwsize[2])) y_training = np.empty((n_training,m)) X_testing = np.empty((n_testing,m,nwsize[0],nwsize[1],nwsize[2])) y_testing = np.empty((n_testing,m)) kdtree = cKDTree(locations) k = 1000 print "processing training data" for i,index in enumerate(train_index): location = locations[index] image = preprocess.create_image_from_neighbours_3d(location,locations,data,k,kdtree,(wsize[0],wsize[1],wsize[2]),(10.0,10.0,10.0),distance=np.inf) X_training[i,:,:,:,:] = image y_training[i,:] = data[index,:] print "processing testing data" for i,index in enumerate(test_index): location = locations[index] image = preprocess.create_image_from_neighbours_3d(location,locations,data,k,kdtree,(wsize[0],wsize[1],wsize[2]),(10.0,10.0,10.0),distance=np.inf) X_testing[i,:,:,:,:] = image y_testing[i,:] = data[index,:] np.save("X_training",X_training) np.save("y_training",y_training) np.save("X_testing",X_testing) np.save("y_testing",y_testing) else: pass history = model.fit(X_training, y_training, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_testing, y_testing)) score = model.predict(X_testing) print np.mean(y_testing[:,0]) print np.mean(score[:,0]) print np.std(y_testing[:,0]) print np.std(score[:,0]) print "Testing values" for true_value, prediction in zip(y_testing[:,0],score[:,0]): print true_value,',', prediction print "Training values" score = model.predict(X_training) for true_value, prediction in zip(y_training[:,0],score[:,0]): print true_value,',', prediction