def train_DBN(self, x):
for index, layer in enumerate(self.layers):
if index == 0:
vn = self.input_size
else:
vn = self.layers[index - 1]
hn = self.layers[index]
rbm = RBM(vn,
hn,
epochs=100,
mode='bernoulli',
lr=0.0005,
k=10,
batch_size=128,
gpu=True,
optimizer='adam',
early_stopping_patience=10)
x_dash = self.generate_input_for_layer(index, x)
rbm.train(x_dash)
self.layer_parameters[index]['W'] = rbm.W.cpu()
self.layer_parameters[index]['hb'] = rbm.hb.cpu()
self.layer_parameters[index]['vb'] = rbm.vb.cpu()
print("Finished Training Layer:", index, "to", index + 1)
if self.savefile is not None:
torch.save(self.layer_parameters, self.savefile)
def main():
learningRate = float(sys.argv[1]) if len(sys.argv) >= 2 else 0.0001
maxIterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 300
# load data
data = scipy.io.loadmat('data/usps_resampled.mat')
train_patterns = data['train_patterns']
train_labels = data['train_labels']
test_patterns = data['test_patterns']
test_labels = data['test_labels']
# initialize and train RBM
rbm = RBM(192, train_patterns, learningRate=learningRate, verbose=True)
rbm.train(convThreshold=0.1, maxIterations=maxIterations)
print 'Autoencoding. . . '
hidden_patterns = rbm.translate(train_patterns)
ae_patterns = rbm.invert(hidden_patterns)
print 'Finished.'
W = rbm.W.get_value()
while True:
while True:
try:
detectorNum = raw_input("Pick a detector image from [0-" +
str(W.shape[0] - 1) +
"] (q to quit): ")
if detectorNum == 'q':
sys.exit(0)
detectorNum = int(detectorNum)
if detectorNum not in range(W.shape[0]):
raise ValueError
except ValueError:
continue
except EOFError:
sys.exit(0)
except KeyboardInterrupt:
sys.exit(0)
break
# show example image
plt.matshow(W[detectorNum, :].reshape(16, 16))
plt.show()
def main():
learningRate = float(sys.argv[1]) if len(sys.argv) >= 2 else 0.0001
maxIterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 300
# load data
data = scipy.io.loadmat('data/usps_resampled.mat')
train_patterns = data['train_patterns']
train_labels = data['train_labels']
test_patterns = data['test_patterns']
test_labels = data['test_labels']
# initialize and train RBM
rbm = RBM(192, train_patterns, learningRate=learningRate, verbose=True)
iterationsCompleted = rbm.train(convThreshold=0.03,
maxIterations=maxIterations)
print 'Autoencoding. . . '
hidden_patterns = rbm.translate(train_patterns)
ae_patterns = rbm.invert(hidden_patterns)
print 'Finished.'
while True:
while True:
try:
sampleImage = raw_input("Pick a sample image from [0-" +
str(train_patterns.shape[1] - 1) +
"] (q to quit): ")
if sampleImage == 'q':
y = raw_input("Save this classifier (y)? ")
fn = 'data/RBM_' + str(
(learningRate, 192, iterationsCompleted))
if y in ['y', '']:
f = open(fn, 'w')
pickle.dump(TrainedRBM(rbm.W, rbm.b_h, rbm.b_v), f)
print "RBM saved as " + fn
sys.exit(0)
sampleImage = int(sampleImage)
if sampleImage not in range(train_patterns.shape[1]):
raise ValueError
except ValueError:
continue
except EOFError:
sys.exit(0)
except KeyboardInterrupt:
sys.exit(0)
break
# show example image
plt.matshow(train_patterns[:, sampleImage].reshape(16, 16))
plt.matshow(ae_patterns[:, sampleImage].reshape(16, 16))
plt.show()
def train(self, trainingData):
rbmList = [] # list RBM's weights
tempData = trainingData
# start RBM's training and get the respective weights
for n in range(self.nEncoderLayers):
if (n == 0 or n == (self.nEncoderLayers - 1)):
rbm = RBM(tempData, self.sEncoderLayers[n], rbmType='GBRBM')
else:
rbm = RBM(tempData, self.sEncoderLayers[n], rbmType='BBRBM')
print('Start %d RBM training' % (n + 1))
rbm.train(batchSize=100)
[weights, visBias, hidBias] = rbm.getWeights()
rbmList.append(RBM_Weights(weights, visBias, hidBias))
data = tf.convert_to_tensor(tempData,
dtype=tf.float32,
name='data')
probHid = tf.sigmoid(tf.matmul(data, weights) + hidBias)
hid = tf.cast(tf.greater(
probHid,
tf.random_uniform(tf.shape(probHid),
minval=0,
maxval=1,
dtype=tf.float32)),
dtype=tf.float32)
with tf.Session() as sess:
if ((self.nEncoderLayers - 1) == n):
tempData = sess.run(probHid)
else:
tempData = sess.run(hid)
# start the fine tuning process
return self.fineTuning(rbmList, trainingData)
rbm1 = RBM(n_hidden=n_hidden1,
n_visible=n_feature,
alpha=0.0001,
datatype="gaussian")
rbm2 = RBM(n_hidden=n_hidden2,
n_visible=n_hidden1,
alpha=0.0001,
datatype="binary")
rbm3 = RBM(n_hidden=n_hidden3,
n_visible=n_hidden2,
alpha=0.0001,
datatype="binary")
# rbm4 = RBM(n_hidden=n_hidden4, n_visible=n_hidden3, alpha=0.0001, datatype="binary")
for num in range(Epoch):
new_w, new_hb, new_vb, ReconErr = rbm1.train(MA_data)
print("Epoch: {}, Reconstruction Error: {}".format(num, ReconErr))
for num in range(Epoch):
batch_xs = rbm1.passThrough(MA_data)
new_w, new_hb, new_vb, ReconErr = rbm2.train(batch_xs)
print("Epoch: {}, Reconstruction Error: {}".format(num, ReconErr))
for num in range(Epoch):
batch_xs = rbm1.passThrough(MA_data)
batch_xs1 = rbm2.passThrough(batch_xs)
new_w, new_hb, new_vb, ReconErr = rbm3.train(batch_xs1)
print("Epoch: {}, Reconstruction Error: {}".format(num, ReconErr))
temp1 = rbm1.passThrough(MA_data)
temp2 = rbm2.passThrough(temp1)
import torch
import numpy as np
import pandas as pd
import os
from RBM import RBM
from load_dataset import MNIST
if __name__ == '__main__':
mnist = MNIST()
train_x, train_y, test_x, test_y = mnist.load_dataset()
print('MAE for all 0 selection:', torch.mean(train_x))
vn = train_x.shape[1]
hn = 2500
rbm = RBM(vn, hn, epochs=100, mode='bernoulli', lr=0.0005, k=10, batch_size=128, gpu=True, optimizer='adam', savefile='mnist_trained_rbm.pt', early_stopping_patience=10)
rbm.train(train_x)
vbias=vbias,
verbose=verbose)
pre_trained = os.path.isfile(model_dir)
if pre_trained:
rbm.load_state_dict(torch.load(model_dir))
# -----------------------------------------------------------------------------
# Training
# -----------------------------------------------------------------------------
if not pre_trained:
validation = data[:10000]
for _ in range(epochs):
train_loader = torch.utils.data.DataLoader(data,
batch_size=batch_size,
shuffle=True)
rbm.train(train_loader)
# A good measure of well-fitting is the free energy difference
# between some known and unknown instances. It is related to the
# log-likelihood difference, but it does not depend on the
# partition function. It should be around 0, and if it grows, it
# might be overfitting to the training data.
# High-probability instances have very negative free energy, so the
# gap becoming very negative is sign of overfitting.
gap = (rbm.free_energy(validation) - rbm.free_energy(test)).mean(0)
print('Gap = {}'.format(gap.item()))
torch.save(rbm.state_dict(), model_dir)
# -----------------------------------------------------------------------------
# Plotting
# -----------------------------------------------------------------------------
# data = data[idx]
# target = mnist.target[idx]
# We keep only the digit "0"
data = data[mnist.target == 2]
np.random.shuffle(data)
train_data = data[:6000, :]
validation_data = data[6000:, :]
rbm = RBM(num_hidden=50, num_visible=784)
n_iter = 200
k = 1
print("Training")
errors = np.zeros(n_iter)
rbm.train(train_data, method="PCD", learning_rate=0.1, num_iter=n_iter, k=k, errors=errors, decrease_eta=False)
# Plot the reconstruction error on the training set
# plt.figure()
# plt.plot(errors)
# Plot the weights of the RBM (one figure per hidden unit)
display_weights(rbm.weights, show=False)
# Plot one sample of visible units (based on hidden units computed from a real sample)
plt.figure()
_, samp = rbm.gibbs_vhv(validation_data[0, :], k=10, binary=False)
sb.heatmap(samp.reshape((28, 28)), cmap='gray')
plt.show()
train_dataset.data = (train_dataset.train_data.type(torch.FloatTensor)/255).bernoulli()
tensor_x = train_dataset.train_data.type(torch.FloatTensor) # transform to torch tensors
tensor_y = train_dataset.train_labels.type(torch.FloatTensor)
_dataset = torch.utils.data.TensorDataset(tensor_x,tensor_y) # create your datset
train_loader = torch.utils.data.DataLoader(_dataset,
batch_size=batch_size, shuffle=True,drop_last = True)
# 加载数据
# train_loader = DataLoader(train_dataset, batch_size, shuffle=True)
# make model
rbm_mnist = RBM(visible_units, hidden_units, k, learning_rate, learning_rate_decay, xavier_init,
increase_to_cd_k, use_gpu)
epochs = 50
rbm_mnist.train(train_loader, epochs, batch_size)
learned_weights = rbm_mnist.W.transpose(0, 1).numpy()
plt.show()
fig = plt.figure(3, figsize=(10, 10))
for i in range(25):
sub = fig.add_subplot(5, 5, i+1)
sub.imshow(learned_weights[i, :].reshape((28, 28)), cmap=plt.cm.gray)
plt.show()
torch.save(rbm_mnist, 'RBM.pth')
learning_rate = 0.01
learning_rate_decay = False
xavier_init = True
increase_to_cd_k = False
use_gpu = False
rbm_mnist = RBM(visible_units, hidden_units, k, learning_rate, learning_rate_decay, xavier_init,
increase_to_cd_k, use_gpu)
# Epochs = [10, 15, 20, 25, 30, 35]
Epochs = [30]
for epochs in Epochs:
print('-----------------epoch:{}----------------'.format(epochs))
rbm_mnist.train(mnist_particular_dataloader, epochs)
# This shows the weights for each of the 64 hidden neurons and give an idea how each neuron is activated.
learned_weights = rbm_mnist.W.transpose(0,1).numpy()
plt.show()
fig = plt.figure(3, figsize=(10, 10))
for i in range(25):
sub = fig.add_subplot(5, 5, i+1)
sub.imshow(learned_weights[i, :].reshape((28, 28)), cmap=plt.cm.gray)
plt.title(epochs)
plt.show()
#Lets try reconstructing a random number from this model which has learned 5
idx = 7
img = train_dataset.train_data[idx]
num_workers=0)
eye_loader = DataLoader(dataset=torch.utils.data.TensorDataset(
torch.Tensor(eye_train_data).bernoulli(), torch.Tensor(train_Y)),
batch_size=32,
shuffle=True,
num_workers=0)
visible__eeg_units = 310
visible__eye_units = 31
hidden_units = 50
k = 2
learning_rate = 0.001
learning_rate_decay = True
xavier_init = True
increas_to_cd_k = False
use_gpu = False
rbm_eeg = RBM(visible__eeg_units, hidden_units, int(k), learning_rate,
learning_rate_decay, xavier_init, increas_to_cd_k, use_gpu)
rbm_eye = RBM(visible__eye_units, hidden_units, int(k), learning_rate,
learning_rate_decay, xavier_init, increas_to_cd_k, use_gpu)
epochs = 30
batch_size = 32
rbm_eeg.train(eeg_loader, epochs, batch_size)
rbm_eye.train(eye_loader, epochs, batch_size)
if not os.path.exists('../../saved_models/RBM'):
os.makedirs('../../saved_models/RBM')
torch.save(rbm_eeg, '../../saved_models/RBM/rbm_eeg')
torch.save(rbm_eye, '../../saved_models/RBM/rbm_eye')
index += 1
""" Show the obtained plot """
plt.show()
train = data[:, 1:]/255
train[np.where(train>0)]=1
#works well for binary images, but features don't get properly extraced for non binary images.
ones = train[np.where(np.where(data[:,0] == 5)[0]<100000)[0]]
rbm = RBM(784, 196)
m = 0.5
for i in range(10):
if i > 5:
m = 0.9
n = 10
for j in range(1000):
rbm.train(train[j*10:j*10+9], momentum=m, w_cost=0.0001)
w = rbm.w#.flatten()
visualizeW1(w.T, 28, 14)
#
# plt.imshow(np.reshape(ones[20], (-1,28)))
# plt.show()
#
rbm.train(ones[:10], momentum= m, w_cost=0.003, beta=0)
plt.imshow(np.reshape(ones[7], (-1,28)))
plt.show()
plt.imshow(np.reshape(rbm.negdata[7],(-1, 28)))
plt.show()
# define
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
d1 = 784
l1 = 256
l2 = 64
l3 = 32
#'''
# RBM-1
BATCH_SIZE1 = 32
EPOCH1 = 20
LR1 = 1e-2
k1 = 2
rbm1 = RBM(d1, l1, DEVICE)
dataset1 = aux_Dataset2(torch.from_numpy(train_data).float())
reps1, loss4_1 = rbm1.train(dataset1, k1, EPOCH1, BATCH_SIZE1, LR=LR1)
# RBM-2
BATCH_SIZE2 = 32
EPOCH2 = 20
LR2 = 1e-2
d2 = l1
k2 = 2
rbm2 = RBM(d2, l2, DEVICE, real=False)
dataset2 = aux_Dataset2(reps1)
reps2, loss4_2 = rbm2.train(dataset2, k2, EPOCH2, BATCH_SIZE2, LR=LR2)
# RBM-3
BATCH_SIZE3 = 32
EPOCH3 = 20
LR3 = 1e-2
k = 3
learning_rate = 0.01
learning_rate_decay = True
xavier_init = True
increase_to_cd_k = False
use_gpu = True
rbm_mnist = RBM(visible_units, hidden_units, k, learning_rate,
learning_rate_decay, xavier_init, increase_to_cd_k,
use_gpu).cuda()
# In[7]:
epochs = 3
rbm_mnist.train(train_loader, epochs, batch_size)
# In[8]:
# learned_weights = rbm_mnist.W.transpose(0,1).numpy()
# plt.show()
# fig = plt.figure(3, figsize=(10,10))
# for i in range(25):
# sub = fig.add_subplot(5, 5, i+1)
# sub.imshow(learned_weights[i,:].reshape((28,28)), cmap=plt.cm.gray)
# plt.show()
# In[9]:
#This is an unsupervised learning algorithm. So let us try training on one particular number.But first
# we need to seperate the data.
