def getHiddenActivations(self, data):
nrRbms = self.nrLayers - 2
activations = data
activationsList = []
# You have to do it in decreasing order
for i in xrange(nrRbms):
# If the network can be initialized from the previous one,
# do so, by using the transpose of the already trained net
weigths = self.classificationWeights[i]
biases = np.array([self.generativeBiases[i], self.biases[i]])
net = rbm.RBM(
self.layerSizes[i],
self.layerSizes[i + 1],
learningRate=self.unsupervisedLearningRate,
visibleActivationFunction=self.rbmActivationFunctionVisible,
hiddenActivationFunction=self.rbmActivationFunctionHidden,
hiddenDropout=1.0,
visibleDropout=1.0,
rmsprop=True, # TODO: argument here as well?
nesterov=self.rbmNesterovMomentum,
initialWeights=weigths,
initialBiases=biases)
# Do pass trough the net
activations = net.hiddenRepresentation(activations)
activationsList += [activations]
return activationsList
def rbmMain(reconstructRandom=True):
trainVectors, trainLabels =\
readmnist.read(0, args.trainSize, digits=None, bTrain=True, path=args.path)
testingVectors, testLabels =\
readmnist.read(0, args.testSize, digits=None, bTrain=False, path=args.path)
trainingScaledVectors = trainVectors / 255.0
testingScaledVectors = testingVectors / 255.0
# Train the network
if args.train:
# The number of hidden units is taken from a deep learning tutorial
# The data are the values of the images have to be normalized before being
# presented to the network
nrVisible = len(trainingScaledVectors[0])
nrHidden = 500
# use 1 dropout to test the rbm for now
net = rbm.RBM(nrVisible, nrHidden, rbm.contrastiveDivergence, 1, 1)
net.train(trainingScaledVectors)
t = visualizeWeights(net.weights.T, (28,28), (10,10))
else:
# Take the saved network and use that for reconstructions
f = open(args.netFile, "rb")
t = pickle.load(f)
net = pickle.load(f)
f.close()
# Reconstruct an image and see that it actually looks like a digit
test = testingScaledVectors[0,:]
# get a random image and see it looks like
if reconstructRandom:
test = np.random.random_sample(test.shape)
# Show the initial image first
plt.imshow(vectorToImage(test, (28,28)), cmap=plt.cm.gray)
plt.show()
# Show the reconstruction
recon = net.reconstruct(test.reshape(1, test.shape[0]))
plt.imshow(vectorToImage(recon, (28,28)), cmap=plt.cm.gray)
plt.axis('off')
plt.savefig('1.png', transparent=True)
# plt.show()
# Show the weights and their form in a tile fashion
# Plot the weights
plt.imshow(t, cmap=plt.cm.gray)
plt.axis('off')
plt.savefig('weights.png', transparent=True)
print "done"
if args.save:
f = open(args.netFile, "wb")
pickle.dump(t, f)
pickle.dump(net, f)
def sample(self, nrSamples):
nrRbms = self.nrLayers - 2
# Create a random samples of the size of the last layer
if self.binary:
samples = np.random.rand(nrSamples, self.layerSizes[-2])
else:
samples = np.random.randint(255,
size=(nrSamples, self.layerSizes[-2]))
# You have to do it in decreasing order
for i in xrange(nrRbms - 1, 0, -1):
# If the network can be initialized from the previous one,
# do so, by using the transpose of the already trained net
weigths = self.classificationWeights[i - 1].T
biases = np.array(
[self.biases[i - 1], self.generativeBiases[i - 1]])
net = rbm.RBM(
self.layerSizes[i],
self.layerSizes[i - 1],
learningRate=self.unsupervisedLearningRate,
visibleActivationFunction=self.rbmActivationFunctionVisible,
hiddenActivationFunction=self.rbmActivationFunctionHidden,
hiddenDropout=1.0,
visibleDropout=1.0,
rmsprop=True, # TODO: argument here as well?
nesterov=self.rbmNesterovMomentum,
initialWeights=weigths,
initialBiases=biases)
# Do 20 layers of gibbs sampling for the last layer
print samples.shape
print biases.shape
print biases[1].shape
if i == nrRbms - 1:
samples = net.reconstruct(samples, cdSteps=20)
# Do pass trough the net
samples = net.hiddenRepresentation(samples)
return samples
def pretrain(self, data, unsupervisedData):
nrRbms = self.nrLayers - 2
self.weights = []
self.biases = []
self.generativeBiases = []
currentData = data
if unsupervisedData is not None:
print "adding unsupervisedData"
currentData = np.vstack((currentData, unsupervisedData))
print "pre-training with a data set of size", len(currentData)
lastRbmBiases = None
lastRbmTrainWeights = None
dropoutList = [self.visibleDropout
] + [self.hiddenDropout] * (self.nrLayers - 1)
for i in xrange(nrRbms):
# If the RBM can be initialized from the previous one,
# do so, by using the transpose of the already trained net
if i > 0 and self.layerSizes[i + 1] == self.layerSizes[
i - 1] and type(self.rbmActivationFunctionVisible) == type(
self.rbmActivationFunctionHidden):
print "compatible rbms: initializing rbm number " + str(
i) + "with the trained weights of rbm " + str(i - 1)
initialWeights = lastRbmTrainWeights.T
initialBiases = lastRbmBiases
else:
initialWeights = None
initialBiases = None
if i == 0 and self.firstRBMheuristic:
print "different learning rate for the first rbm"
# Do not let the learning rate be bigger than 1
unsupervisedLearningRate = min(
self.unsupervisedLearningRate * 10, 1.0)
else:
unsupervisedLearningRate = self.unsupervisedLearningRate
net = rbm.RBM(
self.layerSizes[i],
self.layerSizes[i + 1],
learningRate=unsupervisedLearningRate,
visibleActivationFunction=self.rbmActivationFunctionVisible,
hiddenActivationFunction=self.rbmActivationFunctionHidden,
hiddenDropout=self.rbmHiddenDropout,
visibleDropout=self.rbmVisibleDropout,
rmsprop=self.rmspropRbm,
momentumMax=self.momentumMaxRbm,
momentumFactorForLearningRate=self.
momentumFactorForLearningRateRBM,
nesterov=self.rbmNesterovMomentum,
initialWeights=initialWeights,
initialBiases=initialBiases,
trainingEpochs=self.preTrainEpochs,
sparsityConstraint=self.sparsityConstraintRbm,
sparsityTraget=self.sparsityTragetRbm,
sparsityRegularization=self.sparsityRegularizationRbm)
net.train(currentData)
# Use the test weights from the rbm, the ones the correspond to the incoming
# weights for the hidden units
# Then you have to divide by the dropout
self.weights += [net.testWeights[1] / dropoutList[i]]
# Only add the biases for the hidden unit
b = net.biases[1]
lastRbmBiases = net.biases
# Do not take the test weight, take the training ones
# because you will continue training with them
lastRbmTrainWeights = net.weights
self.biases += [b]
self.generativeBiases += [net.biases[0]]
# Let's update the current representation given to the next RBM
currentData = net.hiddenRepresentation(currentData)
# Average activation
print "average activation after rbm pretraining"
print currentData.mean()
# This depends if you have generative or not
# Initialize the last layer of weights to zero if you have
# a discriminative net
lastLayerWeights = np.zeros(shape=(self.layerSizes[-2],
self.layerSizes[-1]),
dtype=theanoFloat)
lastLayerBiases = np.zeros(shape=(self.layerSizes[-1]),
dtype=theanoFloat)
self.weights += [lastLayerWeights]
self.biases += [lastLayerBiases]
