def getHiddenActivations(self, data): nrRbms = self.nrLayers - 2 activations = data activationsList = [] # You have to do it in decreasing order for i in xrange(nrRbms): # If the network can be initialized from the previous one, # do so, by using the transpose of the already trained net weigths = self.classificationWeights[i] biases = np.array([self.generativeBiases[i], self.biases[i]]) net = rbm.RBM( self.layerSizes[i], self.layerSizes[i + 1], learningRate=self.unsupervisedLearningRate, visibleActivationFunction=self.rbmActivationFunctionVisible, hiddenActivationFunction=self.rbmActivationFunctionHidden, hiddenDropout=1.0, visibleDropout=1.0, rmsprop=True, # TODO: argument here as well? nesterov=self.rbmNesterovMomentum, initialWeights=weigths, initialBiases=biases) # Do pass trough the net activations = net.hiddenRepresentation(activations) activationsList += [activations] return activationsList
def rbmMain(reconstructRandom=True): trainVectors, trainLabels =\ readmnist.read(0, args.trainSize, digits=None, bTrain=True, path=args.path) testingVectors, testLabels =\ readmnist.read(0, args.testSize, digits=None, bTrain=False, path=args.path) trainingScaledVectors = trainVectors / 255.0 testingScaledVectors = testingVectors / 255.0 # Train the network if args.train: # The number of hidden units is taken from a deep learning tutorial # The data are the values of the images have to be normalized before being # presented to the network nrVisible = len(trainingScaledVectors[0]) nrHidden = 500 # use 1 dropout to test the rbm for now net = rbm.RBM(nrVisible, nrHidden, rbm.contrastiveDivergence, 1, 1) net.train(trainingScaledVectors) t = visualizeWeights(net.weights.T, (28,28), (10,10)) else: # Take the saved network and use that for reconstructions f = open(args.netFile, "rb") t = pickle.load(f) net = pickle.load(f) f.close() # Reconstruct an image and see that it actually looks like a digit test = testingScaledVectors[0,:] # get a random image and see it looks like if reconstructRandom: test = np.random.random_sample(test.shape) # Show the initial image first plt.imshow(vectorToImage(test, (28,28)), cmap=plt.cm.gray) plt.show() # Show the reconstruction recon = net.reconstruct(test.reshape(1, test.shape[0])) plt.imshow(vectorToImage(recon, (28,28)), cmap=plt.cm.gray) plt.axis('off') plt.savefig('1.png', transparent=True) # plt.show() # Show the weights and their form in a tile fashion # Plot the weights plt.imshow(t, cmap=plt.cm.gray) plt.axis('off') plt.savefig('weights.png', transparent=True) print "done" if args.save: f = open(args.netFile, "wb") pickle.dump(t, f) pickle.dump(net, f)
def sample(self, nrSamples): nrRbms = self.nrLayers - 2 # Create a random samples of the size of the last layer if self.binary: samples = np.random.rand(nrSamples, self.layerSizes[-2]) else: samples = np.random.randint(255, size=(nrSamples, self.layerSizes[-2])) # You have to do it in decreasing order for i in xrange(nrRbms - 1, 0, -1): # If the network can be initialized from the previous one, # do so, by using the transpose of the already trained net weigths = self.classificationWeights[i - 1].T biases = np.array( [self.biases[i - 1], self.generativeBiases[i - 1]]) net = rbm.RBM( self.layerSizes[i], self.layerSizes[i - 1], learningRate=self.unsupervisedLearningRate, visibleActivationFunction=self.rbmActivationFunctionVisible, hiddenActivationFunction=self.rbmActivationFunctionHidden, hiddenDropout=1.0, visibleDropout=1.0, rmsprop=True, # TODO: argument here as well? nesterov=self.rbmNesterovMomentum, initialWeights=weigths, initialBiases=biases) # Do 20 layers of gibbs sampling for the last layer print samples.shape print biases.shape print biases[1].shape if i == nrRbms - 1: samples = net.reconstruct(samples, cdSteps=20) # Do pass trough the net samples = net.hiddenRepresentation(samples) return samples
def pretrain(self, data, unsupervisedData): nrRbms = self.nrLayers - 2 self.weights = [] self.biases = [] self.generativeBiases = [] currentData = data if unsupervisedData is not None: print "adding unsupervisedData" currentData = np.vstack((currentData, unsupervisedData)) print "pre-training with a data set of size", len(currentData) lastRbmBiases = None lastRbmTrainWeights = None dropoutList = [self.visibleDropout ] + [self.hiddenDropout] * (self.nrLayers - 1) for i in xrange(nrRbms): # If the RBM can be initialized from the previous one, # do so, by using the transpose of the already trained net if i > 0 and self.layerSizes[i + 1] == self.layerSizes[ i - 1] and type(self.rbmActivationFunctionVisible) == type( self.rbmActivationFunctionHidden): print "compatible rbms: initializing rbm number " + str( i) + "with the trained weights of rbm " + str(i - 1) initialWeights = lastRbmTrainWeights.T initialBiases = lastRbmBiases else: initialWeights = None initialBiases = None if i == 0 and self.firstRBMheuristic: print "different learning rate for the first rbm" # Do not let the learning rate be bigger than 1 unsupervisedLearningRate = min( self.unsupervisedLearningRate * 10, 1.0) else: unsupervisedLearningRate = self.unsupervisedLearningRate net = rbm.RBM( self.layerSizes[i], self.layerSizes[i + 1], learningRate=unsupervisedLearningRate, visibleActivationFunction=self.rbmActivationFunctionVisible, hiddenActivationFunction=self.rbmActivationFunctionHidden, hiddenDropout=self.rbmHiddenDropout, visibleDropout=self.rbmVisibleDropout, rmsprop=self.rmspropRbm, momentumMax=self.momentumMaxRbm, momentumFactorForLearningRate=self. momentumFactorForLearningRateRBM, nesterov=self.rbmNesterovMomentum, initialWeights=initialWeights, initialBiases=initialBiases, trainingEpochs=self.preTrainEpochs, sparsityConstraint=self.sparsityConstraintRbm, sparsityTraget=self.sparsityTragetRbm, sparsityRegularization=self.sparsityRegularizationRbm) net.train(currentData) # Use the test weights from the rbm, the ones the correspond to the incoming # weights for the hidden units # Then you have to divide by the dropout self.weights += [net.testWeights[1] / dropoutList[i]] # Only add the biases for the hidden unit b = net.biases[1] lastRbmBiases = net.biases # Do not take the test weight, take the training ones # because you will continue training with them lastRbmTrainWeights = net.weights self.biases += [b] self.generativeBiases += [net.biases[0]] # Let's update the current representation given to the next RBM currentData = net.hiddenRepresentation(currentData) # Average activation print "average activation after rbm pretraining" print currentData.mean() # This depends if you have generative or not # Initialize the last layer of weights to zero if you have # a discriminative net lastLayerWeights = np.zeros(shape=(self.layerSizes[-2], self.layerSizes[-1]), dtype=theanoFloat) lastLayerBiases = np.zeros(shape=(self.layerSizes[-1]), dtype=theanoFloat) self.weights += [lastLayerWeights] self.biases += [lastLayerBiases]