def test_dbn(): import dbn # dbn.classify() rb = dbn.dbn(4, [4, 4, 2], nlabels=2, eta=0.3, momentum=0.5, nCDsteps=1, nepochs=1001) N = 2500 v = np.zeros((2 * N, 4)) l = np.zeros((2 * N, 2)) for n in range(N): r = np.random.rand() if r > 0.666: v[n, :] = [0, 1, 0, 0] l[n, :] = [1, 0] elif r > 0.333: v[n, :] = [1, 0, 0, 0] l[n, :] = [1, 0] for n in range(N): r = np.random.rand() if r > 0.666: v[N + n, :] = [0, 0, 0, 1] l[N + n, :] = [0, 1] elif r > 0.333: v[N + n, :] = [0, 0, 1, 0] l[N + n, :] = [0, 1] rb.greedy(v, l) # , delta=delta, momentum=mom) rb.classify_after_greedy(v, l) rb.updown(v, l) rb.classify(v, l)
def test_dbn(): import dbn #dbn.classify() rb = dbn.dbn(4,[4,4,2],nlabels=2,eta=0.3,momentum=0.5,nCDsteps=1,nepochs=1001) N=2500 v = np.zeros((2*N,4)) l = np.zeros((2*N,2)) for n in range(N): r = np.random.rand() if r>0.666: v[n,:] = [0,1,0,0] l[n,:] = [1,0] elif r>0.333: v[n,:] = [1,0,0,0] l[n,:] = [1,0] for n in range(N): r = np.random.rand() if r>0.666: v[N+n,:] = [0,0,0,1] l[N+n,:] = [0,1] elif r>0.333: v[N+n,:] = [0,0,1,0] l[N+n,:] = [0,1] rb.greedy(v,l) #, delta=delta, momentum=mom) rb.classify_after_greedy(v,l) rb.updown(v,l) rb.classify(v,l)
def preTrain(self,lR,k,nE,method): # initialize deep belief network preModel = dbn(self.net,self.T,lR,k,self.bS,nE,self.roll) # load data preModel.loadData(self.data) # train dbn preModel.preTrain(lR,k,nE,method) # set pre-trained parameters self.w = copy.deepcopy(preModel.w) self.b = copy.deepcopy(preModel.b) self.bR = copy.deepcopy(preModel.bR)
def preTrain(self, lR, k, nE, method): # initialize deep belief network preModel = dbn(self.net, self.T, lR, k, self.bS, nE, self.roll) # load data preModel.loadData(self.data) # train dbn preModel.preTrain(lR, k, nE, method) # set pre-trained parameters self.w = copy.deepcopy(preModel.w) for i in range(self.nL): self.b[i] = 0.5 * (copy.deepcopy(preModel.b[i]) + copy.deepcopy(preModel.bR[i]))
def test_dbn_digs(): import dbn import scipy as scipy import scipy.io as sio tmp = sio.loadmat('binaryalphadigs.mat') NTRAIN = 39 CLASSES = [10, 12, 28] # A, C, S NCLASSES = len(CLASSES) I, L = 20*16, 3 N = NTRAIN*NCLASSES # organize data data = np.zeros((NCLASSES, NTRAIN, I)) labels = np.zeros((NCLASSES, NTRAIN, L)) for k in range(L): for m in range(NTRAIN): data[k,m,:] = (tmp['dat'][CLASSES[k],m].ravel()).astype('d') labels[k,m,k] = 1. # prepare observations, labels perm = np.arange(N) np.random.shuffle(perm) v = data.reshape(N, I)[perm,:] l = labels.reshape(N, L)[perm,:] rb = dbn.dbn(20*16,[100,100,100],nlabels=3,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=401) rb.greedy(v,l) rb.classify_after_greedy(v,l) rb.updown(v,l) rb.classify(v,l) import pylab as pl pl.figure(), pl.imshow(v[5,:].reshape(20,16)) toph, topph, labs = rb.bottom_up(v,l) obs, pobs = rb.top_down(topph) pl.figure(), pl.imshow(pobs[5,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest') return v, l, obs, pobs
def test(): import sys sys.path.insert(0,'digit_reconstructor\\rbm') sys.path.insert(0,'digit_reconstructor\\dbn') print("Loading DBN\n") import cPickle as cP with open("data\\defValues-300-300.pklb","rb") as f: a=cP.load(f) import dbn import rbm0_m as rbm0 r=rbm0.rbm0() r.weights=a[0] r.load_data() d=dbn.dbn([r],30) d.rbmList[1].weights=a[1] print("Loaded DBN\n") epochs = 20 print("Training DBN\n") d.train(epochs) dweights =[] print("Saving DBN weights\n") for x in xrange(len(d.rbmList)): dweights.append(d.rbmList[x].weights) file_write_name="new_weights-784-300-30.pklb" with open(file_write_name,"wb") as f: cP.dump(dweights,f) print("Displaying DBN reconstructed digits") import dbninter dbninter.display(d)
def test_dbn_mnist(): import dbn import pickle, gzip # Load the dataset f = gzip.open('mnist.pkl.gz', 'rb') train_set, valid_set, test_set = pickle.load(f) f.close() indices = np.arange(np.shape(train_set[0])[0]) np.random.shuffle(indices) # indices = indices[:200] t = train_set[0][indices, :] l = train_set[1][indices] labels = np.zeros(((np.shape(t)[0]), 10)) for i in range(20 * 10): labels[i, l[i]] = 1 rb = dbn.dbn(28 * 28, [100, 100, 50], nlabels=10, eta=0.3, momentum=0.4, nCDsteps=3, nepochs=1000) rb.greedy(t, labels) # , delta=delta, momentum=mom) rb.classify_after_greedy(t, labels) rb.updown(t, labels) rb.classify(t, labels) return rb
def test_dbn_mnist(): import dbn import cPickle, gzip # Load the dataset f = gzip.open('mnist.pkl.gz', 'rb') train_set, valid_set, test_set = cPickle.load(f) f.close() indices = np.arange(np.shape(train_set[0])[0]) np.random.shuffle(indices) #indices = indices[:200] t = train_set[0][indices,:] l = train_set[1][indices] labels = np.zeros(((np.shape(t)[0]),10)) for i in range(20*10): labels[i,l[i]] = 1 rb = dbn.dbn(28*28,[100,100,50],nlabels=10,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=1000) rb.greedy(t,labels) #, delta=delta, momentum=mom) rb.classify_after_greedy(t,labels) rb.updown(t,labels) rb.classify(t,labels) return rb
def learn_letters(): import scipy.io as sio nperclass = 39 classes = [10, 11, 28] # classes = [10, 13, 28] # A, C, S nclasses = len(classes) # Read in the data and prepare it data = sio.loadmat('binaryalphadigs.mat') inputs = np.ones((nclasses, nperclass, 20 * 16)) labels = np.zeros((nclasses, nperclass, nclasses)) for k in range(nclasses): for m in range(nperclass): inputs[k, m, :] = (data['dat'][classes[k], m].ravel()).astype('float') labels[k, m, k] = 1. nexamples = 20 v = inputs[:, :nexamples, :].reshape(nclasses * nexamples, 20 * 16) l = labels[:, :nexamples, :].reshape(nclasses * nexamples, nclasses) import pylab as pl # This shows a set of examples from the training set pl.figure() for i in range(60): pl.subplot(6, 10, i + 1), pl.imshow(v[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis( 'off') import dbn rb = dbn.dbn(20 * 16, [100, 100, 100], nlabels=nclasses, eta=0.3, momentum=0.4, nCDsteps=3, nepochs=600) rb.greedy(v, l) rb.classify_after_greedy(v, l) rb.updown(v, l) rb.classify(v, l) toph, topph, labs = rb.bottom_up(v, l) obs, pobs = rb.top_down(topph) pl.figure() for i in range(60): pl.subplot(6, 10, i + 1), pl.imshow(pobs[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off') pl.figure() for i in range(60): pl.subplot(6, 10, i + 1), pl.imshow(obs[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off') newv = inputs[:, nexamples:, :].reshape(nclasses * (39 - nexamples), 20 * 16) newl = labels[:, nexamples:, :].reshape(nclasses * (39 - nexamples), nclasses) rb.classify(newv, newl) toph, topph, labs = rb.bottom_up(newv, newl) obs, pobs = rb.top_down(topph) pl.figure() for i in range(57): pl.subplot(6, 10, i + 1), pl.imshow(newv[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6, 10, i + 1), pl.imshow(pobs[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6, 10, i + 1), pl.imshow(obs[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off') vis = np.random.randn(np.shape(v)[0], np.shape(v)[1]) * 0.05 for i in range(1000): toph, topph, labs = rb.bottom_up(vis, l) vis, pvis = rb.top_down(topph) pl.figure() for i in range(60): pl.subplot(6, 10, i + 1), pl.imshow(vis[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis('off')
#Interpreter python2 #displays the dbn reconstructed digits import sys sys.path.insert(0,'..\\rbm') print("Loading DBN\n") import cPickle as cP with open("..\\..\\data\\defValues-300-300.pklb","rb") as f: a=cP.load(f) import dbn import rbm0_m as rbm0 r=rbm0.rbm0() r.weights=a[0] r.load_data() d=dbn.dbn([r],30) d.rbmList[1].weights=a[1] print("Displaying DBN\n") import dbninter dbninter.display(d)
import dataset_mnist import numpy as np from dbn import dbn x, y, xTest, yTest = dataset_mnist.read(randomShift=False) h=dbn(n_inputs=784, layers=[{'neurons':12, 'activation':'sigmoid'}, {'neurons':3, 'activation':'softmax'}] ) #you cn first do unsupervised learning layer by layer training of #the hidden layers like so. This treats the hidden layers as a stack # of Restricted Boltzmann Machines (RBM) (if you remove this, equivalent to MLP model) h.learn_unsupervised(input=x, alpha=0.01, num_epochs=50) #unsupervised training can be followed by supervised training of the entire model h.learn(input=x, labels=y, cost='ce', alpha=0.01, num_epochs=50) eTrain = h.error(input=x, labels=y) eTest = h.error(input=xTest, labels=yTest) print("Training error: %.2f%% " %eTrain) print("Test error: %.2f%%" % eTest) #To compute the output of the model yhatTest = h.output(input=xTest) #This output is actual output of the network, which you can interpet as the probability of a given neuron being on. #To see how this output classifies a sample of the data, you can invoke the following function which shows 16 images #and their labels dataset_mnist.show(input=xTest, labels=yhatTest) #Extract missclassified examples from a dataset and inspect them. xm,ym= h.extract_misclassified_data(input=xTest, labels=yTest)
def learn_letters(): import scipy.io as sio nperclass = 39 classes = [10, 11, 28] #classes = [10, 13, 28] # A, C, S nclasses = len(classes) # Read in the data and prepare it data = sio.loadmat('binaryalphadigs.mat') inputs = np.ones((nclasses, nperclass, 20*16)) labels = np.zeros((nclasses, nperclass, nclasses)) for k in range(nclasses): for m in range(nperclass): inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float') labels[k,m,k] = 1. nexamples = 20 v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16) l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses) import pylab as pl # This shows a set of examples from the training set pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') import dbn rb = dbn.dbn(20*16,[100,100,100],nlabels=nclasses,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=600) rb.greedy(v,l) rb.classify_after_greedy(v,l) rb.updown(v,l) rb.classify(v,l) toph, topph, labs = rb.bottom_up(v,l) obs, pobs = rb.top_down(topph) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(pobs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(obs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16) newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses) rb.classify(newv,newl) toph, topph, labs = rb.bottom_up(newv,newl) obs, pobs = rb.top_down(topph) pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(pobs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') pl.figure() for i in range(57): pl.subplot(6,10,i+1), pl.imshow(obs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off') vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05 for i in range(1000): toph, topph, labs = rb.bottom_up(vis,l) vis, pvis = rb.top_down(topph) pl.figure() for i in range(60): pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
os.path.join(arguments.output_fldr, "dbn_def.txt")) shutil.copy2(arguments.param_def_file, os.path.join(arguments.output_fldr, "run_params.txt")) run_params = run_params_pb2.params() f = open(arguments.param_def_file) cfg_str = f.read() f.close() Merge(cfg_str, run_params) # No vtlp used in DBN training. data_src = htkdb_cm.htkdb_cm(arguments.db_name, arguments.db_path, run_params.data_params.num_frames_per_pt, run_params.data_params.use_delta) dev_src = htkdb_cm.htkdb_cm(arguments.dev_db_name, arguments.db_path, run_params.data_params.num_frames_per_pt, run_params.data_params.use_delta) cmn = run_params.data_params.normalization == run_params_pb2.DataParams.CMN cmvn = run_params.data_params.normalization == run_params_pb2.DataParams.CMVN normalize = run_params.data_params.normalization == run_params_pb2.DataParams.GLOB_NORMALIZE data_src.setup_data(arguments.num_files_per_load, cmn, cmvn, normalize) dev_src.setup_data(arguments.num_files_per_load, cmn, cmvn, normalize) dbn_train = dbn.dbn(arguments.dbn_def_file) dbn_train.train(data_src, dev_src, run_params.batch_size, arguments.output_fldr, reload_if_exists=True)