def test_dbn():
import dbn
# dbn.classify()
rb = dbn.dbn(4, [4, 4, 2], nlabels=2, eta=0.3, momentum=0.5, nCDsteps=1, nepochs=1001)
N = 2500
v = np.zeros((2 * N, 4))
l = np.zeros((2 * N, 2))
for n in range(N):
r = np.random.rand()
if r > 0.666:
v[n, :] = [0, 1, 0, 0]
l[n, :] = [1, 0]
elif r > 0.333:
v[n, :] = [1, 0, 0, 0]
l[n, :] = [1, 0]
for n in range(N):
r = np.random.rand()
if r > 0.666:
v[N + n, :] = [0, 0, 0, 1]
l[N + n, :] = [0, 1]
elif r > 0.333:
v[N + n, :] = [0, 0, 1, 0]
l[N + n, :] = [0, 1]
rb.greedy(v, l) # , delta=delta, momentum=mom)
rb.classify_after_greedy(v, l)
rb.updown(v, l)
rb.classify(v, l)
def test_dbn():
import dbn
#dbn.classify()
rb = dbn.dbn(4,[4,4,2],nlabels=2,eta=0.3,momentum=0.5,nCDsteps=1,nepochs=1001)
N=2500
v = np.zeros((2*N,4))
l = np.zeros((2*N,2))
for n in range(N):
r = np.random.rand()
if r>0.666:
v[n,:] = [0,1,0,0]
l[n,:] = [1,0]
elif r>0.333:
v[n,:] = [1,0,0,0]
l[n,:] = [1,0]
for n in range(N):
r = np.random.rand()
if r>0.666:
v[N+n,:] = [0,0,0,1]
l[N+n,:] = [0,1]
elif r>0.333:
v[N+n,:] = [0,0,1,0]
l[N+n,:] = [0,1]
rb.greedy(v,l) #, delta=delta, momentum=mom)
rb.classify_after_greedy(v,l)
rb.updown(v,l)
rb.classify(v,l)
def preTrain(self,lR,k,nE,method):
# initialize deep belief network
preModel = dbn(self.net,self.T,lR,k,self.bS,nE,self.roll)
# load data
preModel.loadData(self.data)
# train dbn
preModel.preTrain(lR,k,nE,method)
# set pre-trained parameters
self.w = copy.deepcopy(preModel.w)
self.b = copy.deepcopy(preModel.b)
self.bR = copy.deepcopy(preModel.bR)
def preTrain(self, lR, k, nE, method):
# initialize deep belief network
preModel = dbn(self.net, self.T, lR, k, self.bS, nE, self.roll)
# load data
preModel.loadData(self.data)
# train dbn
preModel.preTrain(lR, k, nE, method)
# set pre-trained parameters
self.w = copy.deepcopy(preModel.w)
for i in range(self.nL):
self.b[i] = 0.5 * (copy.deepcopy(preModel.b[i]) +
copy.deepcopy(preModel.bR[i]))
def test_dbn_digs():
import dbn
import scipy as scipy
import scipy.io as sio
tmp = sio.loadmat('binaryalphadigs.mat')
NTRAIN = 39
CLASSES = [10, 12, 28] # A, C, S
NCLASSES = len(CLASSES)
I, L = 20*16, 3
N = NTRAIN*NCLASSES
# organize data
data = np.zeros((NCLASSES, NTRAIN, I))
labels = np.zeros((NCLASSES, NTRAIN, L))
for k in range(L):
for m in range(NTRAIN):
data[k,m,:] = (tmp['dat'][CLASSES[k],m].ravel()).astype('d')
labels[k,m,k] = 1.
# prepare observations, labels
perm = np.arange(N)
np.random.shuffle(perm)
v = data.reshape(N, I)[perm,:]
l = labels.reshape(N, L)[perm,:]
rb = dbn.dbn(20*16,[100,100,100],nlabels=3,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=401)
rb.greedy(v,l)
rb.classify_after_greedy(v,l)
rb.updown(v,l)
rb.classify(v,l)
import pylab as pl
pl.figure(), pl.imshow(v[5,:].reshape(20,16))
toph, topph, labs = rb.bottom_up(v,l)
obs, pobs = rb.top_down(topph)
pl.figure(), pl.imshow(pobs[5,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest')
return v, l, obs, pobs
def test():
import sys
sys.path.insert(0,'digit_reconstructor\\rbm')
sys.path.insert(0,'digit_reconstructor\\dbn')
print("Loading DBN\n")
import cPickle as cP
with open("data\\defValues-300-300.pklb","rb") as f:
a=cP.load(f)
import dbn
import rbm0_m as rbm0
r=rbm0.rbm0()
r.weights=a[0]
r.load_data()
d=dbn.dbn([r],30)
d.rbmList[1].weights=a[1]
print("Loaded DBN\n")
epochs = 20
print("Training DBN\n")
d.train(epochs)
dweights =[]
print("Saving DBN weights\n")
for x in xrange(len(d.rbmList)):
dweights.append(d.rbmList[x].weights)
file_write_name="new_weights-784-300-30.pklb"
with open(file_write_name,"wb") as f:
cP.dump(dweights,f)
print("Displaying DBN reconstructed digits")
import dbninter
dbninter.display(d)
def test_dbn_mnist():
import dbn
import pickle, gzip
# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = pickle.load(f)
f.close()
indices = np.arange(np.shape(train_set[0])[0])
np.random.shuffle(indices)
# indices = indices[:200]
t = train_set[0][indices, :]
l = train_set[1][indices]
labels = np.zeros(((np.shape(t)[0]), 10))
for i in range(20 * 10):
labels[i, l[i]] = 1
rb = dbn.dbn(28 * 28, [100, 100, 50], nlabels=10, eta=0.3, momentum=0.4, nCDsteps=3, nepochs=1000)
rb.greedy(t, labels) # , delta=delta, momentum=mom)
rb.classify_after_greedy(t, labels)
rb.updown(t, labels)
rb.classify(t, labels)
return rb
def test_dbn_mnist():
import dbn
import cPickle, gzip
# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
indices = np.arange(np.shape(train_set[0])[0])
np.random.shuffle(indices)
#indices = indices[:200]
t = train_set[0][indices,:]
l = train_set[1][indices]
labels = np.zeros(((np.shape(t)[0]),10))
for i in range(20*10):
labels[i,l[i]] = 1
rb = dbn.dbn(28*28,[100,100,50],nlabels=10,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=1000)
rb.greedy(t,labels) #, delta=delta, momentum=mom)
rb.classify_after_greedy(t,labels)
rb.updown(t,labels)
rb.classify(t,labels)
return rb
def learn_letters():
import scipy.io as sio
nperclass = 39
classes = [10, 11, 28]
# classes = [10, 13, 28] # A, C, S
nclasses = len(classes)
# Read in the data and prepare it
data = sio.loadmat('binaryalphadigs.mat')
inputs = np.ones((nclasses, nperclass, 20 * 16))
labels = np.zeros((nclasses, nperclass, nclasses))
for k in range(nclasses):
for m in range(nperclass):
inputs[k, m, :] = (data['dat'][classes[k], m].ravel()).astype('float')
labels[k, m, k] = 1.
nexamples = 20
v = inputs[:, :nexamples, :].reshape(nclasses * nexamples, 20 * 16)
l = labels[:, :nexamples, :].reshape(nclasses * nexamples, nclasses)
import pylab as pl
# This shows a set of examples from the training set
pl.figure()
for i in range(60):
pl.subplot(6, 10, i + 1), pl.imshow(v[i, :].reshape(20, 16), cmap=pl.cm.gray, interpolation='nearest'), pl.axis(
'off')
import dbn
rb = dbn.dbn(20 * 16, [100, 100, 100], nlabels=nclasses, eta=0.3, momentum=0.4, nCDsteps=3, nepochs=600)
rb.greedy(v, l)
rb.classify_after_greedy(v, l)
rb.updown(v, l)
rb.classify(v, l)
toph, topph, labs = rb.bottom_up(v, l)
obs, pobs = rb.top_down(topph)
pl.figure()
for i in range(60):
pl.subplot(6, 10, i + 1), pl.imshow(pobs[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(60):
pl.subplot(6, 10, i + 1), pl.imshow(obs[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
newv = inputs[:, nexamples:, :].reshape(nclasses * (39 - nexamples), 20 * 16)
newl = labels[:, nexamples:, :].reshape(nclasses * (39 - nexamples), nclasses)
rb.classify(newv, newl)
toph, topph, labs = rb.bottom_up(newv, newl)
obs, pobs = rb.top_down(topph)
pl.figure()
for i in range(57):
pl.subplot(6, 10, i + 1), pl.imshow(newv[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6, 10, i + 1), pl.imshow(pobs[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6, 10, i + 1), pl.imshow(obs[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
vis = np.random.randn(np.shape(v)[0], np.shape(v)[1]) * 0.05
for i in range(1000):
toph, topph, labs = rb.bottom_up(vis, l)
vis, pvis = rb.top_down(topph)
pl.figure()
for i in range(60):
pl.subplot(6, 10, i + 1), pl.imshow(vis[i, :].reshape(20, 16), cmap=pl.cm.gray,
interpolation='nearest'), pl.axis('off')
#Interpreter python2
#displays the dbn reconstructed digits
import sys
sys.path.insert(0,'..\\rbm')
print("Loading DBN\n")
import cPickle as cP
with open("..\\..\\data\\defValues-300-300.pklb","rb") as f:
a=cP.load(f)
import dbn
import rbm0_m as rbm0
r=rbm0.rbm0()
r.weights=a[0]
r.load_data()
d=dbn.dbn([r],30)
d.rbmList[1].weights=a[1]
print("Displaying DBN\n")
import dbninter
dbninter.display(d)
import dataset_mnist
import numpy as np
from dbn import dbn
x, y, xTest, yTest = dataset_mnist.read(randomShift=False)
h=dbn(n_inputs=784,
layers=[{'neurons':12, 'activation':'sigmoid'},
{'neurons':3, 'activation':'softmax'}]
)
#you cn first do unsupervised learning layer by layer training of
#the hidden layers like so. This treats the hidden layers as a stack
# of Restricted Boltzmann Machines (RBM) (if you remove this, equivalent to MLP model)
h.learn_unsupervised(input=x, alpha=0.01, num_epochs=50)
#unsupervised training can be followed by supervised training of the entire model
h.learn(input=x, labels=y, cost='ce', alpha=0.01, num_epochs=50)
eTrain = h.error(input=x, labels=y)
eTest = h.error(input=xTest, labels=yTest)
print("Training error: %.2f%% " %eTrain)
print("Test error: %.2f%%" % eTest)
#To compute the output of the model
yhatTest = h.output(input=xTest)
#This output is actual output of the network, which you can interpet as the probability of a given neuron being on.
#To see how this output classifies a sample of the data, you can invoke the following function which shows 16 images
#and their labels
dataset_mnist.show(input=xTest, labels=yhatTest)
#Extract missclassified examples from a dataset and inspect them.
xm,ym= h.extract_misclassified_data(input=xTest, labels=yTest)
def learn_letters():
import scipy.io as sio
nperclass = 39
classes = [10, 11, 28]
#classes = [10, 13, 28] # A, C, S
nclasses = len(classes)
# Read in the data and prepare it
data = sio.loadmat('binaryalphadigs.mat')
inputs = np.ones((nclasses, nperclass, 20*16))
labels = np.zeros((nclasses, nperclass, nclasses))
for k in range(nclasses):
for m in range(nperclass):
inputs[k,m,:] = (data['dat'][classes[k],m].ravel()).astype('float')
labels[k,m,k] = 1.
nexamples = 20
v = inputs[:,:nexamples,:].reshape(nclasses*nexamples, 20*16)
l = labels[:,:nexamples,:].reshape(nclasses*nexamples, nclasses)
import pylab as pl
# This shows a set of examples from the training set
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(v[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
import dbn
rb = dbn.dbn(20*16,[100,100,100],nlabels=nclasses,eta=0.3,momentum=0.4,nCDsteps=3,nepochs=600)
rb.greedy(v,l)
rb.classify_after_greedy(v,l)
rb.updown(v,l)
rb.classify(v,l)
toph, topph, labs = rb.bottom_up(v,l)
obs, pobs = rb.top_down(topph)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(pobs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(obs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
newv = inputs[:,nexamples:,:].reshape(nclasses*(39-nexamples),20*16)
newl = labels[:,nexamples:,:].reshape(nclasses*(39-nexamples),nclasses)
rb.classify(newv,newl)
toph, topph, labs = rb.bottom_up(newv,newl)
obs, pobs = rb.top_down(topph)
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(newv[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(pobs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
pl.figure()
for i in range(57):
pl.subplot(6,10,i+1), pl.imshow(obs[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
vis = np.random.randn(np.shape(v)[0],np.shape(v)[1])*0.05
for i in range(1000):
toph, topph, labs = rb.bottom_up(vis,l)
vis, pvis = rb.top_down(topph)
pl.figure()
for i in range(60):
pl.subplot(6,10,i+1), pl.imshow(vis[i,:].reshape(20,16),cmap=pl.cm.gray,interpolation='nearest'), pl.axis('off')
os.path.join(arguments.output_fldr, "dbn_def.txt"))
shutil.copy2(arguments.param_def_file,
os.path.join(arguments.output_fldr, "run_params.txt"))
run_params = run_params_pb2.params()
f = open(arguments.param_def_file)
cfg_str = f.read()
f.close()
Merge(cfg_str, run_params)
# No vtlp used in DBN training.
data_src = htkdb_cm.htkdb_cm(arguments.db_name, arguments.db_path,
run_params.data_params.num_frames_per_pt,
run_params.data_params.use_delta)
dev_src = htkdb_cm.htkdb_cm(arguments.dev_db_name, arguments.db_path,
run_params.data_params.num_frames_per_pt,
run_params.data_params.use_delta)
cmn = run_params.data_params.normalization == run_params_pb2.DataParams.CMN
cmvn = run_params.data_params.normalization == run_params_pb2.DataParams.CMVN
normalize = run_params.data_params.normalization == run_params_pb2.DataParams.GLOB_NORMALIZE
data_src.setup_data(arguments.num_files_per_load, cmn, cmvn, normalize)
dev_src.setup_data(arguments.num_files_per_load, cmn, cmvn, normalize)
dbn_train = dbn.dbn(arguments.dbn_def_file)
dbn_train.train(data_src,
dev_src,
run_params.batch_size,
arguments.output_fldr,
reload_if_exists=True)
