def train_autoencoder(model):
bigramtrainfeatures = theano.shared(numpy.concatenate(alldata[:numtrain]).astype("float32"))
trainer = train.GraddescentMinibatch(model, bigramtrainfeatures, batchsize=100, learningrate=0.01, normalizefilters=False)
print "training model"
for epoch in xrange(100):
trainer.step()
if epoch % 10 == 0:
pylab.figure(1)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3,3,i*3+j+1)
W = model.wdfh.get_value()[i*3+j,:model.numvisY, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion(); pylab.draw(); pylab.show()
pylab.figure(2)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3,3,i*3+j+1)
W = model.wdfh.get_value()[i*3+j,model.numvisY:, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion(); pylab.draw(); pylab.show()
print "done training; extracting features with bigram model"
allbigramfeatures = []
print "xxxxx",
for i, d in enumerate(alldata):
print "\b\b\b\b\b\b{0:5d}".format(i),
allbigramfeatures.append(model.uncorruptedhiddens(d).mean(0))
return allbigramfeatures
def train_kmeans(featurelearndata):
Rinit = numpy.random.permutation(numhid)
W = featurelearndata[Rinit]
for epoch in range(10):
W = online_kmeans.kmeans(featurelearndata, numhid, Winit=W, numepochs=10, learningrate=0.1*0.8**epoch)
W_ = numpy.dot(pca_forward,W.T).T.reshape(numhid, patchsize, patchsize, numchannels)
dispims_color.dispims_color(W_)
pylab.ion()
pylab.draw()
pylab.show()
pylab.ioff()
print "done"
allbigramfeatures = []
print "xxxxx",
for i, image in enumerate(allims):
print "\b\b\b\b\b\b{0:5d}".format(i),
image = image.reshape(numchannels, inputdim, inputdim).transpose(1,2,0)
prange = numpy.arange(patchsize/2, inputdim-patchsize/2)
meshg = numpy.meshgrid(prange, prange)
keypoints = numpy.array([c.flatten() for c in meshg]).T
patches = crop_patches_color(image, keypoints, patchsize)
patches -= patches.mean(1)[:,None]
patches /= patches.std(1)[:,None] + 0.1 * meanstd
patches = numpy.dot(patches, pca_backward.T).astype("float32")
if pooling==1:
allbigramfeatures.append(online_kmeans.assign_triangle(patches, W).mean(0).astype("float32"))
elif pooling==2:
quadrants = numpy.array([int(str(int(a[0]>=inputdim/2))+str(int(a[1]>=inputdim/2)), 2) for a in keypoints])
features = online_kmeans.assign_triangle(patches, W).astype("float32")
allbigramfeatures.append(numpy.array([(features * (quadrants==i)[:,None]).mean(0) for i in range(4)]).reshape(4*numhid))
return allbigramfeatures
def train_kmeans(featurelearndata):
Rinit = numpy.random.permutation(numhid)
W = featurelearndata[Rinit]
for epoch in range(10):
W = online_kmeans.kmeans(featurelearndata,
numhid,
Winit=W,
numepochs=10,
learningrate=0.1 * 0.8**epoch)
W_ = numpy.dot(pca_forward, W.T).T.reshape(numhid, patchsize,
patchsize, numchannels)
dispims_color.dispims_color(W_)
pylab.ion()
pylab.draw()
pylab.show()
pylab.ioff()
print "done"
allbigramfeatures = []
print "xxxxx",
for i, image in enumerate(allims):
print "\b\b\b\b\b\b{0:5d}".format(i),
image = image.reshape(numchannels, inputdim,
inputdim).transpose(1, 2, 0)
prange = numpy.arange(patchsize / 2, inputdim - patchsize / 2)
meshg = numpy.meshgrid(prange, prange)
keypoints = numpy.array([c.flatten() for c in meshg]).T
patches = crop_patches_color(image, keypoints, patchsize)
patches -= patches.mean(1)[:, None]
patches /= patches.std(1)[:, None] + 0.1 * meanstd
patches = numpy.dot(patches, pca_backward.T).astype("float32")
if pooling == 1:
allbigramfeatures.append(
online_kmeans.assign_triangle(patches,
W).mean(0).astype("float32"))
elif pooling == 2:
quadrants = numpy.array([
int(
str(int(a[0] >= inputdim / 2)) +
str(int(a[1] >= inputdim / 2)), 2) for a in keypoints
])
features = online_kmeans.assign_triangle(patches,
W).astype("float32")
allbigramfeatures.append(
numpy.array([(features * (quadrants == i)[:, None]).mean(0)
for i in range(4)]).reshape(4 * numhid))
return allbigramfeatures
def train_autoencoder(model):
bigramtrainfeatures = theano.shared(
numpy.concatenate(alldata[:numtrain]).astype("float32"))
trainer = train.GraddescentMinibatch(model,
bigramtrainfeatures,
batchsize=100,
learningrate=0.01,
normalizefilters=False)
print "training model"
for epoch in xrange(100):
trainer.step()
if epoch % 10 == 0:
pylab.figure(1)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3, 3, i * 3 + j + 1)
W = model.wdfh.get_value()[i * 3 + j, :model.numvisY, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(
W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion()
pylab.draw()
pylab.show()
pylab.figure(2)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3, 3, i * 3 + j + 1)
W = model.wdfh.get_value()[i * 3 + j, model.numvisY:, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(
W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion()
pylab.draw()
pylab.show()
print "done training; extracting features with bigram model"
allbigramfeatures = []
print "xxxxx",
for i, d in enumerate(alldata):
print "\b\b\b\b\b\b{0:5d}".format(i),
allbigramfeatures.append(model.uncorruptedhiddens(d).mean(0))
return allbigramfeatures
testims = numpy.loadtxt("cifar_mini_images_test.txt")
trainlabels = numpy.loadtxt("cifar_mini_labels_train.txt").astype("int")
testlabels = numpy.loadtxt("cifar_mini_labels_test.txt").astype("int")
allims = numpy.concatenate((trainims, testims), 0)
numtrain = numpy.int(trainims.shape[0] * 0.75) #use 3/4 for training
numvali = numpy.int(trainims.shape[0] * 0.25) #use 1/4 for validation
#SHOW SOME IMAGES
imstoshow = numpy.zeros((30, 32, 32, 3))
for c in range(numclasses):
imstoshow[c *
3:(c + 1) * 3] = trainims[trainlabels[:, c] == 1][:3].reshape(
3, 3, 32, 32).transpose(0, 2, 3, 1)
dispims_color.dispims_color(imstoshow, 1)
#WHITENING
patches = numpy.concatenate([
crop_patches_color(
im.reshape(3, 32, 32).transpose(1, 2, 0),
numpy.array([
rng.randint(patchsize / 2, 32 - patchsize / 2, 20),
rng.randint(patchsize / 2, 32 - patchsize / 2, 20)
]).T, patchsize) for im in trainims
]).astype("float32")
R = rng.permutation(patches.shape[0])
patches = patches[R, :]
meanstd = patches.std()
patches -= patches.mean(1)[:, None]
patches /= patches.std(1)[:, None] + 0.1 * meanstd
patches -= patches.mean(0)[None,:]
patches /= patches.std(0)[None,:]
pcadata, pca_backward, pca_forward = pca(patches.T, .9)
featurelearndata = pcadata.T.astype("float32")
del pcadata
numpatches = featurelearndata.shape[0]
print "numpatches: ", numpatches
print "done"
#TRAIN KMEANS
f1 = pylab.figure()
Rinit = rng.permutation(numhid)
W = featurelearndata[Rinit]
print "training kmeans"
for epoch in range(20):
W = online_kmeans.kmeans(featurelearndata, numhid, Winit=W, numepochs=1, learningrate=0.01*0.8**epoch)
W_ = numpy.dot(pca_forward,W.T).T.reshape(numhid, patchsize, patchsize, 3)
dispims_color(W_)
pylab.draw(); pylab.show()
print "done"
f2 = pylab.figure()
dispims_color(pca_backward.reshape(pca_backward.shape[0], 6, 6, 3))
trainpatches -= trainpatches_mean
trainpatches /= trainpatches_std + 0.1 * meanstd
pca_backward, pca_forward, zca_backward, zca_forward = pca(trainpatches, 0.9, whiten=True)
trainpatches_whitened = numpy.dot(trainpatches, pca_backward.T).astype("float32")
print "done"
#INSTANTIATE THE ZERO-BIAS AUTOENCODER
model = zae.Zae(numvis=trainpatches_whitened.shape[1], numhid=numfeatures, vistype="real", init_features=0.1*trainpatches_whitened[:numfeatures].T, selectionthreshold=1.0*numpy.ones(numfeatures, dtype="float32"))
assert False, "preprocessing is done, may train now"
#DO SOME STEPS WITH SMALL LEARNING RATE TO MAKE SURE THE INITIALIZATION IS IN A REASONABLE RANGE
trainer = GraddescentMinibatchDiscreteThresholdSearch(model, trainpatches_whitened, 100, learningrate=0.0001, momentum=0.9)
trainer.step(); trainer.step(); trainer.step()
#TRAIN THE MODEL FOR REAL, AND SHOW FILTERS
trainer = GraddescentMinibatchDiscreteThresholdSearch(model, trainpatches_whitened, 100, learningrate=0.01, momentum=0.9)
#trainer = graddescent_rewrite.SGD_Trainer(model, trainpatches_whitened, batchsize=128, learningrate=0.1, momentum=0.9, loadsize=30000, gradient_clip_threshold=5.0)
for epoch in xrange(100):
trainer.step()
if epoch % 10 == 0 and epoch > 0:
#trainer.set_learningrate(trainer.learningrate*0.8)
dispims_color(numpy.dot(model.W.get_value().T, pca_forward.T).reshape(-1, patchsize, patchsize, 3), 1)
pylab.draw(); pylab.show()
testims = numpy.loadtxt("cifar_mini_images_test.txt")
trainlabels = numpy.loadtxt("cifar_mini_labels_train.txt").astype("int")
testlabels = numpy.loadtxt("cifar_mini_labels_test.txt").astype("int")
allims = numpy.concatenate((trainims,testims), 0)
numtrain = numpy.int(trainims.shape[0]*0.75) #use 3/4 for training
numvali = numpy.int(trainims.shape[0]*0.25) #use 1/4 for validation
#SHOW SOME IMAGES
imstoshow = numpy.zeros((30, 32, 32, 3))
for c in range(numclasses):
imstoshow[c*3:(c+1)*3] = trainims[trainlabels[:,c]==1][:3].reshape(3,3,32,32).transpose(0,2,3,1)
dispims_color.dispims_color(imstoshow, 1)
#WHITENING
patches = numpy.concatenate([crop_patches_color(im.reshape(3, 32, 32).transpose(1,2,0), numpy.array([rng.randint(patchsize/2, 32-patchsize/2, 20), rng.randint(patchsize/2, 32-patchsize/2, 20)]).T, patchsize) for im in trainims]).astype("float32")
R = rng.permutation(patches.shape[0])
patches = patches[R, :]
meanstd = patches.std()
patches -= patches.mean(1)[:,None]
patches /= patches.std(1)[:,None] + 0.1 * meanstd
#patches -= patches.mean(0)[None,:]
#patches /= patches.std(0)[None,:]
pca_backward, pca_forward = pca(patches.T, .99)
#pcadata, pca_backward, pca_forward = pca(patches.T, .9)
bigramtrainfeatures = theano.shared(numpy.concatenate(alldata[:numtrain]).astype("float32"))
trainer = train.GraddescentMinibatch(model, bigramtrainfeatures, batchsize=100, learningrate=0.01, normalizefilters=False)
print "training model"
for epoch in xrange(100):
trainer.step()
if epoch % 10 == 0:
pylab.figure(1)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3,3,i*3+j+1)
W = model.wdfh.get_value()[i*3+j,:model.numvisY, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion(); pylab.draw(); pylab.show()
pylab.figure(2)
pylab.clf()
for i in range(3):
for j in range(3):
pylab.subplot(3,3,i*3+j+1)
W = model.wdfh.get_value()[i*3+j,model.numvisY:, :]
W_ = numpy.dot(pca_forward, W)
dispims_color.dispims_color(W_.T.reshape(-1, patchsize, patchsize, 3))
pylab.ion(); pylab.draw(); pylab.show()
print "done"
print "extracting features with bigram model"
##trainer = graddescent_rewrite.SGD_Trainer(model, trainpatches_whitened, batchsize=128, learningrate=0.001, momentum=0.9, loadsize=30000, gradient_clip_threshold=5.0)
#trainer.step(); trainer.step(); trainer.step()
#TRAIN THE MODEL FOR REAL, AND SHOW FILTERS
import graddescent_rewrite
#trainer = train.GraddescentMinibatch(model, trainpatches_theano, 100, learningrate=0.01, momentum=0.9)
trainer = graddescent_rewrite.SGD_Trainer(model, trainpatches_whitened, batchsize=128, learningrate=0.1, momentum=0.9, loadsize=30000, gradient_clip_threshold=5.0)
assert False, "preprocessing is done, may train now"
for epoch in xrange(100):
trainer.step()
if epoch % 10 == 0 and epoch > 0:
#trainer.set_learningrate(trainer.learningrate*0.8)
dispims_color(numpy.dot(model.W.get_value().T, pca_forward.T).reshape(-1, patchsize, patchsize, 3), 1)
pylab.draw(); pylab.show()
#SHOW SOME EXAMPLES
subplot(1,2,2); dispims_color(numpy.dot(model.recons_from_prehiddens(trainpatches_whitened[:25], 0.5*randn(25,model.numhid).astype("float32")), pca_forward.T).reshape(-1, patchsize, patchsize, 3), 1)
#AND SOME SAMPLED RECONSTRUCTIONS
subplot(1,2,2); dispims_color(numpy.dot(model.recons_from_prehiddens(trainpatches_whitened[:25], 0.5*randn(25,model.numhid).astype("float32")), pca_forward.T).reshape(-1, patchsize, patchsize, 3), 1)
#OR SOME RANDOM SAMPLES
#subplot(1,2,2); dispims_color(numpy.dot(model.recons_from_prehiddens(1.0*randn(100,92).astype("float32"), 0.24*randn(100,model.numhid).astype("float32")), pca_forward.T).reshape(-1, patchsize, patchsize, 3), 1)
usortind = numpy.argsort(u)[::-1]
v = v[:, usortind]
u = u[usortind]
u = u[u.cumsum()<u.sum()*var_fraction]
numprincomps = u.shape[0]
V = ((u**(-0.5))[:numprincomps][numpy.newaxis,:]*v[:,:numprincomps]).T
W = (u**0.5)[:numprincomps][numpy.newaxis,:]*v[:,:numprincomps]
return numpy.dot(V,data), V, W
#SHOW SOME IMAGES (only works for RGB images with numchannels = 3)
imstoshow = numpy.zeros((30, inputdim, inputdim, 3))
for c in range(numclasses):
imstoshow[c*3:(c+1)*3] = trainims[trainlabels[:,c]==1][:3].reshape(3,3,inputdim,inputdim).transpose(0,2,3,1)
dispims_color.dispims_color(imstoshow, 1)
#WHITENING
rand1 = numpy.random.randint(patchsize/2, inputdim-patchsize/2, 20)
rand2 = numpy.random.randint(patchsize/2, inputdim-patchsize/2, 20)
to_concatenate = [crop_patches_color(
im.reshape(numchannels, inputdim, inputdim).transpose(1,2,0),
numpy.array([rand1, rand2]).T,
patchsize)
for im in trainims]
patches = numpy.concatenate(to_concatenate).astype("float32")
R = numpy.random.permutation(patches.shape[0])
patches = patches[R, :]
usortind = numpy.argsort(u)[::-1]
v = v[:, usortind]
u = u[usortind]
u = u[u.cumsum()<u.sum()*var_fraction]
numprincomps = u.shape[0]
V = ((u**(-0.5))[:numprincomps][numpy.newaxis,:]*v[:,:numprincomps]).T
W = (u**0.5)[:numprincomps][numpy.newaxis,:]*v[:,:numprincomps]
return numpy.dot(V,data), V, W
#SHOW SOME IMAGES (only works for RGB images with numchannels = 3)
imstoshow = numpy.zeros((30, inputdim, inputdim, 3))
for c in range(numclasses):
imstoshow[c*3:(c+1)*3] = trainims[trainlabels[:,c]==1][:3].reshape(3,3,inputdim,inputdim).transpose(0,2,3,1)
dispims_color.dispims_color(imstoshow, 1)
#WHITENING
rand1 = numpy.random.randint(patchsize/2, inputdim-patchsize/2, 20)
rand2 = numpy.random.randint(patchsize/2, inputdim-patchsize/2, 20)
to_concatenate = [crop_patches_color(
im.reshape(numchannels, inputdim, inputdim).transpose(1,2,0),
numpy.array([rand1, rand2]).T,
patchsize)
for im in trainims]
patches = numpy.concatenate(to_concatenate).astype("float32")
R = numpy.random.permutation(patches.shape[0])
patches = patches[R, :]
