def test_SdA(finetune_lr=0.1, pretraining_epochs=2,
pretrain_lr=0.001, training_epochs=2,
dataset='mnist.pkl.gz', batch_size=1):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on MNIST.
:type learning_rate: float
:param learning_rate: learning rate used in the finetune stage
(factor for the stochastic gradient)
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
"""
data_path = '/Applications/MAMP/htdocs/DeepLearningTutorials/data/'
# Use the following command if you want to run the dA in production
# THEANO_FLAGS='floatX=float32,device=gpu0,nvcc.fastmath=True,cuda.root=/usr/local/cuda,mode=FAST_RUN' python SdA_v2.py
#data_path = '/home/ubuntu/DeepLearningTutorials/data/'
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
# sda = SdA(
# numpy_rng=numpy_rng,
# n_ins=128 * 128,
# hidden_layers_sizes=[1000, 1000],
# n_outs=21,
# data_path=data_path
# )
sda = SdA(
numpy_rng=numpy_rng,
n_ins=128 * 128 * 3,
hidden_layers_sizes=[1000, 1000],
n_outs=3,
data_path=data_path
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size)
print '... pre-training the model'
start_time = timeit.default_timer()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = timeit.default_timer()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
from utils import tile_raster_images
try:
import PIL.Image as Image
except ImportError:
import Image
# image = Image.fromarray(tile_raster_images(
# X=sda.dA_layers[0].W.get_value(borrow=True).T,
# img_shape=(128, 128), tile_shape=(10, 10),
# tile_spacing=(1, 1)))
# print sda.dA_layers[1].W.get_value(borrow=True).T.shape
# print sda.dA_layers[0].W.get_value(borrow=True).T.shape
# image = Image.fromarray(tile_raster_images(
# X=sda.dA_layers[1].W.get_value(borrow=True).T,
# img_shape=(36, 36), tile_shape=(10, 10),
# tile_spacing=(1, 1)))
# image.save('filters_corruption_30.png')
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = timeit.default_timer()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print(
(
'Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%'
)
% (best_validation_loss * 100., best_iter + 1, test_score * 100.)
)
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# x = T.matrix('x')
# index_1 = T.lscalar() # index to a [mini]batch
# index_2 = T.lscalar() # index to a [mini]batch
# getHV = sda.dA_layers[0].get_hidden_values(x)
# getHiddenValues = theano.function(
# [index_1,index_2],
# getHV,
# givens={
# x: train_set_x[index_1:index_2]
# }
# )
# print getHiddenValues(0,len(train_set_x.get_value(borrow=True))).shape
# da1output = T.matrix('da1output')
# getHV2 = sda.dA_layers[1].get_hidden_values(da1output)
# getHiddenValues2 = theano.function(
# [da1output],
# getHV2
# )
# #print getHiddenValues2(getHiddenValues(0,1)).shape
# X = getHiddenValues2(getHiddenValues(0,len(train_set_x.get_value(borrow=True))))
sda.save_weights()
# sda2 = SdA(
# numpy_rng=numpy_rng,
# n_ins=128 * 128,
# hidden_layers_sizes=[1000, 1000],
# n_outs=21,
# data_path=data_path
# )
sda2 = SdA(
numpy_rng=numpy_rng,
n_ins=128 * 128 * 3,
hidden_layers_sizes=[1000, 1000],
n_outs=3,
data_path=data_path
)
sda2.load_weights()
#print sda2.dA_layers[1].W.get_value(borrow=True).shape
x = T.matrix('x')
index_1 = T.lscalar() # index to a [mini]batch
index_2 = T.lscalar() # index to a [mini]batch
getHV = sda2.dA_layers[0].get_hidden_values(x)
getHiddenValues = theano.function(
[index_1,index_2],
getHV,
givens={
x: train_set_x[index_1:index_2]
}
)
#print getHiddenValues(0,len(train_set_x.get_value(borrow=True))).shape
print getHiddenValues(0,1)
da1output = T.matrix('da1output')
getHV2 = sda2.dA_layers[1].get_hidden_values(da1output)
getHiddenValues2 = theano.function(
[da1output],
getHV2
)
#print getHiddenValues2(getHiddenValues(0,1)).shape
X = getHiddenValues2(getHiddenValues(0,len(train_set_x.get_value(borrow=True))))
print X.shape
# print X.shape
# da2output = T.matrix('da2output')
# getHV3 = sda.dA_layers[2].get_hidden_values(da2output)
# getHiddenValues3 = theano.function(
# [da2output],
# getHV3
# )
# print getHiddenValues3([getHiddenValues2(0,1)])
from fetex_image import FetexImage
pkl_file = open(data_path + 'im_index.pkl', 'rb')
im_index = cPickle.load(pkl_file)
fe = FetexImage(verbose=True,support_per_class=10000,data_path=data_path, dataset='categories', mode='RGB')
fe.im_index = im_index
# print im_index[0]
# print im_index[1]
#X_compressed = getHiddenValues(0,100)
X_compressed = X
#print X_compressed.shape
#fe.dimReductionSdA(X)
fe.similarImages(X_compressed)
def test_dA(learning_rate=0.1, training_epochs=50,
dataset='mnist.pkl.gz',
batch_size=20, output_folder='dA_plots'):
"""
This demo is tested on MNIST
:type learning_rate: float
:param learning_rate: learning rate used for training the DeNosing
AutoEncoder
:type training_epochs: int
:param training_epochs: number of epochs used for training
:type dataset: string
:param dataset: path to the picked dataset
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# start-snippet-2
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
# end-snippet-2
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
####################################
# BUILDING THE MODEL NO CORRUPTION #
####################################
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
da = dA(
numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=128 * 128,
n_hidden=500
)
cost, updates = da.get_cost_updates(
corruption_level=0.,
learning_rate=learning_rate
)
train_da = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size]
}
)
start_time = timeit.default_timer()
############
# TRAINING #
############
# go through training epochs
for epoch in xrange(training_epochs):
# go through trainng set
c = []
for batch_index in xrange(n_train_batches):
c.append(train_da(batch_index))
print 'Training epoch %d, cost ' % epoch, numpy.mean(c)
end_time = timeit.default_timer()
training_time = (end_time - start_time)
print >> sys.stderr, ('The no corruption code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((training_time) / 60.))
# image = Image.fromarray(
# tile_raster_images(X=da.W.get_value(borrow=True).T,
# img_shape=(128, 128), tile_shape=(10, 10),
# tile_spacing=(1, 1)))
# image.save('filters_corruption_0.png')
# print train_set_x.get_value(borrow=True).shape
# sample = train_set_x.get_value(borrow=True)[0]
# print sample.shape
# print da.get_hidden_values(sample)
# W = da.W.get_value(borrow=True).T
# print da.W.get_value(borrow=True).T.shape
# print da.W.get_value(borrow=True).T[0].shape
#sample = T.ivector('sample')
#sample = T.matrix('sample')
index_1 = T.lscalar() # index to a [mini]batch
index_2 = T.lscalar() # index to a [mini]batch
getHV = da.get_hidden_values(x)
getHiddenValues = theano.function(
[index_1,index_2],
getHV,
givens={
x: train_set_x[index_1:index_2]
}
)
#print getHiddenValues(0,1).shape
import cPickle
from fetex_image import FetexImage
pkl_file = open('/Applications/MAMP/htdocs/DeepLearningTutorials/data/im_index.pkl', 'rb')
im_index = cPickle.load(pkl_file)
data_path = '/Applications/MAMP/htdocs/DeepLearningTutorials/data/'
#store = pd.HDFStore('/Applications/MAMP/htdocs/DeepLearningTutorials/data/df_images.h5', 'r')
fe = FetexImage(verbose=True,support_per_class=100,data_path=data_path, dataset='categories', mode='RGB')
fe.im_index = im_index
# print im_index[0]
# print im_index[1]
X_compressed = getHiddenValues(0,100)
print X_compressed.shape
fe.similarImages(X_compressed,pca=False)
# print getHiddenValues(0,1).shape
# print sum(X_compressed[0])
# print sum(getHiddenValues(1,2)[0])
#print sum(getHiddenValues(100,101)[0])
# start-snippet-3
#####################################
# BUILDING THE MODEL CORRUPTION 30% #
#####################################
# rng = numpy.random.RandomState(123)
# theano_rng = RandomStreams(rng.randint(2 ** 30))
# da = dA(
# numpy_rng=rng,
# theano_rng=theano_rng,
# input=x,
# n_visible=128 * 128,
# n_hidden=500
# )
# cost, updates = da.get_cost_updates(
# corruption_level=0.3,
# learning_rate=learning_rate
# )
# train_da = theano.function(
# [index],
# cost,
# updates=updates,
# givens={
# x: train_set_x[index * batch_size: (index + 1) * batch_size]
# }
# )
# start_time = timeit.default_timer()
# ############
# # TRAINING #
# ############
# # go through training epochs
# for epoch in xrange(training_epochs):
# # go through trainng set
# c = []
# for batch_index in xrange(n_train_batches):
# c.append(train_da(batch_index))
# print 'Training epoch %d, cost ' % epoch, numpy.mean(c)
# end_time = timeit.default_timer()
# training_time = (end_time - start_time)
# print >> sys.stderr, ('The 30% corruption code for file ' +
# os.path.split(__file__)[1] +
# ' ran for %.2fm' % (training_time / 60.))
# # end-snippet-3
# # start-snippet-4
# image = Image.fromarray(tile_raster_images(
# X=da.W.get_value(borrow=True).T,
# img_shape=(128, 128), tile_shape=(10, 10),
# tile_spacing=(1, 1)))
# image.save('filters_corruption_30.png')
# # end-snippet-4
# print da.W.get_value(borrow=True).T
os.chdir('../')
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=poolsize, ignore_border=True)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
if __name__ == '__main__':
from fetex_image import FetexImage
fetex = FetexImage()
folder = '/Applications/MAMP/htdocs/DeepLearningTutorials/data/categories/'
fe = FetexImage(verbose=True,support_per_class=10,folder=folder)
train_set,valid_set,test_set,input = fe.processImagesPipeline()
X_train, Y_train = train_set
X_test, Y_test = test_set
X, Y = input
#print X
cnn = CNN(X_train)
cnn.fit(X_train,Y_train)
max_similarity = 0
max_similarity_pos = -1
#max_similarity_pos = []
#for i in xrange(1,len(train_set_x)):
a = cnn_output_vectors[base_image_index]
#a = X_train[base_image_index]
#print a.shape
for i in xrange(0,64 * 7):
if i != base_image_index:
b = cnn_output_vectors[i]
#b = X_train[i]
d = cosine_distance(a, b)
print d
#if d > max_similarity:
if d > max_similarity:
max_similarity = d
max_similarity_pos = i
#max_similarity_pos.append(i)
print 'max_similarity: {}'.format(max_similarity)
fe = FetexImage(mode='L')
fe.reconstructImage(X_train[base_image_index]).show()
fe.reconstructImage(X_train[max_similarity_pos]).show()
# fe.reconstructImage(X_train[max_similarity_pos[0]]).show()
# fe.reconstructImage(X_train[max_similarity_pos[1]]).show()
# fe.reconstructImage(X_train[max_similarity_pos[2]]).show()
# fe.reconstructImage(X_train[max_similarity_pos[3]]).show()
# print a.shape
# print b.shape
# print cosine_distance(a, b)
