def pretrain_nnet(data_set, n_train_images=100, batch_size=5, learning_rate=0.1):
print '\n', '#'*50
print 'starting Greedy Layerwise Unsupervised Pretraining ...'
layer0 = data_set
'''
layer1, params_1 = pretrain_conv_autoencoder(1, layer0, (batch_size, 1, 96, 336), (2, 1, 3, 3), 5, (1, 1), learning_rate)
dump_params(layer1.params, 'pretrainparams1.pkl')
dump_params(params_1, 'pretrainparams_1.pkl')
print '\n'
layer2, params_2 = pretrain_conv_autoencoder(2, layer1.output, (batch_size, 2, 96, 336), (3, 2, 3, 3), 5, (2, 2), learning_rate)
dump_params(layer2.params, 'pretrainparams2.pkl')
dump_params(params_2, 'pretrainparams_2.pkl')
print '\n'
layer3, params_3 = pretrain_conv_autoencoder(3, layer2.output, (batch_size, 3, 48, 168), (5, 3, 3, 3), 5, (1, 1), learning_rate)
dump_params(layer3.params, 'pretrainparams3.pkl')
dump_params(params_3, 'pretrainparams_3.pkl')
print '\n'
layer4, params_4 = pretrain_conv_autoencoder(4, layer3.output, (batch_size, 5, 48, 168), (8, 5, 3, 3), 5, (2, 2), learning_rate)
dump_params(layer4.params, 'pretrainparams4.pkl')
dump_params(params_4, 'pretrainparams_4.pkl')
print '\n'
layer5, params_5 = pretrain_conv_autoencoder(5, layer4.output, (batch_size, 8, 24, 84), (5, 8, 3, 3), 5, (2, 2), learning_rate)
dump_params(layer5.params, 'pretrainparams5.pkl')
dump_params(params_5, 'pretrainparams_5.pkl')
print '\n'
'''
n_pretrained_layers = 5
n_test = train_set.shape.eval()[0]
img_shapes = [(n_test, 1, 96, 336), (n_test, 2, 96, 336), (n_test, 3, 48, 168), (n_test, 5, 48, 168), (n_test, 8, 24, 84)]
k_shapes = [(2, 1, 3, 3), (3, 2, 3, 3), (5, 3, 3, 3), (8, 5, 3, 3), (5, 8, 3, 3)]
p_shapes = [(1, 1), (2, 2), (1, 1), (2, 2), (2, 2)]
for i in range(n_pretrained_layers):
load_file = open('pretrainparams'+str(i+1)+'.pkl')
W = theano.shared(cPickle.load(load_file), borrow=True)
b = theano.shared(cPickle.load(load_file), borrow=True)
layer0 = conv_pool_layer(rng, layer0, k_shapes[i], img_shapes[i], poolsize=p_shapes[i], read_file=True, W_input=W, b_input=b)
layer5 = layer0
layer0 = layer0.output
load_file.close()
layer6_input = T.reshape(layer5.output, (layer5.output.shape.eval()[0], 5, 504))
layer6, params_6 = pretrain_local_autoencoders(6, layer6_input, 5, 504, 40, 5, learning_rate)
dump_params(layer6.params, 'pretrainparams6.pkl')
dump_params(params_6, 'pretrainparams_6.pkl')
print '\n'
layerwise_params = []
for i in range(6):
load_file = open('pretrainparams'+str(i+1)+'.pkl')
W = theano.shared(cPickle.load(load_file), borrow=True)
b = theano.shared(cPickle.load(load_file), borrow=True)
layerwise_params.append([W, b])
load_file.close()
for i in range(6, 0, -1):
load_file = open('pretrainparams_'+str(i)+'.pkl')
W = theano.shared(cPickle.load(load_file), borrow=True)
b = theano.shared(cPickle.load(load_file), borrow=True)
layerwise_params.append([W, b])
load_file.close()
return layerwise_params
def __init__(self,
rng,
input,
input_dim,
batch_size=5,
init=False,
params=None):
# assume input dimensions is 128x128
self.input = input
self.inp_h = input_dim[0]
self.inp_w = input_dim[1]
self.layer1 = conv_pool_layer(rng,
input=input,
image_shape=(batch_size, 3, self.inp_h,
self.inp_w),
filter_shape=(64, 3, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[0][0],
b_input=params[0][1])
# input is 64x64x64
self.layer2 = conv_pool_layer(rng,
input=self.layer1.output,
image_shape=(batch_size, 64,
self.inp_h / 2,
self.inp_w / 2),
filter_shape=(128, 64, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[1][0],
b_input=params[1][1])
# input is 128x32x32
self.layer3 = conv_pool_layer(rng,
input=self.layer2.output,
image_shape=(batch_size, 128,
self.inp_h / 4,
self.inp_w / 4),
filter_shape=(512, 128, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[2][0],
b_input=params[2][1])
# input is 512x16x16
self.layer4 = conv_pool_layer(rng,
input=self.layer3.output,
image_shape=(batch_size, 512,
self.inp_h / 8,
self.inp_w / 8),
filter_shape=(1024, 512, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[3][0],
b_input=params[3][1])
# decovolution starts here
# input is 1024x8x8
self.layer5 = deconv_unpool_layer(rng,
input=self.layer4.output,
image_shape=(batch_size, 1024,
self.inp_h / 16,
self.inp_w / 16),
filter_shape=(512, 1024, 3, 3),
zero_pad=True,
non_linearity=True,
switch=self.layer4.switch,
unpoolsize=(2, 2),
read_file=init,
W_input=params[4][0],
b_input=params[4][1])
self.layer6 = deconv_unpool_layer(rng,
input=self.layer5.output,
image_shape=(batch_size, 512,
self.inp_h / 8,
self.inp_w / 8),
filter_shape=(128, 512, 3, 3),
zero_pad=True,
non_linearity=True,
switch=self.layer3.switch,
unpoolsize=(2, 2),
read_file=init,
W_input=params[5][0],
b_input=params[5][1])
self.layer7 = deconv_unpool_layer(rng,
input=self.layer6.output,
image_shape=(batch_size, 128,
self.inp_h / 4,
self.inp_w / 4),
filter_shape=(64, 128, 3, 3),
zero_pad=True,
non_linearity=True,
switch=self.layer2.switch,
unpoolsize=(2, 2),
read_file=init,
W_input=params[6][0],
b_input=params[6][1])
self.layer8 = deconv_unpool_layer(rng,
input=self.layer7.output,
image_shape=(batch_size, 64,
self.inp_h / 2,
self.inp_w / 2),
filter_shape=(1, 64, 3, 3),
zero_pad=True,
non_linearity=True,
sigmoid=True,
switch=self.layer1.switch,
unpoolsize=(2, 2),
read_file=init,
W_input=params[7][0],
b_input=params[7][1])
'''
def __init__(self, rng, input, input_dim, batch_size=5, init=False, params=None):
self.input = input
self.inp_h = input_dim[0]
self.inp_w = input_dim[1]
self.layer1 = conv_pool_layer(
rng,
input=input,
image_shape=(batch_size, 1, self.inp_h, self.inp_w),
filter_shape=(2, 1, 3, 3),
poolsize=(1, 1),
zero_pad=True,
read_file=init,
W_input=params[0][0],
b_input=params[0][1],
)
self.layer2 = conv_pool_layer(
rng,
input=self.layer1.output,
image_shape=(batch_size, 2, self.inp_h / 1, self.inp_w / 1),
filter_shape=(3, 2, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[1][0],
b_input=params[1][1],
)
self.layer3 = conv_pool_layer(
rng,
input=self.layer2.output,
image_shape=(batch_size, 3, (self.inp_h / 1) / 2, (self.inp_w / 1) / 2),
filter_shape=(5, 3, 3, 3),
poolsize=(1, 1),
zero_pad=True,
read_file=init,
W_input=params[2][0],
b_input=params[2][1],
)
self.layer4 = conv_pool_layer(
rng,
input=self.layer3.output,
image_shape=(batch_size, 5, (self.inp_h / 2), (self.inp_w) / 2),
filter_shape=(8, 5, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[3][0],
b_input=params[3][1],
)
self.layer5 = conv_pool_layer(
rng,
input=self.layer4.output,
image_shape=(batch_size, 8, (self.inp_h / 2) / 2, (self.inp_w / 2) / 2),
filter_shape=(5, 8, 3, 3),
poolsize=(2, 2),
zero_pad=True,
read_file=init,
W_input=params[4][0],
b_input=params[4][1],
)
# layer 6 is the embedding
n_feature_maps = 5
# embedding space input: batch_size x 5 x linearised vector(504)
# embedding space o/p: batch_size x 5 x 40
embedding_input = T.reshape(self.layer5.output, (batch_size, n_feature_maps, (12 * 42)))
self.layer6 = hidden_layer(
rng,
input=embedding_input,
n_feature_maps=n_feature_maps,
n_in=504,
n_out=40,
b_size=batch_size,
read_file=init,
W=params[5][0],
b=params[5][1],
)
# layer 7 i/p: batch_size x 5 x 40
# layer 7 o/p: batch_size x 5 x linearised vector(504)
# before passing it to layer8 reformat it into batch_size x 5 x (row x col)
self.layer7 = hidden_layer(
rng,
input=self.layer6.output,
n_feature_maps=n_feature_maps,
n_in=40,
n_out=504,
b_size=batch_size,
read_file=init,
W=params[6][0],
b=params[6][1],
)
embedding_output = T.reshape(self.layer7.output, (batch_size, n_feature_maps, 12, 42))
self.layer8 = deconv_unpool_layer(
rng,
input=embedding_output, # self.layer5.output,
image_shape=(batch_size, 5, ((self.inp_h / 2) / 2) / 2, ((self.inp_w / 2) / 2) / 2),
filter_shape=(8, 5, 3, 3),
unpoolsize=(2, 2),
zero_pad=True,
non_linearity=True,
switch=self.layer5.switch,
read_file=init,
W_input=params[7][0],
b_input=params[7][1],
)
self.layer9 = deconv_unpool_layer(
rng,
input=self.layer8.output,
image_shape=(batch_size, 8, (self.inp_h / 2) / 2, (self.inp_w / 2) / 2),
filter_shape=(5, 8, 3, 3),
unpoolsize=(2, 2),
zero_pad=True,
non_linearity=True,
switch=self.layer4.switch,
read_file=init,
W_input=params[8][0],
b_input=params[8][1],
)
self.layer10 = deconv_unpool_layer(
rng,
input=self.layer9.output,
image_shape=(batch_size, 5, self.inp_h / 2, self.inp_w / 2),
filter_shape=(3, 5, 3, 3),
unpoolsize=(1, 1),
switch=None,
non_linearity=True,
read_file=init,
W_input=params[9][0],
b_input=params[9][1],
)
self.layer11 = deconv_unpool_layer(
rng,
input=self.layer10.output,
image_shape=(batch_size, 3, self.inp_h / 2, self.inp_w / 2),
filter_shape=(2, 3, 3, 3),
non_linearity=True,
unpoolsize=(2, 2),
switch=self.layer2.switch,
read_file=init,
W_input=params[10][0],
b_input=params[10][1],
)
self.layer12 = deconv_unpool_layer(
rng,
input=self.layer11.output,
image_shape=(batch_size, 2, self.inp_h, self.inp_w),
filter_shape=(1, 2, 3, 3),
unpoolsize=(1, 1),
switch=None,
read_file=init,
W_input=params[11][0],
b_input=params[11][1],
)
