def __init__(self, np_rng, theano_rng=None, n_ins=784,
hidden_layer_sizes=[500, 500],
n_outs=10,
corruption_rates=[0.1, 0.1]):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes)
if theano_rng is None:
theano_rng = RandomStreams(np_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
layer_input = self.x
else:
input_size = hidden_layer_sizes[i-1]
layer_input = self.sigmoid_layers[-1].output
hidden_layer = HiddenLayer(
np_rng, layer_input, input_size, hidden_layer_sizes[i], sigmoid)
self.sigmoid_layers.append(hidden_layer)
self.params.extend(hidden_layer.params)
# construc the denoising autoencoder layer
da = dA(np_rng, theano_rng, layer_input,
n_visible=input_size, n_hidden=hidden_layer_sizes[i],
W=hidden_layer.W, bhid=hidden_layer.b)
self.dA_layers.append(da)
# LogisticRegression Layer for classification
self.logLayer = LogisticRegressionLayer(self.sigmoid_layers[-1].output, n_in=hidden_layer_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def __init__(self, np_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10):
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
if theano_rng is None:
theano_rng = MRG_RandomStreams(np_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
# end snippet -1
for i in xrange(self.n_layers):
# construct the sigmoidal layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of
# the hidden layer below or the input of the DBN if you
# are on the first layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=np_rng,
input=layer_input, n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the sigmoid layer to layer list
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
# construct an RBM that shared weights with this layer
rbm_layer = RBM( np_rng=np_rng,
theano_rng=theano_rng, input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We need to add a Logistic layer on top of MLP
self.logLayer = LogisticRegressionLayer(input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
# compute the cost for second phase of training, defined
# as the negative log likelihood of the logistic regression
self.finetune_cost = self.logLayer.negative_log_likehihood(self.y)
# compuate the gradientes with respect to the model parameters
# symbolic variable that points to the number of errors
# made on the mibibatch given by self.x and self.y
self.erros = self.logLayer.errors(self.y)
class sdA(object):
"""Stacked denoising_autoencoder"""
def __init__(self, np_rng, theano_rng=None, n_ins=784,
hidden_layer_sizes=[500, 500],
n_outs=10,
corruption_rates=[0.1, 0.1]):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes)
if theano_rng is None:
theano_rng = RandomStreams(np_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
layer_input = self.x
else:
input_size = hidden_layer_sizes[i-1]
layer_input = self.sigmoid_layers[-1].output
hidden_layer = HiddenLayer(
np_rng, layer_input, input_size, hidden_layer_sizes[i], sigmoid)
self.sigmoid_layers.append(hidden_layer)
self.params.extend(hidden_layer.params)
# construc the denoising autoencoder layer
da = dA(np_rng, theano_rng, layer_input,
n_visible=input_size, n_hidden=hidden_layer_sizes[i],
W=hidden_layer.W, bhid=hidden_layer.b)
self.dA_layers.append(da)
# LogisticRegression Layer for classification
self.logLayer = LogisticRegressionLayer(self.sigmoid_layers[-1].output, n_in=hidden_layer_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def pretraining_functions(self, train_set_x, batch_size):
index = T.lscalar('index')
corruption_rate = T.scalar('corruption')
learning_rate = T.scalar('lr')
batch_begin = index * batch_size
batch_end = batch_begin + batch_size
pretrain_fns = []
for da in self.dA_layers:
cost, updates = da.get_cost_updates(corruption_rate,
learning_rate)
fn = theano.function(
inputs=[index, theano.Param(corruption_rate, default=0.2),
theano.Param(learning_rate, default=0.1)],
outputs=cost,
updates=updates,
givens={
self.x: train_set_x[batch_begin:batch_end]
}
)
pretrain_fns.append(fn)
return pretrain_fns
def build_finetune_functions(self, datasets, batch_size, learning_rate):
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[1]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
index = T.lscalar('index')
gparams = T.grad(self.finetune_cost, self.params)
updates = [(p, p - learning_rate*g) for p, g in
zip(self.params, gparams)]
train_fn = theano.function(
inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x: train_set_x[index*batch_size:(index+1)*batch_size],
self.y: train_set_y[index*batch_size:(index+1)*batch_size]
},
name='train'
)
test_score_i = theano.function(
[index],
self.errors,
givens={
self.x: test_set_x[index*batch_size:(index+1)*batch_size],
self.y: test_set_y[index*batch_size:(index+1):batch_size]
},
name='test'
)
valid_score_i = theano.function(
[index],
self.errors,
givens={
self.x: valid_set_x[index*batch_size:(index+1)*batch_size],
self.y: valid_set_y[index*batch_size:(index+1)*batch_size]
},
name='valid'
)
# create a function that scans the entire validation set
def valid_score(self):
return [valid_score_i(i) for i in range(n_valid_batches)]
def test_score(self):
return [test_score_i(i) for i in range(n_test_batches)]
return train_fn, valid_score, test_score
class DBN(object):
"""Deep belief network"""
def __init__(self, np_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10):
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
if theano_rng is None:
theano_rng = MRG_RandomStreams(np_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
# end snippet -1
for i in xrange(self.n_layers):
# construct the sigmoidal layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of
# the hidden layer below or the input of the DBN if you
# are on the first layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=np_rng,
input=layer_input, n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the sigmoid layer to layer list
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
# construct an RBM that shared weights with this layer
rbm_layer = RBM( np_rng=np_rng,
theano_rng=theano_rng, input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We need to add a Logistic layer on top of MLP
self.logLayer = LogisticRegressionLayer(input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
# compute the cost for second phase of training, defined
# as the negative log likelihood of the logistic regression
self.finetune_cost = self.logLayer.negative_log_likehihood(self.y)
# compuate the gradientes with respect to the model parameters
# symbolic variable that points to the number of errors
# made on the mibibatch given by self.x and self.y
self.erros = self.logLayer.errors(self.y)
def pretraining_functions(self, train_set_x, batch_size, k):
index = T.lscalar('index')
learning_rate = T.scalar('lr')
# compute number of batches
batch_begin = index * batch_size
batch_end = batch_begin + batch_size
pretrain_fns = []
for rbm in self.rbm_layers:
# get the cost and the updates list
# using CD-k here
cost, updates = rbm.get_cost_updates(learning_rate,
persistent=None, k=k)
# compile the theano function
fn = theano.function(inputs=[index, theano.Param(learning_rate, default=0.1)],
outputs=cost,
updates=updates,
givens={
self.x: train_set_x[batch_begin:batch_end]
})
pretrain_fns.append(fn)
return pretrain_fns
def build_finetune_functions(self, datasets, batch_size, learning_rate):
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of minibatches for training, validation and testing
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_valid_batches /= batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_test_batches /= batch_size
index = T.lscalar('index')
gparams = T.grad(self.finetune_cost, self.params)
updates = []
for param, gparam in zip(self.params, gparams):
updates.append((param, param - gparam*learning_rate))
train_fn = theano.function(
inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x: train_set_x[index*batch_size:(index+1)*batch_size],
self.y: train_set_y[index*batch_size:(index+1)*batch_size]
}
)
test_score_i = theano.function(
[index],
self.errors,
givens={
self.x: test_set_x[index*batch_size:(index+1)*batch_size],
self.y: test_set_y[index*batch_size:(index+1)*batch_size]
}
)
valid_score_i = theano.function(
[index],
outputs=self.erros,
givens={
self.x: valid_set_x[index*batch_size:(index+1)*batch_size],
self.y: valid_set_y[index*batch_size:(index+1)*batch_size]
}
)
# construct a function that scans the entire validation set
def valid_score():
return [valid_score_i(i) for i in xrange(n_valid_batches)]
def test_score():
return [test_score_i(i) for i in xrange(n_test_batches)]
return train_fn, valid_score, test_score
