def evaluate_lenet5(learning_rate=0.005,
n_epochs=5,
data=None,
nkerns=64,
batch_size=30):
#for i in range(len(x_val)):
#if len(x_val[i]) == 490 and len(x_val[i][0]) == 640:
#x1.append(x_val[i])
#y1.append(y_val[i]-1)
#if len(x1) == 80:
#break
from data_loader import load_data
train, validate, test = load_data()
x_train = np.array(train[0], 'float32')
y_train = train[1]
x_valid = np.array(validate[0], 'float32')
y_valid = validate[1]
x_test = np.array(test[0], 'float32')
y_test = test[1]
x_train2 = theano.shared(numpy.asarray(x_train,
dtype=theano.config.floatX))
y_train_2 = theano.shared(
numpy.asarray(y_train, dtype=theano.config.floatX))
x_valid2 = theano.shared(numpy.asarray(x_valid,
dtype=theano.config.floatX))
y_valid_2 = theano.shared(
numpy.asarray(y_valid, dtype=theano.config.floatX))
x_test2 = theano.shared(numpy.asarray(x_test, dtype=theano.config.floatX))
y_test_2 = theano.shared(numpy.asarray(y_test, dtype=theano.config.floatX))
y_train2 = T.cast(y_train_2, 'int32')
y_test2 = T.cast(y_test_2, 'int32')
y_valid2 = T.cast(y_valid_2, 'int32')
print len(x_train)
print len(y_train)
rng = numpy.random.RandomState(23455)
n_train_batches = len(y_train) / batch_size
n_valid_batches = len(y_valid) / batch_size
n_test_batches = len(y_test) / batch_size
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are p
layer0_input = x.reshape((batch_size, 1, 64, 64))
'''构建第一层网络：
image_shape：输入大小为490*640的特征图，batch_size个训练数据，每个训练数据有1个特征图
filter_shape：卷积核个数为nkernes=64，因此本层每个训练样本即将生成64个特征图
经过卷积操作，图片大小变为(490-7+1 , 640-7+1) = (484, 634)
经过池化操作，图片大小变为 (484/2, 634/2) = (242, 317)
最后生成的本层image_shape为(batch_size, nklearn, 242, 317)'''
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 1, 64, 64),
filter_shape=(nkerns, 1, 7, 7),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (batch_size, nkerns * 7 * 7),
# (100, 64*7*7) with the default values.
layer2_input = layer0.output.flatten(2)
'''全链接：输入layer2_input是一个二维的矩阵，第一维表示样本，第二维表示上面经过卷积下采样后
每个样本所得到的神经元，也就是每个样本的特征，HiddenLayer类是一个单层网络结构
下面的layer2把神经元个数由800个压缩映射为500个'''
layer2 = HiddenLayer(rng,
input=layer2_input,
n_in=nkerns * 29 * 29,
n_out=500,
activation=T.tanh)
layer2.output = dropout_layer(layer2.output, 0.5)
# 最后一层：逻辑回归层分类判别，把500个神经元，压缩映射成10个神经元，分别对应于手写字体的0~9
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=8)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
y: y_test2[index * batch_size:(index + 1) * batch_size],
x: x_test2[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: x_valid2[index * batch_size:(index + 1) * batch_size],
y: y_valid2[index * batch_size:(index + 1) * batch_size]
})
#把所有的参数放在同一个列表里，可直接使用列表相加
params = layer3.params + layer2.params + layer0.params
#梯度求导
grads = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: x_train2[index * batch_size:(index + 1) * batch_size],
y: y_train2[index * batch_size:(index + 1) * batch_size]
})
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.2 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
#while epoch < n_epochs:
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches): #每一批训练数据
cost_ij = train_model(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [
validate_model(i) for i in xrange(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i) for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
with open('param0.pkl', 'wb') as f0:
pickle.dump(layer0.params, f0)
f0.close()
with open('param2.pkl', 'wb') as f2:
pickle.dump(layer2.params, f2)
f2.close()
with open('param3.pkl', 'wb') as f3:
pickle.dump(layer3.params, f3)
f3.close()
end_time = timeit.default_timer()
print('Optimization complete.')
print(
'Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
class ConvolutionalNeuralNetwork(Classifier):
def __init__(self, rng, batch_size, nkerns=(20, 50)):
self.batch_size = batch_size
# 28x28 -> (24x24) // 2 = 12x12
self.layer0 = LeNetConvPoolLayer(
rng=rng,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
)
# 12x12 -> (8x8) // 2 = 4x4
self.layer1 = LeNetConvPoolLayer(rng=rng,
image_shape=(batch_size, nkerns[0],
12, 12),
filter_shape=(nkerns[1], nkerns[0], 5,
5))
# TODO: make this an MLP rather than a hidden layer -> LogReg
# self.layer2 = MLP()
self.layer2 = HiddenLayer(
rng=rng,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh,
)
self.layer3 = LogisticRegression(
n_in=500,
n_out=10,
)
def pre_logreg_output(self, x):
layer0_input = x.reshape((self.batch_size, 1, 28, 28))
l0_output = self.layer0.output(layer0_input)
l1_output = self.layer1.output(l0_output)
l2_input = l1_output.flatten(2)
l2_output = self.layer2.output(l2_input)
return l2_output
def negative_log_likelihood(self, x, y):
output = self.pre_logreg_output(x)
return self.layer3.negative_log_likelihood(output, y)
def pred_label(self, x):
output = self.pre_logreg_output(x)
output = output.flatten(1)
return self.layer3.pred_label(output)
def errors(self, x, y):
output = self.pre_logreg_output(x)
return self.layer3.errors(output, y)
def train(self,
train_x,
train_y,
test_x,
test_y,
valid_x,
valid_y,
alpha=0.13,
batch_size=500,
l1_reg=0.,
l2_reg=0.0,
n_epochs=1000):
x = T.matrix('x')
y = T.ivector('y')
batch_size = self.batch_size
layer0_input = x.reshape((batch_size, 1, 28, 28))
cost = self.negative_log_likelihood(layer0_input, y)
params = self.layer0.params + self.layer1.params + self.layer2.params + self.layer3.params
grads = T.grad(cost, params)
updates = [(param, param - alpha * grad)
for param, grad in zip(params, grads)]
index = T.lscalar()
train_func = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x: train_x[index * batch_size:(index + 1) * batch_size],
y: train_y[index * batch_size:(index + 1) * batch_size],
})
best_loss = self.run_batches(train_x,
train_y,
test_x,
test_y,
valid_x,
valid_y,
x,
y,
train_model_func=train_func,
batch_size=batch_size,
n_epochs=n_epochs)
return best_loss
def evaluate_lenet5(learning_rate=0.005, n_epochs=5,data = None,nkerns= 64, batch_size=30):
#for i in range(len(x_val)):
#if len(x_val[i]) == 490 and len(x_val[i][0]) == 640:
#x1.append(x_val[i])
#y1.append(y_val[i]-1)
#if len(x1) == 80:
#break
from data_loader import load_data
train, validate, test = load_data()
x_train = np.array(train[0],'float32')
y_train = train[1]
x_valid = np.array(validate[0],'float32')
y_valid = validate[1]
x_test = np.array(test[0],'float32')
y_test = test[1]
x_train2 = theano.shared(numpy.asarray(x_train,dtype=theano.config.floatX))
y_train_2 = theano.shared(numpy.asarray(y_train,dtype=theano.config.floatX))
x_valid2 = theano.shared(numpy.asarray(x_valid,dtype=theano.config.floatX))
y_valid_2 = theano.shared(numpy.asarray(y_valid,dtype=theano.config.floatX))
x_test2 = theano.shared(numpy.asarray(x_test,dtype=theano.config.floatX))
y_test_2 = theano.shared(numpy.asarray(y_test,dtype=theano.config.floatX))
y_train2 = T.cast(y_train_2, 'int32')
y_test2 = T.cast(y_test_2, 'int32')
y_valid2 = T.cast(y_valid_2, 'int32')
print len(x_train)
print len(y_train)
rng = numpy.random.RandomState(23455)
n_train_batches = len(y_train)/batch_size
n_valid_batches = len(y_valid)/batch_size
n_test_batches = len(y_test)/batch_size
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are p
layer0_input = x.reshape((batch_size, 1, 64, 64))
'''构建第一层网络：
image_shape：输入大小为490*640的特征图，batch_size个训练数据，每个训练数据有1个特征图
filter_shape：卷积核个数为nkernes=64，因此本层每个训练样本即将生成64个特征图
经过卷积操作，图片大小变为(490-7+1 , 640-7+1) = (484, 634)
经过池化操作，图片大小变为 (484/2, 634/2) = (242, 317)
最后生成的本层image_shape为(batch_size, nklearn, 242, 317)'''
layer0 = LeNetConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 1, 64, 64),
filter_shape=(nkerns, 1, 7, 7),
poolsize=(2, 2)
)
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (batch_size, nkerns * 7 * 7),
# (100, 64*7*7) with the default values.
layer2_input = layer0.output.flatten(2)
'''全链接：输入layer2_input是一个二维的矩阵，第一维表示样本，第二维表示上面经过卷积下采样后
每个样本所得到的神经元，也就是每个样本的特征，HiddenLayer类是一个单层网络结构
下面的layer2把神经元个数由800个压缩映射为500个'''
layer2 = HiddenLayer(
rng,
input=layer2_input,
n_in=nkerns * 29 * 29,
n_out=500,
activation=T.tanh
)
layer2.output = dropout_layer(layer2.output,0.5)
# 最后一层：逻辑回归层分类判别，把500个神经元，压缩映射成10个神经元，分别对应于手写字体的0~9
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=8)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
y: y_test2[index * batch_size: (index + 1) * batch_size],
x: x_test2[index * batch_size: (index + 1) * batch_size]
}
)
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: x_valid2[index * batch_size: (index + 1) * batch_size],
y: y_valid2[index * batch_size: (index + 1) * batch_size]
}
)
#把所有的参数放在同一个列表里，可直接使用列表相加
params = layer3.params + layer2.params + layer0.params
#梯度求导
grads = T.grad(cost, params)
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: x_train2[index * batch_size: (index + 1) * batch_size],
y: y_train2[index * batch_size: (index + 1) * batch_size]
}
)
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.2 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
#while epoch < n_epochs:
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):#每一批训练数据
cost_ij = train_model(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i)
for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
with open('param0.pkl', 'wb') as f0:
pickle.dump(layer0.params, f0)
f0.close()
with open('param2.pkl', 'wb') as f2:
pickle.dump(layer2.params, f2)
f2.close()
with open('param3.pkl', 'wb') as f3:
pickle.dump(layer3.params, f3)
f3.close()
end_time = timeit.default_timer()
print('Optimization complete.')
print('Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
