def classify_lenet5(batch_size=500, output_size=20):
""" Demonstrates lenet on MNIST dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: path to the dataset used for training /testing (MNIST here)
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
"""
rng = numpy.random.RandomState(23455)
# start-snippet-1
x = T.matrix('x') # the data is presented as rasterized images
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28, 28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 1, 37, 23))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 1, 37, 23),
filter_shape=(20, 1, 4, 2),
poolsize=(2, 2),
)
# layer1 = LeNetConvPoolLayer(
# rng,
# input=layer0.output,
# image_shape=(batch_size, 20, 17, 11),
# filter_shape=(50, 20, 4, 2),
# poolsize=(2, 2),
# )
#
# layer4 = LeNetConvPoolLayer(
# rng,
# input=layer1.output,
# image_shape=(batch_size, 50, 7, 5),
# filter_shape=(100, 50, 4, 2),
# poolsize=(2, 2),
# )
layer2_input = layer0.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = HiddenLayer(
rng,
input=layer2_input,
n_in=3740,
n_out=output_size,
activation=T.tanh,
use_bias=True
)
# layer5 = HiddenLayer(
# rng,
# input=layer2.output,
# n_in=200,
# n_out=output_size,
# activation=T.tanh,
# use_bias=True
# )
# classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=output_size, n_out=2)
model_params = pickle.load(open('../model/cnn_dist_'+str(output_size)+'.pkl'))
#
layer0.W = theano.shared(
value=numpy.array(
model_params[2].get_value(True),
dtype=theano.config.floatX
),
name='W',
borrow=True
)
layer0.b = theano.shared(
value=numpy.array(
model_params[3].get_value(True),
dtype=theano.config.floatX
),
name='b',
borrow=True
)
# layer1.W = theano.shared(
# value=numpy.array(
# model_params[-4].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer1.b = theano.shared(
# value=numpy.array(
# model_params[-3].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
#
# layer4.W = theano.shared(
# value=numpy.array(
# model_params[-6].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer4.b = theano.shared(
# value=numpy.array(
# model_params[-5].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
layer2.W = theano.shared(
value=numpy.array(
model_params[0].get_value(True),
dtype=theano.config.floatX
),
name='W',
borrow=True
)
layer2.b = theano.shared(
value=numpy.array(
model_params[1].get_value(True),
dtype=theano.config.floatX
),
name='b',
borrow=True
)
# layer5.W = theano.shared(
# value=numpy.array(
# model_params[-10].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer5.b = theano.shared(
# value=numpy.array(
# model_params[-9].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
layer3.W = theano.shared(
value=numpy.array(
model_params[4].get_value(True),
dtype=theano.config.floatX
),
name='W',
borrow=True
)
layer3.b = theano.shared(
value=numpy.array(
model_params[5].get_value(True),
dtype=theano.config.floatX
),
name='b',
borrow=True
)
# params = layer3.params + layer5.params + layer2.params + layer4.params + layer1.params + layer0.params
datasets = load_data(None)
sets = ['train', 'dev', 'test']
dimension = [20000, 20000, 20000]
for k in range(3):
if k == 0:
classify_set_x, classify_set_y, classify_set_z, classify_set_m, classify_set_c, classify_set_b= datasets[k]
else:
classify_set_x, classify_set_y, classify_set_z= datasets[k]
# compute number of minibatches for training, validation and testing
n_classify_batches = classify_set_x.get_value(borrow=True).shape[0]
n_classify_batches /= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
classify = theano.function(
[index],
layer2.output,
givens={
x: classify_set_x[index * batch_size: (index + 1) * batch_size],
}
)
r = []
for i in xrange(n_classify_batches):
m = classify(i)
r.extend(m)
r = np.array(r)
print r.shape
r = np.append(r, np.reshape(classify_set_y.eval(),(dimension[k], 1)), 1)
numpy.savetxt('../extractedInformation/cnn_dist_'+str(output_size)+'/'+sets[k]+'.csv', r, delimiter=",")
def classify_lenet5(batch_size=500, output_size=20):
""" Demonstrates lenet on MNIST dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: path to the dataset used for training /testing (MNIST here)
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
"""
rng = numpy.random.RandomState(23455)
# start-snippet-1
x = T.matrix('x') # the data is presented as rasterized images
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28, 28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 1, 37, 23))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 1, 37, 23),
filter_shape=(20, 1, 4, 2),
poolsize=(2, 2),
)
# layer1 = LeNetConvPoolLayer(
# rng,
# input=layer0.output,
# image_shape=(batch_size, 20, 17, 11),
# filter_shape=(50, 20, 4, 2),
# poolsize=(2, 2),
# )
#
# layer4 = LeNetConvPoolLayer(
# rng,
# input=layer1.output,
# image_shape=(batch_size, 50, 7, 5),
# filter_shape=(100, 50, 4, 2),
# poolsize=(2, 2),
# )
layer2_input = layer0.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = HiddenLayer(rng,
input=layer2_input,
n_in=3740,
n_out=output_size,
activation=T.tanh,
use_bias=True)
# layer5 = HiddenLayer(
# rng,
# input=layer2.output,
# n_in=200,
# n_out=output_size,
# activation=T.tanh,
# use_bias=True
# )
# classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=output_size, n_out=2)
model_params = pickle.load(
open('../model/cnn_dist_' + str(output_size) + '.pkl'))
#
layer0.W = theano.shared(value=numpy.array(model_params[2].get_value(True),
dtype=theano.config.floatX),
name='W',
borrow=True)
layer0.b = theano.shared(value=numpy.array(model_params[3].get_value(True),
dtype=theano.config.floatX),
name='b',
borrow=True)
# layer1.W = theano.shared(
# value=numpy.array(
# model_params[-4].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer1.b = theano.shared(
# value=numpy.array(
# model_params[-3].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
#
# layer4.W = theano.shared(
# value=numpy.array(
# model_params[-6].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer4.b = theano.shared(
# value=numpy.array(
# model_params[-5].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
layer2.W = theano.shared(value=numpy.array(model_params[0].get_value(True),
dtype=theano.config.floatX),
name='W',
borrow=True)
layer2.b = theano.shared(value=numpy.array(model_params[1].get_value(True),
dtype=theano.config.floatX),
name='b',
borrow=True)
# layer5.W = theano.shared(
# value=numpy.array(
# model_params[-10].get_value(True),
# dtype=theano.config.floatX
# ),
# name='W',
# borrow=True
# )
#
# layer5.b = theano.shared(
# value=numpy.array(
# model_params[-9].get_value(True),
# dtype=theano.config.floatX
# ),
# name='b',
# borrow=True
# )
layer3.W = theano.shared(value=numpy.array(model_params[4].get_value(True),
dtype=theano.config.floatX),
name='W',
borrow=True)
layer3.b = theano.shared(value=numpy.array(model_params[5].get_value(True),
dtype=theano.config.floatX),
name='b',
borrow=True)
# params = layer3.params + layer5.params + layer2.params + layer4.params + layer1.params + layer0.params
datasets = load_data(None)
sets = ['train', 'dev', 'test']
dimension = [20000, 20000, 20000]
for k in range(3):
if k == 0:
classify_set_x, classify_set_y, classify_set_z, classify_set_m, classify_set_c, classify_set_b = datasets[
k]
else:
classify_set_x, classify_set_y, classify_set_z = datasets[k]
# compute number of minibatches for training, validation and testing
n_classify_batches = classify_set_x.get_value(borrow=True).shape[0]
n_classify_batches /= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
classify = theano.function(
[index],
layer2.output,
givens={
x: classify_set_x[index * batch_size:(index + 1) * batch_size],
})
r = []
for i in xrange(n_classify_batches):
m = classify(i)
r.extend(m)
r = np.array(r)
print r.shape
r = np.append(r, np.reshape(classify_set_y.eval(), (dimension[k], 1)),
1)
numpy.savetxt('../extractedInformation/cnn_dist_' + str(output_size) +
'/' + sets[k] + '.csv',
r,
delimiter=",")
def trainword(keyword, window_radius = 3, learning_rate = 0.1, n_epochs = 10,batch_size = 1,filter_height=3,filter_width = 50, pool_height=1,pool_width = 1, loginput_num = 50, vector_size = 50):
print '==training parameters=='
print 'window_radius: '+str(window_radius)
print 'vector_size: '+str(vector_size)
print 'filter_height: '+str(filter_height)
print 'filter_width: '+str(filter_width)
print 'pool_height: '+str(pool_height)
print 'pool_width: '+str(pool_width)
print 'loginput_num: '+str(loginput_num)
print 'learning_rate: '+str(learning_rate)
print 'n_epochs: '+str(n_epochs)
print 'batch_size: '+str(batch_size)
rng = numpy.random.RandomState(23455)
datasets = load_data_word(keyword, window_radius, vector_size)
train_set_x, train_set_y, trainsentence = datasets[0][0]
valid_set_x, valid_set_y, validsentence = datasets[0][1]
test_set_x, test_set_y, testsentence = datasets[0][2]
senselist = datasets[1]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
print n_train_batches, n_valid_batches, n_test_batches
index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
print '... building the model for '+keyword
layer0_input = x.reshape((batch_size, 1, 2*window_radius+1, vector_size))
layer0 = WsdConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 1, 2*window_radius+1, vector_size),
filter_shape=(1, 1, filter_height, filter_width),
poolsize=(pool_height, pool_width)
)
layer1_input = layer0.output.flatten(2)
#layer1_input = layer0_input.flatten(2)
layer1 = HiddenLayer(
rng,
input=layer1_input,
#n_in=(2*window_radius+1)*(vector_size+1-filter_width+1-pool_width),
n_in=int((2*window_radius+2-filter_height)/float(pool_height))*int((vector_size+1-filter_width)/float(pool_width)),
n_out=loginput_num,
activation=T.tanh
)
layer2 = LogisticRegression(input=layer1_input, n_in=int((2*window_radius+2-filter_height)/float(pool_height))*int((vector_size+1-filter_width)/float(pool_width)), n_out=20)
cost = layer2.negative_log_likelihood(y)
test_model = theano.function(
[index],
layer2.errors(y),
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size]
}
)
validate_model = theano.function(
[index],
layer2.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
output_size = theano.function(
[index],
[layer0.output.shape],
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size]
}
)
output_model = theano.function(
[index],
[layer2.y_pred],
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size]
}
)
output_test = theano.function(
[index],
[layer2.y_pred],
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size]
}
)
params = layer2.params + layer0.params
grads = T.grad(cost, params)
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_params = 0
best_iter = 0
test_score = 0.
start_time = time.clock()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print 'training @ iter = ', iter
cost_ij = train_model(minibatch_index)
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
#for index in range(0, n_valid_batches):
# print output_model(index)
# print valid_set_y[index * batch_size: (index + 1) * batch_size].eval()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
best_params = [copy.deepcopy(layer0.params), copy.deepcopy(layer1.params), copy.deepcopy(layer2.params)]
# test it on the test set
test_losses = [
test_model(i)
for i in xrange(n_test_batches)
]
#print params[0].eval()
#print (params[0].eval() == layer2.params[0].eval())
#print validation_losses
for index in range(0, n_valid_batches):
for i in range(0, batch_size):
true_i = batch_size*index+i
#print output_model(index)
print validsentence[true_i], '\t',senselist[output_model(index)[0][i]], '\t', senselist[valid_set_y[true_i].eval()]
#print test_losses
test_score = numpy.mean(test_losses)
for index in range(0, n_test_batches):
for i in range(0, batch_size):
true_i = batch_size*index+i
#print output_model(index)
print testsentence[true_i], '\t',senselist[output_test(index)[0][i]], '\t', senselist[test_set_y[true_i].eval()]
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print('Optimization complete.')
for index in range(0, n_test_batches):
for i in range(0, batch_size):
true_i = batch_size*index+i
#print output_model(index)
print testsentence[true_i], '\t',senselist[output_test(index)[0][i]], '\t', senselist[test_set_y[true_i].eval()]
layer0.W = copy.deepcopy(best_params[0][0])
layer0.b = copy.deepcopy(best_params[0][1])
#layer0.params = [layer0.W, layer0.b]
layer1.W = copy.deepcopy(best_params[1][0])
layer1.b = copy.deepcopy(best_params[1][1])
#layer1.params = [layer1.W, layer1.b]
layer2.W = copy.deepcopy(best_params[2][0])
layer2.b = copy.deepcopy(best_params[2][1])
#layer2.params = [layer2.W, layer2.b]
for index in range(0, n_test_batches):
for i in range(0, batch_size):
true_i = batch_size*index+i
#print output_model(index)
print testsentence[true_i], '\t',senselist[output_test(index)[0][i]], '\t', senselist[test_set_y[true_i].eval()]
print('Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
