def setUp(self):
# Load parameters of RNNConfiger and create RNN
self.configer = RNNConfiger('testrnn.conf')
pprint('Parameters loaded by RNNConfiger: ')
pprint('=' * 50)
pprint('Input dimension of RNN: %d' % self.configer.num_input)
pprint('Hidden dimension of RNN: %d' % self.configer.num_hidden)
pprint('Regularizer parameter for L1-norm: %f' % self.configer.lambda1)
pprint('Regularizer parameter for L2-norm: %f' % self.configer.lambda2)
# Construct RNN
start_time = time.time()
self.rnn = RNN(configs=self.configer, verbose=True)
end_time = time.time()
pprint('Time used to build the architecture of RNN: %f seconds.' %
(end_time - start_time))
qwords = qsent.split()
qwords = [qword.lower() for qword in qwords]
qvectors = np.zeros((len(qwords)+2, edim), dtype=floatX)
qvectors[0, :], qvectors[-1, :] = blank_token, blank_token
qvectors[1:-1, :] = np.asarray([word_embedding.wordvec(qword) for qword in qwords], dtype=floatX)
test_pairs_set.append((pvectors, qvectors))
end_time = time.time()
logger.debug('Training and test data sets building finished...')
logger.debug('Time used to build training and test data set: %f seconds.' % (end_time-start_time))
# Set print precision
# np.set_printoptions(threshold=np.nan)
# config_filename = './brnn_ranking.conf'
config_filename = args.config
start_time = time.time()
configer = RNNConfiger(config_filename)
if args.model == 'NONE':
brnn = BRNNMatchScorer(configer, verbose=True)
else:
brnn = BRNNMatchScorer.load(args.model)
end_time = time.time()
logger.debug('Time used to build/load BRNNMatchScorer: %f seconds.' % (end_time-start_time))
# Define negative/positive sampling ratio
# Begin training
# Using AdaGrad learning algorithm
learn_rate = args.rate
batch_size = args.size
fudge_factor = 1e-6
logger.debug('BRNNMatchScorer.params: {}'.format(brnn.params))
hist_grads = [np.zeros(param.get_value(borrow=True).shape, dtype=floatX) for param in brnn.params]
initial_params = {param.name : param.get_value(borrow=True) for param in brnn.params}
def testTBRNN(self):
# Set print precision
np.set_printoptions(threshold=np.nan)
config_filename = './sp_brnn.conf'
start_time = time.time()
configer = RNNConfiger(config_filename)
brnn = BRNN(configer, verbose=True)
end_time = time.time()
logger.debug('Time used to build TBRNN: %f seconds.' % (end_time-start_time))
# Training
logger.debug('positive labels: %d' % np.sum(self.sp_train_label))
logger.debug('negative labels: %d' % (self.sp_train_label.shape[0]-np.sum(self.sp_train_label)))
start_time = time.time()
## AdaGrad learning algorithm instead of the stochastic gradient descent algorithm
# history_grads = np.zeros(brnn.num_params)
n_epoch = 200
learn_rate = 1e-1
batch_size = 100
fudge_factor = 1e-6
history_grads = np.zeros(brnn.num_params, dtype=floatX)
logger.debug('Number of parameters in the model: {}'.format(brnn.num_params))
for i in xrange(n_epoch):
tot_count = 0
tot_error = 0.0
tot_grads = np.zeros(brnn.num_params, dtype=floatX)
for j, (train_seq, train_label) in enumerate(zip(self.sp_train_set, self.sp_train_label)):
if (j+1) % 1000 == 0:
logger.debug('%4d @ %4d epoch' % (j+1, i))
cost, current_grads = brnn.compute_cost_and_gradient(train_seq, [train_label])
# Accumulate current gradients
tot_grads += current_grads
tot_error += cost
# Accumulate historical gradients
history_grads += np.square(current_grads)
# predict current training label
prediction = brnn.predict(train_seq)[0]
tot_count += prediction == train_label
# Batch updating with AdaGrad
if (j+1) % batch_size == 0 or j == self.train_size-1:
# Adjust gradient for AdaGrad
tot_grads /= batch_size
tot_grads /= fudge_factor + np.sqrt(history_grads)
brnn.update_params(tot_grads, learn_rate)
tot_grads = np.zeros(brnn.num_params, dtype=floatX)
history_grads = np.zeros(brnn.num_params, dtype=floatX)
logger.debug('Training @ %d epoch, total cost = %f, accuracy = %f' % (i, tot_error, tot_count / float(self.train_size)))
# Testing
tot_count = 0
for test_seq, test_label in zip(self.sp_test_set, self.sp_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
end_time = time.time()
logger.debug('Time used for training: %f minutes.' % ((end_time-start_time)/60))
# Testing
tot_count = 0
for test_seq, test_label in zip(self.sp_test_set, self.sp_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.sp_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.sp_test_label==0) / float(self.test_size)))
# Re-testing on training set
tot_count = 0
for train_seq, train_label in zip(self.sp_train_set, self.sp_train_label):
prediction = brnn.predict(train_seq)[0]
tot_count += train_label == prediction
logger.debug('Training accuracy re-testing: %f' % (tot_count / float(self.train_size)))
# Save TBRNN
BRNN.save('sp.brnn.pkl', brnn)
logger.debug('Model successfully saved...')
def testTBRNNwithFineTuning(self):
# Set print precision
np.set_printoptions(threshold=np.nan)
config_filename = './sentiment_brnn.conf'
start_time = time.time()
configer = RNNConfiger(config_filename)
brnn = TBRNN(configer, verbose=True)
# brnn = TBRNN.load('sentiment.brnn.Sep5.pkl')
end_time = time.time()
logger.debug('Time used to load TBRNN: %f seconds.' % (end_time-start_time))
logger.debug('Start training TBRNN with fine-tuning...')
# Training
logger.debug('positive labels: %d' % np.sum(self.senti_train_label))
logger.debug('negative labels: %d' % (self.senti_train_label.shape[0]-np.sum(self.senti_train_label)))
start_time = time.time()
## AdaGrad learning algorithm instead of the stochastic gradient descent algorithm
# history_grads = np.zeros(brnn.num_params)
n_epoch = 2000
learn_rate = 1e-2
embed_learn_rate = 1e-3
fudge_factor = 1e-6
for i in xrange(n_epoch):
tot_count = 0
tot_error = 0.0
conf_matrix = np.zeros((2, 2), dtype=np.int32)
tot_grads = np.zeros(brnn.num_params)
logger.debug('Total number of parameters in TBRNN: %d' % brnn.num_params)
for train_indices, train_label in zip(self.senti_train_words_label, self.senti_train_label):
# Dynamically build training instances
train_seq = self.word_embedding._embedding[train_indices, :]
# Compute cost and gradients with respect to parameters and word-embeddings
cost, current_grads = brnn.compute_cost_and_gradient(train_seq, [train_label])
input_grads = brnn.compute_input_gradient(train_seq, [train_label])
# Accumulating errors and gradients
tot_grads += current_grads
tot_error += cost
# historical gradient accumulation
# history_grads += current_grads ** 2
# predict current training label
prediction = brnn.predict(train_seq)[0]
tot_count += prediction == train_label
conf_matrix[train_label, prediction] += 1
# Update word-embedding
for k, j in enumerate(train_indices):
self.word_embedding._embedding[j, :] -= embed_learn_rate * input_grads[k, :]
# Batch updating
tot_grads /= self.train_size
# Update historical gradient vector
# adjusted_grads = tot_grads / (fudge_factor + np.sqrt(history_grads))
# brnn.update_params(adjusted_grads, learn_rate)
brnn.update_params(tot_grads, learn_rate)
# End of the core AdaGrad updating algorithm
accuracy = tot_count / float(self.train_size)
logger.debug('Epoch %d, total cost: %f, overall accuracy: %f' % (i, tot_error, accuracy))
logger.debug('Confusion matrix: ')
logger.debug(conf_matrix)
logger.debug('-' * 50)
if (i+1) % 100 == 0:
logger.debug('=' * 50)
logger.debug('Test at epoch: %d' % i)
# Testing
tot_count = 0
for test_indices, test_label in zip(self.senti_test_words_label, self.senti_test_label):
# Dynamically build test instances
test_seq = self.word_embedding._embedding[test_indices, :]
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
logger.debug('=' * 50)
end_time = time.time()
logger.debug('Time used for training: %f minutes.' % ((end_time-start_time)/60))
# Testing
tot_count = 0
for test_indices, test_label in zip(self.senti_test_set, self.senti_test_label):
# Dynamically build test instances
test_seq = self.word_embedding._embedding[test_indices, :]
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
# Re-testing on training set
tot_count = 0
for train_seq, train_label in zip(self.senti_train_set, self.senti_train_label):
prediction = brnn.predict(train_seq)[0]
tot_count += train_label == prediction
logger.debug('Training accuracy re-testing: %f' % (tot_count / float(self.train_size)))
# Save new word-embedding on sentiment analysis task
WordEmbedding.save('word-embedding-sentiment.pkl', )
# Save TBRNN
TBRNN.save('sentiment.brnn.finetune.Sep5_1.pkl', brnn)
logger.debug('Model successfully saved...')
def testBRNNonSentiment(self):
# Set print precision
np.set_printoptions(threshold=np.nan)
config_filename = './sentiment_brnn.conf'
start_time = time.time()
configer = RNNConfiger(config_filename)
brnn = BRNN.load('./sentiment.nbrnn.Oct.pkl')
# brnn = BRNN(configer, verbose=True)
end_time = time.time()
logger.debug('Time used to build BRNN: %f seconds.' % (end_time-start_time))
# Training
logger.debug('positive labels: %d' % np.sum(self.senti_train_label))
logger.debug('negative labels: %d' % (self.senti_train_label.shape[0]-np.sum(self.senti_train_label)))
start_time = time.time()
## AdaGrad learning algorithm instead of the stochastic gradient descent algorithm
# history_grads = np.zeros(brnn.num_params, dtype=floatX)
n_epoch = 2000
learn_rate = 0.1
fudge_factor = 1e-6
batch_size = 200
for i in xrange(n_epoch):
learn_rate = 0.1 / (1 + i/100)
tot_count = 0
tot_error = 0.0
conf_matrix = np.zeros((2, 2), dtype=np.int32)
tot_grads = np.zeros(brnn.num_params, dtype=floatX)
for j, (train_seq, train_label) in enumerate(zip(self.senti_train_set, self.senti_train_label)):
# Reach a pre-specified batch size, updating the gradients
if (j+1) % batch_size == 0:
tot_grads /= batch_size
brnn.update_params(tot_grads, learn_rate)
tot_grads = np.zeros(brnn.num_params, dtype=floatX)
cost, current_grads = brnn.compute_cost_and_gradient(train_seq, [train_label])
tot_grads += current_grads
tot_error += cost
# historical gradient accumulation
# history_grads += current_grads ** 2
# predict current training label
prediction = brnn.predict(train_seq)[0]
tot_count += prediction == train_label
conf_matrix[train_label, prediction] += 1
# # Batch updating
# tot_grads /= self.train_size
# # Update historical gradient vector
# adjusted_grads = tot_grads / (fudge_factor + np.sqrt(history_grads))
# brnn.update_params(adjusted_grads, learn_rate)
# End of the core AdaGrad updating algorithm
accuracy = tot_count / float(self.train_size)
logger.debug('Epoch %d, total cost: %f, overall accuracy: %f' % (i, tot_error, accuracy))
logger.debug('Confusion matrix: ')
logger.debug(conf_matrix)
logger.debug('-' * 50)
if (i+1) % 10 == 0:
logger.debug('=' * 50)
logger.debug('Test at epoch: %d' % i)
# Testing
tot_count = 0
for test_seq, test_label in zip(self.senti_test_set, self.senti_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
logger.debug('=' * 50)
BRNN.save('sentiment.nbrnn.Oct.pkl', brnn)
end_time = time.time()
logger.debug('Time used for training: %f minutes.' % ((end_time-start_time)/60))
# Testing
tot_count = 0
for test_seq, test_label in zip(self.senti_test_set, self.senti_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
# Re-testing on training set
tot_count = 0
for train_seq, train_label in zip(self.senti_train_set, self.senti_train_label):
prediction = brnn.predict(train_seq)[0]
tot_count += train_label == prediction
logger.debug('Training accuracy re-testing: %f' % (tot_count / float(self.train_size)))
# Save BRNN
BRNN.save('sentiment.nbrnn.Oct.pkl', brnn)
logger.debug('Model successfully saved...')
def testTBRNNonSentiment(self):
# Set print precision
np.set_printoptions(threshold=np.nan)
config_filename = './sentiment_brnn.conf'
start_time = time.time()
configer = RNNConfiger(config_filename)
brnn = TBRNN(configer, verbose=True)
# brnn = TBRNN.load('sentiment.brnn.Sep5.pkl')
end_time = time.time()
logger.debug('Time used to load TBRNN: %f seconds.' % (end_time-start_time))
# Training
logger.debug('positive labels: %d' % np.sum(self.senti_train_label))
logger.debug('negative labels: %d' % (self.senti_train_label.shape[0]-np.sum(self.senti_train_label)))
start_time = time.time()
## AdaGrad learning algorithm instead of the stochastic gradient descent algorithm
history_grads = np.zeros(brnn.num_params)
n_epoch = 2000
learn_rate = 1
fudge_factor = 1e-6
for i in xrange(n_epoch):
tot_count = 0
tot_error = 0.0
conf_matrix = np.zeros((2, 2), dtype=np.int32)
tot_grads = np.zeros(brnn.num_params)
logger.debug('Total number of parameters in TBRNN: %d' % brnn.num_params)
for train_seq, train_label in zip(self.senti_train_set, self.senti_train_label):
# cost = brnn.train(train_seq, [train_label], learn_rate)
cost, current_grads = brnn.compute_cost_and_gradient(train_seq, [train_label])
tot_grads += current_grads
tot_error += cost
# historical gradient accumulation
history_grads += current_grads ** 2
# predict current training label
prediction = brnn.predict(train_seq)[0]
tot_count += prediction == train_label
conf_matrix[train_label, prediction] += 1
# Batch updating
tot_grads /= self.train_size
# Update historical gradient vector
adjusted_grads = tot_grads / (fudge_factor + np.sqrt(history_grads))
brnn.update_params(adjusted_grads, learn_rate)
# End of the core AdaGrad updating algorithm
accuracy = tot_count / float(self.train_size)
logger.debug('Epoch %d, total cost: %f, overall accuracy: %f' % (i, tot_error, accuracy))
logger.debug('Confusion matrix: ')
logger.debug(conf_matrix)
logger.debug('-' * 50)
if (i+1) % 100 == 0:
logger.debug('=' * 50)
logger.debug('Test at epoch: %d' % i)
# Testing
tot_count = 0
for test_seq, test_label in zip(self.senti_test_set, self.senti_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
logger.debug('=' * 50)
end_time = time.time()
logger.debug('Time used for training: %f minutes.' % ((end_time-start_time)/60))
# Testing
tot_count = 0
for test_seq, test_label in zip(self.senti_test_set, self.senti_test_label):
prediction = brnn.predict(test_seq)[0]
tot_count += test_label == prediction
logger.debug('Test accuracy: %f' % (tot_count / float(self.test_size)))
logger.debug('Percentage of positive in Test data: %f' % (np.sum(self.senti_test_label==1) / float(self.test_size)))
logger.debug('Percentage of negative in Test data: %f' % (np.sum(self.senti_test_label==0) / float(self.test_size)))
# Re-testing on training set
tot_count = 0
for train_seq, train_label in zip(self.senti_train_set, self.senti_train_label):
prediction = brnn.predict(train_seq)[0]
tot_count += train_label == prediction
logger.debug('Training accuracy re-testing: %f' % (tot_count / float(self.train_size)))
# Show representation for training inputs and testing inputs
start_time = time.time()
training_forward_rep = np.zeros((self.train_size, configer.num_hidden))
test_forward_rep = np.zeros((self.test_size, configer.num_hidden))
training_backward_rep = np.zeros((self.train_size, configer.num_hidden))
test_backward_rep = np.zeros((self.test_size, configer.num_hidden))
# Save TBRNN
TBRNN.save('sentiment.brnn.Sep5_2.pkl', brnn)
logger.debug('Model successfully saved...')