def LSTM_question_embedding(self, sentence, sentence_length):
#LSTM processes the input question
lstm_params = "question_lstm"
hidden_vectors = []
sentence = self.batch_question
question_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims],
self.data_type)
question_c_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims],
self.data_type)
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
rnn_dropout_mask = tf.cast(
tf.random_uniform(
tf.shape(question_hidden), minval=0.0, maxval=1.0) <
self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
rnn_dropout_mask = tf.ones_like(question_hidden)
for question_iterator in range(self.question_length):
curr_word = sentence[:, question_iterator]
question_vector = nn_utils.apply_dropout(
nn_utils.get_embedding(curr_word, self.utility, self.params),
self.utility.FLAGS.dropout, self.mode)
question_hidden, question_c_hidden = nn_utils.LSTMCell(
question_vector, question_hidden, question_c_hidden,
lstm_params, self.params)
if (self.utility.FLAGS.rnn_dropout > 0.0):
question_hidden = question_hidden * rnn_dropout_mask
hidden_vectors.append(tf.expand_dims(question_hidden, 0))
hidden_vectors = tf.concat(axis=0, values=hidden_vectors)
return question_hidden, hidden_vectors
def LSTM_question_embedding(self, sentence, sentence_length):
#LSTM processes the input question
lstm_params = "question_lstm"
hidden_vectors = []
sentence = self.batch_question
question_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
question_c_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
rnn_dropout_mask = tf.cast(
tf.random_uniform(
tf.shape(question_hidden), minval=0.0, maxval=1.0) <
self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
rnn_dropout_mask = tf.ones_like(question_hidden)
for question_iterator in range(self.question_length):
curr_word = sentence[:, question_iterator]
question_vector = nn_utils.apply_dropout(
nn_utils.get_embedding(curr_word, self.utility, self.params),
self.utility.FLAGS.dropout, self.mode)
question_hidden, question_c_hidden = nn_utils.LSTMCell(
question_vector, question_hidden, question_c_hidden, lstm_params,
self.params)
if (self.utility.FLAGS.rnn_dropout > 0.0):
question_hidden = question_hidden * rnn_dropout_mask
hidden_vectors.append(tf.expand_dims(question_hidden, 0))
hidden_vectors = tf.concat(axis=0, values=hidden_vectors)
return question_hidden, hidden_vectors
def compute_column_softmax(self, column_controller_vector, time_step):
#compute softmax over all the columns using column controller vector
column_controller_vector = tf.tile(
tf.expand_dims(column_controller_vector, 1),
[1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d
column_controller_vector = nn_utils.apply_dropout(
column_controller_vector, self.utility.FLAGS.dropout, self.mode)
self.full_column_hidden_vectors = tf.concat(
axis=1, values=[self.column_hidden_vectors, self.word_column_hidden_vectors])
self.full_column_hidden_vectors += self.summary_text_entry_embeddings
self.full_column_hidden_vectors = nn_utils.apply_dropout(
self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode)
column_logits = tf.reduce_sum(
column_controller_vector * self.full_column_hidden_vectors, 2) + (
self.params["word_match_feature_column_name"] *
self.batch_column_exact_match) + self.full_column_mask
column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols
return column_softmax
def compute_column_softmax(self, column_controller_vector, time_step):
#compute softmax over all the columns using column controller vector
column_controller_vector = tf.tile(
tf.expand_dims(column_controller_vector, 1),
[1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d
column_controller_vector = nn_utils.apply_dropout(
column_controller_vector, self.utility.FLAGS.dropout, self.mode)
self.full_column_hidden_vectors = tf.concat(
[self.column_hidden_vectors, self.word_column_hidden_vectors], 1)
self.full_column_hidden_vectors += self.summary_text_entry_embeddings
self.full_column_hidden_vectors = nn_utils.apply_dropout(
self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode)
column_logits = tf.reduce_sum(
column_controller_vector * self.full_column_hidden_vectors, 2) + (
self.params["word_match_feature_column_name"] *
self.batch_column_exact_match) + self.full_column_mask
column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols
return column_softmax
def one_pass(self, select, question_embedding, hidden_vectors, hprev,
prev_select_1, curr_pass):
#Performs one timestep which involves selecting an operation and a column
attention_vector = self.perform_attention(
hprev, hidden_vectors, self.question_length,
self.batch_question_attention_mask) #batch_size * embedding_dims
controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul(
tf.concat(axis=1,
values=[question_embedding, attention_vector]),
self.params["controller"]))
column_controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["column_controller_prev"]) +
tf.matmul(
tf.concat(axis=1,
values=[question_embedding, attention_vector]),
self.params["column_controller"]))
controller_vector = nn_utils.apply_dropout(controller_vector,
self.utility.FLAGS.dropout,
self.mode)
self.operation_logits = tf.matmul(controller_vector,
tf.transpose(self.params_unit))
softmax = tf.nn.softmax(self.operation_logits)
soft_softmax = softmax
#compute column softmax: bs * max_columns
weighted_op_representation = tf.transpose(
tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax)))
column_controller_vector = tf.nn.relu(
tf.matmul(
tf.concat(axis=1,
values=[
column_controller_vector,
weighted_op_representation
]), self.params["break_conditional"]))
full_column_softmax = self.compute_column_softmax(
column_controller_vector, curr_pass)
soft_column_softmax = full_column_softmax
if (self.mode == "test" or self.mode == "error-test"
or self.mode == "demo"):
self.debug_soft_ops.append(softmax)
self.debug_soft_cols.append(full_column_softmax)
full_column_softmax = self.make_hard_softmax(full_column_softmax)
softmax = self.make_hard_softmax(softmax)
self.debug_ops.append(softmax)
self.debug_cols.append(full_column_softmax)
output, select = self.perform_operations(softmax, full_column_softmax,
select, prev_select_1,
curr_pass)
self.debug_rows.append(select)
return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
def one_pass(self, select, question_embedding, hidden_vectors, hprev,
prev_select_1, curr_pass):
#Performs one timestep which involves selecting an operation and a column
attention_vector = self.perform_attention(
hprev, hidden_vectors, self.question_length,
self.batch_question_attention_mask) #batch_size * embedding_dims
controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul(
tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[
"controller"]))
column_controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["column_controller_prev"]) + tf.matmul(
tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[
"column_controller"]))
controller_vector = nn_utils.apply_dropout(
controller_vector, self.utility.FLAGS.dropout, self.mode)
self.operation_logits = tf.matmul(controller_vector,
tf.transpose(self.params_unit))
softmax = tf.nn.softmax(self.operation_logits)
soft_softmax = softmax
#compute column softmax: bs * max_columns
weighted_op_representation = tf.transpose(
tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax)))
column_controller_vector = tf.nn.relu(
tf.matmul(
tf.concat(axis=1, values=[
column_controller_vector, weighted_op_representation
]), self.params["break_conditional"]))
full_column_softmax = self.compute_column_softmax(column_controller_vector,
curr_pass)
soft_column_softmax = full_column_softmax
if (self.mode == "test"):
full_column_softmax = self.make_hard_softmax(full_column_softmax)
softmax = self.make_hard_softmax(softmax)
output, select = self.perform_operations(softmax, full_column_softmax,
select, prev_select_1, curr_pass)
return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
def batch_process(self):
#Computes loss and fraction of correct examples in a batch.
self.params_unit = nn_utils.apply_dropout(self.params["unit"],
self.utility.FLAGS.dropout,
self.mode)
batch_size = self.batch_size
max_passes = self.max_passes
num_timesteps = 1
max_elements = self.max_elements
select = tf.cast(tf.fill([self.batch_size, max_elements], 1.0),
self.data_type)
hprev = tf.cast(
tf.fill([self.batch_size, self.embedding_dims], 0.0),
self.data_type) #running sum of the hidden states of the model
output = tf.cast(tf.fill([self.batch_size, 1], 0.0),
self.data_type) #output of the model
correct = tf.cast(
tf.fill([1], 0.0), self.data_type
) #to compute accuracy, returns number of correct examples for this batch
total_error = 0.0
prev_select_1 = tf.zeros_like(select)
self.create_summary_embeddings()
self.get_column_hidden_vectors()
#get question embedding
question_embedding, hidden_vectors = self.LSTM_question_embedding(
self.batch_question, self.question_length)
#compute arguments for comparison operation
greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax(
hidden_vectors)
self.init_select_greater = tf.cast(
tf.greater(self.full_processed_column,
tf.expand_dims(greater_question_number,
2)), self.data_type
) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_lesser = tf.cast(
tf.less(self.full_processed_column,
tf.expand_dims(lesser_question_number, 2)), self.data_type
) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_geq = tf.cast(
tf.greater_equal(self.full_processed_column,
tf.expand_dims(geq_question_number,
2)), self.data_type
) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_leq = tf.cast(
tf.less_equal(self.full_processed_column,
tf.expand_dims(leq_question_number,
2)), self.data_type
) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_word_match = 0
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
history_rnn_dropout_mask = tf.cast(
tf.random_uniform(tf.shape(hprev), minval=0.0, maxval=1.0)
< self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
history_rnn_dropout_mask = tf.ones_like(hprev)
select = select * self.select_whole_mask
self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type)
#Perform max_passes and at each pass select operation and column
for curr_pass in range(max_passes):
print "step: ", curr_pass
output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass(
select, question_embedding, hidden_vectors, hprev,
prev_select_1, curr_pass)
prev_select_1 = select
#compute input to history RNN
input_op = tf.transpose(
tf.matmul(tf.transpose(self.params_unit),
tf.transpose(soft_softmax))
) #weighted average of emebdding of operations
input_col = tf.reduce_sum(
tf.expand_dims(soft_column_softmax, 2) *
self.full_column_hidden_vectors, 1)
history_input = tf.concat(axis=1, values=[input_op, input_col])
history_input = nn_utils.apply_dropout(history_input,
self.utility.FLAGS.dropout,
self.mode)
hprev = self.history_recurrent_step(history_input, hprev)
if (self.utility.FLAGS.rnn_dropout > 0.0):
hprev = hprev * history_rnn_dropout_mask
self.scalar_output = output
error = self.error_computation()
cond = tf.less(error, 0.0001, name="cond")
correct_add = tf.where(cond, tf.fill(tf.shape(cond), 1.0),
tf.fill(tf.shape(cond), 0.0))
correct = tf.reduce_sum(correct_add)
error = error / batch_size
total_error = tf.reduce_sum(error)
total_correct = correct / batch_size
return total_error, total_correct
def batch_process(self):
#Computes loss and fraction of correct examples in a batch.
self.params_unit = nn_utils.apply_dropout(
self.params["unit"], self.utility.FLAGS.dropout, self.mode)
batch_size = self.batch_size
max_passes = self.max_passes
num_timesteps = 1
max_elements = self.max_elements
select = tf.cast(
tf.fill([self.batch_size, max_elements], 1.0), self.data_type)
hprev = tf.cast(
tf.fill([self.batch_size, self.embedding_dims], 0.0),
self.data_type) #running sum of the hidden states of the model
output = tf.cast(tf.fill([self.batch_size, 1], 0.0),
self.data_type) #output of the model
correct = tf.cast(
tf.fill([1], 0.0), self.data_type
) #to compute accuracy, returns number of correct examples for this batch
total_error = 0.0
prev_select_1 = tf.zeros_like(select)
self.create_summary_embeddings()
self.get_column_hidden_vectors()
#get question embedding
question_embedding, hidden_vectors = self.LSTM_question_embedding(
self.batch_question, self.question_length)
#compute arguments for comparison operation
greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax(
hidden_vectors)
self.init_select_greater = tf.cast(
tf.greater(self.full_processed_column,
tf.expand_dims(greater_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_lesser = tf.cast(
tf.less(self.full_processed_column,
tf.expand_dims(lesser_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_geq = tf.cast(
tf.greater_equal(self.full_processed_column,
tf.expand_dims(geq_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_leq = tf.cast(
tf.less_equal(self.full_processed_column,
tf.expand_dims(leq_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_word_match = 0
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
history_rnn_dropout_mask = tf.cast(
tf.random_uniform(
tf.shape(hprev), minval=0.0, maxval=1.0) <
self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
history_rnn_dropout_mask = tf.ones_like(hprev)
select = select * self.select_whole_mask
self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type)
#Perform max_passes and at each pass select operation and column
for curr_pass in range(max_passes):
print "step: ", curr_pass
output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass(
select, question_embedding, hidden_vectors, hprev, prev_select_1,
curr_pass)
prev_select_1 = select
#compute input to history RNN
input_op = tf.transpose(
tf.matmul(
tf.transpose(self.params_unit), tf.transpose(
soft_softmax))) #weighted average of emebdding of operations
input_col = tf.reduce_sum(
tf.expand_dims(soft_column_softmax, 2) *
self.full_column_hidden_vectors, 1)
history_input = tf.concat(axis=1, values=[input_op, input_col])
history_input = nn_utils.apply_dropout(
history_input, self.utility.FLAGS.dropout, self.mode)
hprev = self.history_recurrent_step(history_input, hprev)
if (self.utility.FLAGS.rnn_dropout > 0.0):
hprev = hprev * history_rnn_dropout_mask
self.scalar_output = output
error = self.error_computation()
cond = tf.less(error, 0.0001, name="cond")
correct_add = tf.where(
cond, tf.fill(tf.shape(cond), 1.0), tf.fill(tf.shape(cond), 0.0))
correct = tf.reduce_sum(correct_add)
error = error / batch_size
total_error = tf.reduce_sum(error)
total_correct = correct / batch_size
return total_error, total_correct
