def LSTM_question_embedding(self, sentence, sentence_length): #LSTM processes the input question lstm_params = "question_lstm" hidden_vectors = [] sentence = self.batch_question question_hidden = tf.zeros( [self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type) question_c_hidden = tf.zeros( [self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type) if (self.utility.FLAGS.rnn_dropout > 0.0): if (self.mode == "train"): rnn_dropout_mask = tf.cast( tf.random_uniform( tf.shape(question_hidden), minval=0.0, maxval=1.0) < self.utility.FLAGS.rnn_dropout, self.data_type) / self.utility.FLAGS.rnn_dropout else: rnn_dropout_mask = tf.ones_like(question_hidden) for question_iterator in range(self.question_length): curr_word = sentence[:, question_iterator] question_vector = nn_utils.apply_dropout( nn_utils.get_embedding(curr_word, self.utility, self.params), self.utility.FLAGS.dropout, self.mode) question_hidden, question_c_hidden = nn_utils.LSTMCell( question_vector, question_hidden, question_c_hidden, lstm_params, self.params) if (self.utility.FLAGS.rnn_dropout > 0.0): question_hidden = question_hidden * rnn_dropout_mask hidden_vectors.append(tf.expand_dims(question_hidden, 0)) hidden_vectors = tf.concat(axis=0, values=hidden_vectors) return question_hidden, hidden_vectors
def compute_column_softmax(self, column_controller_vector, time_step): #compute softmax over all the columns using column controller vector column_controller_vector = tf.tile( tf.expand_dims(column_controller_vector, 1), [1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d column_controller_vector = nn_utils.apply_dropout( column_controller_vector, self.utility.FLAGS.dropout, self.mode) self.full_column_hidden_vectors = tf.concat( axis=1, values=[self.column_hidden_vectors, self.word_column_hidden_vectors]) self.full_column_hidden_vectors += self.summary_text_entry_embeddings self.full_column_hidden_vectors = nn_utils.apply_dropout( self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode) column_logits = tf.reduce_sum( column_controller_vector * self.full_column_hidden_vectors, 2) + ( self.params["word_match_feature_column_name"] * self.batch_column_exact_match) + self.full_column_mask column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols return column_softmax
def compute_column_softmax(self, column_controller_vector, time_step): #compute softmax over all the columns using column controller vector column_controller_vector = tf.tile( tf.expand_dims(column_controller_vector, 1), [1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d column_controller_vector = nn_utils.apply_dropout( column_controller_vector, self.utility.FLAGS.dropout, self.mode) self.full_column_hidden_vectors = tf.concat( [self.column_hidden_vectors, self.word_column_hidden_vectors], 1) self.full_column_hidden_vectors += self.summary_text_entry_embeddings self.full_column_hidden_vectors = nn_utils.apply_dropout( self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode) column_logits = tf.reduce_sum( column_controller_vector * self.full_column_hidden_vectors, 2) + ( self.params["word_match_feature_column_name"] * self.batch_column_exact_match) + self.full_column_mask column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols return column_softmax
def one_pass(self, select, question_embedding, hidden_vectors, hprev, prev_select_1, curr_pass): #Performs one timestep which involves selecting an operation and a column attention_vector = self.perform_attention( hprev, hidden_vectors, self.question_length, self.batch_question_attention_mask) #batch_size * embedding_dims controller_vector = tf.nn.relu( tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul( tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params["controller"])) column_controller_vector = tf.nn.relu( tf.matmul(hprev, self.params["column_controller_prev"]) + tf.matmul( tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params["column_controller"])) controller_vector = nn_utils.apply_dropout(controller_vector, self.utility.FLAGS.dropout, self.mode) self.operation_logits = tf.matmul(controller_vector, tf.transpose(self.params_unit)) softmax = tf.nn.softmax(self.operation_logits) soft_softmax = softmax #compute column softmax: bs * max_columns weighted_op_representation = tf.transpose( tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax))) column_controller_vector = tf.nn.relu( tf.matmul( tf.concat(axis=1, values=[ column_controller_vector, weighted_op_representation ]), self.params["break_conditional"])) full_column_softmax = self.compute_column_softmax( column_controller_vector, curr_pass) soft_column_softmax = full_column_softmax if (self.mode == "test" or self.mode == "error-test" or self.mode == "demo"): self.debug_soft_ops.append(softmax) self.debug_soft_cols.append(full_column_softmax) full_column_softmax = self.make_hard_softmax(full_column_softmax) softmax = self.make_hard_softmax(softmax) self.debug_ops.append(softmax) self.debug_cols.append(full_column_softmax) output, select = self.perform_operations(softmax, full_column_softmax, select, prev_select_1, curr_pass) self.debug_rows.append(select) return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
def one_pass(self, select, question_embedding, hidden_vectors, hprev, prev_select_1, curr_pass): #Performs one timestep which involves selecting an operation and a column attention_vector = self.perform_attention( hprev, hidden_vectors, self.question_length, self.batch_question_attention_mask) #batch_size * embedding_dims controller_vector = tf.nn.relu( tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul( tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[ "controller"])) column_controller_vector = tf.nn.relu( tf.matmul(hprev, self.params["column_controller_prev"]) + tf.matmul( tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[ "column_controller"])) controller_vector = nn_utils.apply_dropout( controller_vector, self.utility.FLAGS.dropout, self.mode) self.operation_logits = tf.matmul(controller_vector, tf.transpose(self.params_unit)) softmax = tf.nn.softmax(self.operation_logits) soft_softmax = softmax #compute column softmax: bs * max_columns weighted_op_representation = tf.transpose( tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax))) column_controller_vector = tf.nn.relu( tf.matmul( tf.concat(axis=1, values=[ column_controller_vector, weighted_op_representation ]), self.params["break_conditional"])) full_column_softmax = self.compute_column_softmax(column_controller_vector, curr_pass) soft_column_softmax = full_column_softmax if (self.mode == "test"): full_column_softmax = self.make_hard_softmax(full_column_softmax) softmax = self.make_hard_softmax(softmax) output, select = self.perform_operations(softmax, full_column_softmax, select, prev_select_1, curr_pass) return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
def batch_process(self): #Computes loss and fraction of correct examples in a batch. self.params_unit = nn_utils.apply_dropout(self.params["unit"], self.utility.FLAGS.dropout, self.mode) batch_size = self.batch_size max_passes = self.max_passes num_timesteps = 1 max_elements = self.max_elements select = tf.cast(tf.fill([self.batch_size, max_elements], 1.0), self.data_type) hprev = tf.cast( tf.fill([self.batch_size, self.embedding_dims], 0.0), self.data_type) #running sum of the hidden states of the model output = tf.cast(tf.fill([self.batch_size, 1], 0.0), self.data_type) #output of the model correct = tf.cast( tf.fill([1], 0.0), self.data_type ) #to compute accuracy, returns number of correct examples for this batch total_error = 0.0 prev_select_1 = tf.zeros_like(select) self.create_summary_embeddings() self.get_column_hidden_vectors() #get question embedding question_embedding, hidden_vectors = self.LSTM_question_embedding( self.batch_question, self.question_length) #compute arguments for comparison operation greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax( hidden_vectors) self.init_select_greater = tf.cast( tf.greater(self.full_processed_column, tf.expand_dims(greater_question_number, 2)), self.data_type ) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_lesser = tf.cast( tf.less(self.full_processed_column, tf.expand_dims(lesser_question_number, 2)), self.data_type ) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_geq = tf.cast( tf.greater_equal(self.full_processed_column, tf.expand_dims(geq_question_number, 2)), self.data_type ) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_leq = tf.cast( tf.less_equal(self.full_processed_column, tf.expand_dims(leq_question_number, 2)), self.data_type ) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_word_match = 0 if (self.utility.FLAGS.rnn_dropout > 0.0): if (self.mode == "train"): history_rnn_dropout_mask = tf.cast( tf.random_uniform(tf.shape(hprev), minval=0.0, maxval=1.0) < self.utility.FLAGS.rnn_dropout, self.data_type) / self.utility.FLAGS.rnn_dropout else: history_rnn_dropout_mask = tf.ones_like(hprev) select = select * self.select_whole_mask self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type) #Perform max_passes and at each pass select operation and column for curr_pass in range(max_passes): print "step: ", curr_pass output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass( select, question_embedding, hidden_vectors, hprev, prev_select_1, curr_pass) prev_select_1 = select #compute input to history RNN input_op = tf.transpose( tf.matmul(tf.transpose(self.params_unit), tf.transpose(soft_softmax)) ) #weighted average of emebdding of operations input_col = tf.reduce_sum( tf.expand_dims(soft_column_softmax, 2) * self.full_column_hidden_vectors, 1) history_input = tf.concat(axis=1, values=[input_op, input_col]) history_input = nn_utils.apply_dropout(history_input, self.utility.FLAGS.dropout, self.mode) hprev = self.history_recurrent_step(history_input, hprev) if (self.utility.FLAGS.rnn_dropout > 0.0): hprev = hprev * history_rnn_dropout_mask self.scalar_output = output error = self.error_computation() cond = tf.less(error, 0.0001, name="cond") correct_add = tf.where(cond, tf.fill(tf.shape(cond), 1.0), tf.fill(tf.shape(cond), 0.0)) correct = tf.reduce_sum(correct_add) error = error / batch_size total_error = tf.reduce_sum(error) total_correct = correct / batch_size return total_error, total_correct
def batch_process(self): #Computes loss and fraction of correct examples in a batch. self.params_unit = nn_utils.apply_dropout( self.params["unit"], self.utility.FLAGS.dropout, self.mode) batch_size = self.batch_size max_passes = self.max_passes num_timesteps = 1 max_elements = self.max_elements select = tf.cast( tf.fill([self.batch_size, max_elements], 1.0), self.data_type) hprev = tf.cast( tf.fill([self.batch_size, self.embedding_dims], 0.0), self.data_type) #running sum of the hidden states of the model output = tf.cast(tf.fill([self.batch_size, 1], 0.0), self.data_type) #output of the model correct = tf.cast( tf.fill([1], 0.0), self.data_type ) #to compute accuracy, returns number of correct examples for this batch total_error = 0.0 prev_select_1 = tf.zeros_like(select) self.create_summary_embeddings() self.get_column_hidden_vectors() #get question embedding question_embedding, hidden_vectors = self.LSTM_question_embedding( self.batch_question, self.question_length) #compute arguments for comparison operation greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax( hidden_vectors) self.init_select_greater = tf.cast( tf.greater(self.full_processed_column, tf.expand_dims(greater_question_number, 2)), self. data_type) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_lesser = tf.cast( tf.less(self.full_processed_column, tf.expand_dims(lesser_question_number, 2)), self. data_type) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_geq = tf.cast( tf.greater_equal(self.full_processed_column, tf.expand_dims(geq_question_number, 2)), self. data_type) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_leq = tf.cast( tf.less_equal(self.full_processed_column, tf.expand_dims(leq_question_number, 2)), self. data_type) * self.select_bad_number_mask #bs * max_cols * max_elements self.init_select_word_match = 0 if (self.utility.FLAGS.rnn_dropout > 0.0): if (self.mode == "train"): history_rnn_dropout_mask = tf.cast( tf.random_uniform( tf.shape(hprev), minval=0.0, maxval=1.0) < self.utility.FLAGS.rnn_dropout, self.data_type) / self.utility.FLAGS.rnn_dropout else: history_rnn_dropout_mask = tf.ones_like(hprev) select = select * self.select_whole_mask self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type) #Perform max_passes and at each pass select operation and column for curr_pass in range(max_passes): print "step: ", curr_pass output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass( select, question_embedding, hidden_vectors, hprev, prev_select_1, curr_pass) prev_select_1 = select #compute input to history RNN input_op = tf.transpose( tf.matmul( tf.transpose(self.params_unit), tf.transpose( soft_softmax))) #weighted average of emebdding of operations input_col = tf.reduce_sum( tf.expand_dims(soft_column_softmax, 2) * self.full_column_hidden_vectors, 1) history_input = tf.concat(axis=1, values=[input_op, input_col]) history_input = nn_utils.apply_dropout( history_input, self.utility.FLAGS.dropout, self.mode) hprev = self.history_recurrent_step(history_input, hprev) if (self.utility.FLAGS.rnn_dropout > 0.0): hprev = hprev * history_rnn_dropout_mask self.scalar_output = output error = self.error_computation() cond = tf.less(error, 0.0001, name="cond") correct_add = tf.where( cond, tf.fill(tf.shape(cond), 1.0), tf.fill(tf.shape(cond), 0.0)) correct = tf.reduce_sum(correct_add) error = error / batch_size total_error = tf.reduce_sum(error) total_correct = correct / batch_size return total_error, total_correct